Marmelator/layout-parser
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 289 Wiki Activity

Merge branch 'RED-7759' into 'main'

Browse Source 
RED-7759: Upgraded storage-commons to newest windwos compatible version

See merge request fforesight/layout-parser!76
This commit is contained in:
Kilian Schüttler 2023-10-13 12:21:04 +02:00
commit 3c53772765
4 changed files with 25 additions and 143 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
layoutparser-service/layoutparser-service-server/build.gradle.kts
View File

@ -19,8 +19,8 @@ dependencies {
implementation(project(":layoutparser-service-processor")) implementation(project(":layoutparser-service-processor"))
implementation(project(":layoutparser-service-internal-api")) implementation(project(":layoutparser-service-internal-api"))
implementation("com.iqser.red.commons:storage-commons:2.40.0") implementation("com.iqser.red.commons:storage-commons:2.45.0")
implementation("com.knecon.fforesight:tenant-commons:0.10.0") implementation("com.knecon.fforesight:tenant-commons:0.14.0")
implementation("org.springframework.boot:spring-boot-starter-actuator:3.1.3") implementation("org.springframework.boot:spring-boot-starter-actuator:3.1.3")
implementation("com.amazonaws:aws-java-sdk-s3:1.12.536") implementation("com.amazonaws:aws-java-sdk-s3:1.12.536")
implementation("org.springframework.boot:spring-boot-starter-amqp:3.1.3") implementation("org.springframework.boot:spring-boot-starter-amqp:3.1.3")

4
layoutparser-service/layoutparser-service-server/src/test/java/com/knecon/fforesight/service/layoutparser/server/model/TextPositionSequenceTest.java
View File

@ -27,14 +27,14 @@ public class TextPositionSequenceTest {
+ " \"pageWidth\": 600\n" // + " \"pageWidth\": 600\n" //
+ "}"; + "}";
private final ObjectSerializer objectSerializer = new ObjectSerializer(new ObjectMapper(), new StorageProperties()); private final ObjectSerializer objectSerializer = new ObjectSerializer(new ObjectMapper());
@Test @Test
@SneakyThrows @SneakyThrows
public void testDeserializationWithJackson() { public void testDeserializationWithJackson() {
TextPositionSequence textPositionSequence = objectSerializer.deserializeWithJackson(new ByteArrayInputStream(TEXT_POSITION_SEQUENCE_AS_JSON.getBytes(StandardCharsets.UTF_8)), TextPositionSequence textPositionSequence = objectSerializer.deserialize(new ByteArrayInputStream(TEXT_POSITION_SEQUENCE_AS_JSON.getBytes(StandardCharsets.UTF_8)),
TextPositionSequence.class); TextPositionSequence.class);
assertPropertiesAfterJsonDeserialization(textPositionSequence); assertPropertiesAfterJsonDeserialization(textPositionSequence);

36
layoutparser-service/layoutparser-service-server/src/test/java/com/knecon/fforesight/service/layoutparser/server/utils/AbstractTest.java
View File

@ -1,8 +1,15 @@
package com.knecon.fforesight.service.layoutparser.server.utils; package com.knecon.fforesight.service.layoutparser.server.utils;
import java.io.InputStream; import com.iqser.red.commons.jackson.ObjectMapperFactory;
import java.util.Optional; import com.iqser.red.storage.commons.service.StorageService;
import com.iqser.red.storage.commons.utils.FileSystemBackedStorageService;
import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingRequest;
import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingStorageService;
import com.knecon.fforesight.service.layoutparser.server.Application;
import com.knecon.fforesight.tenantcommons.TenantContext;
import com.knecon.fforesight.tenantcommons.TenantsClient;
import lombok.SneakyThrows;
import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.extension.ExtendWith; import org.junit.jupiter.api.extension.ExtendWith;
@ -12,21 +19,12 @@ import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration; import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration;
import org.springframework.boot.test.context.SpringBootTest; import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.boot.test.mock.mockito.MockBean; import org.springframework.boot.test.mock.mockito.MockBean;
import org.springframework.context.annotation.ComponentScan; import org.springframework.context.annotation.*;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.Import;
import org.springframework.core.io.ClassPathResource; import org.springframework.core.io.ClassPathResource;
import org.springframework.test.context.junit.jupiter.SpringExtension; import org.springframework.test.context.junit.jupiter.SpringExtension;
import com.iqser.red.storage.commons.service.StorageService; import java.io.InputStream;
import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingRequest; import java.util.Optional;
import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingStorageService;
import com.knecon.fforesight.service.layoutparser.server.Application;
import com.knecon.fforesight.tenantcommons.TenantContext;
import com.knecon.fforesight.tenantcommons.TenantsClient;
import lombok.SneakyThrows;
@ExtendWith(SpringExtension.class) @ExtendWith(SpringExtension.class)
@SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) @SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
@ -157,6 +155,14 @@ public abstract class AbstractTest {
@ComponentScan("com.knecon.fforesight.service.layoutparser") @ComponentScan("com.knecon.fforesight.service.layoutparser")
public static class TestConfiguration { public static class TestConfiguration {
@Bean
@Primary
public StorageService inmemoryStorage() {
return new FileSystemBackedStorageService(ObjectMapperFactory.create());
}
} }
} }

124
layoutparser-service/layoutparser-service-server/src/test/java/com/knecon/fforesight/service/layoutparser/server/utils/FileSystemBackedStorageService.java
View File

@ -1,124 +0,0 @@
package com.knecon.fforesight.service.layoutparser.server.utils;
import static java.io.File.createTempFile;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import org.apache.commons.io.IOUtils;
import org.springframework.stereotype.Service;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.iqser.red.commons.jackson.ObjectMapperFactory;
import com.iqser.red.storage.commons.exception.StorageObjectDoesNotExist;
import com.iqser.red.storage.commons.service.StorageService;
import lombok.SneakyThrows;
@Service
public class FileSystemBackedStorageService implements StorageService {
private final Map<String, File> dataMap = new HashMap<>();
public FileSystemBackedStorageService() {
}
@SneakyThrows
@Override
public void downloadTo(String tenantId, String objectId, File destinationFile) {
var res = dataMap.get(objectId);
if (res == null) {
throw new StorageObjectDoesNotExist(new RuntimeException());
}
IOUtils.copy(new FileInputStream(res), new FileOutputStream(destinationFile));
}
@Override
public void deleteObject(String tenantId, String objectId) {
dataMap.remove(objectId);
}
@Override
public boolean objectExists(String tenantId, String objectId) {
return dataMap.containsKey(objectId);
}
@Override
@SneakyThrows
public <T> void storeJSONObject(String tenantId, String objectId, T any) {
File tempFile = createTempFile("test", ".tmp");
getMapper().writeValue(new FileOutputStream(tempFile), any);
dataMap.put(objectId, tempFile);
}
private ObjectMapper getMapper() {
return ObjectMapperFactory.create();
}
@Override
@SneakyThrows
public <T> T readJSONObject(String tenantId, String objectId, Class<T> clazz) {
if (dataMap.get(objectId) == null || !dataMap.get(objectId).exists()) {
throw new StorageObjectDoesNotExist("Stored object not found");
}
return getMapper().readValue(new FileInputStream(dataMap.get(objectId)), clazz);
}
public List<String> listPaths() {
return new ArrayList<>(dataMap.keySet());
}
public List<String> listFilePaths() {
return dataMap.values().stream().map(File::getAbsolutePath).collect(Collectors.toList());
}
@Override
@SneakyThrows
public void storeObject(String tenantId, String objectId, InputStream stream) {
File tempFile = createTempFile("test", ".tmp");
try (var fileOutputStream = new FileOutputStream(tempFile)) {
IOUtils.copy(stream, fileOutputStream);
}
dataMap.put(objectId, tempFile);
}
public void clearStorage() {
this.dataMap.forEach((k, v) -> {
v.delete();
});
this.dataMap.clear();
}
}