Marmelator/redaction-service
1
0
Fork 0
Code Issues Pull Requests 14 Actions Packages Projects Releases 622 Wiki Activity

RED-7759: Cannot resize local manual redactions

Browse Source
This commit is contained in:
Yannik Hampe 2023-10-18 14:53:33 +02:00
parent e4b1824dd4
commit 96d29b9e3a
14 changed files with 83 additions and 147 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
redaction-service-v1/redaction-service-server-v1/build.gradle.kts
View File

@ -12,7 +12,7 @@ plugins {
description = "redaction-service-server-v1"
val layoutParserVersion = "0.71.0"
val layoutParserVersion = "0.72.0"
val jacksonVersion = "2.15.2"
val droolsVersion = "9.44.0.Final"
val pdfBoxVersion = "3.0.0"
@ -35,7 +35,7 @@ dependencies {
implementation("com.iqser.red.commons:dictionary-merge-commons:1.5.0")
implementation("com.iqser.red.commons:storage-commons:2.45.0")
implementation("com.knecon.fforesight:tenant-commons:0.13.0")
implementation("com.knecon.fforesight:tenant-commons:0.14.0")
implementation("com.fasterxml.jackson.module:jackson-module-afterburner:${jacksonVersion}")
implementation("com.fasterxml.jackson.datatype:jackson-datatype-jsr310:${jacksonVersion}")

2
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/ManualChangesApplicationService.java
View File

@ -92,7 +92,7 @@ public class ManualChangesApplicationService {
private static Rectangle2D toRectangle2D(com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle rect) {
return new Rectangle2D.Double(rect.getTopLeftX() - rect.getWidth(), rect.getTopLeftY() - rect.getHeight(), rect.getWidth(), rect.getHeight());
return new Rectangle2D.Double(rect.getTopLeftX(), rect.getTopLeftY(), rect.getWidth(), rect.getHeight());
}
}

1
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AbstractRedactionIntegrationTest.java
View File

@ -17,6 +17,7 @@ import com.iqser.red.service.redaction.v1.server.utils.TextNormalizationUtilitie
import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
import com.iqser.red.service.redaction.v1.server.utils.LayoutParsingRequestProvider;
import com.iqser.red.storage.commons.service.StorageService;
import com.iqser.red.storage.commons.utils.FileSystemBackedStorageService;
import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingFinishedEvent;
import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingPipeline;

4
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DictionaryServiceTest.java
View File

@ -28,6 +28,7 @@ import org.springframework.context.annotation.Primary;
import org.springframework.test.context.junit.jupiter.SpringExtension;
import com.google.common.collect.Sets;
import com.iqser.red.commons.jackson.ObjectMapperFactory;
import com.iqser.red.service.dictionarymerge.commons.DictionaryEntry;
import com.iqser.red.service.dictionarymerge.commons.DictionaryEntryModel;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.configuration.Colors;
@ -36,6 +37,7 @@ import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryVersion;
import com.iqser.red.service.redaction.v1.server.service.DictionaryService;
import com.iqser.red.storage.commons.service.StorageService;
import com.iqser.red.storage.commons.utils.FileSystemBackedStorageService;
import com.knecon.fforesight.tenantcommons.TenantContext;
import com.knecon.fforesight.tenantcommons.TenantsClient;
@ -67,7 +69,7 @@ public class DictionaryServiceTest {
@Primary
public StorageService inmemoryStorage() {
return new FileSystemBackedStorageService();
return new FileSystemBackedStorageService(ObjectMapperFactory.create());
}
}

4
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DocumineFloraTest.java
View File

@ -21,6 +21,7 @@ import org.springframework.context.annotation.Import;
import org.springframework.context.annotation.Primary;
import org.springframework.test.context.junit.jupiter.SpringExtension;
import com.iqser.red.commons.jackson.ObjectMapperFactory;
import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeResult;
import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType;
@ -31,6 +32,7 @@ import com.iqser.red.service.redaction.v1.server.annotate.AnnotateResponse;
import com.iqser.red.service.redaction.v1.server.redaction.utils.OsUtils;
import com.iqser.red.storage.commons.StorageAutoConfiguration;
import com.iqser.red.storage.commons.service.StorageService;
import com.iqser.red.storage.commons.utils.FileSystemBackedStorageService;
import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingServiceProcessorConfiguration;
import com.knecon.fforesight.tenantcommons.TenantContext;
@ -134,7 +136,7 @@ public class DocumineFloraTest extends AbstractRedactionIntegrationTest {
@Primary
public StorageService inmemoryStorage() {
return new FileSystemBackedStorageService();
return new FileSystemBackedStorageService(ObjectMapperFactory.create());
}
}

128
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/FileSystemBackedStorageService.java
View File

@ -1,128 +0,0 @@
package com.iqser.red.service.redaction.v1.server;
import static java.io.File.createTempFile;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import org.apache.commons.io.IOUtils;
import org.springframework.core.io.InputStreamResource;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.iqser.red.commons.jackson.ObjectMapperFactory;
import com.iqser.red.storage.commons.exception.StorageObjectDoesNotExist;
import com.iqser.red.storage.commons.service.StorageService;
import lombok.SneakyThrows;
public class FileSystemBackedStorageService implements StorageService {
private final Map<String, File> dataMap = new HashMap<>();
public FileSystemBackedStorageService() {
}
@SneakyThrows
@Override
public void downloadTo(String tenantId, String objectId, File destinationFile) {
var res = dataMap.get(objectId);
if (res == null) {
throw new StorageObjectDoesNotExist(new RuntimeException());
}
IOUtils.copy(new FileInputStream(res), new FileOutputStream(destinationFile));
}
@SneakyThrows
public InputStreamResource getObject(String tenantId, String objectId) {
return new InputStreamResource(new FileInputStream(dataMap.get(objectId)));
}
@Override
public void deleteObject(String tenantId, String objectId) {
dataMap.remove(objectId);
}
@Override
public boolean objectExists(String tenantId, String objectId) {
return dataMap.containsKey(objectId);
}
@Override
@SneakyThrows
public <T> void storeJSONObject(String tenantId, String objectId, T any) {
File tempFile = createTempFile("test", ".tmp");
getMapper().writeValue(new FileOutputStream(tempFile), any);
dataMap.put(objectId, tempFile);
}
private ObjectMapper getMapper() {
return ObjectMapperFactory.create();
}
@Override
@SneakyThrows
public <T> T readJSONObject(String tenantId, String objectId, Class<T> clazz) {
if (dataMap.get(objectId) == null || !dataMap.get(objectId).exists()) {
throw new StorageObjectDoesNotExist("Stored object not found");
}
return getMapper().readValue(new FileInputStream(dataMap.get(objectId)), clazz);
}
public List<String> listPaths() {
return new ArrayList<>(dataMap.keySet());
}
public List<String> listFilePaths() {
return dataMap.values().stream().map(File::getAbsolutePath).collect(Collectors.toList());
}
@Override
@SneakyThrows
public void storeObject(String tenantId, String objectId, InputStream stream) {
File tempFile = createTempFile("test", ".tmp");
try (var fileOutputStream = new FileOutputStream(tempFile)) {
IOUtils.copy(stream, fileOutputStream);
}
dataMap.put(objectId, tempFile);
}
public void clearStorage() {
this.dataMap.forEach((k, v) -> {
v.delete();
});
this.dataMap.clear();
}
}

4
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionAcceptanceTest.java
View File

@ -24,6 +24,7 @@ import org.springframework.context.annotation.Import;
import org.springframework.context.annotation.Primary;
import org.springframework.test.context.junit.jupiter.SpringExtension;
import com.iqser.red.commons.jackson.ObjectMapperFactory;
import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeResult;
import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType;
@ -41,6 +42,7 @@ import com.iqser.red.service.redaction.v1.server.annotate.AnnotateResponse;
import com.iqser.red.service.redaction.v1.server.redaction.utils.OsUtils;
import com.iqser.red.storage.commons.StorageAutoConfiguration;
import com.iqser.red.storage.commons.service.StorageService;
import com.iqser.red.storage.commons.utils.FileSystemBackedStorageService;
import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingServiceProcessorConfiguration;
import com.knecon.fforesight.tenantcommons.TenantContext;
@ -62,7 +64,7 @@ public class RedactionAcceptanceTest extends AbstractRedactionIntegrationTest {
@Primary
public StorageService inmemoryStorage() {
return new FileSystemBackedStorageService();
return new FileSystemBackedStorageService(ObjectMapperFactory.create());
}
}

55
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java
View File

@ -41,6 +41,7 @@ import org.springframework.core.io.ClassPathResource;
import org.springframework.test.context.junit.jupiter.SpringExtension;
import com.fasterxml.jackson.core.type.TypeReference;
import com.iqser.red.commons.jackson.ObjectMapperFactory;
import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeResult;
import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;
@ -72,6 +73,7 @@ import com.iqser.red.service.redaction.v1.server.service.document.DocumentGraphM
import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
import com.iqser.red.storage.commons.StorageAutoConfiguration;
import com.iqser.red.storage.commons.service.StorageService;
import com.iqser.red.storage.commons.utils.FileSystemBackedStorageService;
import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingServiceProcessorConfiguration;
import com.knecon.fforesight.tenantcommons.TenantContext;
@ -95,7 +97,7 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
@Primary
public StorageService inmemoryStorage() {
return new FileSystemBackedStorageService();
return new FileSystemBackedStorageService(ObjectMapperFactory.create());
}
}
@ -194,6 +196,57 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
}
@Test
public void manualResizeTest() throws IOException {
// c6be5277f5ee60dc3d83527798b7fe02
// Rectangle(topLeft=Point(x=236.8, y=182.90005), width=70.89604, height=12.642, page=7)
// {
// "page": 7,
// "topLeft": {
// "x": 200.8,
// "y": 332.364
// },
// "height": 15.408000000000015,
// "width": 40.584
// }
AnalyzeRequest request = uploadFileToStorage("files/new/crafted document.pdf");
ManualRedactions manualRedactions = ManualRedactions.builder()
.resizeRedactions(Set.of(ManualResizeRedaction.builder()
.annotationId("c6be5277f5ee60dc3d83527798b7fe02")
.value("Dr. Alan")
.positions(List.of(new Rectangle(236.8f,182.90005f,40.584f,12.642f, 7 )))
.status(AnnotationStatus.APPROVED)
.requestDate(OffsetDateTime.now())
.build()
))
.build();
request.setManualRedactions(manualRedactions);
System.out.println("Start Full integration test");
analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request);
System.out.println("Finished structure analysis");
AnalyzeResult result = analyzeService.analyze(request);
System.out.println("Finished analysis");
var redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID);
AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder().dossierId(TEST_DOSSIER_ID).fileId(TEST_FILE_ID).build());
String outputFileName = OsUtils.getTemporaryDirectory() + "/Annotated.pdf";
try (FileOutputStream fileOutputStream = new FileOutputStream(outputFileName)) {
fileOutputStream.write(annotateResponse.getDocument());
}
}
@Test
public void redactionExpansionOverlap() {

4
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationV2Test.java
View File

@ -20,6 +20,7 @@ import org.springframework.context.annotation.Import;
import org.springframework.context.annotation.Primary;
import org.springframework.test.context.junit.jupiter.SpringExtension;
import com.iqser.red.commons.jackson.ObjectMapperFactory;
import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine;
@ -29,6 +30,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSON
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type;
import com.iqser.red.storage.commons.StorageAutoConfiguration;
import com.iqser.red.storage.commons.service.StorageService;
import com.iqser.red.storage.commons.utils.FileSystemBackedStorageService;
import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingServiceProcessorConfiguration;
import com.knecon.fforesight.tenantcommons.TenantContext;
@ -53,7 +55,7 @@ public class RedactionIntegrationV2Test extends AbstractRedactionIntegrationTest
@Primary
public StorageService inmemoryStorage() {
return new FileSystemBackedStorageService();
return new FileSystemBackedStorageService(ObjectMapperFactory.create());
}
}

4
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RulesTest.java
View File

@ -57,6 +57,7 @@ import org.springframework.test.context.junit.jupiter.SpringExtension;
import com.fasterxml.jackson.databind.DeserializationFeature;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
import com.iqser.red.commons.jackson.ObjectMapperFactory;
import com.iqser.red.service.dictionarymerge.commons.DictionaryEntry;
import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType;
@ -85,6 +86,7 @@ import com.iqser.red.service.redaction.v1.server.utils.ResourceLoader;
import com.iqser.red.service.redaction.v1.server.utils.TextNormalizationUtilities;
import com.iqser.red.storage.commons.StorageAutoConfiguration;
import com.iqser.red.storage.commons.service.StorageService;
import com.iqser.red.storage.commons.utils.FileSystemBackedStorageService;
import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingFinishedEvent;
import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingPipeline;
@ -837,7 +839,7 @@ public class RulesTest {
@Primary
public StorageService inmemoryStorage() {
return new FileSystemBackedStorageService();
return new FileSystemBackedStorageService(ObjectMapperFactory.create());
}
}

5
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/BuildDocumentIntegrationTest.java
View File

@ -12,14 +12,15 @@ import org.springframework.context.annotation.Import;
import org.springframework.context.annotation.Primary;
import org.springframework.test.context.junit.jupiter.SpringExtension;
import com.iqser.red.commons.jackson.ObjectMapperFactory;
import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
import com.iqser.red.service.redaction.v1.server.AbstractRedactionIntegrationTest;
import com.iqser.red.service.redaction.v1.server.Application;
import com.iqser.red.service.redaction.v1.server.FileSystemBackedStorageService;
import com.iqser.red.service.redaction.v1.server.service.document.DocumentGraphMapper;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
import com.iqser.red.storage.commons.StorageAutoConfiguration;
import com.iqser.red.storage.commons.service.StorageService;
import com.iqser.red.storage.commons.utils.FileSystemBackedStorageService;
import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingServiceProcessorConfiguration;
@ -40,7 +41,7 @@ public class BuildDocumentIntegrationTest extends AbstractRedactionIntegrationTe
@Primary
public StorageService inmemoryStorage() {
return new FileSystemBackedStorageService();
return new FileSystemBackedStorageService(ObjectMapperFactory.create());
}
}

5
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesEnd2EndTest.java
View File

@ -31,6 +31,7 @@ import org.springframework.context.annotation.Import;
import org.springframework.context.annotation.Primary;
import org.springframework.test.context.junit.jupiter.SpringExtension;
import com.iqser.red.commons.jackson.ObjectMapperFactory;
import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeResult;
import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType;
@ -51,7 +52,6 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSON
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type;
import com.iqser.red.service.redaction.v1.server.AbstractRedactionIntegrationTest;
import com.iqser.red.service.redaction.v1.server.Application;
import com.iqser.red.service.redaction.v1.server.FileSystemBackedStorageService;
import com.iqser.red.service.redaction.v1.server.annotate.AnnotateRequest;
import com.iqser.red.service.redaction.v1.server.annotate.AnnotateResponse;
import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
@ -63,6 +63,7 @@ import com.iqser.red.service.redaction.v1.server.service.document.EntityCreation
import com.iqser.red.service.redaction.v1.server.service.document.EntityEnrichmentService;
import com.iqser.red.storage.commons.StorageAutoConfiguration;
import com.iqser.red.storage.commons.service.StorageService;
import com.iqser.red.storage.commons.utils.FileSystemBackedStorageService;
import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingServiceProcessorConfiguration;
import com.knecon.fforesight.tenantcommons.TenantContext;
@ -91,7 +92,7 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest {
@Primary
public StorageService inmemoryStorage() {
return new FileSystemBackedStorageService();
return new FileSystemBackedStorageService(ObjectMapperFactory.create());
}
}

2
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesIntegrationTest.java
View File

@ -287,7 +287,7 @@ public class ManualChangesIntegrationTest extends BuildDocumentIntegrationTest {
private static Rectangle toAnnotationRectangle(Rectangle2D rectangle2D, Integer number) {
return new Rectangle((float) rectangle2D.getMaxX(), (float) rectangle2D.getMaxY(), (float) rectangle2D.getWidth(), (float) rectangle2D.getHeight(), number);
return new Rectangle((float) rectangle2D.getMinX(), (float) rectangle2D.getMinY(), (float) rectangle2D.getWidth(), (float) rectangle2D.getHeight(), number);
}
}

8
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/realdata/LiveDataIntegrationTest.java
View File

@ -37,13 +37,13 @@ import org.springframework.test.context.junit.jupiter.SpringExtension;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.iqser.red.commons.jackson.ObjectMapperFactory;
import com.iqser.red.service.dictionarymerge.commons.DictionaryEntry;
import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.configuration.Colors;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type;
import com.iqser.red.service.redaction.v1.server.Application;
import com.iqser.red.service.redaction.v1.server.FileSystemBackedStorageService;
import com.iqser.red.service.redaction.v1.server.RedactionServiceSettings;
import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
import com.iqser.red.service.redaction.v1.server.client.FileStatusProcessingUpdateClient;
@ -55,6 +55,7 @@ import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService
import com.iqser.red.service.redaction.v1.server.utils.ExceptionProvider;
import com.iqser.red.storage.commons.StorageAutoConfiguration;
import com.iqser.red.storage.commons.service.StorageService;
import com.iqser.red.storage.commons.utils.FileSystemBackedStorageService;
import com.knecon.fforesight.tenantcommons.TenantContext;
import com.knecon.fforesight.tenantcommons.TenantsClient;
@ -105,9 +106,6 @@ public class LiveDataIntegrationTest {
@Autowired
protected RedactionMessageReceiver redactionMessageReceiver;
@Autowired
protected FileSystemBackedStorageService fileSystemBackedStorageService;
@Autowired
protected RedactionServiceSettings redactionServiceSettings;
@ -125,7 +123,7 @@ public class LiveDataIntegrationTest {
@Primary
public StorageService inmemoryStorage() {
return new FileSystemBackedStorageService();
return new FileSystemBackedStorageService(ObjectMapperFactory.create());
}
}