diff --git a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/AnalyzeRequest.java b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/AnalyzeRequest.java index 40635d5d..4f5c009b 100644 --- a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/AnalyzeRequest.java +++ b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/AnalyzeRequest.java @@ -1,6 +1,8 @@ package com.iqser.red.service.redaction.v1.model; +import com.iqser.red.service.persistence.service.v1.api.model.annotations.ImportedAnnotation; import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions; + import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; @@ -12,7 +14,6 @@ import java.util.HashSet; import java.util.List; import java.util.Set; - @Data @Builder @NoArgsConstructor @@ -36,5 +37,8 @@ public class AnalyzeRequest { @Builder.Default private List fileAttributes = new ArrayList<>(); + @Builder.Default + List importedAnnotations = new ArrayList<>(); + } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/AnalyzeService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/AnalyzeService.java index 05b642f8..fef1d67f 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/AnalyzeService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/AnalyzeService.java @@ -48,6 +48,7 @@ public class AnalyzeService { private final SectionGridCreatorService sectionGridCreatorService; private final NerAnalyserService nerAnalyserService; private final ImageService imageService; + private final ImportedRedactionService importedRedactionService; public void analyzeDocumentStructure(StructureAnalyzeRequest analyzeRequest) { @@ -121,6 +122,7 @@ public class AnalyzeService { redactionLogEntries, legalBasis, dictionary.getVersion().getDossierTemplateVersion(), dictionary.getVersion().getDossierVersion(), rulesVersion, legalBasisClient.getVersion(analyzeRequest.getDossierTemplateId())); + importedRedactionService.processImportedRedactions(redactionLog, analyzeRequest.getImportedAnnotations(), analyzeRequest.getDossierTemplateId()); return finalizeAnalysis(analyzeRequest, startTime, redactionLog, text, dictionary.getVersion(), false); } @@ -170,6 +172,7 @@ public class AnalyzeService { redactionLog.getRedactionLogEntry().removeIf(entry -> sectionsToReanalyse.contains(entry.getSectionNumber())); redactionLog.getRedactionLogEntry().addAll(newRedactionLogEntries); + importedRedactionService.processImportedRedactions(redactionLog, analyzeRequest.getImportedAnnotations(), analyzeRequest.getDossierTemplateId()); return finalizeAnalysis(analyzeRequest, startTime, redactionLog, text, dictionaryIncrement.getDictionaryVersion(), true); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ImportedRedactionService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ImportedRedactionService.java new file mode 100644 index 00000000..7280d0e5 --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ImportedRedactionService.java @@ -0,0 +1,98 @@ +package com.iqser.red.service.redaction.v1.server.redaction.service; + +import java.util.Iterator; +import java.util.List; +import java.util.stream.Collectors; + +import org.springframework.stereotype.Service; + +import com.iqser.red.service.persistence.service.v1.api.model.annotations.ImportedAnnotation; +import com.iqser.red.service.persistence.service.v1.api.model.annotations.ImportedAnnotationStatus; +import com.iqser.red.service.redaction.v1.model.Point; +import com.iqser.red.service.redaction.v1.model.Rectangle; +import com.iqser.red.service.redaction.v1.model.RedactionLog; +import com.iqser.red.service.redaction.v1.model.RedactionLogEntry; + +import lombok.RequiredArgsConstructor; + +@Service +@RequiredArgsConstructor +public class ImportedRedactionService { + + private final DictionaryService dictionaryService; + + public void processImportedRedactions(RedactionLog redactionLog, List importedAnnotations, String dossierTemplateId) { + + Iterator itty = redactionLog.getRedactionLogEntry().iterator(); + while (itty.hasNext()) { + RedactionLogEntry redactionLogEntry = itty.next(); + boolean mustBeRemoved = false; + + if (redactionLogEntry.getType().equals("imported_redaction")) { + itty.remove(); + continue; + } + + loop: + for (ImportedAnnotation importedAnnotation : importedAnnotations) { + for (com.iqser.red.service.persistence.service.v1.api.model.annotations.Rectangle importedPosition : importedAnnotation.getPositions()) { + for (Rectangle rectangle : redactionLogEntry.getPositions()) { + if (intersects(importedPosition, rectangle)) { + mustBeRemoved = true; + break loop; + } + } + } + } + if (mustBeRemoved) { + itty.remove(); + } + } + + for (ImportedAnnotation importedAnnotation : importedAnnotations) { + RedactionLogEntry redactionLogEntry = RedactionLogEntry.builder() + .id(importedAnnotation.getAnnotationId()) + .type("imported_redaction") + .redacted(importedAnnotation.getStatus().equals(ImportedAnnotationStatus.DECLINED) ? false : true) + .positions(importedAnnotation.getPositions() + .stream() + .map(this::convert) + .collect(Collectors.toList())) + .color(getColor("imported_redaction", dossierTemplateId, importedAnnotation.getStatus().equals(ImportedAnnotationStatus.DECLINED) ? false : true)) + .build(); + + redactionLog.getRedactionLogEntry().add(redactionLogEntry); + } + } + + + public Rectangle convert( + com.iqser.red.service.persistence.service.v1.api.model.annotations.Rectangle importedPosition) { + + return new Rectangle(new Point(importedPosition.getTopLeftX(), importedPosition.getTopLeftY()), importedPosition.getWidth(), importedPosition.getHeight(), importedPosition.getPage()); + } + + + public boolean intersects( + com.iqser.red.service.persistence.service.v1.api.model.annotations.Rectangle importedPosition, + Rectangle redactionLogPosition) { + + return importedPosition.getPage() == redactionLogPosition.getPage() && redactionLogPosition.getTopLeft() + .getX() + redactionLogPosition.getWidth() > importedPosition.getTopLeftX() && redactionLogPosition.getTopLeft() + .getY() + redactionLogPosition.getHeight() > importedPosition.getTopLeftY() && redactionLogPosition.getTopLeft() + .getX() < importedPosition.getTopLeftX() + importedPosition.getWidth() && redactionLogPosition.getTopLeft() + .getY() < importedPosition.getTopLeftY() + importedPosition.getHeight(); + } + + + private float[] getColor(String type, String dossierTemplateId, boolean isRedaction) { + + if (!isRedaction) { + return dictionaryService.getNotRedactedColor(dossierTemplateId); + } + return dictionaryService.getColor(type, dossierTemplateId); + } + + + +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java index 16e357d5..916d444c 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java @@ -23,6 +23,8 @@ import java.util.Set; import java.util.UUID; import java.util.stream.Collectors; +import com.fasterxml.jackson.core.type.TypeReference; +import com.iqser.red.service.persistence.service.v1.api.model.annotations.ImportedAnnotation; import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.*; import org.apache.commons.io.IOUtils; import org.junit.After; @@ -108,6 +110,7 @@ public class RedactionIntegrationTest { private static final String FORMULA = "formula"; private static final String OCR = "ocr"; private static final String DOSSIER_REDACTIONS = "dossier_redactions"; + private static final String IMPORTED_REDACTION = "imported_redaction"; private static final String RECOMMENDATION_AUTHOR = "recommendation_CBI_author"; private static final String RECOMMENDATION_ADDRESS = "recommendation_CBI_address"; @@ -248,6 +251,7 @@ public class RedactionIntegrationTest { when(dictionaryClient.getDictionaryForType(SIGNATURE + ":" + TEST_DOSSIER_TEMPLATE_ID)).thenReturn(getDictionaryResponse(SIGNATURE, false)); when(dictionaryClient.getDictionaryForType(FORMULA + ":" + TEST_DOSSIER_TEMPLATE_ID)).thenReturn(getDictionaryResponse(FORMULA, false)); when(dictionaryClient.getDictionaryForType(DOSSIER_REDACTIONS + ":" + TEST_DOSSIER_TEMPLATE_ID)).thenReturn(getDictionaryResponse(DOSSIER_REDACTIONS, true)); + when(dictionaryClient.getDictionaryForType(IMPORTED_REDACTION + ":" + TEST_DOSSIER_TEMPLATE_ID)).thenReturn(getDictionaryResponse(IMPORTED_REDACTION, true)); when(dictionaryClient.getColors(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(colors); } @@ -359,6 +363,7 @@ public class RedactionIntegrationTest { .stream() .map(this::cleanDictionaryEntry) .collect(Collectors.toSet())); + dossierDictionary.put(IMPORTED_REDACTION, new ArrayList<>()); } @@ -390,6 +395,7 @@ public class RedactionIntegrationTest { typeColorMap.put(LOGO, "#ffe187"); typeColorMap.put(FORMULA, "#ffe187"); typeColorMap.put(SIGNATURE, "#ffe187"); + typeColorMap.put(IMPORTED_REDACTION, "#32a852"); hintTypeMap.put(VERTEBRATE, true); hintTypeMap.put(ADDRESS, false); @@ -412,6 +418,7 @@ public class RedactionIntegrationTest { hintTypeMap.put(LOGO, false); hintTypeMap.put(SIGNATURE, false); hintTypeMap.put(DOSSIER_REDACTIONS, false); + hintTypeMap.put(IMPORTED_REDACTION, false); caseInSensitiveMap.put(VERTEBRATE, true); caseInSensitiveMap.put(ADDRESS, false); @@ -434,6 +441,7 @@ public class RedactionIntegrationTest { caseInSensitiveMap.put(LOGO, true); caseInSensitiveMap.put(FORMULA, true); caseInSensitiveMap.put(DOSSIER_REDACTIONS, false); + caseInSensitiveMap.put(IMPORTED_REDACTION, false); recommendationTypeMap.put(VERTEBRATE, false); recommendationTypeMap.put(ADDRESS, false); @@ -456,6 +464,7 @@ public class RedactionIntegrationTest { recommendationTypeMap.put(SIGNATURE, false); recommendationTypeMap.put(LOGO, false); recommendationTypeMap.put(DOSSIER_REDACTIONS, false); + recommendationTypeMap.put(IMPORTED_REDACTION, false); rankTypeMap.put(FALSE_POSITIVE, 160); rankTypeMap.put(PURITY, 155); @@ -478,6 +487,7 @@ public class RedactionIntegrationTest { rankTypeMap.put(SIGNATURE, 27); rankTypeMap.put(FORMULA, 26); rankTypeMap.put(DOSSIER_REDACTIONS, 200); + rankTypeMap.put(IMPORTED_REDACTION, 200); colors.setDefaultColor("#acfc00"); colors.setNotRedacted("#cccccc"); @@ -495,7 +505,7 @@ public class RedactionIntegrationTest { .type(typeColor.getKey()) .dossierTemplateId(TEST_DOSSIER_TEMPLATE_ID) .hexColor(typeColor.getValue()) - .isHint(hintTypeMap.get(typeColor.getKey())) + .isHint(hintTypeMap.get(typeColor.getKey())) .isCaseInsensitive(caseInSensitiveMap.get(typeColor.getKey())) .isRecommendation(recommendationTypeMap.get(typeColor.getKey())) .rank(rankTypeMap.get(typeColor.getKey())) @@ -841,6 +851,7 @@ public class RedactionIntegrationTest { System.out.println("Output file:" + outputFileName); } + @Test public void testChangeComputation() throws IOException { @@ -856,24 +867,27 @@ public class RedactionIntegrationTest { dictionary.get(AUTHOR).add("report"); reanlysisVersions.put("report", 2L); when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(2L); - when(dictionaryClient.getDictionaryForType(AUTHOR +":" + TEST_DOSSIER_TEMPLATE_ID)).thenReturn(getDictionaryResponse(AUTHOR, false)); + when(dictionaryClient.getDictionaryForType(AUTHOR + ":" + TEST_DOSSIER_TEMPLATE_ID)).thenReturn(getDictionaryResponse(AUTHOR, false)); analyzeService.reanalyze(request); dictionary.get(AUTHOR).add("assessment report"); reanlysisVersions.put("assessment report", 3L); when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(3L); - when(dictionaryClient.getDictionaryForType(AUTHOR +":" + TEST_DOSSIER_TEMPLATE_ID)).thenReturn(getDictionaryResponse(AUTHOR, false)); + when(dictionaryClient.getDictionaryForType(AUTHOR + ":" + TEST_DOSSIER_TEMPLATE_ID)).thenReturn(getDictionaryResponse(AUTHOR, false)); analyzeService.reanalyze(request); when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(3L); - when(dictionaryClient.getDictionaryForType(AUTHOR +":" + TEST_DOSSIER_TEMPLATE_ID)).thenReturn(getDictionaryResponse(AUTHOR, false)); + when(dictionaryClient.getDictionaryForType(AUTHOR + ":" + TEST_DOSSIER_TEMPLATE_ID)).thenReturn(getDictionaryResponse(AUTHOR, false)); analyzeService.reanalyze(request); - var redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID); + var redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID); - var changes = redactionLog.getRedactionLogEntry().stream().filter(entry -> - entry.getValue() != null && entry.getValue().equals("report")) - .findFirst().get().getChanges(); + var changes = redactionLog.getRedactionLogEntry() + .stream() + .filter(entry -> entry.getValue() != null && entry.getValue().equals("report")) + .findFirst() + .get() + .getChanges(); assertThat(changes.size()).isEqualTo(2); @@ -887,6 +901,7 @@ public class RedactionIntegrationTest { } } + @Test public void redactionTest() throws IOException { @@ -1556,4 +1571,31 @@ public class RedactionIntegrationTest { } + + @Test + public void testImportedRedactions() throws IOException { + + String outputFileName = OsUtils.getTemporaryDirectory() + "/Annotated.pdf"; + ClassPathResource pdfFileResource = new ClassPathResource("files/ImportedRedactions/S13.pdf"); + ClassPathResource importedRedactions = new ClassPathResource("files/ImportedRedactions/S13.json"); + List importedAnnotations = objectMapper.readValue(importedRedactions.getInputStream().readAllBytes(), new TypeReference>() { + }); + AnalyzeRequest request = prepareStorage(pdfFileResource.getInputStream()); + request.setImportedAnnotations(importedAnnotations); + + analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId())); + AnalyzeResult result = analyzeService.analyze(request); + + var redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID); + + AnnotateResponse annotateResponse = redactionController.annotate(AnnotateRequest.builder() + .dossierId(TEST_DOSSIER_ID) + .fileId(TEST_FILE_ID) + .build()); + + try (FileOutputStream fileOutputStream = new FileOutputStream(outputFileName)) { + fileOutputStream.write(annotateResponse.getDocument()); + } + } + } diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/files/ImportedRedactions/S13.json b/redaction-service-v1/redaction-service-server-v1/src/test/resources/files/ImportedRedactions/S13.json new file mode 100644 index 00000000..7341de2f --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/files/ImportedRedactions/S13.json @@ -0,0 +1,121 @@ +[ + { + "annotationId": "6113d300d4a5cd7e0e91ef692e046fc5", + "fileId": "536bfa2d5ef9aecedda09e300a37cd81", + "positions": [ + { + "topLeftX": 147.86, + "topLeftY": 519.11, + "width": 122.496, + "height": 11.018, + "page": 1 + } + ], + "status": "NEW", + "user": null, + "comment": null, + "processedDate": null + }, + { + "annotationId": "04d043c73922f6e0dc50a1c7484e9425", + "fileId": "536bfa2d5ef9aecedda09e300a37cd81", + "positions": [ + { + "topLeftX": 147.86, + "topLeftY": 544.43, + "width": 83.414, + "height": 11.018, + "page": 1 + } + ], + "status": "NEW", + "user": null, + "comment": null, + "processedDate": null + }, + { + "annotationId": "83f5f2cbe55f2acbbf7adc479cf694cb", + "fileId": "536bfa2d5ef9aecedda09e300a37cd81", + "positions": [ + { + "topLeftX": 147.86, + "topLeftY": 557.03, + "width": 57.624, + "height": 11.018, + "page": 1 + } + ], + "status": "NEW", + "user": null, + "comment": null, + "processedDate": null + }, + { + "annotationId": "4bacdb3675c46181c75295fe8c6e6562", + "fileId": "536bfa2d5ef9aecedda09e300a37cd81", + "positions": [ + { + "topLeftX": 147.86, + "topLeftY": 531.83, + "width": 83.414, + "height": 11.018, + "page": 1 + } + ], + "status": "NEW", + "user": null, + "comment": null, + "processedDate": null + }, + { + "annotationId": "8db925e788f2ab1c9ded097a9de10111", + "fileId": "536bfa2d5ef9aecedda09e300a37cd81", + "positions": [ + { + "topLeftX": 148.94, + "topLeftY": 317.49, + "width": 141.263, + "height": 11.018, + "page": 1 + } + ], + "status": "NEW", + "user": null, + "comment": null, + "processedDate": null + }, + { + "annotationId": "a4d2f29216efe7507bf97d2e55010dd4", + "fileId": "536bfa2d5ef9aecedda09e300a37cd81", + "positions": [ + { + "topLeftX": 148.94, + "topLeftY": 342.81, + "width": 94.389, + "height": 11.018, + "page": 1 + } + ], + "status": "NEW", + "user": null, + "comment": null, + "processedDate": null + }, + { + "annotationId": "9983b5000e8f3540a402178f91b3a2e2", + "fileId": "536bfa2d5ef9aecedda09e300a37cd81", + "positions": [ + { + "topLeftX": 148.94, + "topLeftY": 330.21, + "width": 94.332, + "height": 11.018, + "page": 1 + } + ], + "status": "APPROVED", + "user": "its a me", + "comment": "First Test", + "processedDate": "2022-01-27T12:57:25.48Z" + } +] \ No newline at end of file diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/files/ImportedRedactions/S13.pdf b/redaction-service-v1/redaction-service-server-v1/src/test/resources/files/ImportedRedactions/S13.pdf new file mode 100644 index 00000000..4e3d2969 Binary files /dev/null and b/redaction-service-v1/redaction-service-server-v1/src/test/resources/files/ImportedRedactions/S13.pdf differ