diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java index a04664e8..5843fefd 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java @@ -1,7 +1,5 @@ package com.iqser.red.service.redaction.v1.server.service; -import static com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryType.IMAGE; - import java.util.ArrayList; import java.util.Collections; import java.util.HashSet; @@ -120,7 +118,9 @@ public class EntityLogCreatorService { Set sectionsToReanalyseIds, DictionaryVersion dictionaryVersion) { - List newEntityLogEntries = createEntityLogEntries(document, analyzeRequest.getDossierTemplateId(), notFoundManualRedactionEntries); + List newEntityLogEntries = createEntityLogEntries(document, analyzeRequest.getDossierTemplateId(), notFoundManualRedactionEntries).stream() + .filter(entry -> entry.getContainingNodeId().isEmpty() || sectionsToReanalyseIds.contains(entry.getContainingNodeId().get(0))) + .collect(Collectors.toList()); Set newEntityIds = newEntityLogEntries.stream().map(EntityLogEntry::getId).collect(Collectors.toSet()); List previousEntriesFromReAnalyzedSections = previousEntityLog.getEntityLogEntry() diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java index 1ee51f15..46c47bf7 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java @@ -1185,6 +1185,44 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest { } + @Test + public void signaturesAreRedactionAfterReanalyse() throws IOException { + + AnalyzeRequest request = uploadFileToStorage("files/new/SYNGENTA_EFSA_sanitisation_GFL_v1 (1).pdf"); + ClassPathResource imageServiceResponseFileResource = new ClassPathResource("files/new/SYNGENTA_EFSA_sanitisation_GFL_v1 (1).IMAGE_INFO.json"); + + storageService.storeObject(TenantContext.getTenantId(), + RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.IMAGE_INFO), + imageServiceResponseFileResource.getInputStream()); + + System.out.println("Start Full integration test"); + analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request); + System.out.println("Finished structure analysis"); + analyzeService.analyze(request); + System.out.println("Finished analysis"); + + request.setManualRedactions(ManualRedactions.builder() + .legalBasisChanges(Set.of(ManualLegalBasisChange.builder() + .annotationId("3029651d0842a625f2d23f8375c23600") + .section("[19, 2]: Paragraph: Contact point: LexCo Contact:") + .value("0049 331 441 551 14") + .requestDate(OffsetDateTime.now()) + .status(AnnotationStatus.APPROVED) + .fileId(TEST_FILE_ID) + .legalBasis("Article 39(e)(2) of Regulation (EC) No 178/2002") + .build())) + .build()); + + analyzeService.reanalyze(request); + System.out.println("Finished reanalysis"); + + var entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID); + entityLog.getEntityLogEntry().stream().filter(entityLogEntry -> entityLogEntry.getType().equals("signature")).forEach(entityLogEntry -> { + assertThat(entityLogEntry.getState() == EntryState.APPLIED).isTrue(); + }); + } + + @Test @SneakyThrows public void testRemovingAndAddingRedactionOnTheSameValue() { @@ -1194,8 +1232,12 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest { String manualAddId = UUID.randomUUID().toString(); String manualAddId2 = UUID.randomUUID().toString(); List positions = List.of(Rectangle.builder().topLeftX(305.35f).topLeftY(332.5033f).width(71.40744f).height(13.645125f).page(1).build()); - ManualRedactionEntry manualRedactionEntry = getManualRedactionEntry(manualAddId, positions, "the manufacturing or production process, including the method and innovative aspects thereof, as well as other technical and industrial specifications inherent to that process or method, except for information which is relevant to the assessment of safety"); - ManualRedactionEntry manualRedactionEntry2 = getManualRedactionEntry(manualAddId2, positions, "commercial information revealing sourcing, market shares or business strategy of the applicant"); + ManualRedactionEntry manualRedactionEntry = getManualRedactionEntry(manualAddId, + positions, + "the manufacturing or production process, including the method and innovative aspects thereof, as well as other technical and industrial specifications inherent to that process or method, except for information which is relevant to the assessment of safety"); + ManualRedactionEntry manualRedactionEntry2 = getManualRedactionEntry(manualAddId2, + positions, + "commercial information revealing sourcing, market shares or business strategy of the applicant"); IdRemoval idRemoval = getIdRemoval(manualAddId); IdRemoval idRemoval2 = getIdRemoval(manualAddId2); @@ -1228,7 +1270,10 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest { assertTrue(entityLog.getEntityLogEntry().stream().anyMatch(entityLogEntry -> entityLogEntry.getId().equals(manualAddId2))); assertEquals(entityLog.getEntityLogEntry().stream().filter(entityLogEntry -> entityLogEntry.getId().equals(manualAddId2)).findFirst().get().getState(), EntryState.APPLIED); - request.setManualRedactions(ManualRedactions.builder().entriesToAdd(Set.of(manualRedactionEntry, manualRedactionEntry2)).idsToRemove(Set.of(idRemoval, idRemoval2)).build()); + request.setManualRedactions(ManualRedactions.builder() + .entriesToAdd(Set.of(manualRedactionEntry, manualRedactionEntry2)) + .idsToRemove(Set.of(idRemoval, idRemoval2)) + .build()); analyzeService.reanalyze(request); entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID); @@ -1239,7 +1284,10 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest { assertEquals(entityLog.getEntityLogEntry().stream().filter(entityLogEntry -> entityLogEntry.getId().equals(manualAddId2)).findFirst().get().getState(), EntryState.REMOVED); manualRedactionEntry.setRequestDate(OffsetDateTime.now()); - request.setManualRedactions(ManualRedactions.builder().entriesToAdd(Set.of(manualRedactionEntry, manualRedactionEntry2)).idsToRemove(Set.of(idRemoval, idRemoval2)).build()); + request.setManualRedactions(ManualRedactions.builder() + .entriesToAdd(Set.of(manualRedactionEntry, manualRedactionEntry2)) + .idsToRemove(Set.of(idRemoval, idRemoval2)) + .build()); analyzeService.reanalyze(request); entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/files/new/SYNGENTA_EFSA_sanitisation_GFL_v1 (1).IMAGE_INFO.json b/redaction-service-v1/redaction-service-server-v1/src/test/resources/files/new/SYNGENTA_EFSA_sanitisation_GFL_v1 (1).IMAGE_INFO.json new file mode 100644 index 00000000..79312d23 --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/files/new/SYNGENTA_EFSA_sanitisation_GFL_v1 (1).IMAGE_INFO.json @@ -0,0 +1,849 @@ +{ + "dossierId": "94ce568c-e788-4c1d-ac6b-c1a139e953b3", + "fileId": "f780bb084ac1c5ad2110db68df6a0ce7", + "targetFileExtension": "ORIGIN.pdf.gz", + "responseFileExtension": "IMAGE_INFO.json.gz", + "X-TENANT-ID": "redaction", + "data": [ + { + "classification": { + "label": "signature", + "probabilities": { + "signature": 1.0, + "formula": 0.0, + "logo": 0.0, + "other": 0.0 + } + }, + "representation": "FFFF4F3E1800E1C77FFDF7FFD", + "position": { + "x1": 61, + "x2": 216, + "y1": 523, + "y2": 583, + "pageNumber": 14 + }, + "geometry": { + "width": 155, + "height": 60 + }, + "alpha": false, + "filters": { + "geometry": { + "imageSize": { + "quotient": 0.1385, + "tooLarge": false, + "tooSmall": false + }, + "imageFormat": { + "quotient": 2.5833, + "tooTall": false, + "tooWide": false + } + }, + "probability": { + "unconfident": false + }, + "allPassed": true + } + }, + { + "classification": { + "label": "signature", + "probabilities": { + "signature": 1.0, + "formula": 0.0, + "logo": 0.0, + "other": 0.0 + } + }, + "representation": "FFFEF144801140802FFFFFFFF", + "position": { + "x1": 64, + "x2": 203, + "y1": 619, + "y2": 680, + "pageNumber": 14 + }, + "geometry": { + "width": 139, + "height": 61 + }, + "alpha": true, + "filters": { + "geometry": { + "imageSize": { + "quotient": 0.1323, + "tooLarge": false, + "tooSmall": false + }, + "imageFormat": { + "quotient": 2.2787, + "tooTall": false, + "tooWide": false + } + }, + "probability": { + "unconfident": false + }, + "allPassed": true + } + }, + { + "classification": { + "label": "logo", + "probabilities": { + "logo": 1.0, + "formula": 0.0, + "other": 0.0, + "signature": 0.0 + } + }, + "representation": "FF300FDF3F78F1E78F1EFCFFF", + "position": { + "x1": 71, + "x2": 268, + "y1": 277, + "y2": 419, + "pageNumber": 14 + }, + "geometry": { + "width": 197, + "height": 142 + }, + "alpha": false, + "filters": { + "geometry": { + "imageSize": { + "quotient": 0.2402, + "tooLarge": false, + "tooSmall": false + }, + "imageFormat": { + "quotient": 1.3873, + "tooTall": false, + "tooWide": false + } + }, + "probability": { + "unconfident": false + }, + "allPassed": true + } + }, + { + "classification": { + "label": "logo", + "probabilities": { + "logo": 0.9624, + "other": 0.0221, + "formula": 0.0116, + "signature": 0.0039 + } + }, + "representation": "00CFE000F1CC033CC033CF0E1", + "position": { + "x1": 75, + "x2": 254, + "y1": 117, + "y2": 249, + "pageNumber": 14 + }, + "geometry": { + "width": 179, + "height": 132 + }, + "alpha": false, + "filters": { + "geometry": { + "imageSize": { + "quotient": 0.2208, + "tooLarge": false, + "tooSmall": false + }, + "imageFormat": { + "quotient": 1.3561, + "tooTall": false, + "tooWide": false + } + }, + "probability": { + "unconfident": false + }, + "allPassed": true + } + }, + { + "classification": { + "label": "logo", + "probabilities": { + "logo": 1.0, + "formula": 0.0, + "other": 0.0, + "signature": 0.0 + } + }, + "representation": "FFFFFFF20C00FBFFFFFBFFFFF", + "position": { + "x1": 284, + "x2": 562, + "y1": 140, + "y2": 223, + "pageNumber": 14 + }, + "geometry": { + "width": 278, + "height": 83 + }, + "alpha": false, + "filters": { + "geometry": { + "imageSize": { + "quotient": 0.2182, + "tooLarge": false, + "tooSmall": false + }, + "imageFormat": { + "quotient": 3.3494, + "tooTall": false, + "tooWide": false + } + }, + "probability": { + "unconfident": false + }, + "allPassed": true + } + }, + { + "classification": { + "label": "signature", + "probabilities": { + "signature": 1.0, + "formula": 0.0, + "logo": 0.0, + "other": 0.0 + } + }, + "representation": "FFFFDF7FFFC060C1A7CE3BFFF", + "position": { + "x1": 287, + "x2": 486, + "y1": 613, + "y2": 681, + "pageNumber": 14 + }, + "geometry": { + "width": 199, + "height": 68 + }, + "alpha": true, + "filters": { + "geometry": { + "imageSize": { + "quotient": 0.1671, + "tooLarge": false, + "tooSmall": false + }, + "imageFormat": { + "quotient": 2.9265, + "tooTall": false, + "tooWide": false + } + }, + "probability": { + "unconfident": false + }, + "allPassed": true + } + }, + { + "classification": { + "label": "signature", + "probabilities": { + "signature": 0.9984, + "logo": 0.0016, + "formula": 0.0, + "other": 0.0 + } + }, + "representation": "FFFFBFFEF81078E1874FCFBFF", + "position": { + "x1": 296, + "x2": 462, + "y1": 499, + "y2": 594, + "pageNumber": 14 + }, + "geometry": { + "width": 166, + "height": 95 + }, + "alpha": false, + "filters": { + "geometry": { + "imageSize": { + "quotient": 0.1804, + "tooLarge": false, + "tooSmall": false + }, + "imageFormat": { + "quotient": 1.7474, + "tooTall": false, + "tooWide": false + } + }, + "probability": { + "unconfident": false + }, + "allPassed": true + } + }, + { + "classification": { + "label": "logo", + "probabilities": { + "logo": 1.0, + "formula": 0.0, + "other": 0.0, + "signature": 0.0 + } + }, + "representation": "10C00FFF0C30F0C33FCCB5FFF", + "position": { + "x1": 358, + "x2": 538, + "y1": 274, + "y2": 401, + "pageNumber": 14 + }, + "geometry": { + "width": 180, + "height": 127 + }, + "alpha": false, + "filters": { + "geometry": { + "imageSize": { + "quotient": 0.2172, + "tooLarge": false, + "tooSmall": false + }, + "imageFormat": { + "quotient": 1.4173, + "tooTall": false, + "tooWide": false + } + }, + "probability": { + "unconfident": false + }, + "allPassed": true + } + }, + { + "classification": { + "label": "formula", + "probabilities": { + "formula": 1.0, + "logo": 0.0, + "other": 0.0, + "signature": 0.0 + } + }, + "representation": "FFF3FFDF0C3AF0C35E4FFFFFF", + "position": { + "x1": 58, + "x2": 212, + "y1": 295, + "y2": 448, + "pageNumber": 15 + }, + "geometry": { + "width": 154, + "height": 153 + }, + "alpha": false, + "filters": { + "geometry": { + "imageSize": { + "quotient": 0.2205, + "tooLarge": false, + "tooSmall": false + }, + "imageFormat": { + "quotient": 1.0065, + "tooTall": false, + "tooWide": false + } + }, + "probability": { + "unconfident": false + }, + "allPassed": true + } + }, + { + "classification": { + "label": "formula", + "probabilities": { + "formula": 0.3855, + "other": 0.3396, + "logo": 0.1674, + "signature": 0.1074 + } + }, + "representation": "FA30F080F0C031CE17E8CF237", + "position": { + "x1": 61, + "x2": 210, + "y1": 538, + "y2": 687, + "pageNumber": 15 + }, + "geometry": { + "width": 149, + "height": 149 + }, + "alpha": false, + "filters": { + "geometry": { + "imageSize": { + "quotient": 0.214, + "tooLarge": false, + "tooSmall": false + }, + "imageFormat": { + "quotient": 1.0, + "tooTall": false, + "tooWide": false + } + }, + "probability": { + "unconfident": true + }, + "allPassed": false + } + }, + { + "classification": { + "label": "other", + "probabilities": { + "other": 0.958, + "formula": 0.0272, + "signature": 0.0137, + "logo": 0.001 + } + }, + "representation": "FFCF08C04100000E0CF7FFF3F", + "position": { + "x1": 219, + "x2": 396, + "y1": 537, + "y2": 687, + "pageNumber": 15 + }, + "geometry": { + "width": 177, + "height": 150 + }, + "alpha": false, + "filters": { + "geometry": { + "imageSize": { + "quotient": 0.234, + "tooLarge": false, + "tooSmall": false + }, + "imageFormat": { + "quotient": 1.18, + "tooTall": false, + "tooWide": false + } + }, + "probability": { + "unconfident": false + }, + "allPassed": true + } + }, + { + "classification": { + "label": "formula", + "probabilities": { + "formula": 1.0, + "logo": 0.0, + "other": 0.0, + "signature": 0.0 + } + }, + "representation": "7EFBD3064FBD7084604DF4E33", + "position": { + "x1": 240, + "x2": 384, + "y1": 298, + "y2": 413, + "pageNumber": 15 + }, + "geometry": { + "width": 144, + "height": 115 + }, + "alpha": false, + "filters": { + "geometry": { + "imageSize": { + "quotient": 0.1848, + "tooLarge": false, + "tooSmall": false + }, + "imageFormat": { + "quotient": 1.2522, + "tooTall": false, + "tooWide": false + } + }, + "probability": { + "unconfident": false + }, + "allPassed": true + } + }, + { + "classification": { + "label": "formula", + "probabilities": { + "formula": 0.7103, + "other": 0.1487, + "logo": 0.1336, + "signature": 0.0074 + } + }, + "representation": "FEF9CFAFBE10400DA3BE42FBE", + "position": { + "x1": 399, + "x2": 579, + "y1": 300, + "y2": 422, + "pageNumber": 15 + }, + "geometry": { + "width": 180, + "height": 122 + }, + "alpha": false, + "filters": { + "geometry": { + "imageSize": { + "quotient": 0.2129, + "tooLarge": false, + "tooSmall": false + }, + "imageFormat": { + "quotient": 1.4754, + "tooTall": false, + "tooWide": false + } + }, + "probability": { + "unconfident": false + }, + "allPassed": true + } + }, + { + "classification": { + "label": "other", + "probabilities": { + "other": 0.4883, + "signature": 0.2465, + "formula": 0.1399, + "logo": 0.1253 + } + }, + "representation": "02080770F3CC810403FDC9B7C", + "position": { + "x1": 406, + "x2": 576, + "y1": 522, + "y2": 686, + "pageNumber": 15 + }, + "geometry": { + "width": 170, + "height": 164 + }, + "alpha": false, + "filters": { + "geometry": { + "imageSize": { + "quotient": 0.2398, + "tooLarge": false, + "tooSmall": false + }, + "imageFormat": { + "quotient": 1.0366, + "tooTall": false, + "tooWide": false + } + }, + "probability": { + "unconfident": true + }, + "allPassed": false + } + }, + { + "classification": { + "label": "other", + "probabilities": { + "other": 1.0, + "formula": 0.0, + "logo": 0.0, + "signature": 0.0 + } + }, + "representation": "400C3E39776E9A706954EF1", + "position": { + "x1": 56, + "x2": 252, + "y1": 534, + "y2": 658, + "pageNumber": 16 + }, + "geometry": { + "width": 196, + "height": 124 + }, + "alpha": false, + "filters": { + "geometry": { + "imageSize": { + "quotient": 0.2239, + "tooLarge": false, + "tooSmall": false + }, + "imageFormat": { + "quotient": 1.5806, + "tooTall": false, + "tooWide": false + } + }, + "probability": { + "unconfident": false + }, + "allPassed": true + } + }, + { + "classification": { + "label": "other", + "probabilities": { + "other": 0.9799, + "formula": 0.0162, + "signature": 0.0039, + "logo": 0.0 + } + }, + "representation": "83CE891C0C78B1E4831EBCF3F", + "position": { + "x1": 60, + "x2": 300, + "y1": 105, + "y2": 325, + "pageNumber": 16 + }, + "geometry": { + "width": 240, + "height": 220 + }, + "alpha": false, + "filters": { + "geometry": { + "imageSize": { + "quotient": 0.33, + "tooLarge": false, + "tooSmall": false + }, + "imageFormat": { + "quotient": 1.0909, + "tooTall": false, + "tooWide": false + } + }, + "probability": { + "unconfident": false + }, + "allPassed": true + } + }, + { + "classification": { + "label": "other", + "probabilities": { + "other": 0.9961, + "logo": 0.002, + "formula": 0.001, + "signature": 0.0009 + } + }, + "representation": "FF3000040031C02FBFFEFFF0C", + "position": { + "x1": 126, + "x2": 326, + "y1": 397, + "y2": 509, + "pageNumber": 16 + }, + "geometry": { + "width": 200, + "height": 112 + }, + "alpha": false, + "filters": { + "geometry": { + "imageSize": { + "quotient": 0.215, + "tooLarge": false, + "tooSmall": false + }, + "imageFormat": { + "quotient": 1.7857, + "tooTall": false, + "tooWide": false + } + }, + "probability": { + "unconfident": false + }, + "allPassed": true + } + }, + { + "classification": { + "label": "other", + "probabilities": { + "other": 1.0, + "formula": 0.0, + "logo": 0.0, + "signature": 0.0 + } + }, + "representation": "8C14742030E4CF1F6EBBFFDC3", + "position": { + "x1": 313, + "x2": 549, + "y1": 144, + "y2": 302, + "pageNumber": 16 + }, + "geometry": { + "width": 236, + "height": 158 + }, + "alpha": false, + "filters": { + "geometry": { + "imageSize": { + "quotient": 0.2774, + "tooLarge": false, + "tooSmall": false + }, + "imageFormat": { + "quotient": 1.4937, + "tooTall": false, + "tooWide": false + } + }, + "probability": { + "unconfident": false + }, + "allPassed": true + } + }, + { + "classification": { + "label": "other", + "probabilities": { + "other": 1.0, + "formula": 0.0, + "logo": 0.0, + "signature": 0.0 + } + }, + "representation": "F7404EB6010C067090E0FFDF7", + "position": { + "x1": 342, + "x2": 509, + "y1": 524, + "y2": 662, + "pageNumber": 16 + }, + "geometry": { + "width": 167, + "height": 138 + }, + "alpha": false, + "filters": { + "geometry": { + "imageSize": { + "quotient": 0.2181, + "tooLarge": false, + "tooSmall": false + }, + "imageFormat": { + "quotient": 1.2101, + "tooTall": false, + "tooWide": false + } + }, + "probability": { + "unconfident": false + }, + "allPassed": true + } + }, + { + "classification": { + "label": "other", + "probabilities": { + "other": 0.9704, + "formula": 0.0269, + "signature": 0.0021, + "logo": 0.0007 + } + }, + "representation": "1F72F14C033C30F0F3AF0FFCF", + "position": { + "x1": 376, + "x2": 546, + "y1": 577, + "y2": 679, + "pageNumber": 19 + }, + "geometry": { + "width": 170, + "height": 102 + }, + "alpha": false, + "filters": { + "geometry": { + "imageSize": { + "quotient": 0.1891, + "tooLarge": false, + "tooSmall": false + }, + "imageFormat": { + "quotient": 1.6667, + "tooTall": false, + "tooWide": false + } + }, + "probability": { + "unconfident": false + }, + "allPassed": true + } + } + ] +} \ No newline at end of file diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/files/new/SYNGENTA_EFSA_sanitisation_GFL_v1 (1).pdf b/redaction-service-v1/redaction-service-server-v1/src/test/resources/files/new/SYNGENTA_EFSA_sanitisation_GFL_v1 (1).pdf new file mode 100644 index 00000000..c5d5a2fa Binary files /dev/null and b/redaction-service-v1/redaction-service-server-v1/src/test/resources/files/new/SYNGENTA_EFSA_sanitisation_GFL_v1 (1).pdf differ