diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionService.java index 070b6837..523137ed 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionService.java @@ -82,9 +82,10 @@ public class EntityRedactionService { var idsToRemove = analyzeRequest.getManualRedactions().getIdsToRemove().stream() .filter(idr -> idr.getStatus() == AnnotationStatus.APPROVED && !idr.isRemoveFromDictionary()) .filter(idr -> idr.getRequestDate() != null) - .filter(idr -> approvedForceRedactions.stream().noneMatch(forceRedact -> forceRedact.getRequestDate().isAfter(idr.getRequestDate()))) + .filter(idr -> approvedForceRedactions.stream().noneMatch(forceRedact -> forceRedact.getAnnotationId().equals(idr.getAnnotationId()) && forceRedact.getRequestDate().isAfter(idr.getRequestDate()))) .map(IdRemoval::getAnnotationId).collect(Collectors.toSet()); + if (reanalysisSection.getImages() != null && !reanalysisSection.getImages() .isEmpty() && analyzeRequest.getManualRedactions().getImageRecategorization() != null) { for (Image image : reanalysisSection.getImages()) { diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java index 2403181f..2e74a990 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java @@ -1,6 +1,7 @@ package com.iqser.red.service.redaction.v1.server; import com.amazonaws.services.s3.AmazonS3; +import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus; import com.iqser.red.service.persistence.service.v1.api.model.annotations.Comment; @@ -58,6 +59,7 @@ import java.io.*; import java.net.URL; import java.nio.charset.StandardCharsets; import java.time.OffsetDateTime; +import java.time.ZoneOffset; import java.util.*; import java.util.stream.Collectors; @@ -715,6 +717,72 @@ public class RedactionIntegrationTest { } + @Test + public void testRemovePublishedInformations() throws IOException { + + long start = System.currentTimeMillis(); + + ClassPathResource colorsResource = new ClassPathResource("colors/colors.json"); + var colors = objectMapper.readValue(colorsResource.getInputStream(), Colors.class); + + ClassPathResource typeResource = new ClassPathResource("colors/types.json"); + TypeReference> typeRefForTypes = new TypeReference<>(){}; + List types = objectMapper.readValue(typeResource.getInputStream(), typeRefForTypes); + + + AnalyzeRequest request = prepareStorage("files/new/PublishedInformationTest.pdf"); + analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId())); + ManualRedactions manualRedactions = new ManualRedactions(); + manualRedactions.getIdsToRemove().add(IdRemoval.builder() + .annotationId("308dab9015bfafd911568cffe0a7f7de") + .fileId(TEST_FILE_ID) + .status(AnnotationStatus.APPROVED) + .requestDate(OffsetDateTime.of(2022,05,23,8,30,07,475479, ZoneOffset.UTC)) + .processedDate(OffsetDateTime.of(2022,05,23,8,30,07,483651, ZoneOffset.UTC)) + .build()); + + manualRedactions.getForceRedactions().add(ManualForceRedaction.builder() + .annotationId("0b56ea1a87c83f351df177315af94f0d") + .fileId(TEST_FILE_ID) + .status(AnnotationStatus.APPROVED) + .requestDate(OffsetDateTime.of(2022,05,23,9,30,15,4653, ZoneOffset.UTC)) + .processedDate(OffsetDateTime.of(2022,05,23,9,30,15,794, ZoneOffset.UTC)) + .build()); + + manualRedactions.getIdsToRemove().add(IdRemoval.builder() + .annotationId("0b56ea1a87c83f351df177315af94f0d") + .fileId(TEST_FILE_ID) + .status(AnnotationStatus.APPROVED) + .requestDate(OffsetDateTime.of(2022,05,23,8,30,23,961721, ZoneOffset.UTC)) + .processedDate(OffsetDateTime.of(2022,05,23,8,30,23,96528, ZoneOffset.UTC)) + .build()); + + + + request.setManualRedactions(manualRedactions); + + + AnalyzeResult result = analyzeService.analyze(request); + + + AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder() + .manualRedactions(manualRedactions) + .colors(colors) + .types(types) + .dossierId(TEST_DOSSIER_ID) + .fileId(TEST_FILE_ID) + .build()); + + try (FileOutputStream fileOutputStream = new FileOutputStream(OsUtils.getTemporaryDirectory() + "/Annotated.pdf")) { + fileOutputStream.write(annotateResponse.getDocument()); + } + long end = System.currentTimeMillis(); + + System.out.println("duration: " + (end - start)); + System.out.println("numberOfPages: " + result.getNumberOfPages()); + } + + @Test public void testTableRedaction() throws IOException { diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/annotate/AnnotateRequest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/annotate/AnnotateRequest.java index 9b9dd2b5..910fade9 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/annotate/AnnotateRequest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/annotate/AnnotateRequest.java @@ -1,6 +1,10 @@ package com.iqser.red.service.redaction.v1.server.annotate; +import java.util.List; + import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.configuration.Colors; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.Type; import lombok.AllArgsConstructor; import lombok.Builder; @@ -17,4 +21,6 @@ public class AnnotateRequest { private String dossierTemplateId; private String fileId; private ManualRedactions manualRedactions; + private Colors colors; + private List types; } diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/annotate/AnnotationService.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/annotate/AnnotationService.java index 987da9d5..087f545e 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/annotate/AnnotationService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/annotate/AnnotationService.java @@ -46,6 +46,8 @@ public class AnnotationService { .manualRedactions(annotateRequest.getManualRedactions()) .dossierId(annotateRequest.getDossierId()) .dossierTemplateId(annotateRequest.getDossierTemplateId()) + .colors(annotateRequest.getColors()) + .types(annotateRequest.getTypes()) .build()); var sectionsGrid = redactionStorageService.getSectionGrid(annotateRequest.getDossierId(), annotateRequest.getFileId()); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/colors/colors.json b/redaction-service-v1/redaction-service-server-v1/src/test/resources/colors/colors.json new file mode 100644 index 00000000..d51ddc52 --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/colors/colors.json @@ -0,0 +1,13 @@ +{ + "dossierTemplateId": "31039447-9040-4376-9ca7-614e56b284b9", + "defaultColor": "#9398a0", + "requestAdd": "#04b093", + "requestRemove": "#04b093", + "notRedacted": "#c498fa", + "analysisColor": "#dd4d50", + "updatedColor": "#fdbd00", + "dictionaryRequestColor": "#5b97db", + "manualRedactionColor": "#9398a0", + "previewColor": "#9398a0", + "ignoredHintColor": "#e7d4ff" +} \ No newline at end of file diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/colors/types.json b/redaction-service-v1/redaction-service-server-v1/src/test/resources/colors/types.json new file mode 100644 index 00000000..1a3685c2 --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/colors/types.json @@ -0,0 +1,223 @@ +[ + { + "id": "CBI_address:31039447-9040-4376-9ca7-614e56b284b9", + "type": "CBI_address", + "hexColor": "#9398a0", + "recommendationHexColor": "#8df06c", + "rank": 140, + "isHint": false, + "dossierTemplateId": "31039447-9040-4376-9ca7-614e56b284b9", + "isCaseInsensitive": false, + "isRecommendation": false, + "description": "All site names and addresses, and location (e.g. Syngenta, Monthey, GPS Co-ordinates, Mr Smith of … providing the…). Except addresses in published literature and the applicant address.", + "addToDictionaryAction": true, + "label": "CBI Address", + "hasDictionary": true, + "systemManaged": false, + "autoHideSkipped": false + }, + { + "id": "CBI_author:31039447-9040-4376-9ca7-614e56b284b9", + "type": "CBI_author", + "hexColor": "#9398a0", + "recommendationHexColor": "#8df06c", + "rank": 130, + "isHint": false, + "dossierTemplateId": "31039447-9040-4376-9ca7-614e56b284b9", + "isCaseInsensitive": false, + "isRecommendation": false, + "description": "All authors named in the study documentation. Except names in published literature.", + "addToDictionaryAction": true, + "label": "CBI Author", + "hasDictionary": true, + "systemManaged": false, + "autoHideSkipped": false + }, + { + "id": "PII:31039447-9040-4376-9ca7-614e56b284b9", + "type": "PII", + "hexColor": "#9398a0", + "recommendationHexColor": "#8df06c", + "rank": 150, + "isHint": false, + "dossierTemplateId": "31039447-9040-4376-9ca7-614e56b284b9", + "isCaseInsensitive": false, + "isRecommendation": false, + "description": "Not authors but listed in the document: Names, signatures, telephone, email etc.; e.g. Reg Manager, QA Manager", + "addToDictionaryAction": true, + "label": "PII", + "hasDictionary": true, + "systemManaged": false, + "autoHideSkipped": false + }, + { + "id": "formula:31039447-9040-4376-9ca7-614e56b284b9", + "type": "formula", + "hexColor": "#036ffc", + "recommendationHexColor": "#8df06c", + "rank": 1002, + "isHint": true, + "dossierTemplateId": "31039447-9040-4376-9ca7-614e56b284b9", + "isCaseInsensitive": true, + "isRecommendation": false, + "description": "Empty dictionary used to configure formula colors.", + "addToDictionaryAction": false, + "label": "Formula", + "hasDictionary": false, + "systemManaged": true, + "autoHideSkipped": false + }, + { + "id": "isHint_only:31039447-9040-4376-9ca7-614e56b284b9", + "type": "isHint_only", + "hexColor": "#fa98f7", + "recommendationHexColor": "#8df06c", + "rank": 50, + "isHint": true, + "dossierTemplateId": "31039447-9040-4376-9ca7-614e56b284b9", + "isCaseInsensitive": true, + "isRecommendation": false, + "description": "Entries of this dictionary will be highlighted only", + "addToDictionaryAction": false, + "label": "isHint Only", + "hasDictionary": true, + "systemManaged": false, + "autoHideSkipped": false + }, + { + "id": "image:31039447-9040-4376-9ca7-614e56b284b9", + "type": "image", + "hexColor": "#bdd6ff", + "recommendationHexColor": "#8df06c", + "rank": 999, + "isHint": true, + "dossierTemplateId": "31039447-9040-4376-9ca7-614e56b284b9", + "isCaseInsensitive": true, + "isRecommendation": false, + "description": "Empty dictionary used to configure image colors.", + "addToDictionaryAction": false, + "label": "Image", + "hasDictionary": false, + "systemManaged": true, + "autoHideSkipped": false + }, + { + "id": "logo:31039447-9040-4376-9ca7-614e56b284b9", + "type": "logo", + "hexColor": "#9398a0", + "recommendationHexColor": "#8df06c", + "rank": 1001, + "isHint": false, + "dossierTemplateId": "31039447-9040-4376-9ca7-614e56b284b9", + "isCaseInsensitive": true, + "isRecommendation": false, + "description": "Empty dictionary used to configure logo colors.", + "addToDictionaryAction": false, + "label": "Logo", + "hasDictionary": false, + "systemManaged": true, + "autoHideSkipped": false + }, + { + "id": "must_redact:31039447-9040-4376-9ca7-614e56b284b9", + "type": "must_redact", + "hexColor": "#9398a0", + "recommendationHexColor": "#8df06c", + "rank": 100, + "isHint": false, + "dossierTemplateId": "31039447-9040-4376-9ca7-614e56b284b9", + "isCaseInsensitive": true, + "isRecommendation": false, + "description": "Entries of this dictionary get redacted wherever found.", + "addToDictionaryAction": false, + "label": "Must Redact", + "hasDictionary": true, + "systemManaged": false, + "autoHideSkipped": false + }, + { + "id": "ocr:31039447-9040-4376-9ca7-614e56b284b9", + "type": "ocr", + "hexColor": "#bdd6ff", + "recommendationHexColor": "#8df06c", + "rank": 1000, + "isHint": true, + "dossierTemplateId": "31039447-9040-4376-9ca7-614e56b284b9", + "isCaseInsensitive": true, + "isRecommendation": false, + "description": "Empty dictionary used to configure ocr colors.", + "addToDictionaryAction": false, + "label": "Ocr", + "hasDictionary": false, + "systemManaged": true, + "autoHideSkipped": false + }, + { + "id": "signature:31039447-9040-4376-9ca7-614e56b284b9", + "type": "signature", + "hexColor": "#9398a0", + "recommendationHexColor": "#8df06c", + "rank": 1003, + "isHint": false, + "dossierTemplateId": "31039447-9040-4376-9ca7-614e56b284b9", + "isCaseInsensitive": true, + "isRecommendation": false, + "description": "Empty dictionary used to configure signature colors.", + "addToDictionaryAction": false, + "label": "Signature", + "hasDictionary": false, + "systemManaged": true, + "autoHideSkipped": false + }, + { + "id": "imported_redaction:31039447-9040-4376-9ca7-614e56b284b9", + "type": "imported_redaction", + "hexColor": "#f0f0c0", + "recommendationHexColor": "#8df06c", + "rank": 9999, + "isHint": false, + "dossierTemplateId": "31039447-9040-4376-9ca7-614e56b284b9", + "isCaseInsensitive": false, + "isRecommendation": false, + "description": "Redaction Annotations that were imported from documents", + "addToDictionaryAction": false, + "label": "Imported Redaction", + "hasDictionary": false, + "systemManaged": true, + "autoHideSkipped": true + }, + { + "id": "published_information:31039447-9040-4376-9ca7-614e56b284b9", + "type": "published_information", + "hexColor": "#85ebff", + "recommendationHexColor": "#8df06c", + "rank": 70, + "isHint": true, + "dossierTemplateId": "31039447-9040-4376-9ca7-614e56b284b9", + "isCaseInsensitive": false, + "isRecommendation": false, + "description": "Manual managed list of public journals and papers that need no redaction", + "addToDictionaryAction": true, + "label": "Published Information", + "hasDictionary": true, + "systemManaged": false, + "autoHideSkipped": false + }, + { + "id": "dossier_redaction:31039447-9040-4376-9ca7-614e56b284b9:5dfb2724-74a4-4a1a-a1eb-165e7943ffcd", + "type": "dossier_redaction", + "hexColor": "#9398a0", + "recommendationHexColor": null, + "rank": 1500, + "isHint": false, + "dossierTemplateId": "31039447-9040-4376-9ca7-614e56b284b9", + "isCaseInsensitive": false, + "isRecommendation": false, + "description": "Entries in this dictionary will only be redacted in this dossier", + "addToDictionaryAction": false, + "label": "Dossier Redaction", + "hasDictionary": true, + "systemManaged": true, + "autoHideSkipped": false + } +] \ No newline at end of file diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/published_information.txt b/redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/published_information.txt index b1496e4b..66b5a466 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/published_information.txt +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/published_information.txt @@ -86,3 +86,4 @@ Toxicol Sci Toxicol Sci. Toxicol Sci. 1 Test Ignored Hint Published Information +Workshop diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/files/new/PublishedInformationTest.pdf b/redaction-service-v1/redaction-service-server-v1/src/test/resources/files/new/PublishedInformationTest.pdf new file mode 100644 index 00000000..7ebacaa6 Binary files /dev/null and b/redaction-service-v1/redaction-service-server-v1/src/test/resources/files/new/PublishedInformationTest.pdf differ