RED-3674: Fixed force redactions overrides idRemoval with different id

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionService.java

@@ -82,9 +82,10 @@ public class EntityRedactionService {
                 var idsToRemove = analyzeRequest.getManualRedactions().getIdsToRemove().stream()
                         .filter(idr -> idr.getStatus() == AnnotationStatus.APPROVED && !idr.isRemoveFromDictionary())
                         .filter(idr -> idr.getRequestDate() != null)
-                        .filter(idr -> approvedForceRedactions.stream().noneMatch(forceRedact -> forceRedact.getRequestDate().isAfter(idr.getRequestDate())))
+                        .filter(idr -> approvedForceRedactions.stream().noneMatch(forceRedact -> forceRedact.getAnnotationId().equals(idr.getAnnotationId()) && forceRedact.getRequestDate().isAfter(idr.getRequestDate())))
                         .map(IdRemoval::getAnnotationId).collect(Collectors.toSet());
 
+
                 if (reanalysisSection.getImages() != null && !reanalysisSection.getImages()
                         .isEmpty() && analyzeRequest.getManualRedactions().getImageRecategorization() != null) {
                     for (Image image : reanalysisSection.getImages()) {

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java

@@ -1,6 +1,7 @@
 package com.iqser.red.service.redaction.v1.server;
 
 import com.amazonaws.services.s3.AmazonS3;
+import com.fasterxml.jackson.core.type.TypeReference;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus;
 import com.iqser.red.service.persistence.service.v1.api.model.annotations.Comment;
@@ -58,6 +59,7 @@ import java.io.*;
 import java.net.URL;
 import java.nio.charset.StandardCharsets;
 import java.time.OffsetDateTime;
+import java.time.ZoneOffset;
 import java.util.*;
 import java.util.stream.Collectors;
 
@@ -715,6 +717,72 @@ public class RedactionIntegrationTest {
     }
 
 
+    @Test
+    public void testRemovePublishedInformations() throws IOException {
+
+        long start = System.currentTimeMillis();
+
+        ClassPathResource colorsResource = new ClassPathResource("colors/colors.json");
+        var colors = objectMapper.readValue(colorsResource.getInputStream(), Colors.class);
+
+        ClassPathResource typeResource = new ClassPathResource("colors/types.json");
+        TypeReference<List<Type>> typeRefForTypes = new TypeReference<>(){};
+        List<Type> types = objectMapper.readValue(typeResource.getInputStream(), typeRefForTypes);
+
+
+        AnalyzeRequest request = prepareStorage("files/new/PublishedInformationTest.pdf");
+        analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
+        ManualRedactions manualRedactions = new ManualRedactions();
+        manualRedactions.getIdsToRemove().add(IdRemoval.builder()
+                .annotationId("308dab9015bfafd911568cffe0a7f7de")
+                .fileId(TEST_FILE_ID)
+                .status(AnnotationStatus.APPROVED)
+                        .requestDate(OffsetDateTime.of(2022,05,23,8,30,07,475479,  ZoneOffset.UTC))
+                        .processedDate(OffsetDateTime.of(2022,05,23,8,30,07,483651,  ZoneOffset.UTC))
+                .build());
+
+        manualRedactions.getForceRedactions().add(ManualForceRedaction.builder()
+                .annotationId("0b56ea1a87c83f351df177315af94f0d")
+                .fileId(TEST_FILE_ID)
+                .status(AnnotationStatus.APPROVED)
+                .requestDate(OffsetDateTime.of(2022,05,23,9,30,15,4653, ZoneOffset.UTC))
+                .processedDate(OffsetDateTime.of(2022,05,23,9,30,15,794,  ZoneOffset.UTC))
+                .build());
+
+        manualRedactions.getIdsToRemove().add(IdRemoval.builder()
+                .annotationId("0b56ea1a87c83f351df177315af94f0d")
+                .fileId(TEST_FILE_ID)
+                .status(AnnotationStatus.APPROVED)
+                .requestDate(OffsetDateTime.of(2022,05,23,8,30,23,961721,  ZoneOffset.UTC))
+                .processedDate(OffsetDateTime.of(2022,05,23,8,30,23,96528,  ZoneOffset.UTC))
+                .build());
+
+
+
+        request.setManualRedactions(manualRedactions);
+
+
+        AnalyzeResult result = analyzeService.analyze(request);
+
+
+        AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder()
+                        .manualRedactions(manualRedactions)
+                        .colors(colors)
+                        .types(types)
+                .dossierId(TEST_DOSSIER_ID)
+                .fileId(TEST_FILE_ID)
+                .build());
+
+        try (FileOutputStream fileOutputStream = new FileOutputStream(OsUtils.getTemporaryDirectory() + "/Annotated.pdf")) {
+            fileOutputStream.write(annotateResponse.getDocument());
+        }
+        long end = System.currentTimeMillis();
+
+        System.out.println("duration: " + (end - start));
+        System.out.println("numberOfPages: " + result.getNumberOfPages());
+    }
+
+
     @Test
     public void testTableRedaction() throws IOException {
 

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/annotate/AnnotateRequest.java

@@ -1,6 +1,10 @@
 package com.iqser.red.service.redaction.v1.server.annotate;
 
+import java.util.List;
+
 import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
+import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.configuration.Colors;
+import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.Type;
 
 import lombok.AllArgsConstructor;
 import lombok.Builder;
@@ -17,4 +21,6 @@ public class AnnotateRequest {
     private String dossierTemplateId;
     private String fileId;
     private ManualRedactions manualRedactions;
+    private Colors colors;
+    private List<Type> types;
 }

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/annotate/AnnotationService.java

@@ -46,6 +46,8 @@ public class AnnotationService {
                 .manualRedactions(annotateRequest.getManualRedactions())
                 .dossierId(annotateRequest.getDossierId())
                 .dossierTemplateId(annotateRequest.getDossierTemplateId())
+                        .colors(annotateRequest.getColors())
+                        .types(annotateRequest.getTypes())
                 .build());
         var sectionsGrid = redactionStorageService.getSectionGrid(annotateRequest.getDossierId(), annotateRequest.getFileId());
 

redaction-service-v1/redaction-service-server-v1/src/test/resources/colors/colors.json

@@ -0,0 +1,13 @@
+{
+  "dossierTemplateId": "31039447-9040-4376-9ca7-614e56b284b9",
+  "defaultColor": "#9398a0",
+  "requestAdd": "#04b093",
+  "requestRemove": "#04b093",
+  "notRedacted": "#c498fa",
+  "analysisColor": "#dd4d50",
+  "updatedColor": "#fdbd00",
+  "dictionaryRequestColor": "#5b97db",
+  "manualRedactionColor": "#9398a0",
+  "previewColor": "#9398a0",
+  "ignoredHintColor": "#e7d4ff"
+}

redaction-service-v1/redaction-service-server-v1/src/test/resources/colors/types.json

@@ -0,0 +1,223 @@
+[
+  {
+    "id": "CBI_address:31039447-9040-4376-9ca7-614e56b284b9",
+    "type": "CBI_address",
+    "hexColor": "#9398a0",
+    "recommendationHexColor": "#8df06c",
+    "rank": 140,
+    "isHint": false,
+    "dossierTemplateId": "31039447-9040-4376-9ca7-614e56b284b9",
+    "isCaseInsensitive": false,
+    "isRecommendation": false,
+    "description": "All site names and addresses, and location (e.g. Syngenta, Monthey, GPS Co-ordinates, Mr Smith of … providing the…). Except addresses in published literature and the applicant address.",
+    "addToDictionaryAction": true,
+    "label": "CBI Address",
+    "hasDictionary": true,
+    "systemManaged": false,
+    "autoHideSkipped": false
+  },
+  {
+    "id": "CBI_author:31039447-9040-4376-9ca7-614e56b284b9",
+    "type": "CBI_author",
+    "hexColor": "#9398a0",
+    "recommendationHexColor": "#8df06c",
+    "rank": 130,
+    "isHint": false,
+    "dossierTemplateId": "31039447-9040-4376-9ca7-614e56b284b9",
+    "isCaseInsensitive": false,
+    "isRecommendation": false,
+    "description": "All authors named in the study documentation. Except names in published literature.",
+    "addToDictionaryAction": true,
+    "label": "CBI Author",
+    "hasDictionary": true,
+    "systemManaged": false,
+    "autoHideSkipped": false
+  },
+  {
+    "id": "PII:31039447-9040-4376-9ca7-614e56b284b9",
+    "type": "PII",
+    "hexColor": "#9398a0",
+    "recommendationHexColor": "#8df06c",
+    "rank": 150,
+    "isHint": false,
+    "dossierTemplateId": "31039447-9040-4376-9ca7-614e56b284b9",
+    "isCaseInsensitive": false,
+    "isRecommendation": false,
+    "description": "Not authors but listed in the document: Names, signatures, telephone, email etc.; e.g. Reg Manager, QA Manager",
+    "addToDictionaryAction": true,
+    "label": "PII",
+    "hasDictionary": true,
+    "systemManaged": false,
+    "autoHideSkipped": false
+  },
+  {
+    "id": "formula:31039447-9040-4376-9ca7-614e56b284b9",
+    "type": "formula",
+    "hexColor": "#036ffc",
+    "recommendationHexColor": "#8df06c",
+    "rank": 1002,
+    "isHint": true,
+    "dossierTemplateId": "31039447-9040-4376-9ca7-614e56b284b9",
+    "isCaseInsensitive": true,
+    "isRecommendation": false,
+    "description": "Empty dictionary used to configure formula colors.",
+    "addToDictionaryAction": false,
+    "label": "Formula",
+    "hasDictionary": false,
+    "systemManaged": true,
+    "autoHideSkipped": false
+  },
+  {
+    "id": "isHint_only:31039447-9040-4376-9ca7-614e56b284b9",
+    "type": "isHint_only",
+    "hexColor": "#fa98f7",
+    "recommendationHexColor": "#8df06c",
+    "rank": 50,
+    "isHint": true,
+    "dossierTemplateId": "31039447-9040-4376-9ca7-614e56b284b9",
+    "isCaseInsensitive": true,
+    "isRecommendation": false,
+    "description": "Entries of this dictionary will be highlighted only",
+    "addToDictionaryAction": false,
+    "label": "isHint Only",
+    "hasDictionary": true,
+    "systemManaged": false,
+    "autoHideSkipped": false
+  },
+  {
+    "id": "image:31039447-9040-4376-9ca7-614e56b284b9",
+    "type": "image",
+    "hexColor": "#bdd6ff",
+    "recommendationHexColor": "#8df06c",
+    "rank": 999,
+    "isHint": true,
+    "dossierTemplateId": "31039447-9040-4376-9ca7-614e56b284b9",
+    "isCaseInsensitive": true,
+    "isRecommendation": false,
+    "description": "Empty dictionary used to configure image colors.",
+    "addToDictionaryAction": false,
+    "label": "Image",
+    "hasDictionary": false,
+    "systemManaged": true,
+    "autoHideSkipped": false
+  },
+  {
+    "id": "logo:31039447-9040-4376-9ca7-614e56b284b9",
+    "type": "logo",
+    "hexColor": "#9398a0",
+    "recommendationHexColor": "#8df06c",
+    "rank": 1001,
+    "isHint": false,
+    "dossierTemplateId": "31039447-9040-4376-9ca7-614e56b284b9",
+    "isCaseInsensitive": true,
+    "isRecommendation": false,
+    "description": "Empty dictionary used to configure logo colors.",
+    "addToDictionaryAction": false,
+    "label": "Logo",
+    "hasDictionary": false,
+    "systemManaged": true,
+    "autoHideSkipped": false
+  },
+  {
+    "id": "must_redact:31039447-9040-4376-9ca7-614e56b284b9",
+    "type": "must_redact",
+    "hexColor": "#9398a0",
+    "recommendationHexColor": "#8df06c",
+    "rank": 100,
+    "isHint": false,
+    "dossierTemplateId": "31039447-9040-4376-9ca7-614e56b284b9",
+    "isCaseInsensitive": true,
+    "isRecommendation": false,
+    "description": "Entries of this dictionary get redacted wherever found.",
+    "addToDictionaryAction": false,
+    "label": "Must Redact",
+    "hasDictionary": true,
+    "systemManaged": false,
+    "autoHideSkipped": false
+  },
+  {
+    "id": "ocr:31039447-9040-4376-9ca7-614e56b284b9",
+    "type": "ocr",
+    "hexColor": "#bdd6ff",
+    "recommendationHexColor": "#8df06c",
+    "rank": 1000,
+    "isHint": true,
+    "dossierTemplateId": "31039447-9040-4376-9ca7-614e56b284b9",
+    "isCaseInsensitive": true,
+    "isRecommendation": false,
+    "description": "Empty dictionary used to configure ocr colors.",
+    "addToDictionaryAction": false,
+    "label": "Ocr",
+    "hasDictionary": false,
+    "systemManaged": true,
+    "autoHideSkipped": false
+  },
+  {
+    "id": "signature:31039447-9040-4376-9ca7-614e56b284b9",
+    "type": "signature",
+    "hexColor": "#9398a0",
+    "recommendationHexColor": "#8df06c",
+    "rank": 1003,
+    "isHint": false,
+    "dossierTemplateId": "31039447-9040-4376-9ca7-614e56b284b9",
+    "isCaseInsensitive": true,
+    "isRecommendation": false,
+    "description": "Empty dictionary used to configure signature colors.",
+    "addToDictionaryAction": false,
+    "label": "Signature",
+    "hasDictionary": false,
+    "systemManaged": true,
+    "autoHideSkipped": false
+  },
+  {
+    "id": "imported_redaction:31039447-9040-4376-9ca7-614e56b284b9",
+    "type": "imported_redaction",
+    "hexColor": "#f0f0c0",
+    "recommendationHexColor": "#8df06c",
+    "rank": 9999,
+    "isHint": false,
+    "dossierTemplateId": "31039447-9040-4376-9ca7-614e56b284b9",
+    "isCaseInsensitive": false,
+    "isRecommendation": false,
+    "description": "Redaction Annotations that were imported from documents",
+    "addToDictionaryAction": false,
+    "label": "Imported Redaction",
+    "hasDictionary": false,
+    "systemManaged": true,
+    "autoHideSkipped": true
+  },
+  {
+    "id": "published_information:31039447-9040-4376-9ca7-614e56b284b9",
+    "type": "published_information",
+    "hexColor": "#85ebff",
+    "recommendationHexColor": "#8df06c",
+    "rank": 70,
+    "isHint": true,
+    "dossierTemplateId": "31039447-9040-4376-9ca7-614e56b284b9",
+    "isCaseInsensitive": false,
+    "isRecommendation": false,
+    "description": "Manual managed list of public journals and papers that need no redaction",
+    "addToDictionaryAction": true,
+    "label": "Published Information",
+    "hasDictionary": true,
+    "systemManaged": false,
+    "autoHideSkipped": false
+  },
+  {
+    "id": "dossier_redaction:31039447-9040-4376-9ca7-614e56b284b9:5dfb2724-74a4-4a1a-a1eb-165e7943ffcd",
+    "type": "dossier_redaction",
+    "hexColor": "#9398a0",
+    "recommendationHexColor": null,
+    "rank": 1500,
+    "isHint": false,
+    "dossierTemplateId": "31039447-9040-4376-9ca7-614e56b284b9",
+    "isCaseInsensitive": false,
+    "isRecommendation": false,
+    "description": "Entries in this dictionary will only be redacted in this dossier",
+    "addToDictionaryAction": false,
+    "label": "Dossier Redaction",
+    "hasDictionary": true,
+    "systemManaged": true,
+    "autoHideSkipped": false
+  }
+]

redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/published_information.txt

@@ -86,3 +86,4 @@ Toxicol Sci
 Toxicol Sci.
 Toxicol Sci. 1
 Test Ignored Hint Published Information
+Workshop