Ignore hint feature RED-2867

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Entity.java

@@ -39,6 +39,8 @@ public class Entity implements ReasonHolder {
 
     private boolean isDossierDictionaryEntry;
 
+    private boolean ignored;
+
 
     public Entity(String word, String type, boolean redaction, String redactionReason, List<EntityPositionSequence> positionSequences, String headline, int matchedRule, int sectionNumber, String legalBasis, boolean isDictionaryEntry, String textBefore, String textAfter, Integer start, Integer end, boolean isDossierDictionaryEntry) {
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Image.java

@@ -20,6 +20,7 @@ public class Image implements ReasonHolder {
     private int sectionNumber;
     private String section;
     private int page;
+    private boolean ignored;
     private boolean hasTransparency;
 
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java

@@ -134,13 +134,13 @@ public class Section {
 
     public boolean matchesType(String type) {
 
-        return entities.stream().anyMatch(entity -> entity.getType().equals(type));
+        return entities.stream().anyMatch(entity -> !entity.isIgnored() && entity.getType().equals(type));
     }
 
 
     public boolean matchesImageType(String type) {
 
-        return images.stream().anyMatch(image -> image.getType().equals(type));
+        return images.stream().anyMatch(image -> !image.isIgnored() &&  image.getType().equals(type));
     }
 
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ReanalyzeService.java

@@ -213,19 +213,33 @@ public class ReanalyzeService {
                 surroundingWordsService.addSurroundingText(entities, reanalysisSection.getSearchableText(), dictionary);
             }
 
-            if (!local && reanalysisSection.getImages() != null && !reanalysisSection.getImages()
-                    .isEmpty() && analyzeRequest.getManualRedactions() != null && analyzeRequest.getManualRedactions()
-                    .getImageRecategorizations() != null) {
-                for (Image image : reanalysisSection.getImages()) {
-                    String imageId = IdBuilder.buildId(image.getPosition(), image.getPage());
-                    for (ManualImageRecategorization imageRecategorization : analyzeRequest.getManualRedactions()
-                            .getImageRecategorizations()) {
-                        if (imageRecategorization.getStatus().equals(Status.APPROVED) && imageRecategorization.getId()
-                                .equals(imageId)) {
-                            image.setType(imageRecategorization.getType());
+            if (!local && analyzeRequest.getManualRedactions() != null) {
+
+                var idsToRemove = analyzeRequest.getManualRedactions().getIdsToRemove().stream().map(IdRemoval::getId).collect(Collectors.toSet());
+
+                if (reanalysisSection.getImages() != null && !reanalysisSection.getImages().isEmpty() && analyzeRequest.getManualRedactions().getImageRecategorizations() != null) {
+                    for (Image image : reanalysisSection.getImages()) {
+                        String imageId = IdBuilder.buildId(image.getPosition(), image.getPage());
+                        for (ManualImageRecategorization imageRecategorization : analyzeRequest.getManualRedactions()
+                                .getImageRecategorizations()) {
+                            if (imageRecategorization.getStatus().equals(Status.APPROVED) && imageRecategorization.getId()
+                                    .equals(imageId)) {
+                                image.setType(imageRecategorization.getType());
+                            }
+                        }
+                        if (idsToRemove.contains(imageId)) {
+                            image.setIgnored(true);
                         }
                     }
                 }
+
+                entities.forEach(entity ->
+                        entity.getPositionSequences().forEach(ps -> {
+                            if (idsToRemove.contains(ps.getId())) {
+                                entity.setIgnored(true);
+                            }
+                        }));
+
             }
 
             sectionSearchableTextPairs.add(new SectionSearchableTextPair(Section.builder()

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogMergeService.java

@@ -136,6 +136,7 @@ public class RedactionLogMergeService {
                     manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", removed by manual override");
                     redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, false, redactionLogEntry
                             .isRedacted(), true));
+                    redactionLogEntry.setHint(false);
                     redactionLogEntry.setHasBeenRemovedByManualOverride(true);
                 } else if (manualRemoval.getStatus().equals(Status.REQUESTED)) {
                     manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", requested to remove");

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java

@@ -18,11 +18,10 @@ import com.iqser.red.service.redaction.v1.server.redaction.utils.ResourceLoader;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.TextNormalizationUtilities;
 import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
 import com.iqser.red.storage.commons.service.StorageService;
-
 import lombok.SneakyThrows;
-
 import org.apache.commons.io.IOUtils;
 import org.apache.commons.lang3.StringUtils;
+import org.assertj.core.util.Sets;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Ignore;
@@ -758,6 +757,43 @@ public class RedactionIntegrationTest {
     }
 
 
+    @Test
+    public void testIgnoreHint() {
+
+        System.out.println("testIgnoreHint");
+
+        AnalyzeRequest request = prepareStorage("files/new/test-ignore-hint.pdf");
+        reanalyzeService.analyze(request);
+        var redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID);
+
+        var manualRedactions = ManualRedactions.builder().idsToRemove(Sets.newLinkedHashSet(
+                IdRemoval.builder()
+                        .id("c630599611e6e3db314518374bcf70f7")
+                        .status(Status.APPROVED)
+                        .user("test")
+                        .removeFromDictionary(false)
+                        .processedDate(OffsetDateTime.now())
+                        .requestDate(OffsetDateTime.now())
+                        .build())).build();
+
+        request.setManualRedactions(manualRedactions);
+        reanalyzeService.reanalyze(request);
+
+
+        var mergedRedactionLog = redactionController.getRedactionLog(RedactionRequest.builder().withSectionDataForManualRedactions(true)
+                .manualRedactions(manualRedactions)
+                .dossierTemplateId(TEST_DOSSIER_TEMPLATE_ID)
+                .dossierId(TEST_DOSSIER_ID)
+                .fileId(TEST_FILE_ID)
+                .build());
+
+        var cbiAddressBeforeHintRemoval = redactionLog.getRedactionLogEntry().stream().filter(re -> re.getType().equalsIgnoreCase("CBI_Address")).findAny().get();
+        assertThat(cbiAddressBeforeHintRemoval.isRedacted()).isFalse();
+
+        var cbiAddressAfterHintRemoval = mergedRedactionLog.getRedactionLogEntry().stream().filter(re -> re.getType().equalsIgnoreCase("CBI_Address")).findAny().get();
+        assertThat(cbiAddressAfterHintRemoval.isRedacted()).isTrue();
+    }
+
     @Test
     public void testTableRedaction() throws IOException {
 
@@ -805,9 +841,6 @@ public class RedactionIntegrationTest {
     }
 
 
-
-
-
     @Test
     public void testManualRedaction() throws IOException {
 

redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/CBI_address.txt

@@ -1654,3 +1654,4 @@ Zyma SA
 Zyma SA, Nyon, Switzerland
 Mambo-Tox Ltd. Biomedical Sciences Building Bassett Crescent East Southampton SO16 7PX UK
 Syngenta Environmental Sciences Jealott’s Hill International Research Centre Bracknell, Berkshire RG42 6EY UK
+Test Ignored Hint CBI_ADDRESS

redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/published_information.txt

@@ -85,3 +85,4 @@ Toxicology and Applied Pharmacology
 Toxicol Sci
 Toxicol Sci.
 Toxicol Sci. 1
+Test Ignored Hint Published Information