RED-3025: Ported code to ignore hints

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Entity.java

@@ -42,6 +42,8 @@ public class Entity implements ReasonHolder {
 
     private boolean isDossierDictionaryEntry;
 
+    private boolean ignored;
+
     private Set<Engine> engines = new HashSet<>();
 
     private Set<Entity> references = new HashSet<>();

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Image.java

@@ -20,6 +20,7 @@ public class Image implements ReasonHolder {
     private int sectionNumber;
     private String section;
     private int page;
+    private boolean ignored;
     private boolean hasTransparency;
 
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java

@@ -156,14 +156,14 @@ public class Section {
     @WhenCondition
     public boolean matchesType(@Argument(ArgumentType.TYPE) String type) {
 
-        return entities.stream().anyMatch(entity -> entity.getType().equals(type));
+        return entities.stream().anyMatch(entity -> !entity.isIgnored() && entity.getType().equals(type));
     }
 
 
     @WhenCondition
     public boolean matchesImageType(@Argument(ArgumentType.TYPE) String type) {
 
-        return images.stream().anyMatch(image -> image.getType().equals(type));
+        return images.stream().anyMatch(image -> !image.isIgnored() && image.getType().equals(type));
     }
 
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionService.java

@@ -10,7 +10,9 @@ import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
 import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus;
+import com.iqser.red.service.persistence.service.v1.api.model.annotations.IdRemoval;
 import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualImageRecategorization;
+
 import org.apache.commons.codec.binary.Base64;
 import org.apache.commons.lang3.StringUtils;
 import org.kie.api.runtime.KieContainer;
@@ -82,20 +84,43 @@ public class EntityRedactionService {
                 surroundingWordsService.addSurroundingText(entities, reanalysisSection.getSearchableText(), dictionary);
             }
 
-            if (!local && reanalysisSection.getImages() != null && !reanalysisSection.getImages()
+            if (!local && analyzeRequest.getManualRedactions() != null) {
-                    .isEmpty() && analyzeRequest.getManualRedactions() != null && analyzeRequest.getManualRedactions()
+
-                    .getImageRecategorization() != null) {
+                var approvedForceRedactions = analyzeRequest.getManualRedactions().getForceRedactions().stream()
+                        .filter(fr -> fr.getStatus() == AnnotationStatus.APPROVED)
+                        .filter(fr -> fr.getRequestDate() != null)
+                        .collect(Collectors.toList());
+                // only approved id removals, that haven't been forced back afterwards
+                var idsToRemove = analyzeRequest.getManualRedactions().getIdsToRemove().stream()
+                        .filter(idr -> idr.getStatus() == AnnotationStatus.APPROVED)
+                        .filter(idr -> idr.getRequestDate() != null)
+                        .filter(idr -> approvedForceRedactions.stream().noneMatch(forceRedact -> forceRedact.getRequestDate().isAfter(idr.getRequestDate())))
+                        .map(IdRemoval::getAnnotationId).collect(Collectors.toSet());
+
+                if (reanalysisSection.getImages() != null && !reanalysisSection.getImages()
+                        .isEmpty() && analyzeRequest.getManualRedactions().getImageRecategorization() != null) {
                     for (Image image : reanalysisSection.getImages()) {
                         String imageId = IdBuilder.buildId(image.getPosition(), image.getPage());
                         for (ManualImageRecategorization imageRecategorization : analyzeRequest.getManualRedactions()
                                 .getImageRecategorization()) {
-                        if (imageRecategorization.getStatus().equals(AnnotationStatus.APPROVED) && imageRecategorization.getAnnotationId()
+                            if (imageRecategorization.getStatus()
+                                    .equals(AnnotationStatus.APPROVED) && imageRecategorization.getAnnotationId()
                                     .equals(imageId)) {
                                 image.setType(imageRecategorization.getType());
                             }
                         }
+                        if (idsToRemove.contains(imageId)) {
+                            image.setIgnored(true);
                         }
                     }
+                }
+
+                entities.forEach(entity -> entity.getPositionSequences().forEach(ps -> {
+                    if (idsToRemove.contains(ps.getId())) {
+                        entity.setIgnored(true);
+                    }
+                }));
+            }
 
             sectionSearchableTextPairs.add(new SectionSearchableTextPair(Section.builder()
                     .isLocal(false)
@@ -149,7 +174,8 @@ public class EntityRedactionService {
                         .add(new Entity(entity.getWord(), entity.getType(), entity.isRedaction(), entity.getRedactionReason(), entry
                                 .getValue(), entity.getHeadline(), entity.getMatchedRule(), entity.getSectionNumber(), entity
                                 .getLegalBasis(), entity.isDictionaryEntry(), entity.getTextBefore(), entity.getTextAfter(), entity
-                                .getStart(), entity.getEnd(), entity.isDossierDictionaryEntry(), entity.getEngines(), entity.getReferences()));
+                                .getStart(), entity.getEnd(), entity.isDossierDictionaryEntry(), entity.getEngines(), entity
+                                .getReferences()));
             }
         }
         return entitiesPerPage;

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogMergeService.java

@@ -172,6 +172,7 @@ public class RedactionLogMergeService {
                     redactionLogEntry.setStatus(AnnotationStatus.APPROVED);
                     manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", removed by manual override");
                     redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, false, redactionLogEntry.isRedacted(), true));
+                    redactionLogEntry.setHint(false);
                     redactionLogEntry.setHasBeenRemovedByManualOverride(true);
                 } else if (manualRemoval.getStatus().equals(AnnotationStatus.REQUESTED)) {
                     manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", requested to remove");
@@ -193,7 +194,12 @@ public class RedactionLogMergeService {
                 var manualForceRedact = (ManualForceRedaction) mrw.getItem();
                 String manualOverrideReason = null;
                 if (manualForceRedact.getStatus().equals(AnnotationStatus.APPROVED)) {
+                    // Forcing a skipped hint should result in a hint
+                    if (dictionaryService.isHint(redactionLogEntry.getType(), dossierTemplateId)) {
+                        redactionLogEntry.setHint(true);
+                    } else {
                         redactionLogEntry.setRedacted(true);
+                    }
                     redactionLogEntry.setStatus(AnnotationStatus.APPROVED);
                     redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, false, redactionLogEntry.isRedacted(), false));
                     manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", forced by manual override");

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java

@@ -50,6 +50,7 @@ import org.springframework.test.context.junit4.SpringRunner;
 
 import com.amazonaws.services.s3.AmazonS3;
 import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.collect.Sets;
 import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus;
 import com.iqser.red.service.persistence.service.v1.api.model.annotations.Comment;
 import com.iqser.red.service.persistence.service.v1.api.model.annotations.IdRemoval;
@@ -628,6 +629,52 @@ public class RedactionIntegrationTest {
     }
 
 
+
+    @Test
+    @SneakyThrows
+    public void testIgnoreHint() {
+
+        System.out.println("testIgnoreHint");
+
+        ClassPathResource pdfFileResource = new ClassPathResource("files/new/test-ignore-hint.pdf");
+        AnalyzeRequest request = prepareStorage(pdfFileResource.getInputStream());
+
+        analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
+        analyzeService.analyze(request);
+
+        var redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID);
+
+        var toRemove = IdRemoval.builder()
+                .annotationId("c630599611e6e3db314518374bcf70f7")
+                .status(AnnotationStatus.APPROVED)
+                .user("test")
+                .removeFromDictionary(false)
+                .processedDate(OffsetDateTime.now())
+                .requestDate(OffsetDateTime.now())
+                .build();
+
+        var manualRedactions = ManualRedactions.builder().idsToRemove(Set.of(toRemove)).build();
+        request.setManualRedactions(manualRedactions);
+        analyzeService.reanalyze(request);
+
+        var mergedRedactionLog = redactionController.getRedactionLog(RedactionRequest.builder()
+                .manualRedactions(manualRedactions)
+                .dossierTemplateId(TEST_DOSSIER_TEMPLATE_ID)
+                .dossierId(TEST_DOSSIER_ID)
+                .fileId(TEST_FILE_ID)
+                .build());
+
+
+        var cbiAddressBeforeHintRemoval = redactionLog.getRedactionLogEntry().stream().filter(re -> re.getType().equalsIgnoreCase("CBI_Address")).findAny().get();
+        assertThat(cbiAddressBeforeHintRemoval.isRedacted()).isFalse();
+
+        var cbiAddressAfterHintRemoval = mergedRedactionLog.getRedactionLogEntry().stream().filter(re -> re.getType().equalsIgnoreCase("CBI_Address")).findAny().get();
+        assertThat(cbiAddressAfterHintRemoval.isRedacted()).isTrue();
+    }
+
+
+
+
     @Test
     @Ignore
     public void noExceptionShouldBeThrownForAnyFiles() throws IOException {
@@ -1469,10 +1516,10 @@ public class RedactionIntegrationTest {
 
     private static String getTemporaryDirectory() {
 
-        String tmpdir = System.getProperty("java.io.tmpdir");
+//        String tmpdir = System.getProperty("java.io.tmpdir");
-        if (StringUtils.isNotBlank(tmpdir)) {
+//        if (StringUtils.isNotBlank(tmpdir)) {
-            return tmpdir;
+//            return tmpdir;
-        }
+//        }
         return "/tmp";
     }
 

redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/CBI_address.txt

@@ -1660,3 +1660,4 @@ Zyma SA
 Zyma SA, Nyon, Switzerland
 Mambo-Tox Ltd. Biomedical Sciences Building Bassett Crescent East Southampton SO16 7PX UK
 Syngenta Environmental Sciences Jealott’s Hill International Research Centre Bracknell, Berkshire RG42 6EY UK
+Test Ignored Hint CBI_ADDRESS

redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/published_information.txt

@@ -85,3 +85,4 @@ Toxicology and Applied Pharmacology
 Toxicol Sci
 Toxicol Sci.
 Toxicol Sci. 1
+Test Ignored Hint Published Information