diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Entity.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Entity.java index 2ae553db..2cb82371 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Entity.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Entity.java @@ -39,6 +39,8 @@ public class Entity implements ReasonHolder { private boolean isDossierDictionaryEntry; + private boolean ignored; + public Entity(String word, String type, boolean redaction, String redactionReason, List positionSequences, String headline, int matchedRule, int sectionNumber, String legalBasis, boolean isDictionaryEntry, String textBefore, String textAfter, Integer start, Integer end, boolean isDossierDictionaryEntry) { diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Image.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Image.java index 63d382d5..35317fdf 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Image.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Image.java @@ -20,6 +20,7 @@ public class Image implements ReasonHolder { private int sectionNumber; private String section; private int page; + private boolean ignored; private boolean hasTransparency; } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java index f87e937f..cc267f33 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java @@ -134,13 +134,13 @@ public class Section { public boolean matchesType(String type) { - return entities.stream().anyMatch(entity -> entity.getType().equals(type)); + return entities.stream().anyMatch(entity -> !entity.isIgnored() && entity.getType().equals(type)); } public boolean matchesImageType(String type) { - return images.stream().anyMatch(image -> image.getType().equals(type)); + return images.stream().anyMatch(image -> !image.isIgnored() && image.getType().equals(type)); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ReanalyzeService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ReanalyzeService.java index 4a22087e..172cae85 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ReanalyzeService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ReanalyzeService.java @@ -213,19 +213,33 @@ public class ReanalyzeService { surroundingWordsService.addSurroundingText(entities, reanalysisSection.getSearchableText(), dictionary); } - if (!local && reanalysisSection.getImages() != null && !reanalysisSection.getImages() - .isEmpty() && analyzeRequest.getManualRedactions() != null && analyzeRequest.getManualRedactions() - .getImageRecategorizations() != null) { - for (Image image : reanalysisSection.getImages()) { - String imageId = IdBuilder.buildId(image.getPosition(), image.getPage()); - for (ManualImageRecategorization imageRecategorization : analyzeRequest.getManualRedactions() - .getImageRecategorizations()) { - if (imageRecategorization.getStatus().equals(Status.APPROVED) && imageRecategorization.getId() - .equals(imageId)) { - image.setType(imageRecategorization.getType()); + if (!local && analyzeRequest.getManualRedactions() != null) { + + var idsToRemove = analyzeRequest.getManualRedactions().getIdsToRemove().stream().map(IdRemoval::getId).collect(Collectors.toSet()); + + if (reanalysisSection.getImages() != null && !reanalysisSection.getImages().isEmpty() && analyzeRequest.getManualRedactions().getImageRecategorizations() != null) { + for (Image image : reanalysisSection.getImages()) { + String imageId = IdBuilder.buildId(image.getPosition(), image.getPage()); + for (ManualImageRecategorization imageRecategorization : analyzeRequest.getManualRedactions() + .getImageRecategorizations()) { + if (imageRecategorization.getStatus().equals(Status.APPROVED) && imageRecategorization.getId() + .equals(imageId)) { + image.setType(imageRecategorization.getType()); + } + } + if (idsToRemove.contains(imageId)) { + image.setIgnored(true); } } } + + entities.forEach(entity -> + entity.getPositionSequences().forEach(ps -> { + if (idsToRemove.contains(ps.getId())) { + entity.setIgnored(true); + } + })); + } sectionSearchableTextPairs.add(new SectionSearchableTextPair(Section.builder() diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogMergeService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogMergeService.java index 53e073f7..f0fb70b6 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogMergeService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogMergeService.java @@ -136,6 +136,7 @@ public class RedactionLogMergeService { manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", removed by manual override"); redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, false, redactionLogEntry .isRedacted(), true)); + redactionLogEntry.setHint(false); redactionLogEntry.setHasBeenRemovedByManualOverride(true); } else if (manualRemoval.getStatus().equals(Status.REQUESTED)) { manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", requested to remove"); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java index 7f684a1c..bb0f19bb 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java @@ -18,11 +18,10 @@ import com.iqser.red.service.redaction.v1.server.redaction.utils.ResourceLoader; import com.iqser.red.service.redaction.v1.server.redaction.utils.TextNormalizationUtilities; import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService; import com.iqser.red.storage.commons.service.StorageService; - import lombok.SneakyThrows; - import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.StringUtils; +import org.assertj.core.util.Sets; import org.junit.After; import org.junit.Before; import org.junit.Ignore; @@ -758,6 +757,43 @@ public class RedactionIntegrationTest { } + @Test + public void testIgnoreHint() { + + System.out.println("testIgnoreHint"); + + AnalyzeRequest request = prepareStorage("files/new/test-ignore-hint.pdf"); + reanalyzeService.analyze(request); + var redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID); + + var manualRedactions = ManualRedactions.builder().idsToRemove(Sets.newLinkedHashSet( + IdRemoval.builder() + .id("c630599611e6e3db314518374bcf70f7") + .status(Status.APPROVED) + .user("test") + .removeFromDictionary(false) + .processedDate(OffsetDateTime.now()) + .requestDate(OffsetDateTime.now()) + .build())).build(); + + request.setManualRedactions(manualRedactions); + reanalyzeService.reanalyze(request); + + + var mergedRedactionLog = redactionController.getRedactionLog(RedactionRequest.builder().withSectionDataForManualRedactions(true) + .manualRedactions(manualRedactions) + .dossierTemplateId(TEST_DOSSIER_TEMPLATE_ID) + .dossierId(TEST_DOSSIER_ID) + .fileId(TEST_FILE_ID) + .build()); + + var cbiAddressBeforeHintRemoval = redactionLog.getRedactionLogEntry().stream().filter(re -> re.getType().equalsIgnoreCase("CBI_Address")).findAny().get(); + assertThat(cbiAddressBeforeHintRemoval.isRedacted()).isFalse(); + + var cbiAddressAfterHintRemoval = mergedRedactionLog.getRedactionLogEntry().stream().filter(re -> re.getType().equalsIgnoreCase("CBI_Address")).findAny().get(); + assertThat(cbiAddressAfterHintRemoval.isRedacted()).isTrue(); + } + @Test public void testTableRedaction() throws IOException { @@ -805,9 +841,6 @@ public class RedactionIntegrationTest { } - - - @Test public void testManualRedaction() throws IOException { diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/CBI_address.txt b/redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/CBI_address.txt index b6ad4398..b6acc607 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/CBI_address.txt +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/CBI_address.txt @@ -1654,3 +1654,4 @@ Zyma SA Zyma SA, Nyon, Switzerland Mambo-Tox Ltd. Biomedical Sciences Building Bassett Crescent East Southampton SO16 7PX UK Syngenta Environmental Sciences Jealott’s Hill International Research Centre Bracknell, Berkshire RG42 6EY UK +Test Ignored Hint CBI_ADDRESS diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/published_information.txt b/redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/published_information.txt index 023d801e..b1496e4b 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/published_information.txt +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/published_information.txt @@ -85,3 +85,4 @@ Toxicology and Applied Pharmacology Toxicol Sci Toxicol Sci. Toxicol Sci. 1 +Test Ignored Hint Published Information diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/files/new/test-ignore-hint.pdf b/redaction-service-v1/redaction-service-server-v1/src/test/resources/files/new/test-ignore-hint.pdf new file mode 100644 index 00000000..f63b18ae Binary files /dev/null and b/redaction-service-v1/redaction-service-server-v1/src/test/resources/files/new/test-ignore-hint.pdf differ