diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Entity.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Entity.java index cd90ab6b..5dd418ac 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Entity.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Entity.java @@ -42,6 +42,8 @@ public class Entity implements ReasonHolder { private boolean isDossierDictionaryEntry; + private boolean ignored; + private Set engines = new HashSet<>(); private Set references = new HashSet<>(); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Image.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Image.java index 63d382d5..35317fdf 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Image.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Image.java @@ -20,6 +20,7 @@ public class Image implements ReasonHolder { private int sectionNumber; private String section; private int page; + private boolean ignored; private boolean hasTransparency; } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java index 6b7ed01d..2672a58b 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java @@ -156,14 +156,14 @@ public class Section { @WhenCondition public boolean matchesType(@Argument(ArgumentType.TYPE) String type) { - return entities.stream().anyMatch(entity -> entity.getType().equals(type)); + return entities.stream().anyMatch(entity -> !entity.isIgnored() && entity.getType().equals(type)); } @WhenCondition public boolean matchesImageType(@Argument(ArgumentType.TYPE) String type) { - return images.stream().anyMatch(image -> image.getType().equals(type)); + return images.stream().anyMatch(image -> !image.isIgnored() && image.getType().equals(type)); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionService.java index 8d381880..f1fb0fd4 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionService.java @@ -10,7 +10,9 @@ import java.util.stream.Collectors; import java.util.stream.Stream; import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus; +import com.iqser.red.service.persistence.service.v1.api.model.annotations.IdRemoval; import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualImageRecategorization; + import org.apache.commons.codec.binary.Base64; import org.apache.commons.lang3.StringUtils; import org.kie.api.runtime.KieContainer; @@ -82,19 +84,42 @@ public class EntityRedactionService { surroundingWordsService.addSurroundingText(entities, reanalysisSection.getSearchableText(), dictionary); } - if (!local && reanalysisSection.getImages() != null && !reanalysisSection.getImages() - .isEmpty() && analyzeRequest.getManualRedactions() != null && analyzeRequest.getManualRedactions() - .getImageRecategorization() != null) { - for (Image image : reanalysisSection.getImages()) { - String imageId = IdBuilder.buildId(image.getPosition(), image.getPage()); - for (ManualImageRecategorization imageRecategorization : analyzeRequest.getManualRedactions() - .getImageRecategorization()) { - if (imageRecategorization.getStatus().equals(AnnotationStatus.APPROVED) && imageRecategorization.getAnnotationId() - .equals(imageId)) { - image.setType(imageRecategorization.getType()); + if (!local && analyzeRequest.getManualRedactions() != null) { + + var approvedForceRedactions = analyzeRequest.getManualRedactions().getForceRedactions().stream() + .filter(fr -> fr.getStatus() == AnnotationStatus.APPROVED) + .filter(fr -> fr.getRequestDate() != null) + .collect(Collectors.toList()); + // only approved id removals, that haven't been forced back afterwards + var idsToRemove = analyzeRequest.getManualRedactions().getIdsToRemove().stream() + .filter(idr -> idr.getStatus() == AnnotationStatus.APPROVED) + .filter(idr -> idr.getRequestDate() != null) + .filter(idr -> approvedForceRedactions.stream().noneMatch(forceRedact -> forceRedact.getRequestDate().isAfter(idr.getRequestDate()))) + .map(IdRemoval::getAnnotationId).collect(Collectors.toSet()); + + if (reanalysisSection.getImages() != null && !reanalysisSection.getImages() + .isEmpty() && analyzeRequest.getManualRedactions().getImageRecategorization() != null) { + for (Image image : reanalysisSection.getImages()) { + String imageId = IdBuilder.buildId(image.getPosition(), image.getPage()); + for (ManualImageRecategorization imageRecategorization : analyzeRequest.getManualRedactions() + .getImageRecategorization()) { + if (imageRecategorization.getStatus() + .equals(AnnotationStatus.APPROVED) && imageRecategorization.getAnnotationId() + .equals(imageId)) { + image.setType(imageRecategorization.getType()); + } + } + if (idsToRemove.contains(imageId)) { + image.setIgnored(true); } } } + + entities.forEach(entity -> entity.getPositionSequences().forEach(ps -> { + if (idsToRemove.contains(ps.getId())) { + entity.setIgnored(true); + } + })); } sectionSearchableTextPairs.add(new SectionSearchableTextPair(Section.builder() @@ -149,7 +174,8 @@ public class EntityRedactionService { .add(new Entity(entity.getWord(), entity.getType(), entity.isRedaction(), entity.getRedactionReason(), entry .getValue(), entity.getHeadline(), entity.getMatchedRule(), entity.getSectionNumber(), entity .getLegalBasis(), entity.isDictionaryEntry(), entity.getTextBefore(), entity.getTextAfter(), entity - .getStart(), entity.getEnd(), entity.isDossierDictionaryEntry(), entity.getEngines(), entity.getReferences())); + .getStart(), entity.getEnd(), entity.isDossierDictionaryEntry(), entity.getEngines(), entity + .getReferences())); } } return entitiesPerPage; diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogMergeService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogMergeService.java index 8828cf0b..175947eb 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogMergeService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogMergeService.java @@ -172,6 +172,7 @@ public class RedactionLogMergeService { redactionLogEntry.setStatus(AnnotationStatus.APPROVED); manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", removed by manual override"); redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, false, redactionLogEntry.isRedacted(), true)); + redactionLogEntry.setHint(false); redactionLogEntry.setHasBeenRemovedByManualOverride(true); } else if (manualRemoval.getStatus().equals(AnnotationStatus.REQUESTED)) { manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", requested to remove"); @@ -193,7 +194,12 @@ public class RedactionLogMergeService { var manualForceRedact = (ManualForceRedaction) mrw.getItem(); String manualOverrideReason = null; if (manualForceRedact.getStatus().equals(AnnotationStatus.APPROVED)) { - redactionLogEntry.setRedacted(true); + // Forcing a skipped hint should result in a hint + if (dictionaryService.isHint(redactionLogEntry.getType(), dossierTemplateId)) { + redactionLogEntry.setHint(true); + } else { + redactionLogEntry.setRedacted(true); + } redactionLogEntry.setStatus(AnnotationStatus.APPROVED); redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, false, redactionLogEntry.isRedacted(), false)); manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", forced by manual override"); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java index 1b74e85f..5c1f24e0 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java @@ -50,6 +50,7 @@ import org.springframework.test.context.junit4.SpringRunner; import com.amazonaws.services.s3.AmazonS3; import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.collect.Sets; import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus; import com.iqser.red.service.persistence.service.v1.api.model.annotations.Comment; import com.iqser.red.service.persistence.service.v1.api.model.annotations.IdRemoval; @@ -628,6 +629,52 @@ public class RedactionIntegrationTest { } + + @Test + @SneakyThrows + public void testIgnoreHint() { + + System.out.println("testIgnoreHint"); + + ClassPathResource pdfFileResource = new ClassPathResource("files/new/test-ignore-hint.pdf"); + AnalyzeRequest request = prepareStorage(pdfFileResource.getInputStream()); + + analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId())); + analyzeService.analyze(request); + + var redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID); + + var toRemove = IdRemoval.builder() + .annotationId("c630599611e6e3db314518374bcf70f7") + .status(AnnotationStatus.APPROVED) + .user("test") + .removeFromDictionary(false) + .processedDate(OffsetDateTime.now()) + .requestDate(OffsetDateTime.now()) + .build(); + + var manualRedactions = ManualRedactions.builder().idsToRemove(Set.of(toRemove)).build(); + request.setManualRedactions(manualRedactions); + analyzeService.reanalyze(request); + + var mergedRedactionLog = redactionController.getRedactionLog(RedactionRequest.builder() + .manualRedactions(manualRedactions) + .dossierTemplateId(TEST_DOSSIER_TEMPLATE_ID) + .dossierId(TEST_DOSSIER_ID) + .fileId(TEST_FILE_ID) + .build()); + + + var cbiAddressBeforeHintRemoval = redactionLog.getRedactionLogEntry().stream().filter(re -> re.getType().equalsIgnoreCase("CBI_Address")).findAny().get(); + assertThat(cbiAddressBeforeHintRemoval.isRedacted()).isFalse(); + + var cbiAddressAfterHintRemoval = mergedRedactionLog.getRedactionLogEntry().stream().filter(re -> re.getType().equalsIgnoreCase("CBI_Address")).findAny().get(); + assertThat(cbiAddressAfterHintRemoval.isRedacted()).isTrue(); + } + + + + @Test @Ignore public void noExceptionShouldBeThrownForAnyFiles() throws IOException { @@ -1469,10 +1516,10 @@ public class RedactionIntegrationTest { private static String getTemporaryDirectory() { - String tmpdir = System.getProperty("java.io.tmpdir"); - if (StringUtils.isNotBlank(tmpdir)) { - return tmpdir; - } +// String tmpdir = System.getProperty("java.io.tmpdir"); +// if (StringUtils.isNotBlank(tmpdir)) { +// return tmpdir; +// } return "/tmp"; } diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/CBI_address.txt b/redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/CBI_address.txt index 415b6f41..e308a6f3 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/CBI_address.txt +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/CBI_address.txt @@ -1660,3 +1660,4 @@ Zyma SA Zyma SA, Nyon, Switzerland Mambo-Tox Ltd. Biomedical Sciences Building Bassett Crescent East Southampton SO16 7PX UK Syngenta Environmental Sciences Jealott’s Hill International Research Centre Bracknell, Berkshire RG42 6EY UK +Test Ignored Hint CBI_ADDRESS diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/published_information.txt b/redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/published_information.txt index 023d801e..b1496e4b 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/published_information.txt +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/published_information.txt @@ -85,3 +85,4 @@ Toxicology and Applied Pharmacology Toxicol Sci Toxicol Sci. Toxicol Sci. 1 +Test Ignored Hint Published Information diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/files/new/test-ignore-hint.pdf b/redaction-service-v1/redaction-service-server-v1/src/test/resources/files/new/test-ignore-hint.pdf new file mode 100644 index 00000000..f63b18ae Binary files /dev/null and b/redaction-service-v1/redaction-service-server-v1/src/test/resources/files/new/test-ignore-hint.pdf differ