From e0fba8d38ce9858235ca84182f4587d472890ef1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dominique=20Eifl=C3=A4nder?= Date: Fri, 12 Feb 2021 10:41:32 +0100 Subject: [PATCH] RED-1055: Enabled to force redact ignore annotations --- .../redaction/v1/model/ManualForceRedact.java | 19 +++++++ .../v1/model/ManualRedactionType.java | 2 +- .../redaction/v1/model/ManualRedactions.java | 3 ++ .../service/RedactionLogCreatorService.java | 30 ++++++++++++ .../v1/server/RedactionIntegrationTest.java | 49 ++++++++++++------- 5 files changed, 83 insertions(+), 20 deletions(-) create mode 100644 redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/ManualForceRedact.java diff --git a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/ManualForceRedact.java b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/ManualForceRedact.java new file mode 100644 index 00000000..3af345dd --- /dev/null +++ b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/ManualForceRedact.java @@ -0,0 +1,19 @@ +package com.iqser.red.service.redaction.v1.model; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@Builder +@AllArgsConstructor +@NoArgsConstructor +public class ManualForceRedact { + + private String id; + private String user; + private Status status; + private String legalBasis; + +} \ No newline at end of file diff --git a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/ManualRedactionType.java b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/ManualRedactionType.java index dfc44616..83df7d67 100644 --- a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/ManualRedactionType.java +++ b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/ManualRedactionType.java @@ -1,5 +1,5 @@ package com.iqser.red.service.redaction.v1.model; public enum ManualRedactionType { - ADD, REMOVE + ADD, REMOVE, FORCE_REDACT } diff --git a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/ManualRedactions.java b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/ManualRedactions.java index 9e602a79..ee6e0ad1 100644 --- a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/ManualRedactions.java +++ b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/ManualRedactions.java @@ -20,6 +20,9 @@ public class ManualRedactions { @Builder.Default private Set idsToRemove = new HashSet<>(); + @Builder.Default + private Set forceRedacts = new HashSet<>(); + @Builder.Default private Set entriesToAdd = new HashSet<>(); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java index 19e61ab6..3ff5e131 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java @@ -14,6 +14,7 @@ import org.springframework.stereotype.Service; import com.iqser.red.service.redaction.v1.model.CellRectangle; import com.iqser.red.service.redaction.v1.model.Comment; import com.iqser.red.service.redaction.v1.model.IdRemoval; +import com.iqser.red.service.redaction.v1.model.ManualForceRedact; import com.iqser.red.service.redaction.v1.model.ManualRedactionEntry; import com.iqser.red.service.redaction.v1.model.ManualRedactionType; import com.iqser.red.service.redaction.v1.model.ManualRedactions; @@ -136,9 +137,11 @@ public class RedactionLogCreatorService { redactionLogEntry.setRedacted(false); redactionLogEntry.setStatus(Status.APPROVED); manualOverrideReason = entity.getRedactionReason() + ", removed by manual override"; + redactionLogEntry.setColor(getColor(entity, ruleSetId, false)); } else if (manualRemoval.getStatus().equals(Status.REQUESTED)) { manualOverrideReason = entity.getRedactionReason() + ", requested to remove"; redactionLogEntry.setStatus(Status.REQUESTED); + redactionLogEntry.setColor(getColor(entity, ruleSetId, true)); } else { redactionLogEntry.setStatus(Status.DECLINED); } @@ -149,7 +152,34 @@ public class RedactionLogCreatorService { redactionLogEntry.setManualRedactionType(ManualRedactionType.REMOVE); } } + } + if (manualRedactions != null && !manualRedactions.getForceRedacts().isEmpty()) { + for (ManualForceRedact manualForceRedact : manualRedactions.getForceRedacts()) { + if (manualForceRedact.getId().equals(entityPositionSequence.getId())) { + String manualOverrideReason = null; + if (manualForceRedact.getStatus().equals(Status.APPROVED)) { + entity.setRedaction(true); + redactionLogEntry.setRedacted(true); + redactionLogEntry.setStatus(Status.APPROVED); + redactionLogEntry.setColor(getColor(entity, ruleSetId, false)); + manualOverrideReason = entity.getRedactionReason() + ", forced by manual override"; + redactionLogEntry.setLegalBasis(manualForceRedact.getLegalBasis()); + } else if (manualForceRedact.getStatus().equals(Status.REQUESTED)) { + manualOverrideReason = entity.getRedactionReason() + ", requested to force redact"; + redactionLogEntry.setStatus(Status.REQUESTED); + redactionLogEntry.setColor(getColor(entity, ruleSetId, true)); + redactionLogEntry.setLegalBasis(manualForceRedact.getLegalBasis()); + } else { + redactionLogEntry.setStatus(Status.DECLINED); + } + + entity.setRedactionReason(manualOverrideReason != null ? manualOverrideReason : entity.getRedactionReason()); + redactionLogEntry.setReason(manualOverrideReason); + redactionLogEntry.setManual(true); + redactionLogEntry.setManualRedactionType(ManualRedactionType.FORCE_REDACT); + } + } } if (CollectionUtils.isNotEmpty(entityPositionSequence.getSequences())) { diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java index 75a60c07..f025dbf1 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java @@ -51,6 +51,7 @@ import com.iqser.red.service.redaction.v1.model.AnnotateRequest; import com.iqser.red.service.redaction.v1.model.AnnotateResponse; import com.iqser.red.service.redaction.v1.model.Comment; import com.iqser.red.service.redaction.v1.model.IdRemoval; +import com.iqser.red.service.redaction.v1.model.ManualForceRedact; import com.iqser.red.service.redaction.v1.model.ManualRedactionEntry; import com.iqser.red.service.redaction.v1.model.ManualRedactions; import com.iqser.red.service.redaction.v1.model.Point; @@ -324,7 +325,6 @@ public class RedactionIntegrationTest { recommendationTypeMap.put(PURITY, false); recommendationTypeMap.put(IMAGE, false); - rankTypeMap.put(FALSE_POSITIVE, 160); rankTypeMap.put(PURITY, 155); rankTypeMap.put(PII, 150); @@ -342,7 +342,6 @@ public class RedactionIntegrationTest { rankTypeMap.put(RECOMMENDATION_ADDRESS, 30); rankTypeMap.put(IMAGE, 30); - colors.setDefaultColor("#acfc00"); colors.setNotRedacted("#cccccc"); colors.setRequestAdd("#04b093"); @@ -418,7 +417,6 @@ public class RedactionIntegrationTest { } - private List getPathsRecursively(File path) { List result = new ArrayList<>(); @@ -446,7 +444,6 @@ public class RedactionIntegrationTest { // 91 Trinexapac-ethyl_RAR_01_Volume_1_2018-02-23.pdf // 95 Trinexapac-ethyl_RAR_08_Volume_3CA_B-6_2018-01-10.pdf - System.out.println("redactionTest"); long start = System.currentTimeMillis(); ClassPathResource pdfFileResource = new ClassPathResource("files/Cyprodinil/49 Cyprodinil - EU AIR3 - MCA Section 8 Supplement - Ecotoxicological studies on the active substance.pdf"); @@ -456,11 +453,13 @@ public class RedactionIntegrationTest { .document(IOUtils.toByteArray(pdfFileResource.getInputStream())) .build(); - AnalyzeResult result = redactionController.analyze(request); - AnnotateResponse annotateResponse = redactionController.annotate(AnnotateRequest.builder().document(IOUtils.toByteArray(pdfFileResource.getInputStream())).redactionLog(result.getRedactionLog()).sectionGrid(result.getSectionGrid()).build()); - + AnnotateResponse annotateResponse = redactionController.annotate(AnnotateRequest.builder() + .document(IOUtils.toByteArray(pdfFileResource.getInputStream())) + .redactionLog(result.getRedactionLog()) + .sectionGrid(result.getSectionGrid()) + .build()); try (FileOutputStream fileOutputStream = new FileOutputStream("/tmp/Annotated.pdf")) { fileOutputStream.write(annotateResponse.getDocument()); @@ -485,11 +484,13 @@ public class RedactionIntegrationTest { .document(IOUtils.toByteArray(pdfFileResource.getInputStream())) .build(); - AnalyzeResult result = redactionController.analyze(request); - AnnotateResponse annotateResponse = redactionController.annotate(AnnotateRequest.builder().document(IOUtils.toByteArray(pdfFileResource.getInputStream())).redactionLog(result.getRedactionLog()).sectionGrid(result.getSectionGrid()).build()); - + AnnotateResponse annotateResponse = redactionController.annotate(AnnotateRequest.builder() + .document(IOUtils.toByteArray(pdfFileResource.getInputStream())) + .redactionLog(result.getRedactionLog()) + .sectionGrid(result.getSectionGrid()) + .build()); try (FileOutputStream fileOutputStream = new FileOutputStream("/tmp/Annotated.pdf")) { fileOutputStream.write(annotateResponse.getDocument()); @@ -504,6 +505,8 @@ public class RedactionIntegrationTest { @Test public void testManualRedaction() throws IOException { +// 675eba69b0c2917de55462c817adaa05 + System.out.println("testManualRedaction"); long start = System.currentTimeMillis(); ClassPathResource pdfFileResource = new ClassPathResource("files/Minimal Examples/Single Table.pdf"); @@ -518,8 +521,13 @@ public class RedactionIntegrationTest { .text("This is a comment test") .build(); manualRedactions.setIdsToRemove(Set.of(IdRemoval.builder() - .id("0836727c3508a0b2ea271da69c04cc2f") - .status(Status.REQUESTED) + .id("5b940b2cb401ed9f5be6fc24f6e77bcf") + .status(Status.DECLINED) + .build())); + manualRedactions.setForceRedacts(Set.of(ManualForceRedact.builder() + .id("675eba69b0c2917de55462c817adaa05") + .legalBasis("Something") + .status(Status.APPROVED) .build())); manualRedactions.getComments().put("e5be0f1d941bbb92a068e198648d06c4", List.of(comment)); @@ -542,11 +550,13 @@ public class RedactionIntegrationTest { .manualRedactions(manualRedactions) .build(); - AnalyzeResult result = redactionController.analyze(request); - AnnotateResponse annotateResponse = redactionController.annotate(AnnotateRequest.builder().document(IOUtils.toByteArray(pdfFileResource.getInputStream())).redactionLog(result.getRedactionLog()).sectionGrid(result.getSectionGrid()).build()); - + AnnotateResponse annotateResponse = redactionController.annotate(AnnotateRequest.builder() + .document(IOUtils.toByteArray(pdfFileResource.getInputStream())) + .redactionLog(result.getRedactionLog()) + .sectionGrid(result.getSectionGrid()) + .build()); try (FileOutputStream fileOutputStream = new FileOutputStream("/tmp/Annotated.pdf")) { fileOutputStream.write(annotateResponse.getDocument()); @@ -640,7 +650,6 @@ public class RedactionIntegrationTest { .document(IOUtils.toByteArray(pdfFileResource.getInputStream())) .build(); - AnalyzeResult result = redactionController.analyze(request); result.getRedactionLog().getRedactionLogEntry().forEach(entry -> { @@ -662,11 +671,13 @@ public class RedactionIntegrationTest { .document(IOUtils.toByteArray(pdfFileResource.getInputStream())) .build(); - AnalyzeResult result = redactionController.analyze(request); - AnnotateResponse annotateResponse = redactionController.annotate(AnnotateRequest.builder().document(IOUtils.toByteArray(pdfFileResource.getInputStream())).redactionLog(result.getRedactionLog()).sectionGrid(result.getSectionGrid()).build()); - + AnnotateResponse annotateResponse = redactionController.annotate(AnnotateRequest.builder() + .document(IOUtils.toByteArray(pdfFileResource.getInputStream())) + .redactionLog(result.getRedactionLog()) + .sectionGrid(result.getSectionGrid()) + .build()); try (FileOutputStream fileOutputStream = new FileOutputStream("/tmp/Annotated.pdf")) { fileOutputStream.write(annotateResponse.getDocument());