Marmelator/redaction-service
1
0
Fork 0
Code Issues Pull Requests 14 Actions Packages Projects Releases 622 Wiki Activity

Pull request #122: RED-1055: Enabled to force redact ignore annotations

Browse Source 
Merge in RED/redaction-service from RED-1055 to master

* commit 'e0fba8d38ce9858235ca84182f4587d472890ef1':
  RED-1055: Enabled to force redact ignore annotations
This commit is contained in:
Dominique Eiflaender 2021-02-12 10:55:04 +01:00
commit 53509f072e
5 changed files with 83 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/ManualForceRedact.java Normal file
View File

@ -0,0 +1,19 @@
package com.iqser.red.service.redaction.v1.model;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@Builder
@AllArgsConstructor
@NoArgsConstructor
public class ManualForceRedact {
private String id;
private String user;
private Status status;
private String legalBasis;
}

2
redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/ManualRedactionType.java
View File

@ -1,5 +1,5 @@
package com.iqser.red.service.redaction.v1.model;
public enum ManualRedactionType {
ADD, REMOVE
ADD, REMOVE, FORCE_REDACT
}

3
redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/ManualRedactions.java
View File

@ -20,6 +20,9 @@ public class ManualRedactions {
@Builder.Default
private Set<IdRemoval> idsToRemove = new HashSet<>();
@Builder.Default
private Set<ManualForceRedact> forceRedacts = new HashSet<>();
@Builder.Default
private Set<ManualRedactionEntry> entriesToAdd = new HashSet<>();

30
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java
View File

@ -14,6 +14,7 @@ import org.springframework.stereotype.Service;
import com.iqser.red.service.redaction.v1.model.CellRectangle;
import com.iqser.red.service.redaction.v1.model.Comment;
import com.iqser.red.service.redaction.v1.model.IdRemoval;
import com.iqser.red.service.redaction.v1.model.ManualForceRedact;
import com.iqser.red.service.redaction.v1.model.ManualRedactionEntry;
import com.iqser.red.service.redaction.v1.model.ManualRedactionType;
import com.iqser.red.service.redaction.v1.model.ManualRedactions;
@ -136,9 +137,11 @@ public class RedactionLogCreatorService {
redactionLogEntry.setRedacted(false);
redactionLogEntry.setStatus(Status.APPROVED);
manualOverrideReason = entity.getRedactionReason() + ", removed by manual override";
redactionLogEntry.setColor(getColor(entity, ruleSetId, false));
} else if (manualRemoval.getStatus().equals(Status.REQUESTED)) {
manualOverrideReason = entity.getRedactionReason() + ", requested to remove";
redactionLogEntry.setStatus(Status.REQUESTED);
redactionLogEntry.setColor(getColor(entity, ruleSetId, true));
} else {
redactionLogEntry.setStatus(Status.DECLINED);
}
@ -149,7 +152,34 @@ public class RedactionLogCreatorService {
redactionLogEntry.setManualRedactionType(ManualRedactionType.REMOVE);
}
}
}
if (manualRedactions != null && !manualRedactions.getForceRedacts().isEmpty()) {
for (ManualForceRedact manualForceRedact : manualRedactions.getForceRedacts()) {
if (manualForceRedact.getId().equals(entityPositionSequence.getId())) {
String manualOverrideReason = null;
if (manualForceRedact.getStatus().equals(Status.APPROVED)) {
entity.setRedaction(true);
redactionLogEntry.setRedacted(true);
redactionLogEntry.setStatus(Status.APPROVED);
redactionLogEntry.setColor(getColor(entity, ruleSetId, false));
manualOverrideReason = entity.getRedactionReason() + ", forced by manual override";
redactionLogEntry.setLegalBasis(manualForceRedact.getLegalBasis());
} else if (manualForceRedact.getStatus().equals(Status.REQUESTED)) {
manualOverrideReason = entity.getRedactionReason() + ", requested to force redact";
redactionLogEntry.setStatus(Status.REQUESTED);
redactionLogEntry.setColor(getColor(entity, ruleSetId, true));
redactionLogEntry.setLegalBasis(manualForceRedact.getLegalBasis());
} else {
redactionLogEntry.setStatus(Status.DECLINED);
}
entity.setRedactionReason(manualOverrideReason != null ? manualOverrideReason : entity.getRedactionReason());
redactionLogEntry.setReason(manualOverrideReason);
redactionLogEntry.setManual(true);
redactionLogEntry.setManualRedactionType(ManualRedactionType.FORCE_REDACT);
}
}
}
if (CollectionUtils.isNotEmpty(entityPositionSequence.getSequences())) {

49
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java
View File

@ -51,6 +51,7 @@ import com.iqser.red.service.redaction.v1.model.AnnotateRequest;
import com.iqser.red.service.redaction.v1.model.AnnotateResponse;
import com.iqser.red.service.redaction.v1.model.Comment;
import com.iqser.red.service.redaction.v1.model.IdRemoval;
import com.iqser.red.service.redaction.v1.model.ManualForceRedact;
import com.iqser.red.service.redaction.v1.model.ManualRedactionEntry;
import com.iqser.red.service.redaction.v1.model.ManualRedactions;
import com.iqser.red.service.redaction.v1.model.Point;
@ -324,7 +325,6 @@ public class RedactionIntegrationTest {
recommendationTypeMap.put(PURITY, false);
recommendationTypeMap.put(IMAGE, false);
rankTypeMap.put(FALSE_POSITIVE, 160);
rankTypeMap.put(PURITY, 155);
rankTypeMap.put(PII, 150);
@ -342,7 +342,6 @@ public class RedactionIntegrationTest {
rankTypeMap.put(RECOMMENDATION_ADDRESS, 30);
rankTypeMap.put(IMAGE, 30);
colors.setDefaultColor("#acfc00");
colors.setNotRedacted("#cccccc");
colors.setRequestAdd("#04b093");
@ -418,7 +417,6 @@ public class RedactionIntegrationTest {
}
private List<File> getPathsRecursively(File path) {
List<File> result = new ArrayList<>();
@ -446,7 +444,6 @@ public class RedactionIntegrationTest {
// 91 Trinexapac-ethyl_RAR_01_Volume_1_2018-02-23.pdf
// 95 Trinexapac-ethyl_RAR_08_Volume_3CA_B-6_2018-01-10.pdf
System.out.println("redactionTest");
long start = System.currentTimeMillis();
ClassPathResource pdfFileResource = new ClassPathResource("files/Cyprodinil/49 Cyprodinil - EU AIR3 - MCA Section 8 Supplement - Ecotoxicological studies on the active substance.pdf");
@ -456,11 +453,13 @@ public class RedactionIntegrationTest {
.document(IOUtils.toByteArray(pdfFileResource.getInputStream()))
.build();
AnalyzeResult result = redactionController.analyze(request);
AnnotateResponse annotateResponse = redactionController.annotate(AnnotateRequest.builder().document(IOUtils.toByteArray(pdfFileResource.getInputStream())).redactionLog(result.getRedactionLog()).sectionGrid(result.getSectionGrid()).build());
AnnotateResponse annotateResponse = redactionController.annotate(AnnotateRequest.builder()
.document(IOUtils.toByteArray(pdfFileResource.getInputStream()))
.redactionLog(result.getRedactionLog())
.sectionGrid(result.getSectionGrid())
.build());
try (FileOutputStream fileOutputStream = new FileOutputStream("/tmp/Annotated.pdf")) {
fileOutputStream.write(annotateResponse.getDocument());
@ -485,11 +484,13 @@ public class RedactionIntegrationTest {
.document(IOUtils.toByteArray(pdfFileResource.getInputStream()))
.build();
AnalyzeResult result = redactionController.analyze(request);
AnnotateResponse annotateResponse = redactionController.annotate(AnnotateRequest.builder().document(IOUtils.toByteArray(pdfFileResource.getInputStream())).redactionLog(result.getRedactionLog()).sectionGrid(result.getSectionGrid()).build());
AnnotateResponse annotateResponse = redactionController.annotate(AnnotateRequest.builder()
.document(IOUtils.toByteArray(pdfFileResource.getInputStream()))
.redactionLog(result.getRedactionLog())
.sectionGrid(result.getSectionGrid())
.build());
try (FileOutputStream fileOutputStream = new FileOutputStream("/tmp/Annotated.pdf")) {
fileOutputStream.write(annotateResponse.getDocument());
@ -504,6 +505,8 @@ public class RedactionIntegrationTest {
@Test
public void testManualRedaction() throws IOException {
// 675eba69b0c2917de55462c817adaa05
System.out.println("testManualRedaction");
long start = System.currentTimeMillis();
ClassPathResource pdfFileResource = new ClassPathResource("files/Minimal Examples/Single Table.pdf");
@ -518,8 +521,13 @@ public class RedactionIntegrationTest {
.text("This is a comment test")
.build();
manualRedactions.setIdsToRemove(Set.of(IdRemoval.builder()
.id("0836727c3508a0b2ea271da69c04cc2f")
.status(Status.REQUESTED)
.id("5b940b2cb401ed9f5be6fc24f6e77bcf")
.status(Status.DECLINED)
.build()));
manualRedactions.setForceRedacts(Set.of(ManualForceRedact.builder()
.id("675eba69b0c2917de55462c817adaa05")
.legalBasis("Something")
.status(Status.APPROVED)
.build()));
manualRedactions.getComments().put("e5be0f1d941bbb92a068e198648d06c4", List.of(comment));
@ -542,11 +550,13 @@ public class RedactionIntegrationTest {
.manualRedactions(manualRedactions)
.build();
AnalyzeResult result = redactionController.analyze(request);
AnnotateResponse annotateResponse = redactionController.annotate(AnnotateRequest.builder().document(IOUtils.toByteArray(pdfFileResource.getInputStream())).redactionLog(result.getRedactionLog()).sectionGrid(result.getSectionGrid()).build());
AnnotateResponse annotateResponse = redactionController.annotate(AnnotateRequest.builder()
.document(IOUtils.toByteArray(pdfFileResource.getInputStream()))
.redactionLog(result.getRedactionLog())
.sectionGrid(result.getSectionGrid())
.build());
try (FileOutputStream fileOutputStream = new FileOutputStream("/tmp/Annotated.pdf")) {
fileOutputStream.write(annotateResponse.getDocument());
@ -640,7 +650,6 @@ public class RedactionIntegrationTest {
.document(IOUtils.toByteArray(pdfFileResource.getInputStream()))
.build();
AnalyzeResult result = redactionController.analyze(request);
result.getRedactionLog().getRedactionLogEntry().forEach(entry -> {
@ -662,11 +671,13 @@ public class RedactionIntegrationTest {
.document(IOUtils.toByteArray(pdfFileResource.getInputStream()))
.build();
AnalyzeResult result = redactionController.analyze(request);
AnnotateResponse annotateResponse = redactionController.annotate(AnnotateRequest.builder().document(IOUtils.toByteArray(pdfFileResource.getInputStream())).redactionLog(result.getRedactionLog()).sectionGrid(result.getSectionGrid()).build());
AnnotateResponse annotateResponse = redactionController.annotate(AnnotateRequest.builder()
.document(IOUtils.toByteArray(pdfFileResource.getInputStream()))
.redactionLog(result.getRedactionLog())
.sectionGrid(result.getSectionGrid())
.build());
try (FileOutputStream fileOutputStream = new FileOutputStream("/tmp/Annotated.pdf")) {
fileOutputStream.write(annotateResponse.getDocument());