Marmelator/redaction-service
1
0
Fork 0
Code Issues Pull Requests 14 Actions Packages Projects Releases 622 Wiki Activity

RED-2429: As a user I want to resize a redaction

Browse Source
This commit is contained in:
aoezyetimoglu 2021-10-22 12:05:31 +02:00
parent 0cc71897b8
commit d8f8702b2a
6 changed files with 209 additions and 61 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
redaction-service-v1/redaction-service-api-v1/pom.xml
View File

@ -20,7 +20,7 @@
<dependency>
<groupId>com.iqser.red.service</groupId>
<artifactId>persistence-service-api-v1</artifactId>
<version>0.44.0</version>
<version>0.77.0</version>
</dependency>
</dependencies>
</project>

3
redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/AnnotateRequest.java
View File

@ -1,5 +1,7 @@
package com.iqser.red.service.redaction.v1.model;
import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
@ -14,4 +16,5 @@ public class AnnotateRequest {
private String dossierId;
private String dossierTemplateId;
private String fileId;
private ManualRedactions manualRedactions;
}

2
redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/ManualRedactionType.java
View File

@ -1,5 +1,5 @@
package com.iqser.red.service.redaction.v1.model;
public enum ManualRedactionType {
ADD, REMOVE, FORCE_REDACT, RECATEGORIZE, LEGAL_BASIS_CHANGE
ADD, REMOVE, FORCE_REDACT, RECATEGORIZE, LEGAL_BASIS_CHANGE, RESIZE
}

4
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RedactionController.java
View File

@ -42,14 +42,14 @@ public class RedactionController implements RedactionResource {
public AnnotateResponse annotate(@RequestBody AnnotateRequest annotateRequest) {
var storedObjectStream = redactionStorageService.getStoredObject(RedactionStorageService.StorageIdUtils.getStorageId(annotateRequest.getDossierId(), annotateRequest.getFileId(), FileType.ORIGIN));
var redactionLog = redactionStorageService.getRedactionLog(annotateRequest.getDossierId(), annotateRequest.getFileId());
var mergedRedactionLog = getRedactionLog(RedactionRequest.builder().fileId(annotateRequest.getFileId()).manualRedactions(annotateRequest.getManualRedactions()).dossierId(annotateRequest.getDossierId()).dossierTemplateId(annotateRequest.getDossierTemplateId()).build());
var sectionsGrid = redactionStorageService.getSectionGrid(annotateRequest.getDossierId(), annotateRequest.getFileId());
try (PDDocument pdDocument = PDDocument.load(storedObjectStream, MemoryUsageSetting.setupTempFileOnly())) {
pdDocument.setAllSecurityToBeRemoved(true);
dictionaryService.updateDictionary(annotateRequest.getDossierTemplateId(), annotateRequest.getDossierId());
annotationService.annotate(pdDocument, redactionLog, sectionsGrid);
annotationService.annotate(pdDocument, mergedRedactionLog, sectionsGrid);
try (ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream()) {
pdDocument.save(byteArrayOutputStream);

33
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogMergeService.java
View File

@ -94,6 +94,12 @@ public class RedactionLogMergeService {
}
});
manualRedactions.getResizeRedactions().forEach(item -> {
if (item.getSoftDeletedTime() == null) {
manualRedactionWrappers.add(new ManualRedactionWrapper(item.getAnnotationId(), item.getRequestDate(), item));
}
});
Collections.sort(manualRedactionWrappers);
return manualRedactionWrappers;
@ -204,6 +210,33 @@ public class RedactionLogMergeService {
redactionLogEntry.setManualRedactionType(ManualRedactionType.LEGAL_BASIS_CHANGE);
}
if (mrw.getItem() instanceof ManualResizeRedaction) {
var manualResizeRedact = (ManualResizeRedaction) mrw.getItem();
String manualOverrideReason = null;
if (manualResizeRedact.getStatus().equals(AnnotationStatus.APPROVED)) {
redactionLogEntry.setStatus(AnnotationStatus.APPROVED);
redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, false, redactionLogEntry
.isRedacted(), false));
redactionLogEntry.setPositions(convertPositions(manualResizeRedact.getPositions()));
redactionLogEntry.setValue(manualResizeRedact.getValue());
manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", resized by manual override");
} else if (manualResizeRedact.getStatus().equals(AnnotationStatus.REQUESTED)) {
manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", requested to resize redact");
redactionLogEntry.setStatus(AnnotationStatus.REQUESTED);
redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, true, redactionLogEntry
.isRedacted(), false));
redactionLogEntry.setPositions(convertPositions(manualResizeRedact.getPositions()));
} else {
redactionLogEntry.setStatus(AnnotationStatus.DECLINED);
}
redactionLogEntry.setManualRedactionUserId(manualResizeRedact.getUser());
redactionLogEntry.setReason(manualOverrideReason);
redactionLogEntry.setManual(true);
redactionLogEntry.setManualRedactionType(ManualRedactionType.RESIZE);
}
});
}

226
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java
View File

@ -23,7 +23,9 @@ import com.iqser.red.service.redaction.v1.server.redaction.utils.TextNormalizati
import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
import com.iqser.red.storage.commons.StorageAutoConfiguration;
import com.iqser.red.storage.commons.service.StorageService;
import lombok.SneakyThrows;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.StringUtils;
import org.junit.After;
@ -201,42 +203,26 @@ public class RedactionIntegrationTest {
.rank(rankTypeMap.get(DOSSIER_REDACTIONS))
.build()));
when(dictionaryClient.getDictionaryForType(VERTEBRATE + ":" + TEST_DOSSIER_TEMPLATE_ID))
.thenReturn(getDictionaryResponse(VERTEBRATE, false));
when(dictionaryClient.getDictionaryForType(ADDRESS + ":" + TEST_DOSSIER_TEMPLATE_ID))
.thenReturn(getDictionaryResponse(ADDRESS, false));
when(dictionaryClient.getDictionaryForType(AUTHOR + ":" + TEST_DOSSIER_TEMPLATE_ID))
.thenReturn(getDictionaryResponse(AUTHOR, false));
when(dictionaryClient.getDictionaryForType(SPONSOR + ":" + TEST_DOSSIER_TEMPLATE_ID))
.thenReturn(getDictionaryResponse(SPONSOR, false));
when(dictionaryClient.getDictionaryForType(NO_REDACTION_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID))
.thenReturn(getDictionaryResponse(NO_REDACTION_INDICATOR, false));
when(dictionaryClient.getDictionaryForType(REDACTION_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID))
.thenReturn(getDictionaryResponse(REDACTION_INDICATOR, false));
when(dictionaryClient.getDictionaryForType(HINT_ONLY + ":" + TEST_DOSSIER_TEMPLATE_ID))
.thenReturn(getDictionaryResponse(HINT_ONLY, false));
when(dictionaryClient.getDictionaryForType(MUST_REDACT + ":" + TEST_DOSSIER_TEMPLATE_ID))
.thenReturn(getDictionaryResponse(MUST_REDACT, false));
when(dictionaryClient.getDictionaryForType(PUBLISHED_INFORMATION + ":" + TEST_DOSSIER_TEMPLATE_ID))
.thenReturn(getDictionaryResponse(PUBLISHED_INFORMATION, false));
when(dictionaryClient.getDictionaryForType(TEST_METHOD + ":" + TEST_DOSSIER_TEMPLATE_ID))
.thenReturn(getDictionaryResponse(TEST_METHOD, false));
when(dictionaryClient.getDictionaryForType(VERTEBRATE + ":" + TEST_DOSSIER_TEMPLATE_ID)).thenReturn(getDictionaryResponse(VERTEBRATE, false));
when(dictionaryClient.getDictionaryForType(ADDRESS + ":" + TEST_DOSSIER_TEMPLATE_ID)).thenReturn(getDictionaryResponse(ADDRESS, false));
when(dictionaryClient.getDictionaryForType(AUTHOR + ":" + TEST_DOSSIER_TEMPLATE_ID)).thenReturn(getDictionaryResponse(AUTHOR, false));
when(dictionaryClient.getDictionaryForType(SPONSOR + ":" + TEST_DOSSIER_TEMPLATE_ID)).thenReturn(getDictionaryResponse(SPONSOR, false));
when(dictionaryClient.getDictionaryForType(NO_REDACTION_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID)).thenReturn(getDictionaryResponse(NO_REDACTION_INDICATOR, false));
when(dictionaryClient.getDictionaryForType(REDACTION_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID)).thenReturn(getDictionaryResponse(REDACTION_INDICATOR, false));
when(dictionaryClient.getDictionaryForType(HINT_ONLY + ":" + TEST_DOSSIER_TEMPLATE_ID)).thenReturn(getDictionaryResponse(HINT_ONLY, false));
when(dictionaryClient.getDictionaryForType(MUST_REDACT + ":" + TEST_DOSSIER_TEMPLATE_ID)).thenReturn(getDictionaryResponse(MUST_REDACT, false));
when(dictionaryClient.getDictionaryForType(PUBLISHED_INFORMATION + ":" + TEST_DOSSIER_TEMPLATE_ID)).thenReturn(getDictionaryResponse(PUBLISHED_INFORMATION, false));
when(dictionaryClient.getDictionaryForType(TEST_METHOD + ":" + TEST_DOSSIER_TEMPLATE_ID)).thenReturn(getDictionaryResponse(TEST_METHOD, false));
when(dictionaryClient.getDictionaryForType(PII + ":" + TEST_DOSSIER_TEMPLATE_ID)).thenReturn(getDictionaryResponse(PII, false));
when(dictionaryClient.getDictionaryForType(RECOMMENDATION_AUTHOR + ":" + TEST_DOSSIER_TEMPLATE_ID))
.thenReturn(getDictionaryResponse(RECOMMENDATION_AUTHOR, false));
when(dictionaryClient.getDictionaryForType(RECOMMENDATION_ADDRESS + ":" + TEST_DOSSIER_TEMPLATE_ID))
.thenReturn(getDictionaryResponse(RECOMMENDATION_ADDRESS, false));
when(dictionaryClient.getDictionaryForType(FALSE_POSITIVE + ":" + TEST_DOSSIER_TEMPLATE_ID))
.thenReturn(getDictionaryResponse(FALSE_POSITIVE, false));
when(dictionaryClient.getDictionaryForType(PURITY + ":" + TEST_DOSSIER_TEMPLATE_ID))
.thenReturn(getDictionaryResponse(PURITY, false));
when(dictionaryClient.getDictionaryForType(RECOMMENDATION_AUTHOR + ":" + TEST_DOSSIER_TEMPLATE_ID)).thenReturn(getDictionaryResponse(RECOMMENDATION_AUTHOR, false));
when(dictionaryClient.getDictionaryForType(RECOMMENDATION_ADDRESS + ":" + TEST_DOSSIER_TEMPLATE_ID)).thenReturn(getDictionaryResponse(RECOMMENDATION_ADDRESS, false));
when(dictionaryClient.getDictionaryForType(FALSE_POSITIVE + ":" + TEST_DOSSIER_TEMPLATE_ID)).thenReturn(getDictionaryResponse(FALSE_POSITIVE, false));
when(dictionaryClient.getDictionaryForType(PURITY + ":" + TEST_DOSSIER_TEMPLATE_ID)).thenReturn(getDictionaryResponse(PURITY, false));
when(dictionaryClient.getDictionaryForType(IMAGE + ":" + TEST_DOSSIER_TEMPLATE_ID)).thenReturn(getDictionaryResponse(IMAGE, false));
when(dictionaryClient.getDictionaryForType(OCR + ":" + TEST_DOSSIER_TEMPLATE_ID)).thenReturn(getDictionaryResponse(OCR, false));
when(dictionaryClient.getDictionaryForType(LOGO + ":" + TEST_DOSSIER_TEMPLATE_ID)).thenReturn(getDictionaryResponse(LOGO, false));
when(dictionaryClient.getDictionaryForType(SIGNATURE + ":" + TEST_DOSSIER_TEMPLATE_ID))
.thenReturn(getDictionaryResponse(SIGNATURE, false));
when(dictionaryClient.getDictionaryForType(FORMULA + ":" + TEST_DOSSIER_TEMPLATE_ID))
.thenReturn(getDictionaryResponse(FORMULA, false));
when(dictionaryClient.getDictionaryForType(SIGNATURE + ":" + TEST_DOSSIER_TEMPLATE_ID)).thenReturn(getDictionaryResponse(SIGNATURE, false));
when(dictionaryClient.getDictionaryForType(FORMULA + ":" + TEST_DOSSIER_TEMPLATE_ID)).thenReturn(getDictionaryResponse(FORMULA, false));
when(dictionaryClient.getDictionaryForType(DOSSIER_REDACTIONS + ":" + TEST_DOSSIER_TEMPLATE_ID)).thenReturn(getDictionaryResponse(DOSSIER_REDACTIONS, true));
when(dictionaryClient.getColors(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(colors);
}
@ -500,8 +486,7 @@ public class RedactionIntegrationTest {
return Type.builder()
.id(type + ":" + TEST_DOSSIER_TEMPLATE_ID)
.hexColor(typeColorMap.get(type))
.entries(isDossierDictionary ? toDictionaryEntry(dossierDictionary.get(type)) : toDictionaryEntry(dictionary
.get(type)))
.entries(isDossierDictionary ? toDictionaryEntry(dossierDictionary.get(type)) : toDictionaryEntry(dictionary.get(type)))
.isHint(hintTypeMap.get(type))
.isCaseInsensitive(caseInSensitiveMap.get(type))
.isRecommendation(recommendationTypeMap.get(type))
@ -517,7 +502,8 @@ public class RedactionIntegrationTest {
dictionaryEntries.add(DictionaryEntry.builder()
.value(entry)
.version(reanlysisVersions.getOrDefault(entry, 0L))
.deleted(deleted.contains(entry)).build());
.deleted(deleted.contains(entry))
.build());
});
return dictionaryEntries;
}
@ -729,11 +715,9 @@ public class RedactionIntegrationTest {
when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(3L);
when(dictionaryClient.getDictionaryForType(VERTEBRATE))
.thenReturn(getDictionaryResponse(VERTEBRATE, false));
when(dictionaryClient.getDictionaryForType(VERTEBRATE)).thenReturn(getDictionaryResponse(VERTEBRATE, false));
when(dictionaryClient.getDictionaryForType(FALSE_POSITIVE))
.thenReturn(getDictionaryResponse(FALSE_POSITIVE, false));
when(dictionaryClient.getDictionaryForType(FALSE_POSITIVE)).thenReturn(getDictionaryResponse(FALSE_POSITIVE, false));
start = System.currentTimeMillis();
@ -764,14 +748,12 @@ public class RedactionIntegrationTest {
fileOutputStream.write(annotateResponse.getDocument());
}
deleted.remove("mouse");
reanlysisVersions.put("mouse", 4L);
when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(4L);
when(dictionaryClient.getDictionaryForType(VERTEBRATE))
.thenReturn(getDictionaryResponse(VERTEBRATE, false));
when(dictionaryClient.getDictionaryForType(VERTEBRATE)).thenReturn(getDictionaryResponse(VERTEBRATE, false));
analyzeService.reanalyze(request);
@ -847,21 +829,19 @@ public class RedactionIntegrationTest {
manualRedactionEntry.setType("name");
manualRedactionEntry.setValue("O'Loughlin C.K.");
manualRedactionEntry.setReason("Manual Redaction");
manualRedactionEntry.setPositions(List.of(Rectangle
.builder()
.topLeftX(375.61096f)
.topLeftY(241.282f)
.width(7.648041f)
.height(43.72262f)
.page(1).build()
, Rectangle
.builder()
.topLeftX(384.83517f)
.topLeftY(241.282f)
.width(7.648041f)
.height(17.043358f)
.page(1).build())
);
manualRedactionEntry.setPositions(List.of(Rectangle.builder()
.topLeftX(375.61096f)
.topLeftY(241.282f)
.width(7.648041f)
.height(43.72262f)
.page(1)
.build(), Rectangle.builder()
.topLeftX(384.83517f)
.topLeftY(241.282f)
.width(7.648041f)
.height(17.043358f)
.page(1)
.build()));
// manualRedactions.getEntriesToAdd().add(manualRedactionEntry);
@ -1064,6 +1044,136 @@ public class RedactionIntegrationTest {
}
@Test
public void resizeRedactionTest() throws IOException {
ClassPathResource pdfFileResource = new ClassPathResource("files/Minimal Examples/Single Table.pdf");
ManualRedactions manualRedactions = new ManualRedactions();
String manualAddId = UUID.randomUUID().toString();
Comment comment = Comment.builder()
.date(OffsetDateTime.now())
.user("TEST_USER")
.text("This is a comment test")
.build();
manualRedactions.setIdsToRemove(Set.of(IdRemoval.builder()
.annotationId("5b940b2cb401ed9f5be6fc24f6e77bcf")
.fileId("fileId")
.status(AnnotationStatus.DECLINED)
.processedDate(OffsetDateTime.now())
.requestDate(OffsetDateTime.now())
.build()));
manualRedactions.setForceRedactions(Set.of(ManualForceRedaction.builder()
.annotationId("675eba69b0c2917de55462c817adaa05")
.fileId("fileId")
.legalBasis("Something")
.status(AnnotationStatus.APPROVED)
.requestDate(OffsetDateTime.now())
.processedDate(OffsetDateTime.now())
.build()));
manualRedactions.getComments().put("e5be0f1d941bbb92a068e198648d06c4", List.of(comment));
manualRedactions.getComments().put("0836727c3508a0b2ea271da69c04cc2f", List.of(comment));
manualRedactions.getComments().put(manualAddId, List.of(comment));
ManualRedactionEntry manualRedactionEntry = new ManualRedactionEntry();
manualRedactionEntry.setAnnotationId(manualAddId);
manualRedactionEntry.setFileId("fileId");
manualRedactionEntry.setStatus(AnnotationStatus.APPROVED);
manualRedactionEntry.setType("CBI_author");
manualRedactionEntry.setValue("O'Loughlin C.K.");
manualRedactionEntry.setReason("Manual Redaction");
manualRedactionEntry.setProcessedDate(OffsetDateTime.now());
manualRedactionEntry.setRequestDate(OffsetDateTime.now());
manualRedactionEntry.setPositions(List.of(Rectangle.builder()
.topLeftX(375.61096f)
.topLeftY(241.282f)
.width(7.648041f)
.height(43.72262f)
.page(1)
.build(), Rectangle.builder()
.topLeftX(384.83517f)
.topLeftY(241.282f)
.width(7.648041f)
.height(17.043358f)
.page(1)
.build()));
// manualRedactions.getEntriesToAdd().add(manualRedactionEntry);
AnalyzeRequest request = prepareStorage(pdfFileResource.getInputStream());
request.setManualRedactions(manualRedactions);
analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
AnalyzeResult result = analyzeService.analyze(request);
manualRedactions.getEntriesToAdd().add(manualRedactionEntry);
manualRedactions.setIdsToRemove(Set.of(IdRemoval.builder()
.annotationId("5b940b2cb401ed9f5be6fc24f6e77bcf")
.fileId("fileId")
.status(AnnotationStatus.APPROVED)
.requestDate(OffsetDateTime.now())
.processedDate(OffsetDateTime.now())
.build()));
manualRedactions.setLegalBasisChanges((Set.of(ManualLegalBasisChange.builder()
.annotationId("675eba69b0c2917de55462c817adaa05")
.fileId("fileId")
.legalBasis("Manual Legal Basis Change")
.status(AnnotationStatus.APPROVED)
.processedDate(OffsetDateTime.now())
.requestDate(OffsetDateTime.now())
.build())));
manualRedactions.setResizeRedactions(Set.of(ManualResizeRedaction.builder()
.annotationId("fc287b74be2421156ab2895c7474ccdd")
.fileId("fileId")
.processedDate(OffsetDateTime.now())
.requestDate(OffsetDateTime.now())
.value("Syngenta Crop Protection AG, Basel, Switzerland RCC Ltd., Itingen, Switzerland")
.positions(List.of(Rectangle.builder()
.topLeftX(289.44595f)
.topLeftY(327.567f)
.width(7.648041f)
.height(82.51475f)
.page(1)
.build(), Rectangle.builder()
.topLeftX(298.67056f)
.topLeftY(327.567f)
.width(7.648041f)
.height(75.32377f)
.page(1)
.build(), Rectangle.builder()
.topLeftX(307.89517f)
.topLeftY(327.567f)
.width(7.648041f)
.height(61.670967f)
.page(1)
.build(), Rectangle.builder()
.topLeftX(316.99985f)
.topLeftY(327.567f)
.width(7.648041f)
.height(38.104286f)
.page(1)
.build()))
.status(AnnotationStatus.APPROVED)
.build()));
analyzeService.reanalyze(request);
AnnotateResponse annotateResponse = redactionController.annotate(AnnotateRequest.builder()
.dossierId(TEST_DOSSIER_ID)
.fileId(TEST_FILE_ID)
.dossierTemplateId(TEST_DOSSIER_TEMPLATE_ID)
.manualRedactions(manualRedactions)
.build());
try (FileOutputStream fileOutputStream = new FileOutputStream(getTemporaryDirectory() + "/Annotated.pdf")) {
fileOutputStream.write(annotateResponse.getDocument());
}
}
private static String loadFromClassPath(String path) {
URL resource = ResourceLoader.class.getClassLoader().getResource(path);
@ -1082,7 +1192,9 @@ public class RedactionIntegrationTest {
}
}
private static String getTemporaryDirectory() {
String tmpdir = System.getProperty("java.io.tmpdir");
if (StringUtils.isNotBlank(tmpdir)) {
return tmpdir;