RED-3239: Protoyped logic for imported redactions
This commit is contained in:
deiflaender
parent
0da2a1ab26
commit
26661fa87b
@ -1,6 +1,8 @@
|
||||
package com.iqser.red.service.redaction.v1.model;
|
||||
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.ImportedAnnotation;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
import lombok.Data;
|
||||
@ -12,7 +14,6 @@ import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
|
||||
|
||||
@Data
|
||||
@Builder
|
||||
@NoArgsConstructor
|
||||
@ -36,5 +37,8 @@ public class AnalyzeRequest {
|
||||
@Builder.Default
|
||||
private List<FileAttribute> fileAttributes = new ArrayList<>();
|
||||
|
||||
@Builder.Default
|
||||
List<ImportedAnnotation> importedAnnotations = new ArrayList<>();
|
||||
|
||||
}
|
||||
|
||||
|
||||
@ -48,6 +48,7 @@ public class AnalyzeService {
|
||||
private final SectionGridCreatorService sectionGridCreatorService;
|
||||
private final NerAnalyserService nerAnalyserService;
|
||||
private final ImageService imageService;
|
||||
private final ImportedRedactionService importedRedactionService;
|
||||
|
||||
|
||||
public void analyzeDocumentStructure(StructureAnalyzeRequest analyzeRequest) {
|
||||
@ -121,6 +122,7 @@ public class AnalyzeService {
|
||||
redactionLogEntries, legalBasis, dictionary.getVersion().getDossierTemplateVersion(), dictionary.getVersion().getDossierVersion(),
|
||||
rulesVersion, legalBasisClient.getVersion(analyzeRequest.getDossierTemplateId()));
|
||||
|
||||
importedRedactionService.processImportedRedactions(redactionLog, analyzeRequest.getImportedAnnotations(), analyzeRequest.getDossierTemplateId());
|
||||
return finalizeAnalysis(analyzeRequest, startTime, redactionLog, text, dictionary.getVersion(), false);
|
||||
}
|
||||
|
||||
@ -170,6 +172,7 @@ public class AnalyzeService {
|
||||
|
||||
redactionLog.getRedactionLogEntry().removeIf(entry -> sectionsToReanalyse.contains(entry.getSectionNumber()));
|
||||
redactionLog.getRedactionLogEntry().addAll(newRedactionLogEntries);
|
||||
importedRedactionService.processImportedRedactions(redactionLog, analyzeRequest.getImportedAnnotations(), analyzeRequest.getDossierTemplateId());
|
||||
return finalizeAnalysis(analyzeRequest, startTime, redactionLog, text, dictionaryIncrement.getDictionaryVersion(), true);
|
||||
}
|
||||
|
||||
|
||||
@ -0,0 +1,98 @@
|
||||
package com.iqser.red.service.redaction.v1.server.redaction.service;
|
||||
|
||||
import java.util.Iterator;
|
||||
import java.util.List;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.ImportedAnnotation;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.ImportedAnnotationStatus;
|
||||
import com.iqser.red.service.redaction.v1.model.Point;
|
||||
import com.iqser.red.service.redaction.v1.model.Rectangle;
|
||||
import com.iqser.red.service.redaction.v1.model.RedactionLog;
|
||||
import com.iqser.red.service.redaction.v1.model.RedactionLogEntry;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
public class ImportedRedactionService {
|
||||
|
||||
private final DictionaryService dictionaryService;
|
||||
|
||||
public void processImportedRedactions(RedactionLog redactionLog, List<ImportedAnnotation> importedAnnotations, String dossierTemplateId) {
|
||||
|
||||
Iterator<RedactionLogEntry> itty = redactionLog.getRedactionLogEntry().iterator();
|
||||
while (itty.hasNext()) {
|
||||
RedactionLogEntry redactionLogEntry = itty.next();
|
||||
boolean mustBeRemoved = false;
|
||||
|
||||
if (redactionLogEntry.getType().equals("imported_redaction")) {
|
||||
itty.remove();
|
||||
continue;
|
||||
}
|
||||
|
||||
loop:
|
||||
for (ImportedAnnotation importedAnnotation : importedAnnotations) {
|
||||
for (com.iqser.red.service.persistence.service.v1.api.model.annotations.Rectangle importedPosition : importedAnnotation.getPositions()) {
|
||||
for (Rectangle rectangle : redactionLogEntry.getPositions()) {
|
||||
if (intersects(importedPosition, rectangle)) {
|
||||
mustBeRemoved = true;
|
||||
break loop;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if (mustBeRemoved) {
|
||||
itty.remove();
|
||||
}
|
||||
}
|
||||
|
||||
for (ImportedAnnotation importedAnnotation : importedAnnotations) {
|
||||
RedactionLogEntry redactionLogEntry = RedactionLogEntry.builder()
|
||||
.id(importedAnnotation.getAnnotationId())
|
||||
.type("imported_redaction")
|
||||
.redacted(importedAnnotation.getStatus().equals(ImportedAnnotationStatus.DECLINED) ? false : true)
|
||||
.positions(importedAnnotation.getPositions()
|
||||
.stream()
|
||||
.map(this::convert)
|
||||
.collect(Collectors.toList()))
|
||||
.color(getColor("imported_redaction", dossierTemplateId, importedAnnotation.getStatus().equals(ImportedAnnotationStatus.DECLINED) ? false : true))
|
||||
.build();
|
||||
|
||||
redactionLog.getRedactionLogEntry().add(redactionLogEntry);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
public Rectangle convert(
|
||||
com.iqser.red.service.persistence.service.v1.api.model.annotations.Rectangle importedPosition) {
|
||||
|
||||
return new Rectangle(new Point(importedPosition.getTopLeftX(), importedPosition.getTopLeftY()), importedPosition.getWidth(), importedPosition.getHeight(), importedPosition.getPage());
|
||||
}
|
||||
|
||||
|
||||
public boolean intersects(
|
||||
com.iqser.red.service.persistence.service.v1.api.model.annotations.Rectangle importedPosition,
|
||||
Rectangle redactionLogPosition) {
|
||||
|
||||
return importedPosition.getPage() == redactionLogPosition.getPage() && redactionLogPosition.getTopLeft()
|
||||
.getX() + redactionLogPosition.getWidth() > importedPosition.getTopLeftX() && redactionLogPosition.getTopLeft()
|
||||
.getY() + redactionLogPosition.getHeight() > importedPosition.getTopLeftY() && redactionLogPosition.getTopLeft()
|
||||
.getX() < importedPosition.getTopLeftX() + importedPosition.getWidth() && redactionLogPosition.getTopLeft()
|
||||
.getY() < importedPosition.getTopLeftY() + importedPosition.getHeight();
|
||||
}
|
||||
|
||||
|
||||
private float[] getColor(String type, String dossierTemplateId, boolean isRedaction) {
|
||||
|
||||
if (!isRedaction) {
|
||||
return dictionaryService.getNotRedactedColor(dossierTemplateId);
|
||||
}
|
||||
return dictionaryService.getColor(type, dossierTemplateId);
|
||||
}
|
||||
|
||||
|
||||
|
||||
}
|
||||
@ -23,6 +23,8 @@ import java.util.Set;
|
||||
import java.util.UUID;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import com.fasterxml.jackson.core.type.TypeReference;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.ImportedAnnotation;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.*;
|
||||
import org.apache.commons.io.IOUtils;
|
||||
import org.junit.After;
|
||||
@ -108,6 +110,7 @@ public class RedactionIntegrationTest {
|
||||
private static final String FORMULA = "formula";
|
||||
private static final String OCR = "ocr";
|
||||
private static final String DOSSIER_REDACTIONS = "dossier_redactions";
|
||||
private static final String IMPORTED_REDACTION = "imported_redaction";
|
||||
|
||||
private static final String RECOMMENDATION_AUTHOR = "recommendation_CBI_author";
|
||||
private static final String RECOMMENDATION_ADDRESS = "recommendation_CBI_address";
|
||||
@ -248,6 +251,7 @@ public class RedactionIntegrationTest {
|
||||
when(dictionaryClient.getDictionaryForType(SIGNATURE + ":" + TEST_DOSSIER_TEMPLATE_ID)).thenReturn(getDictionaryResponse(SIGNATURE, false));
|
||||
when(dictionaryClient.getDictionaryForType(FORMULA + ":" + TEST_DOSSIER_TEMPLATE_ID)).thenReturn(getDictionaryResponse(FORMULA, false));
|
||||
when(dictionaryClient.getDictionaryForType(DOSSIER_REDACTIONS + ":" + TEST_DOSSIER_TEMPLATE_ID)).thenReturn(getDictionaryResponse(DOSSIER_REDACTIONS, true));
|
||||
when(dictionaryClient.getDictionaryForType(IMPORTED_REDACTION + ":" + TEST_DOSSIER_TEMPLATE_ID)).thenReturn(getDictionaryResponse(IMPORTED_REDACTION, true));
|
||||
when(dictionaryClient.getColors(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(colors);
|
||||
}
|
||||
|
||||
@ -359,6 +363,7 @@ public class RedactionIntegrationTest {
|
||||
.stream()
|
||||
.map(this::cleanDictionaryEntry)
|
||||
.collect(Collectors.toSet()));
|
||||
dossierDictionary.put(IMPORTED_REDACTION, new ArrayList<>());
|
||||
}
|
||||
|
||||
|
||||
@ -390,6 +395,7 @@ public class RedactionIntegrationTest {
|
||||
typeColorMap.put(LOGO, "#ffe187");
|
||||
typeColorMap.put(FORMULA, "#ffe187");
|
||||
typeColorMap.put(SIGNATURE, "#ffe187");
|
||||
typeColorMap.put(IMPORTED_REDACTION, "#32a852");
|
||||
|
||||
hintTypeMap.put(VERTEBRATE, true);
|
||||
hintTypeMap.put(ADDRESS, false);
|
||||
@ -412,6 +418,7 @@ public class RedactionIntegrationTest {
|
||||
hintTypeMap.put(LOGO, false);
|
||||
hintTypeMap.put(SIGNATURE, false);
|
||||
hintTypeMap.put(DOSSIER_REDACTIONS, false);
|
||||
hintTypeMap.put(IMPORTED_REDACTION, false);
|
||||
|
||||
caseInSensitiveMap.put(VERTEBRATE, true);
|
||||
caseInSensitiveMap.put(ADDRESS, false);
|
||||
@ -434,6 +441,7 @@ public class RedactionIntegrationTest {
|
||||
caseInSensitiveMap.put(LOGO, true);
|
||||
caseInSensitiveMap.put(FORMULA, true);
|
||||
caseInSensitiveMap.put(DOSSIER_REDACTIONS, false);
|
||||
caseInSensitiveMap.put(IMPORTED_REDACTION, false);
|
||||
|
||||
recommendationTypeMap.put(VERTEBRATE, false);
|
||||
recommendationTypeMap.put(ADDRESS, false);
|
||||
@ -456,6 +464,7 @@ public class RedactionIntegrationTest {
|
||||
recommendationTypeMap.put(SIGNATURE, false);
|
||||
recommendationTypeMap.put(LOGO, false);
|
||||
recommendationTypeMap.put(DOSSIER_REDACTIONS, false);
|
||||
recommendationTypeMap.put(IMPORTED_REDACTION, false);
|
||||
|
||||
rankTypeMap.put(FALSE_POSITIVE, 160);
|
||||
rankTypeMap.put(PURITY, 155);
|
||||
@ -478,6 +487,7 @@ public class RedactionIntegrationTest {
|
||||
rankTypeMap.put(SIGNATURE, 27);
|
||||
rankTypeMap.put(FORMULA, 26);
|
||||
rankTypeMap.put(DOSSIER_REDACTIONS, 200);
|
||||
rankTypeMap.put(IMPORTED_REDACTION, 200);
|
||||
|
||||
colors.setDefaultColor("#acfc00");
|
||||
colors.setNotRedacted("#cccccc");
|
||||
@ -495,7 +505,7 @@ public class RedactionIntegrationTest {
|
||||
.type(typeColor.getKey())
|
||||
.dossierTemplateId(TEST_DOSSIER_TEMPLATE_ID)
|
||||
.hexColor(typeColor.getValue())
|
||||
.isHint(hintTypeMap.get(typeColor.getKey()))
|
||||
.isHint(hintTypeMap.get(typeColor.getKey()))
|
||||
.isCaseInsensitive(caseInSensitiveMap.get(typeColor.getKey()))
|
||||
.isRecommendation(recommendationTypeMap.get(typeColor.getKey()))
|
||||
.rank(rankTypeMap.get(typeColor.getKey()))
|
||||
@ -841,6 +851,7 @@ public class RedactionIntegrationTest {
|
||||
System.out.println("Output file:" + outputFileName);
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
public void testChangeComputation() throws IOException {
|
||||
|
||||
@ -856,24 +867,27 @@ public class RedactionIntegrationTest {
|
||||
dictionary.get(AUTHOR).add("report");
|
||||
reanlysisVersions.put("report", 2L);
|
||||
when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(2L);
|
||||
when(dictionaryClient.getDictionaryForType(AUTHOR +":" + TEST_DOSSIER_TEMPLATE_ID)).thenReturn(getDictionaryResponse(AUTHOR, false));
|
||||
when(dictionaryClient.getDictionaryForType(AUTHOR + ":" + TEST_DOSSIER_TEMPLATE_ID)).thenReturn(getDictionaryResponse(AUTHOR, false));
|
||||
analyzeService.reanalyze(request);
|
||||
|
||||
dictionary.get(AUTHOR).add("assessment report");
|
||||
reanlysisVersions.put("assessment report", 3L);
|
||||
when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(3L);
|
||||
when(dictionaryClient.getDictionaryForType(AUTHOR +":" + TEST_DOSSIER_TEMPLATE_ID)).thenReturn(getDictionaryResponse(AUTHOR, false));
|
||||
when(dictionaryClient.getDictionaryForType(AUTHOR + ":" + TEST_DOSSIER_TEMPLATE_ID)).thenReturn(getDictionaryResponse(AUTHOR, false));
|
||||
analyzeService.reanalyze(request);
|
||||
|
||||
when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(3L);
|
||||
when(dictionaryClient.getDictionaryForType(AUTHOR +":" + TEST_DOSSIER_TEMPLATE_ID)).thenReturn(getDictionaryResponse(AUTHOR, false));
|
||||
when(dictionaryClient.getDictionaryForType(AUTHOR + ":" + TEST_DOSSIER_TEMPLATE_ID)).thenReturn(getDictionaryResponse(AUTHOR, false));
|
||||
analyzeService.reanalyze(request);
|
||||
|
||||
var redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID);
|
||||
var redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID);
|
||||
|
||||
var changes = redactionLog.getRedactionLogEntry().stream().filter(entry ->
|
||||
entry.getValue() != null && entry.getValue().equals("report"))
|
||||
.findFirst().get().getChanges();
|
||||
var changes = redactionLog.getRedactionLogEntry()
|
||||
.stream()
|
||||
.filter(entry -> entry.getValue() != null && entry.getValue().equals("report"))
|
||||
.findFirst()
|
||||
.get()
|
||||
.getChanges();
|
||||
|
||||
assertThat(changes.size()).isEqualTo(2);
|
||||
|
||||
@ -887,6 +901,7 @@ public class RedactionIntegrationTest {
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
public void redactionTest() throws IOException {
|
||||
|
||||
@ -1556,4 +1571,31 @@ public class RedactionIntegrationTest {
|
||||
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
public void testImportedRedactions() throws IOException {
|
||||
|
||||
String outputFileName = OsUtils.getTemporaryDirectory() + "/Annotated.pdf";
|
||||
ClassPathResource pdfFileResource = new ClassPathResource("files/ImportedRedactions/S13.pdf");
|
||||
ClassPathResource importedRedactions = new ClassPathResource("files/ImportedRedactions/S13.json");
|
||||
List<ImportedAnnotation> importedAnnotations = objectMapper.readValue(importedRedactions.getInputStream().readAllBytes(), new TypeReference<List<ImportedAnnotation>>() {
|
||||
});
|
||||
AnalyzeRequest request = prepareStorage(pdfFileResource.getInputStream());
|
||||
request.setImportedAnnotations(importedAnnotations);
|
||||
|
||||
analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
|
||||
AnalyzeResult result = analyzeService.analyze(request);
|
||||
|
||||
var redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID);
|
||||
|
||||
AnnotateResponse annotateResponse = redactionController.annotate(AnnotateRequest.builder()
|
||||
.dossierId(TEST_DOSSIER_ID)
|
||||
.fileId(TEST_FILE_ID)
|
||||
.build());
|
||||
|
||||
try (FileOutputStream fileOutputStream = new FileOutputStream(outputFileName)) {
|
||||
fileOutputStream.write(annotateResponse.getDocument());
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -0,0 +1,121 @@
|
||||
[
|
||||
{
|
||||
"annotationId": "6113d300d4a5cd7e0e91ef692e046fc5",
|
||||
"fileId": "536bfa2d5ef9aecedda09e300a37cd81",
|
||||
"positions": [
|
||||
{
|
||||
"topLeftX": 147.86,
|
||||
"topLeftY": 519.11,
|
||||
"width": 122.496,
|
||||
"height": 11.018,
|
||||
"page": 1
|
||||
}
|
||||
],
|
||||
"status": "NEW",
|
||||
"user": null,
|
||||
"comment": null,
|
||||
"processedDate": null
|
||||
},
|
||||
{
|
||||
"annotationId": "04d043c73922f6e0dc50a1c7484e9425",
|
||||
"fileId": "536bfa2d5ef9aecedda09e300a37cd81",
|
||||
"positions": [
|
||||
{
|
||||
"topLeftX": 147.86,
|
||||
"topLeftY": 544.43,
|
||||
"width": 83.414,
|
||||
"height": 11.018,
|
||||
"page": 1
|
||||
}
|
||||
],
|
||||
"status": "NEW",
|
||||
"user": null,
|
||||
"comment": null,
|
||||
"processedDate": null
|
||||
},
|
||||
{
|
||||
"annotationId": "83f5f2cbe55f2acbbf7adc479cf694cb",
|
||||
"fileId": "536bfa2d5ef9aecedda09e300a37cd81",
|
||||
"positions": [
|
||||
{
|
||||
"topLeftX": 147.86,
|
||||
"topLeftY": 557.03,
|
||||
"width": 57.624,
|
||||
"height": 11.018,
|
||||
"page": 1
|
||||
}
|
||||
],
|
||||
"status": "NEW",
|
||||
"user": null,
|
||||
"comment": null,
|
||||
"processedDate": null
|
||||
},
|
||||
{
|
||||
"annotationId": "4bacdb3675c46181c75295fe8c6e6562",
|
||||
"fileId": "536bfa2d5ef9aecedda09e300a37cd81",
|
||||
"positions": [
|
||||
{
|
||||
"topLeftX": 147.86,
|
||||
"topLeftY": 531.83,
|
||||
"width": 83.414,
|
||||
"height": 11.018,
|
||||
"page": 1
|
||||
}
|
||||
],
|
||||
"status": "NEW",
|
||||
"user": null,
|
||||
"comment": null,
|
||||
"processedDate": null
|
||||
},
|
||||
{
|
||||
"annotationId": "8db925e788f2ab1c9ded097a9de10111",
|
||||
"fileId": "536bfa2d5ef9aecedda09e300a37cd81",
|
||||
"positions": [
|
||||
{
|
||||
"topLeftX": 148.94,
|
||||
"topLeftY": 317.49,
|
||||
"width": 141.263,
|
||||
"height": 11.018,
|
||||
"page": 1
|
||||
}
|
||||
],
|
||||
"status": "NEW",
|
||||
"user": null,
|
||||
"comment": null,
|
||||
"processedDate": null
|
||||
},
|
||||
{
|
||||
"annotationId": "a4d2f29216efe7507bf97d2e55010dd4",
|
||||
"fileId": "536bfa2d5ef9aecedda09e300a37cd81",
|
||||
"positions": [
|
||||
{
|
||||
"topLeftX": 148.94,
|
||||
"topLeftY": 342.81,
|
||||
"width": 94.389,
|
||||
"height": 11.018,
|
||||
"page": 1
|
||||
}
|
||||
],
|
||||
"status": "NEW",
|
||||
"user": null,
|
||||
"comment": null,
|
||||
"processedDate": null
|
||||
},
|
||||
{
|
||||
"annotationId": "9983b5000e8f3540a402178f91b3a2e2",
|
||||
"fileId": "536bfa2d5ef9aecedda09e300a37cd81",
|
||||
"positions": [
|
||||
{
|
||||
"topLeftX": 148.94,
|
||||
"topLeftY": 330.21,
|
||||
"width": 94.332,
|
||||
"height": 11.018,
|
||||
"page": 1
|
||||
}
|
||||
],
|
||||
"status": "APPROVED",
|
||||
"user": "its a me",
|
||||
"comment": "First Test",
|
||||
"processedDate": "2022-01-27T12:57:25.48Z"
|
||||
}
|
||||
]
|
||||
Binary file not shown.
Loading…
x
Reference in New Issue
Block a user