From 4e87e188eecf2f86d20bae6821cd4333ba86685d Mon Sep 17 00:00:00 2001 From: Timo Bejan Date: Mon, 1 Nov 2021 18:55:56 +0200 Subject: [PATCH] RED-2620/RED-2621/RED-2622 - Manual Redaction Section Text --- .../v1/model/ManualLegalBasisChange.java | 1 + .../v1/model/ManualRedactionEntry.java | 1 + .../redaction/v1/model/RedactionRequest.java | 2 + .../redaction/v1/model/SectionGrid.java | 2 + .../redaction/v1/model/SectionRectangle.java | 4 ++ .../controller/RedactionController.java | 25 +++++-- .../redaction/service/ReanalyzeService.java | 27 +++---- .../service/RedactionLogCreatorService.java | 4 +- .../service/RedactionLogMergeService.java | 70 ++++++++++++------- .../storage/RedactionStorageService.java | 1 - 10 files changed, 94 insertions(+), 43 deletions(-) diff --git a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/ManualLegalBasisChange.java b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/ManualLegalBasisChange.java index 39210d07..8b352e9a 100644 --- a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/ManualLegalBasisChange.java +++ b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/ManualLegalBasisChange.java @@ -17,6 +17,7 @@ public class ManualLegalBasisChange { private String user; private Status status; private String legalBasis; + private String section; private OffsetDateTime requestDate; private OffsetDateTime processedDate; diff --git a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/ManualRedactionEntry.java b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/ManualRedactionEntry.java index 69eb03cf..46941ece 100644 --- a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/ManualRedactionEntry.java +++ b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/ManualRedactionEntry.java @@ -23,6 +23,7 @@ public class ManualRedactionEntry { private String legalBasis; private List positions = new ArrayList<>(); private Status status; + private String section; private boolean addToDictionary; private boolean addToDossierDictionary; diff --git a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionRequest.java b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionRequest.java index d92f0edf..9b2953f7 100644 --- a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionRequest.java +++ b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionRequest.java @@ -20,4 +20,6 @@ public class RedactionRequest { private ManualRedactions manualRedactions; @Builder.Default private Set excludedPages = new HashSet<>(); + + private boolean withSectionDataForManualRedactions = false; } diff --git a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/SectionGrid.java b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/SectionGrid.java index 362b5c5c..3da17db0 100644 --- a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/SectionGrid.java +++ b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/SectionGrid.java @@ -15,4 +15,6 @@ public class SectionGrid { private Map> rectanglesPerPage = new HashMap<>(); + private Map sectionHeadlines = new HashMap<>(); + } diff --git a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/SectionRectangle.java b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/SectionRectangle.java index 3a21bb47..9ae38163 100644 --- a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/SectionRectangle.java +++ b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/SectionRectangle.java @@ -31,4 +31,8 @@ public class SectionRectangle { private List tableCells; + + public boolean contains(Rectangle other) { + return this.topLeft.getX() <= other.getTopLeft().getX() && this.topLeft.getX() + this.getWidth() >= other.getTopLeft().getX() + other.getWidth() && this.getTopLeft().getY() <= other.getTopLeft().getY() && this.getTopLeft().getY() + this.getHeight() >= other.getTopLeft().getY() + other.getHeight(); + } } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RedactionController.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RedactionController.java index 2ea691d7..e4592e29 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RedactionController.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RedactionController.java @@ -11,7 +11,6 @@ import com.iqser.red.service.redaction.v1.server.redaction.service.DictionarySer import com.iqser.red.service.redaction.v1.server.redaction.service.DroolsExecutionService; import com.iqser.red.service.redaction.v1.server.redaction.service.RedactionLogMergeService; import com.iqser.red.service.redaction.v1.server.segmentation.PdfSegmentationService; -import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings; import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService; import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer; import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table; @@ -20,7 +19,6 @@ import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.apache.pdfbox.io.MemoryUsageSetting; import org.apache.pdfbox.pdmodel.PDDocument; -import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RestController; @@ -147,19 +145,38 @@ public class RedactionController implements RedactionResource { } @Override - public RedactionLog getRedactionLog(RedactionRequest redactionRequest) { + public RedactionLog getRedactionLog(@RequestBody RedactionRequest redactionRequest) { log.info("Requested preview for: {}", redactionRequest); dictionaryService.updateDictionary(redactionRequest.getDossierTemplateId(), redactionRequest.getDossierId()); + + SectionGrid sectionGrid = null; var redactionLog = redactionStorageService.getRedactionLog(redactionRequest.getDossierId(), redactionRequest.getFileId()); + + if(redactionRequest.isWithSectionDataForManualRedactions()) { + sectionGrid = redactionStorageService.getSectionGrid(redactionRequest.getDossierId(), redactionRequest.getFileId()); + if (sectionGrid.getSectionHeadlines().isEmpty()) { + + log.info("SectionGrid does not have headlines set. Computing headlines now!"); + var text = redactionStorageService.getText(redactionRequest.getDossierId(), redactionRequest.getFileId()); + + // enhance section grid with headline data + for(var sectionText: text.getSectionTexts()){ + sectionGrid.getSectionHeadlines().put(sectionText.getSectionNumber(), sectionText.getHeadline()); + } + redactionStorageService.storeObject(redactionRequest.getDossierId(), redactionRequest.getFileId(), FileType.SECTION_GRID, sectionGrid); + } + } + + log.info("Loaded redaction log with computationalVersion: {}", redactionLog.getAnalysisVersion()); if (redactionLog.getAnalysisVersion() == 0) { // old redaction logs are returned directly return redactionLog; } else { - return redactionLogMergeService.mergeRedactionLogData(redactionLog, redactionRequest.getDossierTemplateId(), redactionRequest.getManualRedactions(), redactionRequest.getExcludedPages()); + return redactionLogMergeService.mergeRedactionLogData(redactionLog, sectionGrid, redactionRequest.getDossierTemplateId(), redactionRequest.getManualRedactions(), redactionRequest.getExcludedPages()); } } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ReanalyzeService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ReanalyzeService.java index 03fecb0c..3d017856 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ReanalyzeService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ReanalyzeService.java @@ -14,11 +14,9 @@ import com.iqser.red.service.redaction.v1.server.redaction.utils.IdBuilder; import com.iqser.red.service.redaction.v1.server.segmentation.PdfSegmentationService; import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings; import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService; - import lombok.RequiredArgsConstructor; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; - import org.kie.api.runtime.KieContainer; import org.springframework.stereotype.Service; import org.springframework.web.bind.annotation.RequestBody; @@ -84,6 +82,9 @@ public class ReanalyzeService { redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.REDACTION_LOG, redactionLog); redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.TEXT, new Text(pageCount, classifiedDoc .getSectionText())); + + // enhance section grid with headline data + classifiedDoc.getSectionText().forEach(st -> classifiedDoc.getSectionGrid().getSectionHeadlines().put(st.getSectionNumber(), st.getHeadline())); redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.SECTION_GRID, classifiedDoc .getSectionGrid()); @@ -283,12 +284,12 @@ public class ReanalyzeService { } return Stream.concat(manualRedactions.getManualLegalBasisChanges() - .stream() - .map(ManualLegalBasisChange::getId), Stream.concat(manualRedactions.getImageRecategorizations() - .stream() - .map(ManualImageRecategorization::getId), Stream.concat(manualRedactions.getIdsToRemove() - .stream() - .map(IdRemoval::getId), manualRedactions.getForceRedacts().stream().map(ManualForceRedact::getId)))) + .stream() + .map(ManualLegalBasisChange::getId), Stream.concat(manualRedactions.getImageRecategorizations() + .stream() + .map(ManualImageRecategorization::getId), Stream.concat(manualRedactions.getIdsToRemove() + .stream() + .map(IdRemoval::getId), manualRedactions.getForceRedacts().stream().map(ManualForceRedact::getId)))) .collect(Collectors.toSet()); } @@ -311,10 +312,12 @@ public class ReanalyzeService { private void excludeExcludedPages(RedactionLog redactionLog, Set excludedPages) { - if(excludedPages != null && !excludedPages.isEmpty()) { - redactionLog.getRedactionLogEntry().forEach(entry -> entry.getPositions().forEach(pos -> { if (excludedPages.contains(pos.getPage())) { - entry.setExcluded(true); - }})); + if (excludedPages != null && !excludedPages.isEmpty()) { + redactionLog.getRedactionLogEntry().forEach(entry -> entry.getPositions().forEach(pos -> { + if (excludedPages.contains(pos.getPage())) { + entry.setExcluded(true); + } + })); } } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java index 7fdc89fd..de13e302 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java @@ -79,7 +79,9 @@ public class RedactionLogCreatorService { .getX(), (float) image.getPosition().getY()), (float) image.getPosition() .getWidth(), (float) image.getPosition().getHeight(), pageNumber))) .sectionNumber(image.getSectionNumber()) - .section(image.getSection()) + //.section(image.getSection()) + // RED-2622 + .section("Image:"+image.getType()) .imageHasTransparency(image.isHasTransparency()) .build(); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogMergeService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogMergeService.java index 7888226d..da0837d6 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogMergeService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogMergeService.java @@ -1,31 +1,15 @@ package com.iqser.red.service.redaction.v1.server.redaction.service; -import java.time.OffsetDateTime; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.stream.Collectors; - -import org.springframework.stereotype.Service; - -import com.iqser.red.service.redaction.v1.model.Comment; -import com.iqser.red.service.redaction.v1.model.IdRemoval; -import com.iqser.red.service.redaction.v1.model.ManualForceRedact; -import com.iqser.red.service.redaction.v1.model.ManualImageRecategorization; -import com.iqser.red.service.redaction.v1.model.ManualLegalBasisChange; -import com.iqser.red.service.redaction.v1.model.ManualRedactionEntry; -import com.iqser.red.service.redaction.v1.model.ManualRedactionType; -import com.iqser.red.service.redaction.v1.model.ManualRedactions; -import com.iqser.red.service.redaction.v1.model.RedactionLog; -import com.iqser.red.service.redaction.v1.model.RedactionLogEntry; -import com.iqser.red.service.redaction.v1.model.Status; - +import com.iqser.red.service.redaction.v1.model.*; import lombok.AllArgsConstructor; import lombok.Data; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; + +import java.time.OffsetDateTime; +import java.util.*; +import java.util.stream.Collectors; @Slf4j @Service @@ -35,13 +19,13 @@ public class RedactionLogMergeService { private final DictionaryService dictionaryService; - public RedactionLog mergeRedactionLogData(RedactionLog redactionLog, String dossierTemplateId, + public RedactionLog mergeRedactionLogData(RedactionLog redactionLog, SectionGrid sectionGrid, String dossierTemplateId, ManualRedactions manualRedactions, Set excludedPages) { log.info("Merging Redaction log with manual redactions {}", manualRedactions); if (manualRedactions != null) { - var manualRedactionLogEntries = addManualAddEntries(manualRedactions.getEntriesToAdd(), manualRedactions.getComments(), dossierTemplateId); + var manualRedactionLogEntries = addManualAddEntries(sectionGrid, manualRedactions.getEntriesToAdd(), manualRedactions.getComments(), dossierTemplateId); redactionLog.getRedactionLogEntry().addAll(manualRedactionLogEntries); @@ -115,6 +99,8 @@ public class RedactionLogMergeService { if (imageRecategorization.getStatus().equals(Status.APPROVED)) { redactionLogEntry.setStatus(Status.APPROVED); redactionLogEntry.setType(imageRecategorization.getType()); + // RED-2622 + redactionLogEntry.setSection("Image:" + redactionLogEntry.getType()); manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", recategorized by manual override"); } else if (imageRecategorization.getStatus().equals(Status.REQUESTED)) { manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", requested to recategorize"); @@ -193,6 +179,9 @@ public class RedactionLogMergeService { redactionLogEntry.setRedacted(true); manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", legal basis was manually changed"); redactionLogEntry.setLegalBasis(manualLegalBasisChange.getLegalBasis()); + if (manualLegalBasisChange.getSection() != null) { + redactionLogEntry.setSection(manualLegalBasisChange.getSection()); + } } else if (manualLegalBasisChange.getStatus().equals(Status.REQUESTED)) { manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", legal basis change requested"); redactionLogEntry.setStatus(Status.REQUESTED); @@ -203,6 +192,7 @@ public class RedactionLogMergeService { redactionLogEntry.setStatus(Status.DECLINED); } + redactionLogEntry.setManualRedactionUserId(manualLegalBasisChange.getUser()); redactionLogEntry.setReason(manualOverrideReason); redactionLogEntry.setManual(true); @@ -227,7 +217,8 @@ public class RedactionLogMergeService { } - public List addManualAddEntries(Set manualAdds, + public List addManualAddEntries(SectionGrid sectionGrid, + Set manualAdds, Map> comments, String dossierTemplateId) { List redactionLogEntries = new ArrayList<>(); @@ -235,17 +226,45 @@ public class RedactionLogMergeService { for (ManualRedactionEntry manualRedactionEntry : manualAdds) { if (!approvedAndShouldBeInDictionary(manualRedactionEntry)) { + RedactionLogEntry redactionLogEntry = createRedactionLogEntry(manualRedactionEntry, manualRedactionEntry .getId(), dossierTemplateId); redactionLogEntry.setPositions(manualRedactionEntry.getPositions()); redactionLogEntry.setComments(comments.get(manualRedactionEntry.getId())); + + handleSectionText(sectionGrid, redactionLogEntry); redactionLogEntries.add(redactionLogEntry); + + } } return redactionLogEntries; } + private void handleSectionText(SectionGrid sectionGrid, RedactionLogEntry redactionLogEntry) { + + if (redactionLogEntry.getSection() != null) { + // set by UI + return; + } + + if (sectionGrid != null) { + var firstPosition = !redactionLogEntry.getPositions().isEmpty() ? redactionLogEntry.getPositions().iterator().next() : null; + + if (firstPosition != null) { + List pageRectangles = sectionGrid.getRectanglesPerPage().get(firstPosition.getPage()); + + for (var rectangle : pageRectangles) { + if (rectangle.contains(firstPosition)) { + redactionLogEntry.setSection(sectionGrid.getSectionHeadlines().get(rectangle.getPart())); + } + } + + } + } + } + private boolean approvedAndShouldBeInDictionary(ManualRedactionEntry manualRedactionEntry) { @@ -265,6 +284,7 @@ public class RedactionLogMergeService { .isDossierDictionaryEntry(manualRedactionEntry.isAddToDossierDictionary()) .legalBasis(manualRedactionEntry.getLegalBasis()) .value(manualRedactionEntry.getValue()) + .section(manualRedactionEntry.getSection()) .type(manualRedactionEntry.getType()) .redacted(true) .isHint(false) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/RedactionStorageService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/RedactionStorageService.java index 021beeff..0ebc6265 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/RedactionStorageService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/RedactionStorageService.java @@ -83,7 +83,6 @@ public class RedactionStorageService { } } - @RequiredArgsConstructor public enum StorageType { PARSED_DOCUMENT(".json");