From 1ac294fc0197990936ba62b3889474f63844bc21 Mon Sep 17 00:00:00 2001 From: Timo Date: Wed, 21 Jul 2021 15:58:52 +0300 Subject: [PATCH] redaction-log preview generation and removal of manual redactions from redaction-log --- .../redaction/v1/model/AnalyzeRequest.java | 2 - .../v1/model/ManualRedactionEntry.java | 5 +- .../redaction/v1/model/RedactionLog.java | 7 + .../redaction/v1/model/RedactionLogEntry.java | 2 + .../redaction/v1/model/SectionRectangle.java | 1 + .../v1/resources/RedactionResource.java | 2 +- .../controller/RedactionController.java | 8 +- .../redaction/service/DictionaryService.java | 2 +- .../service/EntityRedactionService.java | 40 +- .../redaction/service/ReanalyzeService.java | 102 +--- .../service/RedactionLogCreatorService.java | 547 +----------------- .../service/RedactionLogMergeService.java | 241 ++++++++ 12 files changed, 312 insertions(+), 647 deletions(-) create mode 100644 redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogMergeService.java diff --git a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/AnalyzeRequest.java b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/AnalyzeRequest.java index 7a891277..4a7be09b 100644 --- a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/AnalyzeRequest.java +++ b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/AnalyzeRequest.java @@ -7,9 +7,7 @@ import lombok.NoArgsConstructor; import java.time.OffsetDateTime; import java.util.ArrayList; -import java.util.HashMap; import java.util.List; -import java.util.Map; import java.util.Set; @Data diff --git a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/ManualRedactionEntry.java b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/ManualRedactionEntry.java index df14617b..69eb03cf 100644 --- a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/ManualRedactionEntry.java +++ b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/ManualRedactionEntry.java @@ -23,11 +23,8 @@ public class ManualRedactionEntry { private String legalBasis; private List positions = new ArrayList<>(); private Status status; + private boolean addToDictionary; - - private String section; - private int sectionNumber; - private boolean addToDossierDictionary; private OffsetDateTime requestDate; diff --git a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionLog.java b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionLog.java index 71a8413e..276363f8 100644 --- a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionLog.java +++ b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionLog.java @@ -10,6 +10,13 @@ import java.util.List; @AllArgsConstructor public class RedactionLog { + + /** + * Version 0 Redaction Logs have manual redactions merged inside them + * Version 1 Redaction Logs only contain system ( rule/dictionary ) redactions. Manual Redactions are merged in at runtime. + */ + private long computationalVersion; + private List redactionLogEntry; private List legalBasis; diff --git a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionLogEntry.java b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionLogEntry.java index e837ddf9..9f7a0146 100644 --- a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionLogEntry.java +++ b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionLogEntry.java @@ -9,6 +9,8 @@ import lombok.NoArgsConstructor; import java.util.ArrayList; import java.util.List; + + @Data @Builder @NoArgsConstructor diff --git a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/SectionRectangle.java b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/SectionRectangle.java index 38031b36..3a21bb47 100644 --- a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/SectionRectangle.java +++ b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/SectionRectangle.java @@ -30,4 +30,5 @@ public class SectionRectangle { private int numberOfParts; private List tableCells; + } diff --git a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/resources/RedactionResource.java b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/resources/RedactionResource.java index 0a6037a6..3abe15f5 100644 --- a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/resources/RedactionResource.java +++ b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/resources/RedactionResource.java @@ -31,6 +31,6 @@ public interface RedactionResource { void testRules(@RequestBody String rules); @PostMapping(value = "/redaction-log/preview", consumes = MediaType.APPLICATION_JSON_VALUE) - RedactionLog getRedactionLogPreview(@RequestBody RedactionRequest redactionRequest); + RedactionLog getRedactionLog(@RequestBody RedactionRequest redactionRequest); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RedactionController.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RedactionController.java index ebbaffc2..047538cf 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RedactionController.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RedactionController.java @@ -13,7 +13,7 @@ import com.iqser.red.service.redaction.v1.server.exception.RedactionException; import com.iqser.red.service.redaction.v1.server.redaction.service.AnnotationService; import com.iqser.red.service.redaction.v1.server.redaction.service.DictionaryService; import com.iqser.red.service.redaction.v1.server.redaction.service.DroolsExecutionService; -import com.iqser.red.service.redaction.v1.server.redaction.service.RedactionLogCreatorService; +import com.iqser.red.service.redaction.v1.server.redaction.service.RedactionLogMergeService; import com.iqser.red.service.redaction.v1.server.segmentation.PdfSegmentationService; import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService; import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer; @@ -41,7 +41,7 @@ public class RedactionController implements RedactionResource { private final AnnotationService annotationService; private final PdfSegmentationService pdfSegmentationService; private final RedactionStorageService redactionStorageService; - private final RedactionLogCreatorService redactionLogCreatorService; + private final RedactionLogMergeService redactionLogMergeService; public AnnotateResponse annotate(@RequestBody AnnotateRequest annotateRequest) { @@ -158,13 +158,13 @@ public class RedactionController implements RedactionResource { } @Override - public RedactionLog getRedactionLogPreview(RedactionRequest redactionRequest) { + public RedactionLog getRedactionLog(RedactionRequest redactionRequest) { log.info("Requested preview for: {}", redactionRequest); dictionaryService.updateDictionary(redactionRequest.getDossierTemplateId(), redactionRequest.getDossierId()); var redactionLog = redactionStorageService.getRedactionLog(redactionRequest.getDossierId(), redactionRequest.getFileId()); - return redactionLogCreatorService.getRedactionLogPreview(redactionLog, redactionRequest.getDossierTemplateId(), redactionRequest.getManualRedactions()); + return redactionLogMergeService.mergeRedactionLogData(redactionLog, redactionRequest.getDossierTemplateId(), redactionRequest.getManualRedactions()); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DictionaryService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DictionaryService.java index fd460dbd..5afff568 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DictionaryService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DictionaryService.java @@ -218,7 +218,7 @@ public class DictionaryService { public float[] getRequestRemoveColor(String dossierTemplateId) { - return dictionariesByDossierTemplate.get(dossierTemplateId).getRequestAddColor(); + return dictionariesByDossierTemplate.get(dossierTemplateId).getRequestRemoveColor(); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionService.java index 389b7186..ea1079a4 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionService.java @@ -1,13 +1,6 @@ package com.iqser.red.service.redaction.v1.server.redaction.service; -import com.iqser.red.service.redaction.v1.model.FileAttribute; -import com.iqser.red.service.redaction.v1.model.ManualImageRecategorization; -import com.iqser.red.service.redaction.v1.model.ManualRedactionEntry; -import com.iqser.red.service.redaction.v1.model.ManualRedactions; -import com.iqser.red.service.redaction.v1.model.Point; -import com.iqser.red.service.redaction.v1.model.Rectangle; -import com.iqser.red.service.redaction.v1.model.SectionArea; -import com.iqser.red.service.redaction.v1.model.Status; +import com.iqser.red.service.redaction.v1.model.*; import com.iqser.red.service.redaction.v1.server.classification.model.*; import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary; import com.iqser.red.service.redaction.v1.server.redaction.model.*; @@ -15,10 +8,8 @@ import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUti import com.iqser.red.service.redaction.v1.server.redaction.utils.IdBuilder; import com.iqser.red.service.redaction.v1.server.tableextraction.model.Cell; import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table; - import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; - import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.StringUtils; import org.kie.api.runtime.KieContainer; @@ -103,9 +94,9 @@ public class EntityRedactionService { List tables = paragraph.getTables(); for (Table table : tables) { if (table.getColCount() == 2) { - sectionSearchableTextPairs.addAll(processTableAsOneText(classifiedDoc, table, manualRedactions, sectionNumber, dictionary, local, hintsPerSectionNumber, fileAttributes)); + sectionSearchableTextPairs.addAll(processTableAsOneText(classifiedDoc, table, sectionNumber, dictionary, local, hintsPerSectionNumber, fileAttributes)); } else { - sectionSearchableTextPairs.addAll(processTablePerRow(classifiedDoc, table, manualRedactions, sectionNumber, dictionary, local, hintsPerSectionNumber, fileAttributes)); + sectionSearchableTextPairs.addAll(processTablePerRow(classifiedDoc, table, sectionNumber, dictionary, local, hintsPerSectionNumber, fileAttributes)); } sectionNumber.incrementAndGet(); } @@ -169,7 +160,6 @@ public class EntityRedactionService { private List processTablePerRow(Document classifiedDoc, Table table, - ManualRedactions manualRedactions, AtomicInteger sectionNumber, Dictionary dictionary, boolean local, Map> hintsPerSectionNumber, @@ -198,7 +188,6 @@ public class EntityRedactionService { sectionText.getSectionAreas().add(sectionArea); sectionText.getTextBlocks().addAll(cell.getTextBlocks()); - addSectionToManualRedactions(cell.getTextBlocks(), manualRedactions, table.getHeadline(), sectionNumber.intValue()); int cellStart = start; if (!cell.isHeaderCell()) { @@ -259,7 +248,6 @@ public class EntityRedactionService { private List processTableAsOneText(Document classifiedDoc, Table table, - ManualRedactions manualRedactions, AtomicInteger sectionNumber, Dictionary dictionary, boolean local, Map> hintsPerSectionNumber, @@ -288,7 +276,6 @@ public class EntityRedactionService { for (TextBlock textBlock : cell.getTextBlocks()) { entireTableText.addAll(textBlock.getSequences()); } - addSectionToManualRedactions(cell.getTextBlocks(), manualRedactions, table.getHeadline(), sectionNumber.intValue()); } } @@ -348,7 +335,6 @@ public class EntityRedactionService { classifiedDoc.getSectionText().add(sectionText); } - addSectionToManualRedactions(paragraphTextBlocks, manualRedactions, headline, sectionNumber.intValue()); Set entities = findEntities(searchableText, headline, sectionNumber.intValue(), dictionary, local); surroundingWordsService.addSurroundingText(entities, searchableText, dictionary); @@ -396,26 +382,6 @@ public class EntityRedactionService { } - private void addSectionToManualRedactions(List textBlocks, ManualRedactions manualRedactions, - String section, int sectionNumber) { - - if (manualRedactions == null || manualRedactions.getEntriesToAdd().isEmpty()) { - return; - } - - for (TextBlock textBlock : textBlocks) { - for (ManualRedactionEntry manualRedactionEntry : manualRedactions.getEntriesToAdd()) { - for (Rectangle rectangle : manualRedactionEntry.getPositions()) { - if (textBlock.contains(rectangle)) { - manualRedactionEntry.setSection(section); - manualRedactionEntry.setSectionNumber(sectionNumber); - } - } - } - } - } - - private Image convertAndRecategorize(PdfImage pdfImage, int sectionNumber, String headline, ManualRedactions manualRedactions) { Image image = Image.builder() diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ReanalyzeService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ReanalyzeService.java index fa3f388c..17dfdf4b 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ReanalyzeService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ReanalyzeService.java @@ -1,55 +1,28 @@ package com.iqser.red.service.redaction.v1.server.redaction.service; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -import org.kie.api.runtime.KieContainer; -import org.springframework.stereotype.Service; -import org.springframework.web.bind.annotation.RequestBody; - import com.iqser.red.service.file.management.v1.api.model.FileType; -import com.iqser.red.service.redaction.v1.model.AnalyzeRequest; -import com.iqser.red.service.redaction.v1.model.AnalyzeResult; -import com.iqser.red.service.redaction.v1.model.Comment; -import com.iqser.red.service.redaction.v1.model.IdRemoval; -import com.iqser.red.service.redaction.v1.model.ManualForceRedact; -import com.iqser.red.service.redaction.v1.model.ManualImageRecategorization; -import com.iqser.red.service.redaction.v1.model.ManualLegalBasisChange; -import com.iqser.red.service.redaction.v1.model.ManualRedactionEntry; -import com.iqser.red.service.redaction.v1.model.ManualRedactions; -import com.iqser.red.service.redaction.v1.model.Rectangle; -import com.iqser.red.service.redaction.v1.model.RedactionLog; -import com.iqser.red.service.redaction.v1.model.RedactionLogEntry; -import com.iqser.red.service.redaction.v1.model.SectionArea; -import com.iqser.red.service.redaction.v1.model.Status; +import com.iqser.red.service.redaction.v1.model.*; import com.iqser.red.service.redaction.v1.server.classification.model.Document; import com.iqser.red.service.redaction.v1.server.classification.model.SectionText; import com.iqser.red.service.redaction.v1.server.classification.model.Text; import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient; import com.iqser.red.service.redaction.v1.server.exception.RedactionException; import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary; -import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryIncrement; -import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryVersion; -import com.iqser.red.service.redaction.v1.server.redaction.model.Entity; -import com.iqser.red.service.redaction.v1.server.redaction.model.EntityPositionSequence; -import com.iqser.red.service.redaction.v1.server.redaction.model.Image; -import com.iqser.red.service.redaction.v1.server.redaction.model.RedRectangle2D; -import com.iqser.red.service.redaction.v1.server.redaction.model.Section; -import com.iqser.red.service.redaction.v1.server.redaction.model.SectionSearchableTextPair; +import com.iqser.red.service.redaction.v1.server.redaction.model.*; import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUtils; import com.iqser.red.service.redaction.v1.server.redaction.utils.IdBuilder; import com.iqser.red.service.redaction.v1.server.segmentation.PdfSegmentationService; import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService; - import lombok.RequiredArgsConstructor; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; +import org.kie.api.runtime.KieContainer; +import org.springframework.stereotype.Service; +import org.springframework.web.bind.annotation.RequestBody; + +import java.util.*; +import java.util.stream.Collectors; +import java.util.stream.Stream; @Slf4j @Service @@ -87,13 +60,13 @@ public class ReanalyzeService { entityRedactionService.processDocument(classifiedDoc, analyzeRequest.getDossierTemplateId(), analyzeRequest.getManualRedactions(), analyzeRequest .getDossierId(), analyzeRequest.getFileAttributes()); - redactionLogCreatorService.createRedactionLog(classifiedDoc, pageCount, analyzeRequest.getManualRedactions(), analyzeRequest + redactionLogCreatorService.createRedactionLog(classifiedDoc, pageCount, analyzeRequest .getDossierTemplateId()); log.info("Redaction analysis successful..."); var legalBasis = legalBasisClient.getLegalBasisMapping(analyzeRequest.getDossierTemplateId()); - var redactionLog = new RedactionLog(classifiedDoc.getRedactionLogEntities(), legalBasis, classifiedDoc.getDictionaryVersion() + var redactionLog = new RedactionLog(1, classifiedDoc.getRedactionLogEntities(), legalBasis, classifiedDoc.getDictionaryVersion() .getDossierTemplateVersion(), classifiedDoc.getDictionaryVersion() .getDossierVersion(), classifiedDoc.getRulesVersion(), legalBasisClient.getVersion(analyzeRequest.getDossierTemplateId())); @@ -132,20 +105,12 @@ public class ReanalyzeService { DictionaryIncrement dictionaryIncrement = dictionaryService.getDictionaryIncrements(analyzeRequest.getDossierTemplateId(), new DictionaryVersion(redactionLog .getDictionaryVersion(), redactionLog.getDossierDictionaryVersion()), analyzeRequest.getDossierId()); - Set manualForceAndRemoveIds = getForceAndRemoveIds(analyzeRequest.getManualRedactions()); - Map> comments = null; - Set manualAdds = null; - - if (analyzeRequest.getManualRedactions() != null) { - // TODO comments will be removed from redactionLog, so we ignore this first. - comments = analyzeRequest.getManualRedactions().getComments(); - manualAdds = analyzeRequest.getManualRedactions().getEntriesToAdd(); - } + Set relevantManuallyModifiedAnnotationIds = getRelevantManuallyModifiedAnnotationIds(analyzeRequest.getManualRedactions()); Set sectionsToReanalyse = new HashSet<>(); Map> imageEntries = new HashMap<>(); for (RedactionLogEntry entry : redactionLog.getRedactionLogEntry()) { - if (entry.isManual() || manualForceAndRemoveIds.contains(entry.getId())) { + if (entry.isManual() || relevantManuallyModifiedAnnotationIds.contains(entry.getId())) { sectionsToReanalyse.add(entry.getSectionNumber()); } if (entry.isImage() || entry.getType().equals("image")) { @@ -159,23 +124,11 @@ public class ReanalyzeService { sectionsToReanalyse.add(sectionText.getSectionNumber()); } - if (manualAdds != null) { - for (SectionArea sectionArea : sectionText.getSectionAreas()) { - for (ManualRedactionEntry manualAdd : manualAdds) { - for (Rectangle manualPosition : manualAdd.getPositions()) { - if (sectionArea.contains(manualPosition)) { - manualAdd.setSection(sectionText.getHeadline()); - manualAdd.setSectionNumber(sectionText.getSectionNumber()); - } - } - } - } - } } log.info("Should reanalyze {} sections for request: {}", sectionsToReanalyse.size(), analyzeRequest); - if (sectionsToReanalyse.isEmpty() && (manualAdds == null || manualAdds.isEmpty())) { + if (sectionsToReanalyse.isEmpty()) { return finalizeAnalysis(analyzeRequest, startTime, redactionLog, text, dictionaryIncrement); } @@ -212,9 +165,9 @@ public class ReanalyzeService { .getImageRecategorizations() != null) { for (Image image : reanalysisSection.getImages()) { String imageId = IdBuilder.buildId(image.getPosition(), image.getPage()); - for(ManualImageRecategorization imageRecategorization: analyzeRequest.getManualRedactions() - .getImageRecategorizations()){ - if(imageRecategorization.getStatus().equals(Status.APPROVED) && imageRecategorization.getId().equals(imageId)){ + for (ManualImageRecategorization imageRecategorization : analyzeRequest.getManualRedactions() + .getImageRecategorizations()) { + if (imageRecategorization.getStatus().equals(Status.APPROVED) && imageRecategorization.getId().equals(imageId)) { image.setType(imageRecategorization.getType()); } } @@ -270,17 +223,15 @@ public class ReanalyzeService { List newRedactionLogEntries = new ArrayList<>(); for (int page = 1; page <= text.getNumberOfPages(); page++) { if (entitiesPerPage.get(page) != null) { - newRedactionLogEntries.addAll(redactionLogCreatorService.addEntries(entitiesPerPage, analyzeRequest.getManualRedactions(), page, analyzeRequest + newRedactionLogEntries.addAll(redactionLogCreatorService.addEntries(entitiesPerPage, page, analyzeRequest .getDossierTemplateId())); } if (imagesPerPage.get(page) != null) { - newRedactionLogEntries.addAll(redactionLogCreatorService.addImageEntries(imagesPerPage, analyzeRequest.getManualRedactions(), page, analyzeRequest + newRedactionLogEntries.addAll(redactionLogCreatorService.addImageEntries(imagesPerPage, page, analyzeRequest .getDossierTemplateId())); } - newRedactionLogEntries.addAll(redactionLogCreatorService.addManualAddEntries(manualAdds, comments, page, analyzeRequest - .getDossierTemplateId())); } redactionLog.getRedactionLogEntry().removeIf(entry -> sectionsToReanalyse.contains(entry.getSectionNumber())); @@ -310,7 +261,7 @@ public class ReanalyzeService { } - private Set getForceAndRemoveIds(ManualRedactions manualRedactions) { + private Set getRelevantManuallyModifiedAnnotationIds(ManualRedactions manualRedactions) { if (manualRedactions == null) { return new HashSet<>(); @@ -345,15 +296,10 @@ public class ReanalyzeService { private void excludeExcludedPages(RedactionLog redactionLog, Set excludedPages) { - redactionLog.getRedactionLogEntry().forEach(entry -> { - entry.getPositions().forEach(pos -> { - if (excludedPages != null && excludedPages.contains(pos.getPage())) { - entry.setExcluded(true); - } else { - entry.setExcluded(false); - } - }); - }); + redactionLog.getRedactionLogEntry().forEach(entry -> + entry.getPositions().forEach(pos -> + entry.setExcluded(excludedPages != null && excludedPages.contains(pos.getPage())) + )); } } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java index 885326c9..ec1f78db 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java @@ -1,6 +1,10 @@ package com.iqser.red.service.redaction.v1.server.redaction.service; -import com.iqser.red.service.redaction.v1.model.*; +import com.iqser.red.service.redaction.v1.model.CellRectangle; +import com.iqser.red.service.redaction.v1.model.Point; +import com.iqser.red.service.redaction.v1.model.Rectangle; +import com.iqser.red.service.redaction.v1.model.RedactionLogEntry; +import com.iqser.red.service.redaction.v1.model.SectionRectangle; import com.iqser.red.service.redaction.v1.server.classification.model.Document; import com.iqser.red.service.redaction.v1.server.classification.model.Paragraph; import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock; @@ -9,17 +13,19 @@ import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSeque import com.iqser.red.service.redaction.v1.server.redaction.model.Entity; import com.iqser.red.service.redaction.v1.server.redaction.model.EntityPositionSequence; import com.iqser.red.service.redaction.v1.server.redaction.model.Image; -import com.iqser.red.service.redaction.v1.server.redaction.model.ReasonHolder; import com.iqser.red.service.redaction.v1.server.redaction.utils.IdBuilder; import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer; import com.iqser.red.service.redaction.v1.server.tableextraction.model.Cell; import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table; import lombok.RequiredArgsConstructor; import org.apache.commons.collections4.CollectionUtils; -import org.springframework.beans.BeanUtils; import org.springframework.stereotype.Service; -import java.util.*; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; import java.util.stream.Collectors; @Service @@ -29,10 +35,7 @@ public class RedactionLogCreatorService { private final DictionaryService dictionaryService; - public void createRedactionLog(Document classifiedDoc, int numberOfPages, ManualRedactions manualRedactions, - String dossierTemplateId) { - - Set manualRedactionPages = getManualRedactionPages(manualRedactions); + public void createRedactionLog(Document classifiedDoc, int numberOfPages, String dossierTemplateId) { for (int page = 1; page <= numberOfPages; page++) { @@ -40,24 +43,18 @@ public class RedactionLogCreatorService { if (classifiedDoc.getEntities().get(page) != null) { classifiedDoc.getRedactionLogEntities() - .addAll(addEntries(classifiedDoc.getEntities(), manualRedactions, page, dossierTemplateId)); - } - - if (manualRedactionPages.contains(page)) { - classifiedDoc.getRedactionLogEntities() - .addAll(addManualAddEntries(manualRedactions.getEntriesToAdd(), manualRedactions.getComments(), page, dossierTemplateId)); + .addAll(addEntries(classifiedDoc.getEntities(), page, dossierTemplateId)); } if (classifiedDoc.getImages().get(page) != null && !classifiedDoc.getImages().get(page).isEmpty()) { classifiedDoc.getRedactionLogEntities() - .addAll(addImageEntries(classifiedDoc.getImages(), manualRedactions, page, dossierTemplateId)); + .addAll(addImageEntries(classifiedDoc.getImages(), page, dossierTemplateId)); } } } - public List addImageEntries(Map> images, ManualRedactions manualRedactions, - int pageNumber, String dossierTemplateId) { + public List addImageEntries(Map> images, int pageNumber, String dossierTemplateId) { List redactionLogEntities = new ArrayList<>(); @@ -67,7 +64,7 @@ public class RedactionLogCreatorService { RedactionLogEntry redactionLogEntry = RedactionLogEntry.builder() .id(id) - .color(getColorForImage(image.getType(), dossierTemplateId, false, image.isRedaction())) + .color(getColor(image.getType(), dossierTemplateId, image.isRedaction())) .isImage(true) .type(image.getType()) .redacted(image.isRedaction()) @@ -86,7 +83,6 @@ public class RedactionLogCreatorService { .imageHasTransparency(image.isHasTransparency()) .build(); - processImageEntry(manualRedactions, dossierTemplateId, image, redactionLogEntry); redactionLogEntities.add(redactionLogEntry); } @@ -94,135 +90,8 @@ public class RedactionLogCreatorService { return redactionLogEntities; } - private void processImageEntry(ManualRedactions manualRedactions, String dossierTemplateId, ReasonHolder image, RedactionLogEntry redactionLogEntry) { - if (manualRedactions != null && !manualRedactions.getImageRecategorizations().isEmpty()) { - for (ManualImageRecategorization recategorization : manualRedactions.getImageRecategorizations()) { - if (recategorization.getId().equals(redactionLogEntry.getId())) { - String manualOverrideReason = null; - if (recategorization.getStatus().equals(Status.APPROVED)) { - redactionLogEntry.setStatus(Status.APPROVED); - redactionLogEntry.setType(recategorization.getType()); - manualOverrideReason = mergeReasonIfNecessary(image.getRedactionReason(), ", recategorized by manual override"); - } else if (recategorization.getStatus().equals(Status.REQUESTED)) { - manualOverrideReason = mergeReasonIfNecessary(image.getRedactionReason(), ", requested to recategorize"); - redactionLogEntry.setStatus(Status.REQUESTED); - redactionLogEntry.setColor(getColorForImage(redactionLogEntry.getType(), dossierTemplateId, true, redactionLogEntry.isRedacted())); - redactionLogEntry.setRecategorizationType(recategorization.getType()); - } else { - redactionLogEntry.setStatus(Status.DECLINED); - } - image.setRedactionReason(manualOverrideReason != null ? manualOverrideReason : image.getRedactionReason()); - redactionLogEntry.setReason(manualOverrideReason); - redactionLogEntry.setManual(true); - redactionLogEntry.setManualRedactionType(ManualRedactionType.RECATEGORIZE); - } - } - } - - if (manualRedactions != null && !manualRedactions.getIdsToRemove().isEmpty()) { - for (IdRemoval manualRemoval : manualRedactions.getIdsToRemove()) { - if (manualRemoval.getId().equals(redactionLogEntry.getId())) { - String manualOverrideReason = null; - if (manualRemoval.getStatus().equals(Status.APPROVED)) { - image.setRedaction(false); - redactionLogEntry.setRedacted(false); - redactionLogEntry.setStatus(Status.APPROVED); - manualOverrideReason = mergeReasonIfNecessary(image.getRedactionReason(), ", removed by manual override"); - redactionLogEntry.setColor(getColorForImage(redactionLogEntry.getType(), dossierTemplateId, false, redactionLogEntry.isRedacted())); - } else if (manualRemoval.getStatus().equals(Status.REQUESTED)) { - manualOverrideReason = mergeReasonIfNecessary(image.getRedactionReason(), ", requested to remove"); - redactionLogEntry.setStatus(Status.REQUESTED); - redactionLogEntry.setColor(getColorForImage(redactionLogEntry.getType(), dossierTemplateId, true, redactionLogEntry.isRedacted())); - } else { - redactionLogEntry.setStatus(Status.DECLINED); - } - - image.setRedactionReason(manualOverrideReason != null ? manualOverrideReason : image.getRedactionReason()); - redactionLogEntry.setReason(manualOverrideReason); - redactionLogEntry.setManual(true); - redactionLogEntry.setManualRedactionType(ManualRedactionType.REMOVE); - redactionLogEntry.setDictionaryEntry(manualRemoval.isRemoveFromDictionary()); - redactionLogEntry.setDossierDictionaryEntry(manualRemoval.isRemoveFromDictionary()); - } - } - } - - if (manualRedactions != null && !manualRedactions.getForceRedacts().isEmpty()) { - for (ManualForceRedact manualForceRedact : manualRedactions.getForceRedacts()) { - if (manualForceRedact.getId().equals(redactionLogEntry.getId())) { - String manualOverrideReason = null; - if (manualForceRedact.getStatus().equals(Status.APPROVED)) { - image.setRedaction(true); - redactionLogEntry.setRedacted(true); - redactionLogEntry.setStatus(Status.APPROVED); - redactionLogEntry.setColor(getColorForImage(redactionLogEntry.getType(), dossierTemplateId, false, redactionLogEntry.isRedacted())); - manualOverrideReason = mergeReasonIfNecessary(image.getRedactionReason(), ", forced by manual override"); - redactionLogEntry.setLegalBasis(manualForceRedact.getLegalBasis()); - } else if (manualForceRedact.getStatus().equals(Status.REQUESTED)) { - manualOverrideReason = mergeReasonIfNecessary(image.getRedactionReason(), ", requested to force redact"); - redactionLogEntry.setStatus(Status.REQUESTED); - redactionLogEntry.setColor(getColorForImage(redactionLogEntry.getType(), dossierTemplateId, true, redactionLogEntry.isRedacted())); - redactionLogEntry.setLegalBasis(manualForceRedact.getLegalBasis()); - } else { - redactionLogEntry.setStatus(Status.DECLINED); - } - - image.setRedactionReason(manualOverrideReason != null ? manualOverrideReason : image.getRedactionReason()); - redactionLogEntry.setReason(manualOverrideReason); - redactionLogEntry.setManual(true); - redactionLogEntry.setManualRedactionType(ManualRedactionType.FORCE_REDACT); - } - } - } - - - if (manualRedactions != null && !manualRedactions.getManualLegalBasisChanges().isEmpty()) { - for (ManualLegalBasisChange manualLegalBasisChange : manualRedactions.getManualLegalBasisChanges()) { - if (manualLegalBasisChange.getId().equals(redactionLogEntry.getId())) { - String manualOverrideReason = null; - if (manualLegalBasisChange.getStatus().equals(Status.APPROVED)) { - redactionLogEntry.setStatus(Status.APPROVED); - manualOverrideReason = mergeReasonIfNecessary(image.getRedactionReason(), ", legal basis was manually changed"); - redactionLogEntry.setLegalBasis(manualLegalBasisChange.getLegalBasis()); - } else if (manualLegalBasisChange.getStatus().equals(Status.REQUESTED)) { - manualOverrideReason = mergeReasonIfNecessary(image.getRedactionReason(), ", legal basis change requested"); - redactionLogEntry.setStatus(Status.REQUESTED); - redactionLogEntry.setColor(getColorForImage(redactionLogEntry.getType(), dossierTemplateId, true, redactionLogEntry.isRedacted())); - redactionLogEntry.setLegalBasisChangeValue(manualLegalBasisChange.getLegalBasis()); - } else { - redactionLogEntry.setStatus(Status.DECLINED); - } - - image.setRedactionReason(manualOverrideReason != null ? manualOverrideReason : image.getRedactionReason()); - redactionLogEntry.setReason(manualOverrideReason); - redactionLogEntry.setManual(true); - redactionLogEntry.setManualRedactionType(ManualRedactionType.LEGAL_BASIS_CHANGE); - } - } - } - } - - - private Set getManualRedactionPages(ManualRedactions manualRedactions) { - - Set manualRedactionPages = new HashSet<>(); - - if (manualRedactions == null) { - return manualRedactionPages; - } - - manualRedactions.getEntriesToAdd().forEach(entry -> { - entry.getPositions().forEach(pos -> { - manualRedactionPages.add(pos.getPage()); - }); - }); - return manualRedactionPages; - } - - - public List addEntries(Map> entities, ManualRedactions manualRedactions, - int page, String dossierTemplateId) { + public List addEntries(Map> entities, int page, String dossierTemplateId) { List redactionLogEntities = new ArrayList<>(); @@ -245,7 +114,6 @@ public class RedactionLogCreatorService { } redactionLogEntry.setId(entityPositionSequence.getId()); - processRedactionLogEntry(manualRedactions, dossierTemplateId, redactionLogEntry, entity); if (CollectionUtils.isNotEmpty(entityPositionSequence.getSequences())) { List rectanglesPerLine = getRectanglesPerLine(entityPositionSequence.getSequences() @@ -269,112 +137,6 @@ public class RedactionLogCreatorService { return redactionLogEntities; } - private void processRedactionLogEntry(ManualRedactions manualRedactions, String dossierTemplateId, RedactionLogEntry redactionLogEntry, ReasonHolder reasonHolder) { - - List comments = null; - - if (manualRedactions != null && !manualRedactions.getIdsToRemove().isEmpty()) { - for (IdRemoval manualRemoval : manualRedactions.getIdsToRemove()) { - if (manualRemoval.getId().equals(redactionLogEntry.getId())) { - comments = manualRedactions.getComments().get(manualRemoval.getId()); - String manualOverrideReason = null; - if (manualRemoval.getStatus().equals(Status.APPROVED)) { - reasonHolder.setRedaction(false); - redactionLogEntry.setRedacted(false); - redactionLogEntry.setStatus(Status.APPROVED); - manualOverrideReason = mergeReasonIfNecessary(reasonHolder.getRedactionReason(), ", removed by manual override"); - redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, false, redactionLogEntry.isRedacted())); - } else if (manualRemoval.getStatus().equals(Status.REQUESTED)) { - manualOverrideReason = mergeReasonIfNecessary(reasonHolder.getRedactionReason(), ", requested to remove"); - redactionLogEntry.setStatus(Status.REQUESTED); - redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, true, redactionLogEntry.isRedacted())); - } else { - redactionLogEntry.setStatus(Status.DECLINED); - } - - reasonHolder.setRedactionReason(manualOverrideReason != null ? manualOverrideReason : reasonHolder.getRedactionReason()); - redactionLogEntry.setReason(manualOverrideReason); - redactionLogEntry.setManual(true); - redactionLogEntry.setManualRedactionType(ManualRedactionType.REMOVE); - redactionLogEntry.setDictionaryEntry(manualRemoval.isRemoveFromDictionary()); - redactionLogEntry.setDossierDictionaryEntry(manualRemoval.isRemoveFromDictionary()); - } - } - } - - if (manualRedactions != null && !manualRedactions.getForceRedacts().isEmpty()) { - for (ManualForceRedact manualForceRedact : manualRedactions.getForceRedacts()) { - if (manualForceRedact.getId().equals(redactionLogEntry.getId())) { - String manualOverrideReason = null; - if (manualForceRedact.getStatus().equals(Status.APPROVED)) { - reasonHolder.setRedaction(true); - redactionLogEntry.setRedacted(true); - redactionLogEntry.setStatus(Status.APPROVED); - redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, false, redactionLogEntry.isRedacted())); - manualOverrideReason = mergeReasonIfNecessary(reasonHolder.getRedactionReason(), ", forced by manual override"); - redactionLogEntry.setLegalBasis(manualForceRedact.getLegalBasis()); - } else if (manualForceRedact.getStatus().equals(Status.REQUESTED)) { - manualOverrideReason = mergeReasonIfNecessary(reasonHolder.getRedactionReason(), ", requested to force redact"); - redactionLogEntry.setStatus(Status.REQUESTED); - redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, true, redactionLogEntry.isRedacted())); - redactionLogEntry.setLegalBasis(manualForceRedact.getLegalBasis()); - } else { - redactionLogEntry.setStatus(Status.DECLINED); - } - - reasonHolder.setRedactionReason(manualOverrideReason != null ? manualOverrideReason : reasonHolder.getRedactionReason()); - redactionLogEntry.setReason(manualOverrideReason); - redactionLogEntry.setManual(true); - redactionLogEntry.setManualRedactionType(ManualRedactionType.FORCE_REDACT); - - } - } - } - - if (manualRedactions != null && !manualRedactions.getManualLegalBasisChanges().isEmpty()) { - for (ManualLegalBasisChange manualLegalBasisChange : manualRedactions.getManualLegalBasisChanges()) { - if (manualLegalBasisChange.getId().equals(redactionLogEntry.getId())) { - String manualOverrideReason = null; - if (manualLegalBasisChange.getStatus().equals(Status.APPROVED)) { - redactionLogEntry.setStatus(Status.APPROVED); - manualOverrideReason = mergeReasonIfNecessary(reasonHolder.getRedactionReason(), ", legal basis was manually changed"); - redactionLogEntry.setLegalBasis(manualLegalBasisChange.getLegalBasis()); - } else if (manualLegalBasisChange.getStatus().equals(Status.REQUESTED)) { - manualOverrideReason = mergeReasonIfNecessary(reasonHolder.getRedactionReason(), ", legal basis change requested"); - redactionLogEntry.setStatus(Status.REQUESTED); - redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, true, redactionLogEntry.isRedacted())); - redactionLogEntry.setLegalBasisChangeValue(manualLegalBasisChange.getLegalBasis()); - } else { - redactionLogEntry.setStatus(Status.DECLINED); - } - - reasonHolder.setRedactionReason(manualOverrideReason != null ? manualOverrideReason : reasonHolder.getRedactionReason()); - redactionLogEntry.setReason(manualOverrideReason); - redactionLogEntry.setManual(true); - redactionLogEntry.setManualRedactionType(ManualRedactionType.LEGAL_BASIS_CHANGE); - } - } - } - - - if (manualRedactions != null) { - comments = manualRedactions.getComments().get(redactionLogEntry.getId()); - } - - redactionLogEntry.setComments(comments); - } - - private String mergeReasonIfNecessary(String currentReason, String addition) { - if (currentReason != null) { - if (!currentReason.contains(addition)) { - return currentReason + addition; - } - return currentReason; - } else { - return ""; - } - } - private List getRectanglesPerLine(List textPositions, int page) { @@ -403,73 +165,10 @@ public class RedactionLogCreatorService { } - public List addManualAddEntries(Set manualAdds, - Map> comments, int page, - String dossierTemplateId) { - - List redactionLogEntities = new ArrayList<>(); - - if (manualAdds == null) { - return redactionLogEntities; - } - - for (ManualRedactionEntry manualRedactionEntry : manualAdds) { - - String id = manualRedactionEntry.getId(); - - RedactionLogEntry redactionLogEntry = createRedactionLogEntry(manualRedactionEntry, id, dossierTemplateId); - - List rectanglesOnPage = new ArrayList<>(); - for (Rectangle rectangle : manualRedactionEntry.getPositions()) { - if (page == rectangle.getPage()) { - rectanglesOnPage.add(rectangle); - redactionLogEntry.getPositions().add(rectangle); - } - } - - redactionLogEntry.setComments(comments.get(id)); - if (!rectanglesOnPage.isEmpty() && !approvedAndShouldBeInDictionary(manualRedactionEntry)) { - redactionLogEntities.add(redactionLogEntry); - } - } - - return redactionLogEntities; - } - - - private boolean approvedAndShouldBeInDictionary(ManualRedactionEntry manualRedactionEntry) { - - return manualRedactionEntry.getStatus().equals(Status.APPROVED) && manualRedactionEntry.isAddToDictionary(); - } - - - private RedactionLogEntry createRedactionLogEntry(ManualRedactionEntry manualRedactionEntry, String id, - String dossierTemplateId) { - - return RedactionLogEntry.builder() - .id(id) - .color(getColorForManualAdd(manualRedactionEntry.getType(), dossierTemplateId, manualRedactionEntry.getStatus())) - .reason(manualRedactionEntry.getReason()) - .legalBasis(manualRedactionEntry.getLegalBasis()) - .value(manualRedactionEntry.getValue()) - .type(manualRedactionEntry.getType()) - .redacted(true) - .isHint(false) - .section(manualRedactionEntry.getSection()) - .sectionNumber(manualRedactionEntry.getSectionNumber()) - .manual(true) - .status(manualRedactionEntry.getStatus()) - .manualRedactionType(ManualRedactionType.ADD) - .isDictionaryEntry(false) - .isDossierDictionaryEntry(manualRedactionEntry.isAddToDossierDictionary()) - .build(); - } - - private RedactionLogEntry createRedactionLogEntry(Entity entity, String dossierTemplateId) { return RedactionLogEntry.builder() - .color(getColor(entity.getType(), dossierTemplateId, false, entity.isRedaction())) + .color(getColor(entity.getType(), dossierTemplateId, entity.isRedaction())) .reason(entity.getRedactionReason()) .legalBasis(entity.getLegalBasis()) .value(entity.getWord()) @@ -490,59 +189,6 @@ public class RedactionLogCreatorService { } - private float[] getColor(String type, String dossierTemplateId, boolean requestedToRemove, boolean isRedaction) { - - if (requestedToRemove) { - return dictionaryService.getRequestRemoveColor(dossierTemplateId); - } - if (!isRedaction && !isHint(type, dossierTemplateId)) { - return dictionaryService.getNotRedactedColor(dossierTemplateId); - } - return dictionaryService.getColor(type, dossierTemplateId); - } - - - private float[] getColorForManualAdd(String type, String dossierTemplateId, Status status) { - - if (status.equals(Status.REQUESTED)) { - return dictionaryService.getRequestAddColor(dossierTemplateId); - } else if (status.equals(Status.DECLINED)) { - return dictionaryService.getNotRedactedColor(dossierTemplateId); - } - return getColor(type, dossierTemplateId); - } - - - private float[] getColor(String type, String dossierTemplateId) { - - return dictionaryService.getColor(type, dossierTemplateId); - } - - - private float[] getColorForImage(String type, String dossierTemplateId, boolean requestedToRemove, boolean isRedaction) { - - if (requestedToRemove) { - return dictionaryService.getRequestRemoveColor(dossierTemplateId); - } - if (!isRedaction && !dictionaryService.isHint(type, dossierTemplateId)) { - return dictionaryService.getNotRedactedColor(dossierTemplateId); - } - return dictionaryService.getColor(type, dossierTemplateId); - } - - - private boolean isHint(String type, String dossierTemplateId) { - - return dictionaryService.isHint(type, dossierTemplateId); - } - - - private boolean isRecommendation(String type, String dossierTemplateId) { - - return dictionaryService.isRecommendation(type, dossierTemplateId); - } - - private void addSectionGrid(Document classifiedDoc, int page) { for (Paragraph paragraph : classifiedDoc.getParagraphs()) { @@ -586,165 +232,26 @@ public class RedactionLogCreatorService { } } - public RedactionLog getRedactionLogPreview(RedactionLog redactionLog, String dossierTemplateId, ManualRedactions manualRedactions) { + private float[] getColor(String type, String dossierTemplateId, boolean isRedaction) { - var manualRedactionPages = getManualRedactionPages(manualRedactions); - - // generate all manual entries - var manualRedactionLogEntries = new HashMap(); - for (var page : manualRedactionPages) { - - var pageEntries = addManualAddEntries(manualRedactions.getEntriesToAdd(), manualRedactions.getComments(), page, dossierTemplateId); - - for (var entry : pageEntries) { - manualRedactionLogEntries.put(entry.getId(), entry); - } + if (!isRedaction && !isHint(type, dossierTemplateId)) { + return dictionaryService.getNotRedactedColor(dossierTemplateId); } - - for (var manualEntry : manualRedactionLogEntries.values()) { - var existingEntry = redactionLog.getRedactionLogEntry().stream().filter(e -> e.getId().equals(manualEntry.getId())).findAny(); - if (existingEntry.isPresent()) { - // if it has already been processed of sorts, update it - BeanUtils.copyProperties(manualEntry, existingEntry.get()); - } else { - // not yet in the redaction-log - add it - redactionLog.getRedactionLogEntry().add(manualEntry); - } - } - - - Set entriesToRemoveBecauseOfUndoActions = new HashSet<>(); - for (RedactionLogEntry entry : redactionLog.getRedactionLogEntry()) { - - var reasonHolder = new PreviewReasonHolder(entry); - - if (entry.isImage()) { - processImageEntry(manualRedactions, dossierTemplateId, reasonHolder, entry); - } - - processRedactionLogEntry(manualRedactions, dossierTemplateId, entry, reasonHolder); - var shouldRemove = postProcessRedactionLogEntryForDeletedManualRedactions(manualRedactions, entry); - if (shouldRemove) { - entriesToRemoveBecauseOfUndoActions.add(entry.getId()); - } - - - entry.setComments(manualRedactions.getComments().get(entry.getId())); - } - - redactionLog.setRedactionLogEntry(redactionLog.getRedactionLogEntry() - .stream().filter(entry -> !entriesToRemoveBecauseOfUndoActions.contains(entry.getId())).collect(Collectors.toList())); - - - handleAddToDictionary(redactionLog, manualRedactions, dossierTemplateId); - - return redactionLog; - } - - private boolean postProcessRedactionLogEntryForDeletedManualRedactions(ManualRedactions manualRedactions, RedactionLogEntry entry) { - if (entry.isManual()) { - - if (entry.getManualRedactionType() == ManualRedactionType.ADD) { - var manualRedactionExists = manualRedactions.getEntriesToAdd().stream().anyMatch(entryToAdd -> entryToAdd.getId().equalsIgnoreCase(entry.getId())); - return !manualRedactionExists; - } - - if (entry.getManualRedactionType() == ManualRedactionType.FORCE_REDACT) { - var forceRedactExists = manualRedactions.getForceRedacts().stream().anyMatch(entryToAdd -> entryToAdd.getId().equalsIgnoreCase(entry.getId())); - if (!forceRedactExists) { - entry.setRedacted(false); - entry.setManual(false); - entry.setStatus(null); - } - } - - if (entry.getManualRedactionType() == ManualRedactionType.REMOVE) { - var removeIdExists = manualRedactions.getIdsToRemove().stream().anyMatch(entryToAdd -> entryToAdd.getId().equalsIgnoreCase(entry.getId())); - if (!removeIdExists) { - entry.setRedacted(true); - entry.setManual(false); - entry.setStatus(null); - } - } - - // Cannot undo Already Approved change because UI won't allow it for now - if (Status.REQUESTED.equals(entry.getStatus())) { - if (entry.getManualRedactionType() == ManualRedactionType.RECATEGORIZE) { - var recategorizeExists = manualRedactions.getImageRecategorizations().stream().anyMatch(entryToAdd -> entryToAdd.getId().equalsIgnoreCase(entry.getId())); - if (!recategorizeExists) { - entry.setManual(false); - entry.setStatus(null); - entry.setRecategorizationType(null); - } - } - if (entry.getManualRedactionType() == ManualRedactionType.LEGAL_BASIS_CHANGE) { - var legalBasisChangeExists = manualRedactions.getImageRecategorizations().stream().anyMatch(entryToAdd -> entryToAdd.getId().equalsIgnoreCase(entry.getId())); - if (!legalBasisChangeExists) { - entry.setManual(false); - entry.setStatus(null); - entry.setLegalBasisChangeValue(null); - } - } - } - } - - return false; + return dictionaryService.getColor(type, dossierTemplateId); } - private void handleAddToDictionary(RedactionLog redactionLog, ManualRedactions manualRedactions, String dossierTemplateId) { + private boolean isHint(String type, String dossierTemplateId) { - for (var manualRedaction : manualRedactions.getEntriesToAdd()) { - - // not yet processed, and dictionary modifying, show in redaction-log preview - if (manualRedaction.getProcessedDate() == null && manualRedaction.isAddToDictionary() || manualRedaction.isAddToDossierDictionary()) { - var redactionLogEntry = createRedactionLogEntry(manualRedaction, manualRedaction.getId(), dossierTemplateId); - redactionLogEntry.setDictionaryEntry(manualRedaction.isAddToDictionary()); - redactionLogEntry.setDossierDictionaryEntry(manualRedaction.isAddToDossierDictionary()); - redactionLogEntry.setPositions(manualRedaction.getPositions()); - - - var found = redactionLog.getRedactionLogEntry().stream().filter(r -> r.getId().equalsIgnoreCase(redactionLogEntry.getId())).findAny(); - if (found.isPresent()) { - found.get().setDictionaryEntry(manualRedaction.isAddToDictionary()); - found.get().setDossierDictionaryEntry(manualRedaction.isAddToDossierDictionary()); - found.get().setPositions(manualRedaction.getPositions()); - } else { - redactionLog.getRedactionLogEntry().add(redactionLogEntry); - } - - - } - } + return dictionaryService.isHint(type, dossierTemplateId); } - public static class PreviewReasonHolder implements ReasonHolder { - private final RedactionLogEntry entry; + private boolean isRecommendation(String type, String dossierTemplateId) { - public PreviewReasonHolder(RedactionLogEntry entry) { - this.entry = entry; - } - - @Override - public String getRedactionReason() { - return entry.getReason(); - } - - @Override - public void setRedactionReason(String reason) { - entry.setReason(reason); - } - - @Override - public boolean isRedaction() { - return entry.isRedacted(); - } - - @Override - public void setRedaction(boolean value) { - entry.setRedacted(value); - } + return dictionaryService.isRecommendation(type, dossierTemplateId); } + + } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogMergeService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogMergeService.java new file mode 100644 index 00000000..ac19c236 --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogMergeService.java @@ -0,0 +1,241 @@ +package com.iqser.red.service.redaction.v1.server.redaction.service; + +import com.iqser.red.service.redaction.v1.model.*; +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Service; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Set; + +@Service +@RequiredArgsConstructor +public class RedactionLogMergeService { + + private final DictionaryService dictionaryService; + + + public RedactionLog mergeRedactionLogData(RedactionLog redactionLog, String dossierTemplateId, ManualRedactions manualRedactions) { + + if (manualRedactions != null) { + + var manualRedactionLogEntries = addManualAddEntries(manualRedactions.getEntriesToAdd(), manualRedactions.getComments(), dossierTemplateId); + + redactionLog.getRedactionLogEntry().addAll(manualRedactionLogEntries); + + for (RedactionLogEntry entry : redactionLog.getRedactionLogEntry()) { + processRedactionLogEntry(manualRedactions, dossierTemplateId, entry); + entry.setComments(manualRedactions.getComments().get(entry.getId())); + } + + } + + return redactionLog; + } + + + private void processRedactionLogEntry(ManualRedactions manualRedactions, String dossierTemplateId, RedactionLogEntry redactionLogEntry) { + + List comments = null; + + + if (manualRedactions != null && !manualRedactions.getImageRecategorizations().isEmpty()) { + for (ManualImageRecategorization recategorization : manualRedactions.getImageRecategorizations()) { + if (recategorization.getId().equals(redactionLogEntry.getId())) { + String manualOverrideReason = null; + if (recategorization.getStatus().equals(Status.APPROVED)) { + redactionLogEntry.setStatus(Status.APPROVED); + redactionLogEntry.setType(recategorization.getType()); + manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", recategorized by manual override"); + } else if (recategorization.getStatus().equals(Status.REQUESTED)) { + manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", requested to recategorize"); + redactionLogEntry.setStatus(Status.REQUESTED); + redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, false, redactionLogEntry.isRedacted(), false)); + redactionLogEntry.setRecategorizationType(recategorization.getType()); + } else { + redactionLogEntry.setStatus(Status.DECLINED); + } + + redactionLogEntry.setReason(manualOverrideReason); + redactionLogEntry.setManual(true); + redactionLogEntry.setManualRedactionType(ManualRedactionType.RECATEGORIZE); + } + } + } + + if (manualRedactions != null && !manualRedactions.getIdsToRemove().isEmpty()) { + for (IdRemoval manualRemoval : manualRedactions.getIdsToRemove()) { + if (manualRemoval.getId().equals(redactionLogEntry.getId())) { + comments = manualRedactions.getComments().get(manualRemoval.getId()); + String manualOverrideReason = null; + if (manualRemoval.getStatus().equals(Status.APPROVED)) { + redactionLogEntry.setRedacted(false); + redactionLogEntry.setStatus(Status.APPROVED); + manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", removed by manual override"); + redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, false, redactionLogEntry.isRedacted(), true)); + } else if (manualRemoval.getStatus().equals(Status.REQUESTED)) { + manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", requested to remove"); + redactionLogEntry.setStatus(Status.REQUESTED); + redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, true, redactionLogEntry.isRedacted(), false)); + } else { + redactionLogEntry.setStatus(Status.DECLINED); + } + + redactionLogEntry.setReason(manualOverrideReason); + redactionLogEntry.setManual(true); + redactionLogEntry.setManualRedactionType(ManualRedactionType.REMOVE); + redactionLogEntry.setDictionaryEntry(manualRemoval.isRemoveFromDictionary()); + redactionLogEntry.setDossierDictionaryEntry(manualRemoval.isRemoveFromDictionary()); + } + } + } + + if (manualRedactions != null && !manualRedactions.getForceRedacts().isEmpty()) { + for (ManualForceRedact manualForceRedact : manualRedactions.getForceRedacts()) { + if (manualForceRedact.getId().equals(redactionLogEntry.getId())) { + String manualOverrideReason = null; + if (manualForceRedact.getStatus().equals(Status.APPROVED)) { + redactionLogEntry.setRedacted(true); + redactionLogEntry.setStatus(Status.APPROVED); + redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, false, redactionLogEntry.isRedacted(), false)); + manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", forced by manual override"); + redactionLogEntry.setLegalBasis(manualForceRedact.getLegalBasis()); + } else if (manualForceRedact.getStatus().equals(Status.REQUESTED)) { + manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", requested to force redact"); + redactionLogEntry.setStatus(Status.REQUESTED); + redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, true, redactionLogEntry.isRedacted(), false)); + redactionLogEntry.setLegalBasis(manualForceRedact.getLegalBasis()); + } else { + redactionLogEntry.setStatus(Status.DECLINED); + } + + redactionLogEntry.setReason(manualOverrideReason); + redactionLogEntry.setManual(true); + redactionLogEntry.setManualRedactionType(ManualRedactionType.FORCE_REDACT); + + } + } + } + + if (manualRedactions != null && !manualRedactions.getManualLegalBasisChanges().isEmpty()) { + for (ManualLegalBasisChange manualLegalBasisChange : manualRedactions.getManualLegalBasisChanges()) { + if (manualLegalBasisChange.getId().equals(redactionLogEntry.getId())) { + String manualOverrideReason = null; + if (manualLegalBasisChange.getStatus().equals(Status.APPROVED)) { + redactionLogEntry.setStatus(Status.APPROVED); + manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", legal basis was manually changed"); + redactionLogEntry.setLegalBasis(manualLegalBasisChange.getLegalBasis()); + } else if (manualLegalBasisChange.getStatus().equals(Status.REQUESTED)) { + manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", legal basis change requested"); + redactionLogEntry.setStatus(Status.REQUESTED); + redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, true, redactionLogEntry.isRedacted(), false)); + redactionLogEntry.setLegalBasisChangeValue(manualLegalBasisChange.getLegalBasis()); + } else { + redactionLogEntry.setStatus(Status.DECLINED); + } + + redactionLogEntry.setReason(manualOverrideReason); + redactionLogEntry.setManual(true); + redactionLogEntry.setManualRedactionType(ManualRedactionType.LEGAL_BASIS_CHANGE); + } + } + } + + + if (manualRedactions != null) { + comments = manualRedactions.getComments().get(redactionLogEntry.getId()); + } + + redactionLogEntry.setComments(comments); + } + + private String mergeReasonIfNecessary(String currentReason, String addition) { + if (currentReason != null) { + if (!currentReason.contains(addition)) { + return currentReason + addition; + } + return currentReason; + } else { + return ""; + } + } + + + public List addManualAddEntries(Set manualAdds, Map> comments, String dossierTemplateId) { + + List redactionLogEntries = new ArrayList<>(); + + for (ManualRedactionEntry manualRedactionEntry : manualAdds) { + + if (!approvedAndShouldBeInDictionary(manualRedactionEntry)) { + RedactionLogEntry redactionLogEntry = createRedactionLogEntry(manualRedactionEntry, manualRedactionEntry.getId(), dossierTemplateId); + redactionLogEntry.setPositions(manualRedactionEntry.getPositions()); + redactionLogEntry.setComments(comments.get(manualRedactionEntry.getId())); + } + } + + return redactionLogEntries; + } + + + private boolean approvedAndShouldBeInDictionary(ManualRedactionEntry manualRedactionEntry) { + + return manualRedactionEntry.getStatus().equals(Status.APPROVED) && (manualRedactionEntry.isAddToDictionary() || manualRedactionEntry.isAddToDossierDictionary()); + } + + + private RedactionLogEntry createRedactionLogEntry(ManualRedactionEntry manualRedactionEntry, String id, + String dossierTemplateId) { + + return RedactionLogEntry.builder() + .id(id) + .color(getColorForManualAdd(manualRedactionEntry.getType(), dossierTemplateId, manualRedactionEntry.getStatus())) + .reason(manualRedactionEntry.getReason()) + .isDictionaryEntry(manualRedactionEntry.isAddToDictionary()) + .isDossierDictionaryEntry(manualRedactionEntry.isAddToDossierDictionary()) + .legalBasis(manualRedactionEntry.getLegalBasis()) + .value(manualRedactionEntry.getValue()) + .type(manualRedactionEntry.getType()) + .redacted(true) + .isHint(false) + .section(null) + .sectionNumber(-1) + .manual(true) + .status(manualRedactionEntry.getStatus()) + .manualRedactionType(ManualRedactionType.ADD) + .isDictionaryEntry(false) + .isDossierDictionaryEntry(manualRedactionEntry.isAddToDossierDictionary()) + .build(); + } + + + private float[] getColor(String type, String dossierTemplateId, boolean requested, boolean isRedaction, boolean skipped) { + if (requested) { + return dictionaryService.getRequestRemoveColor(dossierTemplateId); + } + if ((!isRedaction && !dictionaryService.isHint(type, dossierTemplateId)) || skipped) { + return dictionaryService.getNotRedactedColor(dossierTemplateId); + } + return dictionaryService.getColor(type, dossierTemplateId); + } + + + private float[] getColorForManualAdd(String type, String dossierTemplateId, Status status) { + + if (status.equals(Status.REQUESTED)) { + return dictionaryService.getRequestAddColor(dossierTemplateId); + } else if (status.equals(Status.DECLINED)) { + return dictionaryService.getNotRedactedColor(dossierTemplateId); + } + return getColor(type, dossierTemplateId); + } + + + private float[] getColor(String type, String dossierTemplateId) { + + return dictionaryService.getColor(type, dossierTemplateId); + } + + +}