Marmelator/redaction-service
1
0
Fork 0
Code Issues Pull Requests 14 Actions Packages Projects Releases 622 Wiki Activity

redaction-log preview generation and removal of manual redactions from redaction-log

Browse Source
This commit is contained in:
Timo 2021-07-21 15:58:52 +03:00
parent b75dbde563
commit 1ac294fc01
12 changed files with 312 additions and 647 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/AnalyzeRequest.java
View File

@ -7,9 +7,7 @@ import lombok.NoArgsConstructor;
import java.time.OffsetDateTime;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
@Data

5
redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/ManualRedactionEntry.java
View File

@ -23,11 +23,8 @@ public class ManualRedactionEntry {
private String legalBasis;
private List<Rectangle> positions = new ArrayList<>();
private Status status;
private boolean addToDictionary;
private String section;
private int sectionNumber;
private boolean addToDossierDictionary;
private OffsetDateTime requestDate;

7
redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionLog.java
View File

@ -10,6 +10,13 @@ import java.util.List;
@AllArgsConstructor
public class RedactionLog {
/**
* Version 0 Redaction Logs have manual redactions merged inside them
* Version 1 Redaction Logs only contain system ( rule/dictionary ) redactions. Manual Redactions are merged in at runtime.
*/
private long computationalVersion;
private List<RedactionLogEntry> redactionLogEntry;
private List<LegalBasisMapping> legalBasis;

2
redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionLogEntry.java
View File

@ -9,6 +9,8 @@ import lombok.NoArgsConstructor;
import java.util.ArrayList;
import java.util.List;
@Data
@Builder
@NoArgsConstructor

1
redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/SectionRectangle.java
View File

@ -30,4 +30,5 @@ public class SectionRectangle {
private int numberOfParts;
private List<CellRectangle> tableCells;
}

2
redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/resources/RedactionResource.java
View File

@ -31,6 +31,6 @@ public interface RedactionResource {
void testRules(@RequestBody String rules);
@PostMapping(value = "/redaction-log/preview", consumes = MediaType.APPLICATION_JSON_VALUE)
RedactionLog getRedactionLogPreview(@RequestBody RedactionRequest redactionRequest);
RedactionLog getRedactionLog(@RequestBody RedactionRequest redactionRequest);
}

8
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RedactionController.java
View File

@ -13,7 +13,7 @@ import com.iqser.red.service.redaction.v1.server.exception.RedactionException;
import com.iqser.red.service.redaction.v1.server.redaction.service.AnnotationService;
import com.iqser.red.service.redaction.v1.server.redaction.service.DictionaryService;
import com.iqser.red.service.redaction.v1.server.redaction.service.DroolsExecutionService;
import com.iqser.red.service.redaction.v1.server.redaction.service.RedactionLogCreatorService;
import com.iqser.red.service.redaction.v1.server.redaction.service.RedactionLogMergeService;
import com.iqser.red.service.redaction.v1.server.segmentation.PdfSegmentationService;
import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
@ -41,7 +41,7 @@ public class RedactionController implements RedactionResource {
private final AnnotationService annotationService;
private final PdfSegmentationService pdfSegmentationService;
private final RedactionStorageService redactionStorageService;
private final RedactionLogCreatorService redactionLogCreatorService;
private final RedactionLogMergeService redactionLogMergeService;
public AnnotateResponse annotate(@RequestBody AnnotateRequest annotateRequest) {
@ -158,13 +158,13 @@ public class RedactionController implements RedactionResource {
}
@Override
public RedactionLog getRedactionLogPreview(RedactionRequest redactionRequest) {
public RedactionLog getRedactionLog(RedactionRequest redactionRequest) {
log.info("Requested preview for: {}", redactionRequest);
dictionaryService.updateDictionary(redactionRequest.getDossierTemplateId(), redactionRequest.getDossierId());
var redactionLog = redactionStorageService.getRedactionLog(redactionRequest.getDossierId(), redactionRequest.getFileId());
return redactionLogCreatorService.getRedactionLogPreview(redactionLog, redactionRequest.getDossierTemplateId(), redactionRequest.getManualRedactions());
return redactionLogMergeService.mergeRedactionLogData(redactionLog, redactionRequest.getDossierTemplateId(), redactionRequest.getManualRedactions());
}

2
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DictionaryService.java
View File

@ -218,7 +218,7 @@ public class DictionaryService {
public float[] getRequestRemoveColor(String dossierTemplateId) {
return dictionariesByDossierTemplate.get(dossierTemplateId).getRequestAddColor();
return dictionariesByDossierTemplate.get(dossierTemplateId).getRequestRemoveColor();
}

40
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionService.java
View File

@ -1,13 +1,6 @@
package com.iqser.red.service.redaction.v1.server.redaction.service;
import com.iqser.red.service.redaction.v1.model.FileAttribute;
import com.iqser.red.service.redaction.v1.model.ManualImageRecategorization;
import com.iqser.red.service.redaction.v1.model.ManualRedactionEntry;
import com.iqser.red.service.redaction.v1.model.ManualRedactions;
import com.iqser.red.service.redaction.v1.model.Point;
import com.iqser.red.service.redaction.v1.model.Rectangle;
import com.iqser.red.service.redaction.v1.model.SectionArea;
import com.iqser.red.service.redaction.v1.model.Status;
import com.iqser.red.service.redaction.v1.model.*;
import com.iqser.red.service.redaction.v1.server.classification.model.*;
import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
import com.iqser.red.service.redaction.v1.server.redaction.model.*;
@ -15,10 +8,8 @@ import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUti
import com.iqser.red.service.redaction.v1.server.redaction.utils.IdBuilder;
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Cell;
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.kie.api.runtime.KieContainer;
@ -103,9 +94,9 @@ public class EntityRedactionService {
List<Table> tables = paragraph.getTables();
for (Table table : tables) {
if (table.getColCount() == 2) {
sectionSearchableTextPairs.addAll(processTableAsOneText(classifiedDoc, table, manualRedactions, sectionNumber, dictionary, local, hintsPerSectionNumber, fileAttributes));
sectionSearchableTextPairs.addAll(processTableAsOneText(classifiedDoc, table, sectionNumber, dictionary, local, hintsPerSectionNumber, fileAttributes));
} else {
sectionSearchableTextPairs.addAll(processTablePerRow(classifiedDoc, table, manualRedactions, sectionNumber, dictionary, local, hintsPerSectionNumber, fileAttributes));
sectionSearchableTextPairs.addAll(processTablePerRow(classifiedDoc, table, sectionNumber, dictionary, local, hintsPerSectionNumber, fileAttributes));
}
sectionNumber.incrementAndGet();
}
@ -169,7 +160,6 @@ public class EntityRedactionService {
private List<SectionSearchableTextPair> processTablePerRow(Document classifiedDoc, Table table,
ManualRedactions manualRedactions,
AtomicInteger sectionNumber, Dictionary dictionary,
boolean local,
Map<Integer, Set<Entity>> hintsPerSectionNumber,
@ -198,7 +188,6 @@ public class EntityRedactionService {
sectionText.getSectionAreas().add(sectionArea);
sectionText.getTextBlocks().addAll(cell.getTextBlocks());
addSectionToManualRedactions(cell.getTextBlocks(), manualRedactions, table.getHeadline(), sectionNumber.intValue());
int cellStart = start;
if (!cell.isHeaderCell()) {
@ -259,7 +248,6 @@ public class EntityRedactionService {
private List<SectionSearchableTextPair> processTableAsOneText(Document classifiedDoc, Table table,
ManualRedactions manualRedactions,
AtomicInteger sectionNumber, Dictionary dictionary,
boolean local,
Map<Integer, Set<Entity>> hintsPerSectionNumber,
@ -288,7 +276,6 @@ public class EntityRedactionService {
for (TextBlock textBlock : cell.getTextBlocks()) {
entireTableText.addAll(textBlock.getSequences());
}
addSectionToManualRedactions(cell.getTextBlocks(), manualRedactions, table.getHeadline(), sectionNumber.intValue());
}
}
@ -348,7 +335,6 @@ public class EntityRedactionService {
classifiedDoc.getSectionText().add(sectionText);
}
addSectionToManualRedactions(paragraphTextBlocks, manualRedactions, headline, sectionNumber.intValue());
Set<Entity> entities = findEntities(searchableText, headline, sectionNumber.intValue(), dictionary, local);
surroundingWordsService.addSurroundingText(entities, searchableText, dictionary);
@ -396,26 +382,6 @@ public class EntityRedactionService {
}
private void addSectionToManualRedactions(List<TextBlock> textBlocks, ManualRedactions manualRedactions,
String section, int sectionNumber) {
if (manualRedactions == null || manualRedactions.getEntriesToAdd().isEmpty()) {
return;
}
for (TextBlock textBlock : textBlocks) {
for (ManualRedactionEntry manualRedactionEntry : manualRedactions.getEntriesToAdd()) {
for (Rectangle rectangle : manualRedactionEntry.getPositions()) {
if (textBlock.contains(rectangle)) {
manualRedactionEntry.setSection(section);
manualRedactionEntry.setSectionNumber(sectionNumber);
}
}
}
}
}
private Image convertAndRecategorize(PdfImage pdfImage, int sectionNumber, String headline, ManualRedactions manualRedactions) {
Image image = Image.builder()

96
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ReanalyzeService.java
View File

@ -1,55 +1,28 @@
package com.iqser.red.service.redaction.v1.server.redaction.service;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import org.kie.api.runtime.KieContainer;
import org.springframework.stereotype.Service;
import org.springframework.web.bind.annotation.RequestBody;
import com.iqser.red.service.file.management.v1.api.model.FileType;
import com.iqser.red.service.redaction.v1.model.AnalyzeRequest;
import com.iqser.red.service.redaction.v1.model.AnalyzeResult;
import com.iqser.red.service.redaction.v1.model.Comment;
import com.iqser.red.service.redaction.v1.model.IdRemoval;
import com.iqser.red.service.redaction.v1.model.ManualForceRedact;
import com.iqser.red.service.redaction.v1.model.ManualImageRecategorization;
import com.iqser.red.service.redaction.v1.model.ManualLegalBasisChange;
import com.iqser.red.service.redaction.v1.model.ManualRedactionEntry;
import com.iqser.red.service.redaction.v1.model.ManualRedactions;
import com.iqser.red.service.redaction.v1.model.Rectangle;
import com.iqser.red.service.redaction.v1.model.RedactionLog;
import com.iqser.red.service.redaction.v1.model.RedactionLogEntry;
import com.iqser.red.service.redaction.v1.model.SectionArea;
import com.iqser.red.service.redaction.v1.model.Status;
import com.iqser.red.service.redaction.v1.model.*;
import com.iqser.red.service.redaction.v1.server.classification.model.Document;
import com.iqser.red.service.redaction.v1.server.classification.model.SectionText;
import com.iqser.red.service.redaction.v1.server.classification.model.Text;
import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient;
import com.iqser.red.service.redaction.v1.server.exception.RedactionException;
import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryIncrement;
import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryVersion;
import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
import com.iqser.red.service.redaction.v1.server.redaction.model.EntityPositionSequence;
import com.iqser.red.service.redaction.v1.server.redaction.model.Image;
import com.iqser.red.service.redaction.v1.server.redaction.model.RedRectangle2D;
import com.iqser.red.service.redaction.v1.server.redaction.model.Section;
import com.iqser.red.service.redaction.v1.server.redaction.model.SectionSearchableTextPair;
import com.iqser.red.service.redaction.v1.server.redaction.model.*;
import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUtils;
import com.iqser.red.service.redaction.v1.server.redaction.utils.IdBuilder;
import com.iqser.red.service.redaction.v1.server.segmentation.PdfSegmentationService;
import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
import lombok.RequiredArgsConstructor;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.kie.api.runtime.KieContainer;
import org.springframework.stereotype.Service;
import org.springframework.web.bind.annotation.RequestBody;
import java.util.*;
import java.util.stream.Collectors;
import java.util.stream.Stream;
@Slf4j
@Service
@ -87,13 +60,13 @@ public class ReanalyzeService {
entityRedactionService.processDocument(classifiedDoc, analyzeRequest.getDossierTemplateId(), analyzeRequest.getManualRedactions(), analyzeRequest
.getDossierId(), analyzeRequest.getFileAttributes());
redactionLogCreatorService.createRedactionLog(classifiedDoc, pageCount, analyzeRequest.getManualRedactions(), analyzeRequest
redactionLogCreatorService.createRedactionLog(classifiedDoc, pageCount, analyzeRequest
.getDossierTemplateId());
log.info("Redaction analysis successful...");
var legalBasis = legalBasisClient.getLegalBasisMapping(analyzeRequest.getDossierTemplateId());
var redactionLog = new RedactionLog(classifiedDoc.getRedactionLogEntities(), legalBasis, classifiedDoc.getDictionaryVersion()
var redactionLog = new RedactionLog(1, classifiedDoc.getRedactionLogEntities(), legalBasis, classifiedDoc.getDictionaryVersion()
.getDossierTemplateVersion(), classifiedDoc.getDictionaryVersion()
.getDossierVersion(), classifiedDoc.getRulesVersion(), legalBasisClient.getVersion(analyzeRequest.getDossierTemplateId()));
@ -132,20 +105,12 @@ public class ReanalyzeService {
DictionaryIncrement dictionaryIncrement = dictionaryService.getDictionaryIncrements(analyzeRequest.getDossierTemplateId(), new DictionaryVersion(redactionLog
.getDictionaryVersion(), redactionLog.getDossierDictionaryVersion()), analyzeRequest.getDossierId());
Set<String> manualForceAndRemoveIds = getForceAndRemoveIds(analyzeRequest.getManualRedactions());
Map<String, List<Comment>> comments = null;
Set<ManualRedactionEntry> manualAdds = null;
if (analyzeRequest.getManualRedactions() != null) {
// TODO comments will be removed from redactionLog, so we ignore this first.
comments = analyzeRequest.getManualRedactions().getComments();
manualAdds = analyzeRequest.getManualRedactions().getEntriesToAdd();
}
Set<String> relevantManuallyModifiedAnnotationIds = getRelevantManuallyModifiedAnnotationIds(analyzeRequest.getManualRedactions());
Set<Integer> sectionsToReanalyse = new HashSet<>();
Map<Integer, Set<Image>> imageEntries = new HashMap<>();
for (RedactionLogEntry entry : redactionLog.getRedactionLogEntry()) {
if (entry.isManual() || manualForceAndRemoveIds.contains(entry.getId())) {
if (entry.isManual() || relevantManuallyModifiedAnnotationIds.contains(entry.getId())) {
sectionsToReanalyse.add(entry.getSectionNumber());
}
if (entry.isImage() || entry.getType().equals("image")) {
@ -159,23 +124,11 @@ public class ReanalyzeService {
sectionsToReanalyse.add(sectionText.getSectionNumber());
}
if (manualAdds != null) {
for (SectionArea sectionArea : sectionText.getSectionAreas()) {
for (ManualRedactionEntry manualAdd : manualAdds) {
for (Rectangle manualPosition : manualAdd.getPositions()) {
if (sectionArea.contains(manualPosition)) {
manualAdd.setSection(sectionText.getHeadline());
manualAdd.setSectionNumber(sectionText.getSectionNumber());
}
}
}
}
}
}
log.info("Should reanalyze {} sections for request: {}", sectionsToReanalyse.size(), analyzeRequest);
if (sectionsToReanalyse.isEmpty() && (manualAdds == null || manualAdds.isEmpty())) {
if (sectionsToReanalyse.isEmpty()) {
return finalizeAnalysis(analyzeRequest, startTime, redactionLog, text, dictionaryIncrement);
}
@ -270,17 +223,15 @@ public class ReanalyzeService {
List<RedactionLogEntry> newRedactionLogEntries = new ArrayList<>();
for (int page = 1; page <= text.getNumberOfPages(); page++) {
if (entitiesPerPage.get(page) != null) {
newRedactionLogEntries.addAll(redactionLogCreatorService.addEntries(entitiesPerPage, analyzeRequest.getManualRedactions(), page, analyzeRequest
newRedactionLogEntries.addAll(redactionLogCreatorService.addEntries(entitiesPerPage, page, analyzeRequest
.getDossierTemplateId()));
}
if (imagesPerPage.get(page) != null) {
newRedactionLogEntries.addAll(redactionLogCreatorService.addImageEntries(imagesPerPage, analyzeRequest.getManualRedactions(), page, analyzeRequest
newRedactionLogEntries.addAll(redactionLogCreatorService.addImageEntries(imagesPerPage, page, analyzeRequest
.getDossierTemplateId()));
}
newRedactionLogEntries.addAll(redactionLogCreatorService.addManualAddEntries(manualAdds, comments, page, analyzeRequest
.getDossierTemplateId()));
}
redactionLog.getRedactionLogEntry().removeIf(entry -> sectionsToReanalyse.contains(entry.getSectionNumber()));
@ -310,7 +261,7 @@ public class ReanalyzeService {
}
private Set<String> getForceAndRemoveIds(ManualRedactions manualRedactions) {
private Set<String> getRelevantManuallyModifiedAnnotationIds(ManualRedactions manualRedactions) {
if (manualRedactions == null) {
return new HashSet<>();
@ -345,15 +296,10 @@ public class ReanalyzeService {
private void excludeExcludedPages(RedactionLog redactionLog, Set<Integer> excludedPages) {
redactionLog.getRedactionLogEntry().forEach(entry -> {
entry.getPositions().forEach(pos -> {
if (excludedPages != null && excludedPages.contains(pos.getPage())) {
entry.setExcluded(true);
} else {
entry.setExcluded(false);
}
});
});
redactionLog.getRedactionLogEntry().forEach(entry ->
entry.getPositions().forEach(pos ->
entry.setExcluded(excludedPages != null && excludedPages.contains(pos.getPage()))
));
}
}

543
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java
View File

@ -1,6 +1,10 @@
package com.iqser.red.service.redaction.v1.server.redaction.service;
import com.iqser.red.service.redaction.v1.model.*;
import com.iqser.red.service.redaction.v1.model.CellRectangle;
import com.iqser.red.service.redaction.v1.model.Point;
import com.iqser.red.service.redaction.v1.model.Rectangle;
import com.iqser.red.service.redaction.v1.model.RedactionLogEntry;
import com.iqser.red.service.redaction.v1.model.SectionRectangle;
import com.iqser.red.service.redaction.v1.server.classification.model.Document;
import com.iqser.red.service.redaction.v1.server.classification.model.Paragraph;
import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
@ -9,17 +13,19 @@ import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSeque
import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
import com.iqser.red.service.redaction.v1.server.redaction.model.EntityPositionSequence;
import com.iqser.red.service.redaction.v1.server.redaction.model.Image;
import com.iqser.red.service.redaction.v1.server.redaction.model.ReasonHolder;
import com.iqser.red.service.redaction.v1.server.redaction.utils.IdBuilder;
import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Cell;
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;
import lombok.RequiredArgsConstructor;
import org.apache.commons.collections4.CollectionUtils;
import org.springframework.beans.BeanUtils;
import org.springframework.stereotype.Service;
import java.util.*;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
@Service
@ -29,10 +35,7 @@ public class RedactionLogCreatorService {
private final DictionaryService dictionaryService;
public void createRedactionLog(Document classifiedDoc, int numberOfPages, ManualRedactions manualRedactions,
String dossierTemplateId) {
Set<Integer> manualRedactionPages = getManualRedactionPages(manualRedactions);
public void createRedactionLog(Document classifiedDoc, int numberOfPages, String dossierTemplateId) {
for (int page = 1; page <= numberOfPages; page++) {
@ -40,24 +43,18 @@ public class RedactionLogCreatorService {
if (classifiedDoc.getEntities().get(page) != null) {
classifiedDoc.getRedactionLogEntities()
.addAll(addEntries(classifiedDoc.getEntities(), manualRedactions, page, dossierTemplateId));
}
if (manualRedactionPages.contains(page)) {
classifiedDoc.getRedactionLogEntities()
.addAll(addManualAddEntries(manualRedactions.getEntriesToAdd(), manualRedactions.getComments(), page, dossierTemplateId));
.addAll(addEntries(classifiedDoc.getEntities(), page, dossierTemplateId));
}
if (classifiedDoc.getImages().get(page) != null && !classifiedDoc.getImages().get(page).isEmpty()) {
classifiedDoc.getRedactionLogEntities()
.addAll(addImageEntries(classifiedDoc.getImages(), manualRedactions, page, dossierTemplateId));
.addAll(addImageEntries(classifiedDoc.getImages(), page, dossierTemplateId));
}
}
}
public List<RedactionLogEntry> addImageEntries(Map<Integer, Set<Image>> images, ManualRedactions manualRedactions,
int pageNumber, String dossierTemplateId) {
public List<RedactionLogEntry> addImageEntries(Map<Integer, Set<Image>> images, int pageNumber, String dossierTemplateId) {
List<RedactionLogEntry> redactionLogEntities = new ArrayList<>();
@ -67,7 +64,7 @@ public class RedactionLogCreatorService {
RedactionLogEntry redactionLogEntry = RedactionLogEntry.builder()
.id(id)
.color(getColorForImage(image.getType(), dossierTemplateId, false, image.isRedaction()))
.color(getColor(image.getType(), dossierTemplateId, image.isRedaction()))
.isImage(true)
.type(image.getType())
.redacted(image.isRedaction())
@ -86,7 +83,6 @@ public class RedactionLogCreatorService {
.imageHasTransparency(image.isHasTransparency())
.build();
processImageEntry(manualRedactions, dossierTemplateId, image, redactionLogEntry);
redactionLogEntities.add(redactionLogEntry);
}
@ -94,135 +90,8 @@ public class RedactionLogCreatorService {
return redactionLogEntities;
}
private void processImageEntry(ManualRedactions manualRedactions, String dossierTemplateId, ReasonHolder image, RedactionLogEntry redactionLogEntry) {
if (manualRedactions != null && !manualRedactions.getImageRecategorizations().isEmpty()) {
for (ManualImageRecategorization recategorization : manualRedactions.getImageRecategorizations()) {
if (recategorization.getId().equals(redactionLogEntry.getId())) {
String manualOverrideReason = null;
if (recategorization.getStatus().equals(Status.APPROVED)) {
redactionLogEntry.setStatus(Status.APPROVED);
redactionLogEntry.setType(recategorization.getType());
manualOverrideReason = mergeReasonIfNecessary(image.getRedactionReason(), ", recategorized by manual override");
} else if (recategorization.getStatus().equals(Status.REQUESTED)) {
manualOverrideReason = mergeReasonIfNecessary(image.getRedactionReason(), ", requested to recategorize");
redactionLogEntry.setStatus(Status.REQUESTED);
redactionLogEntry.setColor(getColorForImage(redactionLogEntry.getType(), dossierTemplateId, true, redactionLogEntry.isRedacted()));
redactionLogEntry.setRecategorizationType(recategorization.getType());
} else {
redactionLogEntry.setStatus(Status.DECLINED);
}
image.setRedactionReason(manualOverrideReason != null ? manualOverrideReason : image.getRedactionReason());
redactionLogEntry.setReason(manualOverrideReason);
redactionLogEntry.setManual(true);
redactionLogEntry.setManualRedactionType(ManualRedactionType.RECATEGORIZE);
}
}
}
if (manualRedactions != null && !manualRedactions.getIdsToRemove().isEmpty()) {
for (IdRemoval manualRemoval : manualRedactions.getIdsToRemove()) {
if (manualRemoval.getId().equals(redactionLogEntry.getId())) {
String manualOverrideReason = null;
if (manualRemoval.getStatus().equals(Status.APPROVED)) {
image.setRedaction(false);
redactionLogEntry.setRedacted(false);
redactionLogEntry.setStatus(Status.APPROVED);
manualOverrideReason = mergeReasonIfNecessary(image.getRedactionReason(), ", removed by manual override");
redactionLogEntry.setColor(getColorForImage(redactionLogEntry.getType(), dossierTemplateId, false, redactionLogEntry.isRedacted()));
} else if (manualRemoval.getStatus().equals(Status.REQUESTED)) {
manualOverrideReason = mergeReasonIfNecessary(image.getRedactionReason(), ", requested to remove");
redactionLogEntry.setStatus(Status.REQUESTED);
redactionLogEntry.setColor(getColorForImage(redactionLogEntry.getType(), dossierTemplateId, true, redactionLogEntry.isRedacted()));
} else {
redactionLogEntry.setStatus(Status.DECLINED);
}
image.setRedactionReason(manualOverrideReason != null ? manualOverrideReason : image.getRedactionReason());
redactionLogEntry.setReason(manualOverrideReason);
redactionLogEntry.setManual(true);
redactionLogEntry.setManualRedactionType(ManualRedactionType.REMOVE);
redactionLogEntry.setDictionaryEntry(manualRemoval.isRemoveFromDictionary());
redactionLogEntry.setDossierDictionaryEntry(manualRemoval.isRemoveFromDictionary());
}
}
}
if (manualRedactions != null && !manualRedactions.getForceRedacts().isEmpty()) {
for (ManualForceRedact manualForceRedact : manualRedactions.getForceRedacts()) {
if (manualForceRedact.getId().equals(redactionLogEntry.getId())) {
String manualOverrideReason = null;
if (manualForceRedact.getStatus().equals(Status.APPROVED)) {
image.setRedaction(true);
redactionLogEntry.setRedacted(true);
redactionLogEntry.setStatus(Status.APPROVED);
redactionLogEntry.setColor(getColorForImage(redactionLogEntry.getType(), dossierTemplateId, false, redactionLogEntry.isRedacted()));
manualOverrideReason = mergeReasonIfNecessary(image.getRedactionReason(), ", forced by manual override");
redactionLogEntry.setLegalBasis(manualForceRedact.getLegalBasis());
} else if (manualForceRedact.getStatus().equals(Status.REQUESTED)) {
manualOverrideReason = mergeReasonIfNecessary(image.getRedactionReason(), ", requested to force redact");
redactionLogEntry.setStatus(Status.REQUESTED);
redactionLogEntry.setColor(getColorForImage(redactionLogEntry.getType(), dossierTemplateId, true, redactionLogEntry.isRedacted()));
redactionLogEntry.setLegalBasis(manualForceRedact.getLegalBasis());
} else {
redactionLogEntry.setStatus(Status.DECLINED);
}
image.setRedactionReason(manualOverrideReason != null ? manualOverrideReason : image.getRedactionReason());
redactionLogEntry.setReason(manualOverrideReason);
redactionLogEntry.setManual(true);
redactionLogEntry.setManualRedactionType(ManualRedactionType.FORCE_REDACT);
}
}
}
if (manualRedactions != null && !manualRedactions.getManualLegalBasisChanges().isEmpty()) {
for (ManualLegalBasisChange manualLegalBasisChange : manualRedactions.getManualLegalBasisChanges()) {
if (manualLegalBasisChange.getId().equals(redactionLogEntry.getId())) {
String manualOverrideReason = null;
if (manualLegalBasisChange.getStatus().equals(Status.APPROVED)) {
redactionLogEntry.setStatus(Status.APPROVED);
manualOverrideReason = mergeReasonIfNecessary(image.getRedactionReason(), ", legal basis was manually changed");
redactionLogEntry.setLegalBasis(manualLegalBasisChange.getLegalBasis());
} else if (manualLegalBasisChange.getStatus().equals(Status.REQUESTED)) {
manualOverrideReason = mergeReasonIfNecessary(image.getRedactionReason(), ", legal basis change requested");
redactionLogEntry.setStatus(Status.REQUESTED);
redactionLogEntry.setColor(getColorForImage(redactionLogEntry.getType(), dossierTemplateId, true, redactionLogEntry.isRedacted()));
redactionLogEntry.setLegalBasisChangeValue(manualLegalBasisChange.getLegalBasis());
} else {
redactionLogEntry.setStatus(Status.DECLINED);
}
image.setRedactionReason(manualOverrideReason != null ? manualOverrideReason : image.getRedactionReason());
redactionLogEntry.setReason(manualOverrideReason);
redactionLogEntry.setManual(true);
redactionLogEntry.setManualRedactionType(ManualRedactionType.LEGAL_BASIS_CHANGE);
}
}
}
}
private Set<Integer> getManualRedactionPages(ManualRedactions manualRedactions) {
Set<Integer> manualRedactionPages = new HashSet<>();
if (manualRedactions == null) {
return manualRedactionPages;
}
manualRedactions.getEntriesToAdd().forEach(entry -> {
entry.getPositions().forEach(pos -> {
manualRedactionPages.add(pos.getPage());
});
});
return manualRedactionPages;
}
public List<RedactionLogEntry> addEntries(Map<Integer, List<Entity>> entities, ManualRedactions manualRedactions,
int page, String dossierTemplateId) {
public List<RedactionLogEntry> addEntries(Map<Integer, List<Entity>> entities, int page, String dossierTemplateId) {
List<RedactionLogEntry> redactionLogEntities = new ArrayList<>();
@ -245,7 +114,6 @@ public class RedactionLogCreatorService {
}
redactionLogEntry.setId(entityPositionSequence.getId());
processRedactionLogEntry(manualRedactions, dossierTemplateId, redactionLogEntry, entity);
if (CollectionUtils.isNotEmpty(entityPositionSequence.getSequences())) {
List<Rectangle> rectanglesPerLine = getRectanglesPerLine(entityPositionSequence.getSequences()
@ -269,112 +137,6 @@ public class RedactionLogCreatorService {
return redactionLogEntities;
}
private void processRedactionLogEntry(ManualRedactions manualRedactions, String dossierTemplateId, RedactionLogEntry redactionLogEntry, ReasonHolder reasonHolder) {
List<Comment> comments = null;
if (manualRedactions != null && !manualRedactions.getIdsToRemove().isEmpty()) {
for (IdRemoval manualRemoval : manualRedactions.getIdsToRemove()) {
if (manualRemoval.getId().equals(redactionLogEntry.getId())) {
comments = manualRedactions.getComments().get(manualRemoval.getId());
String manualOverrideReason = null;
if (manualRemoval.getStatus().equals(Status.APPROVED)) {
reasonHolder.setRedaction(false);
redactionLogEntry.setRedacted(false);
redactionLogEntry.setStatus(Status.APPROVED);
manualOverrideReason = mergeReasonIfNecessary(reasonHolder.getRedactionReason(), ", removed by manual override");
redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, false, redactionLogEntry.isRedacted()));
} else if (manualRemoval.getStatus().equals(Status.REQUESTED)) {
manualOverrideReason = mergeReasonIfNecessary(reasonHolder.getRedactionReason(), ", requested to remove");
redactionLogEntry.setStatus(Status.REQUESTED);
redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, true, redactionLogEntry.isRedacted()));
} else {
redactionLogEntry.setStatus(Status.DECLINED);
}
reasonHolder.setRedactionReason(manualOverrideReason != null ? manualOverrideReason : reasonHolder.getRedactionReason());
redactionLogEntry.setReason(manualOverrideReason);
redactionLogEntry.setManual(true);
redactionLogEntry.setManualRedactionType(ManualRedactionType.REMOVE);
redactionLogEntry.setDictionaryEntry(manualRemoval.isRemoveFromDictionary());
redactionLogEntry.setDossierDictionaryEntry(manualRemoval.isRemoveFromDictionary());
}
}
}
if (manualRedactions != null && !manualRedactions.getForceRedacts().isEmpty()) {
for (ManualForceRedact manualForceRedact : manualRedactions.getForceRedacts()) {
if (manualForceRedact.getId().equals(redactionLogEntry.getId())) {
String manualOverrideReason = null;
if (manualForceRedact.getStatus().equals(Status.APPROVED)) {
reasonHolder.setRedaction(true);
redactionLogEntry.setRedacted(true);
redactionLogEntry.setStatus(Status.APPROVED);
redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, false, redactionLogEntry.isRedacted()));
manualOverrideReason = mergeReasonIfNecessary(reasonHolder.getRedactionReason(), ", forced by manual override");
redactionLogEntry.setLegalBasis(manualForceRedact.getLegalBasis());
} else if (manualForceRedact.getStatus().equals(Status.REQUESTED)) {
manualOverrideReason = mergeReasonIfNecessary(reasonHolder.getRedactionReason(), ", requested to force redact");
redactionLogEntry.setStatus(Status.REQUESTED);
redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, true, redactionLogEntry.isRedacted()));
redactionLogEntry.setLegalBasis(manualForceRedact.getLegalBasis());
} else {
redactionLogEntry.setStatus(Status.DECLINED);
}
reasonHolder.setRedactionReason(manualOverrideReason != null ? manualOverrideReason : reasonHolder.getRedactionReason());
redactionLogEntry.setReason(manualOverrideReason);
redactionLogEntry.setManual(true);
redactionLogEntry.setManualRedactionType(ManualRedactionType.FORCE_REDACT);
}
}
}
if (manualRedactions != null && !manualRedactions.getManualLegalBasisChanges().isEmpty()) {
for (ManualLegalBasisChange manualLegalBasisChange : manualRedactions.getManualLegalBasisChanges()) {
if (manualLegalBasisChange.getId().equals(redactionLogEntry.getId())) {
String manualOverrideReason = null;
if (manualLegalBasisChange.getStatus().equals(Status.APPROVED)) {
redactionLogEntry.setStatus(Status.APPROVED);
manualOverrideReason = mergeReasonIfNecessary(reasonHolder.getRedactionReason(), ", legal basis was manually changed");
redactionLogEntry.setLegalBasis(manualLegalBasisChange.getLegalBasis());
} else if (manualLegalBasisChange.getStatus().equals(Status.REQUESTED)) {
manualOverrideReason = mergeReasonIfNecessary(reasonHolder.getRedactionReason(), ", legal basis change requested");
redactionLogEntry.setStatus(Status.REQUESTED);
redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, true, redactionLogEntry.isRedacted()));
redactionLogEntry.setLegalBasisChangeValue(manualLegalBasisChange.getLegalBasis());
} else {
redactionLogEntry.setStatus(Status.DECLINED);
}
reasonHolder.setRedactionReason(manualOverrideReason != null ? manualOverrideReason : reasonHolder.getRedactionReason());
redactionLogEntry.setReason(manualOverrideReason);
redactionLogEntry.setManual(true);
redactionLogEntry.setManualRedactionType(ManualRedactionType.LEGAL_BASIS_CHANGE);
}
}
}
if (manualRedactions != null) {
comments = manualRedactions.getComments().get(redactionLogEntry.getId());
}
redactionLogEntry.setComments(comments);
}
private String mergeReasonIfNecessary(String currentReason, String addition) {
if (currentReason != null) {
if (!currentReason.contains(addition)) {
return currentReason + addition;
}
return currentReason;
} else {
return "";
}
}
private List<Rectangle> getRectanglesPerLine(List<RedTextPosition> textPositions, int page) {
@ -403,73 +165,10 @@ public class RedactionLogCreatorService {
}
public List<RedactionLogEntry> addManualAddEntries(Set<ManualRedactionEntry> manualAdds,
Map<String, List<Comment>> comments, int page,
String dossierTemplateId) {
List<RedactionLogEntry> redactionLogEntities = new ArrayList<>();
if (manualAdds == null) {
return redactionLogEntities;
}
for (ManualRedactionEntry manualRedactionEntry : manualAdds) {
String id = manualRedactionEntry.getId();
RedactionLogEntry redactionLogEntry = createRedactionLogEntry(manualRedactionEntry, id, dossierTemplateId);
List<Rectangle> rectanglesOnPage = new ArrayList<>();
for (Rectangle rectangle : manualRedactionEntry.getPositions()) {
if (page == rectangle.getPage()) {
rectanglesOnPage.add(rectangle);
redactionLogEntry.getPositions().add(rectangle);
}
}
redactionLogEntry.setComments(comments.get(id));
if (!rectanglesOnPage.isEmpty() && !approvedAndShouldBeInDictionary(manualRedactionEntry)) {
redactionLogEntities.add(redactionLogEntry);
}
}
return redactionLogEntities;
}
private boolean approvedAndShouldBeInDictionary(ManualRedactionEntry manualRedactionEntry) {
return manualRedactionEntry.getStatus().equals(Status.APPROVED) && manualRedactionEntry.isAddToDictionary();
}
private RedactionLogEntry createRedactionLogEntry(ManualRedactionEntry manualRedactionEntry, String id,
String dossierTemplateId) {
return RedactionLogEntry.builder()
.id(id)
.color(getColorForManualAdd(manualRedactionEntry.getType(), dossierTemplateId, manualRedactionEntry.getStatus()))
.reason(manualRedactionEntry.getReason())
.legalBasis(manualRedactionEntry.getLegalBasis())
.value(manualRedactionEntry.getValue())
.type(manualRedactionEntry.getType())
.redacted(true)
.isHint(false)
.section(manualRedactionEntry.getSection())
.sectionNumber(manualRedactionEntry.getSectionNumber())
.manual(true)
.status(manualRedactionEntry.getStatus())
.manualRedactionType(ManualRedactionType.ADD)
.isDictionaryEntry(false)
.isDossierDictionaryEntry(manualRedactionEntry.isAddToDossierDictionary())
.build();
}
private RedactionLogEntry createRedactionLogEntry(Entity entity, String dossierTemplateId) {
return RedactionLogEntry.builder()
.color(getColor(entity.getType(), dossierTemplateId, false, entity.isRedaction()))
.color(getColor(entity.getType(), dossierTemplateId, entity.isRedaction()))
.reason(entity.getRedactionReason())
.legalBasis(entity.getLegalBasis())
.value(entity.getWord())
@ -490,59 +189,6 @@ public class RedactionLogCreatorService {
}
private float[] getColor(String type, String dossierTemplateId, boolean requestedToRemove, boolean isRedaction) {
if (requestedToRemove) {
return dictionaryService.getRequestRemoveColor(dossierTemplateId);
}
if (!isRedaction && !isHint(type, dossierTemplateId)) {
return dictionaryService.getNotRedactedColor(dossierTemplateId);
}
return dictionaryService.getColor(type, dossierTemplateId);
}
private float[] getColorForManualAdd(String type, String dossierTemplateId, Status status) {
if (status.equals(Status.REQUESTED)) {
return dictionaryService.getRequestAddColor(dossierTemplateId);
} else if (status.equals(Status.DECLINED)) {
return dictionaryService.getNotRedactedColor(dossierTemplateId);
}
return getColor(type, dossierTemplateId);
}
private float[] getColor(String type, String dossierTemplateId) {
return dictionaryService.getColor(type, dossierTemplateId);
}
private float[] getColorForImage(String type, String dossierTemplateId, boolean requestedToRemove, boolean isRedaction) {
if (requestedToRemove) {
return dictionaryService.getRequestRemoveColor(dossierTemplateId);
}
if (!isRedaction && !dictionaryService.isHint(type, dossierTemplateId)) {
return dictionaryService.getNotRedactedColor(dossierTemplateId);
}
return dictionaryService.getColor(type, dossierTemplateId);
}
private boolean isHint(String type, String dossierTemplateId) {
return dictionaryService.isHint(type, dossierTemplateId);
}
private boolean isRecommendation(String type, String dossierTemplateId) {
return dictionaryService.isRecommendation(type, dossierTemplateId);
}
private void addSectionGrid(Document classifiedDoc, int page) {
for (Paragraph paragraph : classifiedDoc.getParagraphs()) {
@ -586,165 +232,26 @@ public class RedactionLogCreatorService {
}
}
public RedactionLog getRedactionLogPreview(RedactionLog redactionLog, String dossierTemplateId, ManualRedactions manualRedactions) {
private float[] getColor(String type, String dossierTemplateId, boolean isRedaction) {
var manualRedactionPages = getManualRedactionPages(manualRedactions);
// generate all manual entries
var manualRedactionLogEntries = new HashMap<String, RedactionLogEntry>();
for (var page : manualRedactionPages) {
var pageEntries = addManualAddEntries(manualRedactions.getEntriesToAdd(), manualRedactions.getComments(), page, dossierTemplateId);
for (var entry : pageEntries) {
manualRedactionLogEntries.put(entry.getId(), entry);
}
}
for (var manualEntry : manualRedactionLogEntries.values()) {
var existingEntry = redactionLog.getRedactionLogEntry().stream().filter(e -> e.getId().equals(manualEntry.getId())).findAny();
if (existingEntry.isPresent()) {
// if it has already been processed of sorts, update it
BeanUtils.copyProperties(manualEntry, existingEntry.get());
} else {
// not yet in the redaction-log - add it
redactionLog.getRedactionLogEntry().add(manualEntry);
if (!isRedaction && !isHint(type, dossierTemplateId)) {
return dictionaryService.getNotRedactedColor(dossierTemplateId);
}
return dictionaryService.getColor(type, dossierTemplateId);
}
Set<String> entriesToRemoveBecauseOfUndoActions = new HashSet<>();
for (RedactionLogEntry entry : redactionLog.getRedactionLogEntry()) {
private boolean isHint(String type, String dossierTemplateId) {
var reasonHolder = new PreviewReasonHolder(entry);
if (entry.isImage()) {
processImageEntry(manualRedactions, dossierTemplateId, reasonHolder, entry);
}
processRedactionLogEntry(manualRedactions, dossierTemplateId, entry, reasonHolder);
var shouldRemove = postProcessRedactionLogEntryForDeletedManualRedactions(manualRedactions, entry);
if (shouldRemove) {
entriesToRemoveBecauseOfUndoActions.add(entry.getId());
return dictionaryService.isHint(type, dossierTemplateId);
}
entry.setComments(manualRedactions.getComments().get(entry.getId()));
}
private boolean isRecommendation(String type, String dossierTemplateId) {
redactionLog.setRedactionLogEntry(redactionLog.getRedactionLogEntry()
.stream().filter(entry -> !entriesToRemoveBecauseOfUndoActions.contains(entry.getId())).collect(Collectors.toList()));
handleAddToDictionary(redactionLog, manualRedactions, dossierTemplateId);
return redactionLog;
}
private boolean postProcessRedactionLogEntryForDeletedManualRedactions(ManualRedactions manualRedactions, RedactionLogEntry entry) {
if (entry.isManual()) {
if (entry.getManualRedactionType() == ManualRedactionType.ADD) {
var manualRedactionExists = manualRedactions.getEntriesToAdd().stream().anyMatch(entryToAdd -> entryToAdd.getId().equalsIgnoreCase(entry.getId()));
return !manualRedactionExists;
}
if (entry.getManualRedactionType() == ManualRedactionType.FORCE_REDACT) {
var forceRedactExists = manualRedactions.getForceRedacts().stream().anyMatch(entryToAdd -> entryToAdd.getId().equalsIgnoreCase(entry.getId()));
if (!forceRedactExists) {
entry.setRedacted(false);
entry.setManual(false);
entry.setStatus(null);
}
}
if (entry.getManualRedactionType() == ManualRedactionType.REMOVE) {
var removeIdExists = manualRedactions.getIdsToRemove().stream().anyMatch(entryToAdd -> entryToAdd.getId().equalsIgnoreCase(entry.getId()));
if (!removeIdExists) {
entry.setRedacted(true);
entry.setManual(false);
entry.setStatus(null);
}
}
// Cannot undo Already Approved change because UI won't allow it for now
if (Status.REQUESTED.equals(entry.getStatus())) {
if (entry.getManualRedactionType() == ManualRedactionType.RECATEGORIZE) {
var recategorizeExists = manualRedactions.getImageRecategorizations().stream().anyMatch(entryToAdd -> entryToAdd.getId().equalsIgnoreCase(entry.getId()));
if (!recategorizeExists) {
entry.setManual(false);
entry.setStatus(null);
entry.setRecategorizationType(null);
}
}
if (entry.getManualRedactionType() == ManualRedactionType.LEGAL_BASIS_CHANGE) {
var legalBasisChangeExists = manualRedactions.getImageRecategorizations().stream().anyMatch(entryToAdd -> entryToAdd.getId().equalsIgnoreCase(entry.getId()));
if (!legalBasisChangeExists) {
entry.setManual(false);
entry.setStatus(null);
entry.setLegalBasisChangeValue(null);
}
}
}
}
return false;
}
private void handleAddToDictionary(RedactionLog redactionLog, ManualRedactions manualRedactions, String dossierTemplateId) {
for (var manualRedaction : manualRedactions.getEntriesToAdd()) {
// not yet processed, and dictionary modifying, show in redaction-log preview
if (manualRedaction.getProcessedDate() == null && manualRedaction.isAddToDictionary() || manualRedaction.isAddToDossierDictionary()) {
var redactionLogEntry = createRedactionLogEntry(manualRedaction, manualRedaction.getId(), dossierTemplateId);
redactionLogEntry.setDictionaryEntry(manualRedaction.isAddToDictionary());
redactionLogEntry.setDossierDictionaryEntry(manualRedaction.isAddToDossierDictionary());
redactionLogEntry.setPositions(manualRedaction.getPositions());
var found = redactionLog.getRedactionLogEntry().stream().filter(r -> r.getId().equalsIgnoreCase(redactionLogEntry.getId())).findAny();
if (found.isPresent()) {
found.get().setDictionaryEntry(manualRedaction.isAddToDictionary());
found.get().setDossierDictionaryEntry(manualRedaction.isAddToDossierDictionary());
found.get().setPositions(manualRedaction.getPositions());
} else {
redactionLog.getRedactionLogEntry().add(redactionLogEntry);
return dictionaryService.isRecommendation(type, dossierTemplateId);
}
}
}
}
public static class PreviewReasonHolder implements ReasonHolder {
private final RedactionLogEntry entry;
public PreviewReasonHolder(RedactionLogEntry entry) {
this.entry = entry;
}
@Override
public String getRedactionReason() {
return entry.getReason();
}
@Override
public void setRedactionReason(String reason) {
entry.setReason(reason);
}
@Override
public boolean isRedaction() {
return entry.isRedacted();
}
@Override
public void setRedaction(boolean value) {
entry.setRedacted(value);
}
}
}

241
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogMergeService.java Normal file
View File

@ -0,0 +1,241 @@
package com.iqser.red.service.redaction.v1.server.redaction.service;
import com.iqser.red.service.redaction.v1.model.*;
import lombok.RequiredArgsConstructor;
import org.springframework.stereotype.Service;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Set;
@Service
@RequiredArgsConstructor
public class RedactionLogMergeService {
private final DictionaryService dictionaryService;
public RedactionLog mergeRedactionLogData(RedactionLog redactionLog, String dossierTemplateId, ManualRedactions manualRedactions) {
if (manualRedactions != null) {
var manualRedactionLogEntries = addManualAddEntries(manualRedactions.getEntriesToAdd(), manualRedactions.getComments(), dossierTemplateId);
redactionLog.getRedactionLogEntry().addAll(manualRedactionLogEntries);
for (RedactionLogEntry entry : redactionLog.getRedactionLogEntry()) {
processRedactionLogEntry(manualRedactions, dossierTemplateId, entry);
entry.setComments(manualRedactions.getComments().get(entry.getId()));
}
}
return redactionLog;
}
private void processRedactionLogEntry(ManualRedactions manualRedactions, String dossierTemplateId, RedactionLogEntry redactionLogEntry) {
List<Comment> comments = null;
if (manualRedactions != null && !manualRedactions.getImageRecategorizations().isEmpty()) {
for (ManualImageRecategorization recategorization : manualRedactions.getImageRecategorizations()) {
if (recategorization.getId().equals(redactionLogEntry.getId())) {
String manualOverrideReason = null;
if (recategorization.getStatus().equals(Status.APPROVED)) {
redactionLogEntry.setStatus(Status.APPROVED);
redactionLogEntry.setType(recategorization.getType());
manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", recategorized by manual override");
} else if (recategorization.getStatus().equals(Status.REQUESTED)) {
manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", requested to recategorize");
redactionLogEntry.setStatus(Status.REQUESTED);
redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, false, redactionLogEntry.isRedacted(), false));
redactionLogEntry.setRecategorizationType(recategorization.getType());
} else {
redactionLogEntry.setStatus(Status.DECLINED);
}
redactionLogEntry.setReason(manualOverrideReason);
redactionLogEntry.setManual(true);
redactionLogEntry.setManualRedactionType(ManualRedactionType.RECATEGORIZE);
}
}
}
if (manualRedactions != null && !manualRedactions.getIdsToRemove().isEmpty()) {
for (IdRemoval manualRemoval : manualRedactions.getIdsToRemove()) {
if (manualRemoval.getId().equals(redactionLogEntry.getId())) {
comments = manualRedactions.getComments().get(manualRemoval.getId());
String manualOverrideReason = null;
if (manualRemoval.getStatus().equals(Status.APPROVED)) {
redactionLogEntry.setRedacted(false);
redactionLogEntry.setStatus(Status.APPROVED);
manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", removed by manual override");
redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, false, redactionLogEntry.isRedacted(), true));
} else if (manualRemoval.getStatus().equals(Status.REQUESTED)) {
manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", requested to remove");
redactionLogEntry.setStatus(Status.REQUESTED);
redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, true, redactionLogEntry.isRedacted(), false));
} else {
redactionLogEntry.setStatus(Status.DECLINED);
}
redactionLogEntry.setReason(manualOverrideReason);
redactionLogEntry.setManual(true);
redactionLogEntry.setManualRedactionType(ManualRedactionType.REMOVE);
redactionLogEntry.setDictionaryEntry(manualRemoval.isRemoveFromDictionary());
redactionLogEntry.setDossierDictionaryEntry(manualRemoval.isRemoveFromDictionary());
}
}
}
if (manualRedactions != null && !manualRedactions.getForceRedacts().isEmpty()) {
for (ManualForceRedact manualForceRedact : manualRedactions.getForceRedacts()) {
if (manualForceRedact.getId().equals(redactionLogEntry.getId())) {
String manualOverrideReason = null;
if (manualForceRedact.getStatus().equals(Status.APPROVED)) {
redactionLogEntry.setRedacted(true);
redactionLogEntry.setStatus(Status.APPROVED);
redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, false, redactionLogEntry.isRedacted(), false));
manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", forced by manual override");
redactionLogEntry.setLegalBasis(manualForceRedact.getLegalBasis());
} else if (manualForceRedact.getStatus().equals(Status.REQUESTED)) {
manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", requested to force redact");
redactionLogEntry.setStatus(Status.REQUESTED);
redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, true, redactionLogEntry.isRedacted(), false));
redactionLogEntry.setLegalBasis(manualForceRedact.getLegalBasis());
} else {
redactionLogEntry.setStatus(Status.DECLINED);
}
redactionLogEntry.setReason(manualOverrideReason);
redactionLogEntry.setManual(true);
redactionLogEntry.setManualRedactionType(ManualRedactionType.FORCE_REDACT);
}
}
}
if (manualRedactions != null && !manualRedactions.getManualLegalBasisChanges().isEmpty()) {
for (ManualLegalBasisChange manualLegalBasisChange : manualRedactions.getManualLegalBasisChanges()) {
if (manualLegalBasisChange.getId().equals(redactionLogEntry.getId())) {
String manualOverrideReason = null;
if (manualLegalBasisChange.getStatus().equals(Status.APPROVED)) {
redactionLogEntry.setStatus(Status.APPROVED);
manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", legal basis was manually changed");
redactionLogEntry.setLegalBasis(manualLegalBasisChange.getLegalBasis());
} else if (manualLegalBasisChange.getStatus().equals(Status.REQUESTED)) {
manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", legal basis change requested");
redactionLogEntry.setStatus(Status.REQUESTED);
redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, true, redactionLogEntry.isRedacted(), false));
redactionLogEntry.setLegalBasisChangeValue(manualLegalBasisChange.getLegalBasis());
} else {
redactionLogEntry.setStatus(Status.DECLINED);
}
redactionLogEntry.setReason(manualOverrideReason);
redactionLogEntry.setManual(true);
redactionLogEntry.setManualRedactionType(ManualRedactionType.LEGAL_BASIS_CHANGE);
}
}
}
if (manualRedactions != null) {
comments = manualRedactions.getComments().get(redactionLogEntry.getId());
}
redactionLogEntry.setComments(comments);
}
private String mergeReasonIfNecessary(String currentReason, String addition) {
if (currentReason != null) {
if (!currentReason.contains(addition)) {
return currentReason + addition;
}
return currentReason;
} else {
return "";
}
}
public List<RedactionLogEntry> addManualAddEntries(Set<ManualRedactionEntry> manualAdds, Map<String, List<Comment>> comments, String dossierTemplateId) {
List<RedactionLogEntry> redactionLogEntries = new ArrayList<>();
for (ManualRedactionEntry manualRedactionEntry : manualAdds) {
if (!approvedAndShouldBeInDictionary(manualRedactionEntry)) {
RedactionLogEntry redactionLogEntry = createRedactionLogEntry(manualRedactionEntry, manualRedactionEntry.getId(), dossierTemplateId);
redactionLogEntry.setPositions(manualRedactionEntry.getPositions());
redactionLogEntry.setComments(comments.get(manualRedactionEntry.getId()));
}
}
return redactionLogEntries;
}
private boolean approvedAndShouldBeInDictionary(ManualRedactionEntry manualRedactionEntry) {
return manualRedactionEntry.getStatus().equals(Status.APPROVED) && (manualRedactionEntry.isAddToDictionary() || manualRedactionEntry.isAddToDossierDictionary());
}
private RedactionLogEntry createRedactionLogEntry(ManualRedactionEntry manualRedactionEntry, String id,
String dossierTemplateId) {
return RedactionLogEntry.builder()
.id(id)
.color(getColorForManualAdd(manualRedactionEntry.getType(), dossierTemplateId, manualRedactionEntry.getStatus()))
.reason(manualRedactionEntry.getReason())
.isDictionaryEntry(manualRedactionEntry.isAddToDictionary())
.isDossierDictionaryEntry(manualRedactionEntry.isAddToDossierDictionary())
.legalBasis(manualRedactionEntry.getLegalBasis())
.value(manualRedactionEntry.getValue())
.type(manualRedactionEntry.getType())
.redacted(true)
.isHint(false)
.section(null)
.sectionNumber(-1)
.manual(true)
.status(manualRedactionEntry.getStatus())
.manualRedactionType(ManualRedactionType.ADD)
.isDictionaryEntry(false)
.isDossierDictionaryEntry(manualRedactionEntry.isAddToDossierDictionary())
.build();
}
private float[] getColor(String type, String dossierTemplateId, boolean requested, boolean isRedaction, boolean skipped) {
if (requested) {
return dictionaryService.getRequestRemoveColor(dossierTemplateId);
}
if ((!isRedaction && !dictionaryService.isHint(type, dossierTemplateId)) || skipped) {
return dictionaryService.getNotRedactedColor(dossierTemplateId);
}
return dictionaryService.getColor(type, dossierTemplateId);
}
private float[] getColorForManualAdd(String type, String dossierTemplateId, Status status) {
if (status.equals(Status.REQUESTED)) {
return dictionaryService.getRequestAddColor(dossierTemplateId);
} else if (status.equals(Status.DECLINED)) {
return dictionaryService.getNotRedactedColor(dossierTemplateId);
}
return getColor(type, dossierTemplateId);
}
private float[] getColor(String type, String dossierTemplateId) {
return dictionaryService.getColor(type, dossierTemplateId);
}
}