diff --git a/redaction-service-v1/redaction-service-api-v1/pom.xml b/redaction-service-v1/redaction-service-api-v1/pom.xml index 4036fd1d..673abe7a 100644 --- a/redaction-service-v1/redaction-service-api-v1/pom.xml +++ b/redaction-service-v1/redaction-service-api-v1/pom.xml @@ -12,7 +12,7 @@ redaction-service-api-v1 - 0.151.0 + 1.23.0 diff --git a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/ManualChange.java b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/ManualChange.java new file mode 100644 index 00000000..1f6d0a4c --- /dev/null +++ b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/ManualChange.java @@ -0,0 +1,48 @@ +package com.iqser.red.service.redaction.v1.model; + +import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus; +import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.BaseAnnotation; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.time.OffsetDateTime; +import java.util.HashMap; +import java.util.Map; + +@Data +@AllArgsConstructor +@NoArgsConstructor +@Builder +public class ManualChange { + + private AnnotationStatus annotationStatus; + private ManualRedactionType manualRedactionType; + private OffsetDateTime processedDate; + private String userId; + private Map propertyChanges = new HashMap<>(); + + public boolean isProcessed() { + return processedDate != null; + } + + public static ManualChange from(BaseAnnotation baseAnnotation) { + ManualChange manualChange = new ManualChange(); + manualChange.annotationStatus = baseAnnotation.getStatus(); + manualChange.processedDate = baseAnnotation.getProcessedDate(); + manualChange.userId = baseAnnotation.getUser(); + return manualChange; + } + + public ManualChange withManualRedactionType(ManualRedactionType manualRedactionType) { + this.manualRedactionType = manualRedactionType; + return this; + } + + public ManualChange withChange(String property, Object value) { + this.propertyChanges.put(property, value); + return this; + } + +} diff --git a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/ManualRedactionType.java b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/ManualRedactionType.java index 4162c029..6c894f92 100644 --- a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/ManualRedactionType.java +++ b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/ManualRedactionType.java @@ -1,5 +1,13 @@ package com.iqser.red.service.redaction.v1.model; public enum ManualRedactionType { - ADD, REMOVE, FORCE_REDACT, RECATEGORIZE, LEGAL_BASIS_CHANGE, RESIZE + ADD_LOCALLY, + ADD_TO_DICTIONARY, + REMOVE_LOCALLY, + REMOVE_FROM_DICTIONARY, + FORCE_REDACT, + FORCE_HINT, + RECATEGORIZE, + LEGAL_BASIS_CHANGE, + RESIZE } diff --git a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionLogEntry.java b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionLogEntry.java index b23d7b59..cf94e561 100644 --- a/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionLogEntry.java +++ b/redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionLogEntry.java @@ -2,16 +2,9 @@ package com.iqser.red.service.redaction.v1.model; import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus; import com.iqser.red.service.persistence.service.v1.api.model.annotations.Comment; -import lombok.AllArgsConstructor; -import lombok.Builder; -import lombok.Data; -import lombok.EqualsAndHashCode; -import lombok.NoArgsConstructor; +import lombok.*; -import java.util.ArrayList; -import java.util.HashSet; -import java.util.List; -import java.util.Set; +import java.util.*; @Data @@ -28,27 +21,18 @@ public class RedactionLogEntry { private int matchedRule; private boolean rectangle; private String legalBasis; + private boolean redacted; private boolean isHint; private boolean isRecommendation; - private boolean hasBeenResized; - private boolean hasBeenRecategorized; - private boolean hasLegalBasisChanged; - private boolean hasBeenForced; - private boolean hasBeenRemovedByManualOverride; - private String section; private float[] color; @Builder.Default private List positions = new ArrayList<>(); private int sectionNumber; - private boolean manual; - private AnnotationStatus status; - private ManualRedactionType manualRedactionType; - private String manualRedactionUserId; - private boolean isDictionaryEntry; + private String textBefore; private String textAfter; @@ -62,21 +46,39 @@ public class RedactionLogEntry { private boolean isImage; private boolean imageHasTransparency; + private boolean isDictionaryEntry; private boolean isDossierDictionaryEntry; private boolean excluded; - private String recategorizationType; - private String legalBasisChangeValue; - @EqualsAndHashCode.Exclude @Builder.Default private List changes = new ArrayList<>(); - private Set engines= new HashSet<>(); + @EqualsAndHashCode.Exclude + @Builder.Default + private List manualChanges = new ArrayList<>(); + + private Set engines = new HashSet<>(); private Set reference = new HashSet<>(); + public boolean lastChangeIsRemoved() { + return last(changes).map(c -> c.getType() == ChangeType.REMOVED).orElse(false); + } + public boolean isLocalManualRedaction() { + return manualChanges.stream().anyMatch(mc -> mc.getManualRedactionType() == ManualRedactionType.ADD_LOCALLY && + mc.getAnnotationStatus() == AnnotationStatus.APPROVED); + } + + public boolean isManuallyRemoved() { + return manualChanges.stream().anyMatch(mc -> mc.getManualRedactionType() == ManualRedactionType.REMOVE_LOCALLY && + mc.getAnnotationStatus() == AnnotationStatus.APPROVED); + } + + private Optional last(List list) { + return list.isEmpty() ? Optional.empty() : Optional.of(list.get(list.size() - 1)); + } } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/AnalyzeService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/AnalyzeService.java index 32d4bf17..05b642f8 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/AnalyzeService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/AnalyzeService.java @@ -1,8 +1,11 @@ package com.iqser.red.service.redaction.v1.server.redaction.service; -import com.iqser.red.service.persistence.service.v1.api.model.annotations.*; +import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions; +import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.IdRemoval; +import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualForceRedaction; +import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualImageRecategorization; +import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualLegalBasisChange; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType; -import com.iqser.red.service.redaction.v1.model.Rectangle; import com.iqser.red.service.redaction.v1.model.*; import com.iqser.red.service.redaction.v1.server.classification.model.Document; import com.iqser.red.service.redaction.v1.server.classification.model.SectionText; @@ -180,7 +183,7 @@ public class AnalyzeService { Set sectionsToReanalyse = new HashSet<>(); Map> imageEntries = new HashMap<>(); for (RedactionLogEntry entry : redactionLog.getRedactionLogEntry()) { - if (entry.isManual() || relevantManuallyModifiedAnnotationIds.contains(entry.getId())) { + if (entry.isLocalManualRedaction() || relevantManuallyModifiedAnnotationIds.contains(entry.getId())) { sectionsToReanalyse.add(entry.getSectionNumber()); } if (entry.isImage()) { @@ -231,6 +234,7 @@ public class AnalyzeService { .legalBasisVersion(redactionLog.getLegalBasisVersion()) .dossierDictionaryVersion(redactionLog.getDossierDictionaryVersion()) .wasReanalyzed(isReanalysis) + .manualRedactions(analyzeRequest.getManualRedactions()) .build(); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/AnnotationService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/AnnotationService.java index c0376d89..52cce463 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/AnnotationService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/AnnotationService.java @@ -1,6 +1,7 @@ package com.iqser.red.service.redaction.v1.server.redaction.service; import com.iqser.red.service.persistence.service.v1.api.model.annotations.Comment; +import com.iqser.red.service.redaction.v1.model.Rectangle; import com.iqser.red.service.redaction.v1.model.*; import lombok.RequiredArgsConstructor; import org.apache.pdfbox.pdmodel.PDDocument; @@ -15,13 +16,10 @@ import org.apache.pdfbox.pdmodel.interactive.annotation.PDAnnotationText; import org.apache.pdfbox.pdmodel.interactive.annotation.PDAnnotationTextMarkup; import org.springframework.stereotype.Service; -import java.awt.Color; +import java.awt.*; import java.io.IOException; -import java.util.ArrayList; -import java.util.GregorianCalendar; -import java.util.HashMap; import java.util.List; -import java.util.Map; +import java.util.*; import java.util.stream.Collectors; @Service @@ -107,7 +105,7 @@ public class AnnotationService { private String createAnnotationContent(RedactionLogEntry redactionLogEntry) { - if (redactionLogEntry.isManual()) { + if (redactionLogEntry.isLocalManualRedaction()) { return "\nManual Redaction\n\nIn Section : \"" + redactionLogEntry.getSection() + "\""; } return redactionLogEntry.getType() + " \nRule " + redactionLogEntry.getMatchedRule() + " matched\n\n" + redactionLogEntry.getReason() + "\n\nLegal basis:" + redactionLogEntry diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DictionaryService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DictionaryService.java index ae4d1d63..8cc6c80f 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DictionaryService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DictionaryService.java @@ -6,7 +6,6 @@ import com.iqser.red.service.redaction.v1.server.client.DictionaryClient; import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary; import com.iqser.red.service.redaction.v1.server.redaction.model.*; import com.iqser.red.service.redaction.v1.server.redaction.utils.IdBuilder; - import feign.FeignException; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; @@ -14,7 +13,8 @@ import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.SerializationUtils; import org.springframework.stereotype.Service; -import java.awt.Color; +import java.awt.*; +import java.util.List; import java.util.*; import java.util.stream.Collectors; @@ -63,7 +63,7 @@ public class DictionaryService { }); }); - if(dictionariesByDossier.containsKey(dossierId)) { + if (dictionariesByDossier.containsKey(dossierId)) { dictionaryModels = dictionariesByDossier.get(dossierId).getDictionary(); dictionaryModels.forEach(dictionaryModel -> { dictionaryModel.getEntries().forEach(dictionaryEntry -> { @@ -105,7 +105,7 @@ public class DictionaryService { dictionaryRepresentation.setDictionaryVersion(version); dictionaryRepresentation.setDictionary(dictionary); - if(dossierId == null) { + if (dossierId == null) { dictionariesByDossierTemplate.put(dossierTemplateId, dictionaryRepresentation); } else { dictionariesByDossier.put(dossierId, dictionaryRepresentation); @@ -204,7 +204,7 @@ public class DictionaryService { //TODO merge dictionaries if they have same names long dossierDictionaryVersion = -1; - if(dictionariesByDossier.containsKey(dossierId)) { + if (dictionariesByDossier.containsKey(dossierId)) { var dossierRepresentation = dictionariesByDossier.get(dossierId); dossierRepresentation.getDictionary().forEach(dm -> { copy.add(SerializationUtils.clone(dm)); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionService.java index 68522fe8..4f1a3324 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionService.java @@ -1,43 +1,27 @@ package com.iqser.red.service.redaction.v1.server.redaction.service; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.stream.Collectors; -import java.util.stream.Stream; - import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus; -import com.iqser.red.service.persistence.service.v1.api.model.annotations.IdRemoval; -import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualImageRecategorization; - -import org.apache.commons.codec.binary.Base64; -import org.apache.commons.lang3.StringUtils; -import org.kie.api.runtime.KieContainer; -import org.springframework.stereotype.Service; - +import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.IdRemoval; +import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualImageRecategorization; import com.iqser.red.service.redaction.v1.model.AnalyzeRequest; import com.iqser.red.service.redaction.v1.model.Engine; import com.iqser.red.service.redaction.v1.server.classification.model.SectionText; import com.iqser.red.service.redaction.v1.server.client.model.NerEntities; import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary; -import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryModel; -import com.iqser.red.service.redaction.v1.server.redaction.model.Entities; -import com.iqser.red.service.redaction.v1.server.redaction.model.Entity; -import com.iqser.red.service.redaction.v1.server.redaction.model.EntityPositionSequence; -import com.iqser.red.service.redaction.v1.server.redaction.model.Image; -import com.iqser.red.service.redaction.v1.server.redaction.model.PageEntities; -import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText; -import com.iqser.red.service.redaction.v1.server.redaction.model.Section; -import com.iqser.red.service.redaction.v1.server.redaction.model.SectionSearchableTextPair; +import com.iqser.red.service.redaction.v1.server.redaction.model.*; import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUtils; import com.iqser.red.service.redaction.v1.server.redaction.utils.IdBuilder; import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings; - import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.codec.binary.Base64; +import org.apache.commons.lang3.StringUtils; +import org.kie.api.runtime.KieContainer; +import org.springframework.stereotype.Service; + +import java.util.*; +import java.util.stream.Collectors; +import java.util.stream.Stream; @Slf4j @Service @@ -249,12 +233,12 @@ public class EntityRedactionService { nerFound.addAll(getNerValues(sectionNumber, nerEntities, cellstarts, headline)); } - return new Entities(EntitySearchUtils.clearAndFindPositions(found, searchableText, dictionary), nerFound) ; + return new Entities(EntitySearchUtils.clearAndFindPositions(found, searchableText, dictionary), nerFound); } private Set getNerValues(int sectionNumber, NerEntities nerEntities, - List cellstarts, String headline) { + List cellstarts, String headline) { Set entities = new HashSet<>(); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ManualRedactionSurroundingTextService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ManualRedactionSurroundingTextService.java index 89fc057d..f0983b2c 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ManualRedactionSurroundingTextService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ManualRedactionSurroundingTextService.java @@ -1,16 +1,9 @@ package com.iqser.red.service.redaction.v1.server.redaction.service; -import java.util.ArrayList; -import java.util.List; -import java.util.Set; - -import org.apache.commons.lang3.tuple.Pair; -import org.springframework.stereotype.Service; - -import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactionEntry; import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions; -import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualResizeRedaction; import com.iqser.red.service.persistence.service.v1.api.model.annotations.Rectangle; +import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualRedactionEntry; +import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualResizeRedaction; import com.iqser.red.service.redaction.v1.model.Engine; import com.iqser.red.service.redaction.v1.model.SectionArea; import com.iqser.red.service.redaction.v1.server.classification.model.SectionText; @@ -20,9 +13,14 @@ import com.iqser.red.service.redaction.v1.server.redaction.model.Entity; import com.iqser.red.service.redaction.v1.server.redaction.model.EntityPositionSequence; import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUtils; import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService; - import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.tuple.Pair; +import org.springframework.stereotype.Service; + +import java.util.ArrayList; +import java.util.List; +import java.util.Set; @Slf4j @Service @@ -90,9 +88,9 @@ public class ManualRedactionSurroundingTextService { Entity correctEntity = getEntityOnCorrectPosition(entitiesWithPositions, toFindPositions); - if(correctEntity == null){ + if (correctEntity == null) { log.warn("Could not calculate surrounding text"); - return Pair.of("",""); + return Pair.of("", ""); } if (sectionText.getCellStarts() != null && !sectionText.getCellStarts().isEmpty()) { diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionChangeLogService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionChangeLogService.java index 9ebb5f95..5f75bdce 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionChangeLogService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionChangeLogService.java @@ -33,17 +33,11 @@ public class RedactionChangeLogService { List notRemovedPreviousEntries = previousRedactionLog.getRedactionLogEntry() .stream() - .filter(entry -> !entry.getChanges() - .get(entry.getChanges().size() - 1) - .getType() - .equals(ChangeType.REMOVED)) + .filter(entry -> !entry.lastChangeIsRemoved()) .collect(Collectors.toList()); Set added = currentRedactionLog.getRedactionLogEntry().stream() - .filter(entry -> entry.getChanges().isEmpty() || !entry.getChanges() - .get(entry.getChanges().size() - 1) - .getType() - .equals(ChangeType.REMOVED)).collect(Collectors.toSet()); + .filter(entry -> entry.getChanges().isEmpty() || !entry.lastChangeIsRemoved()).collect(Collectors.toSet()); notRemovedPreviousEntries.forEach(added::remove); @@ -87,7 +81,7 @@ public class RedactionChangeLogService { addedIds.forEach((k, v) -> { if (v.getChanges().isEmpty()) { - v.getChanges().add(new Change(analysisNumber,ChangeType.ADDED, OffsetDateTime.now())); + v.getChanges().add(new Change(analysisNumber, ChangeType.ADDED, OffsetDateTime.now())); } newRedactionLogEntries.add(v); }); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java index 6be2e194..9cce2ab9 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java @@ -1,15 +1,5 @@ package com.iqser.red.service.redaction.v1.server.redaction.service; -import java.util.ArrayList; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.stream.Collectors; - -import org.apache.commons.collections4.CollectionUtils; -import org.springframework.stereotype.Service; - import com.iqser.red.service.redaction.v1.model.Point; import com.iqser.red.service.redaction.v1.model.Rectangle; import com.iqser.red.service.redaction.v1.model.RedactionLogEntry; @@ -20,9 +10,13 @@ import com.iqser.red.service.redaction.v1.server.redaction.model.EntityPositionS import com.iqser.red.service.redaction.v1.server.redaction.model.Image; import com.iqser.red.service.redaction.v1.server.redaction.model.PageEntities; import com.iqser.red.service.redaction.v1.server.redaction.utils.IdBuilder; - import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.CollectionUtils; +import org.springframework.stereotype.Service; + +import java.util.*; +import java.util.stream.Collectors; @Service @Slf4j @@ -70,7 +64,6 @@ public class RedactionLogCreatorService { .legalBasis(image.getLegalBasis()) .matchedRule(image.getMatchedRule()) .isHint(dictionaryService.isHint(image.getType(), dossierTemplateId)) - .manual(false) .isDictionaryEntry(false) .isRecommendation(false) .positions(List.of(new Rectangle(new Point((float) image.getPosition() @@ -173,10 +166,7 @@ public class RedactionLogCreatorService { private boolean isCharClose(float x, float xDirAdj, float width) { float max = x + (5 * width); - if (xDirAdj < max) { - return true; - } - return false; + return xDirAdj < max; } @@ -190,11 +180,7 @@ public class RedactionLogCreatorService { float minY = y - offsetY; float maxY = y + offsetY; - if (yCompare > minY && yCompare < maxY && heightCompare > minHeight && heightCompare < maxHeight) { - return true; - } - - return false; + return yCompare > minY && yCompare < maxY && heightCompare > minHeight && heightCompare < maxHeight; } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogMergeService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogMergeService.java index 28b240e1..1311b545 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogMergeService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogMergeService.java @@ -1,7 +1,9 @@ package com.iqser.red.service.redaction.v1.server.redaction.service; -import com.iqser.red.service.persistence.service.v1.api.model.annotations.*; -import com.iqser.red.service.redaction.v1.model.Rectangle; +import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus; +import com.iqser.red.service.persistence.service.v1.api.model.annotations.Comment; +import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions; +import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.*; import com.iqser.red.service.redaction.v1.model.*; import lombok.AllArgsConstructor; import lombok.Data; @@ -46,7 +48,7 @@ public class RedactionLogMergeService { if (excludedPages != null && !excludedPages.isEmpty()) { entry.getPositions().forEach(pos -> { - if (!entry.isManual() && excludedPages.contains(pos.getPage())) { + if (!entry.isLocalManualRedaction() && excludedPages.contains(pos.getPage())) { entry.setExcluded(true); } }); @@ -117,9 +119,9 @@ public class RedactionLogMergeService { var imageRecategorization = (ManualImageRecategorization) mrw.getItem(); String manualOverrideReason = null; if (imageRecategorization.getStatus().equals(AnnotationStatus.APPROVED)) { - redactionLogEntry.setStatus(AnnotationStatus.APPROVED); + + redactionLogEntry.setType(imageRecategorization.getType()); - redactionLogEntry.setHasBeenRecategorized(true); redactionLogEntry.setSection("Image:" + redactionLogEntry.getType()); if (dictionaryService.isHint(imageRecategorization.getType(), dossierTemplateId)) { @@ -132,77 +134,65 @@ public class RedactionLogMergeService { manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", recategorized by manual override"); } else if (imageRecategorization.getStatus().equals(AnnotationStatus.REQUESTED)) { manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", requested to recategorize"); - redactionLogEntry.setStatus(AnnotationStatus.REQUESTED); redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, false, redactionLogEntry.isRedacted(), false)); - redactionLogEntry.setRecategorizationType(imageRecategorization.getType()); - } else { - redactionLogEntry.setStatus(AnnotationStatus.DECLINED); } - redactionLogEntry.setManualRedactionUserId(imageRecategorization.getUser()); redactionLogEntry.setReason(manualOverrideReason); - redactionLogEntry.setManual(true); - redactionLogEntry.setManualRedactionType(ManualRedactionType.RECATEGORIZE); + + redactionLogEntry.getManualChanges().add(ManualChange.from(imageRecategorization) + .withManualRedactionType(ManualRedactionType.RECATEGORIZE) + .withChange("type", imageRecategorization.getType())); } if (mrw.getItem() instanceof IdRemoval) { var manualRemoval = (IdRemoval) mrw.getItem(); if (manualRemoval.getStatus().equals(AnnotationStatus.APPROVED) && manualRemoval.isRemoveFromDictionary()) { - log.info("Skipping merge for dictionary-modifying entry"); + log.debug("Skipping merge for dictionary-modifying entry"); } else { String manualOverrideReason = null; if (manualRemoval.getStatus().equals(AnnotationStatus.APPROVED)) { redactionLogEntry.setRedacted(false); - redactionLogEntry.setStatus(AnnotationStatus.APPROVED); manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", removed by manual override"); redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, false, redactionLogEntry.isRedacted(), true)); redactionLogEntry.setHint(false); - redactionLogEntry.setHasBeenRemovedByManualOverride(true); } else if (manualRemoval.getStatus().equals(AnnotationStatus.REQUESTED)) { manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", requested to remove"); - redactionLogEntry.setStatus(AnnotationStatus.REQUESTED); redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, true, redactionLogEntry.isRedacted(), false)); - } else { - redactionLogEntry.setStatus(AnnotationStatus.DECLINED); } redactionLogEntry.setReason(manualOverrideReason); - redactionLogEntry.setManual(true); - redactionLogEntry.setManualRedactionUserId(manualRemoval.getUser()); - redactionLogEntry.setManualRedactionType(ManualRedactionType.REMOVE); - redactionLogEntry.setDictionaryEntry(manualRemoval.isRemoveFromDictionary()); - redactionLogEntry.setDossierDictionaryEntry(manualRemoval.isRemoveFromDictionary()); + + redactionLogEntry.getManualChanges().add(ManualChange.from(manualRemoval) + .withManualRedactionType(manualRemoval.isRemoveFromDictionary() ? + ManualRedactionType.REMOVE_FROM_DICTIONARY : ManualRedactionType.REMOVE_LOCALLY)); } } if (mrw.getItem() instanceof ManualForceRedaction) { var manualForceRedact = (ManualForceRedaction) mrw.getItem(); String manualOverrideReason = null; + var dictionaryIsHint = dictionaryService.isHint(redactionLogEntry.getType(), dossierTemplateId); if (manualForceRedact.getStatus().equals(AnnotationStatus.APPROVED)) { // Forcing a skipped hint should result in a hint - if (dictionaryService.isHint(redactionLogEntry.getType(), dossierTemplateId)) { + if (dictionaryIsHint) { redactionLogEntry.setHint(true); } else { redactionLogEntry.setRedacted(true); } - redactionLogEntry.setStatus(AnnotationStatus.APPROVED); redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, false, redactionLogEntry.isRedacted(), false)); manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", forced by manual override"); - redactionLogEntry.setHasBeenForced(true); redactionLogEntry.setLegalBasis(manualForceRedact.getLegalBasis()); } else if (manualForceRedact.getStatus().equals(AnnotationStatus.REQUESTED)) { - manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", requested to force redact"); - redactionLogEntry.setStatus(AnnotationStatus.REQUESTED); + manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", requested to force " + (dictionaryIsHint ? "hint" : "redact")); redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, true, redactionLogEntry.isRedacted(), false)); redactionLogEntry.setLegalBasis(manualForceRedact.getLegalBasis()); - } else { - redactionLogEntry.setStatus(AnnotationStatus.DECLINED); } - redactionLogEntry.setManualRedactionUserId(manualForceRedact.getUser()); redactionLogEntry.setReason(manualOverrideReason); - redactionLogEntry.setManual(true); - redactionLogEntry.setManualRedactionType(ManualRedactionType.FORCE_REDACT); + var manualChange = ManualChange.from(manualForceRedact) + .withManualRedactionType(dictionaryIsHint ? ManualRedactionType.FORCE_HINT : ManualRedactionType.FORCE_REDACT); + + redactionLogEntry.getManualChanges().add(manualChange); } @@ -210,11 +200,9 @@ public class RedactionLogMergeService { var manualLegalBasisChange = (ManualLegalBasisChange) mrw.getItem(); String manualOverrideReason = null; if (manualLegalBasisChange.getStatus().equals(AnnotationStatus.APPROVED)) { - redactionLogEntry.setStatus(AnnotationStatus.APPROVED); manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", legal basis was manually changed"); redactionLogEntry.setLegalBasis(manualLegalBasisChange.getLegalBasis()); redactionLogEntry.setRedacted(true); - redactionLogEntry.setHasLegalBasisChanged(true); if (manualLegalBasisChange.getSection() != null) { redactionLogEntry.setSection(manualLegalBasisChange.getSection()); } @@ -223,47 +211,45 @@ public class RedactionLogMergeService { } } else if (manualLegalBasisChange.getStatus().equals(AnnotationStatus.REQUESTED)) { manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", legal basis change requested"); - redactionLogEntry.setStatus(AnnotationStatus.REQUESTED); redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, true, redactionLogEntry.isRedacted(), false)); - redactionLogEntry.setLegalBasisChangeValue(manualLegalBasisChange.getLegalBasis()); - } else { - redactionLogEntry.setStatus(AnnotationStatus.DECLINED); } - redactionLogEntry.setManualRedactionUserId(manualLegalBasisChange.getUser()); redactionLogEntry.setReason(manualOverrideReason); - redactionLogEntry.setManual(true); - redactionLogEntry.setManualRedactionType(ManualRedactionType.LEGAL_BASIS_CHANGE); + + var manualChange = ManualChange.from(manualLegalBasisChange) + .withManualRedactionType(ManualRedactionType.LEGAL_BASIS_CHANGE); + manualChange.withChange("legalBasis", manualLegalBasisChange.getLegalBasis()); + if (manualLegalBasisChange.getSection() != null) { + manualChange.withChange("section", manualLegalBasisChange.getSection()); + } + if (redactionLogEntry.isRectangle() && manualLegalBasisChange.getValue() != null) { + manualChange.withChange("value", manualLegalBasisChange.getValue()); + } + redactionLogEntry.getManualChanges().add(manualChange); } if (mrw.getItem() instanceof ManualResizeRedaction) { var manualResizeRedact = (ManualResizeRedaction) mrw.getItem(); String manualOverrideReason = null; if (manualResizeRedact.getStatus().equals(AnnotationStatus.APPROVED)) { - redactionLogEntry.setStatus(AnnotationStatus.APPROVED); redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, false, redactionLogEntry.isRedacted(), false)); redactionLogEntry.setPositions(convertPositions(manualResizeRedact.getPositions())); redactionLogEntry.setValue(manualResizeRedact.getValue()); - redactionLogEntry.setHasBeenResized(true); redactionLogEntry.setTextBefore(manualResizeRedact.getTextBefore()); redactionLogEntry.setTextAfter(manualResizeRedact.getTextAfter()); manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", resized by manual override"); } else if (manualResizeRedact.getStatus().equals(AnnotationStatus.REQUESTED)) { manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", requested to resize redact"); - redactionLogEntry.setStatus(AnnotationStatus.REQUESTED); redactionLogEntry.setColor(getColor(redactionLogEntry.getType(), dossierTemplateId, true, redactionLogEntry.isRedacted(), false)); redactionLogEntry.setPositions(convertPositions(manualResizeRedact.getPositions())); redactionLogEntry.setTextBefore(manualResizeRedact.getTextBefore()); redactionLogEntry.setTextAfter(manualResizeRedact.getTextAfter()); - } else { - redactionLogEntry.setStatus(AnnotationStatus.DECLINED); } - redactionLogEntry.setManualRedactionUserId(manualResizeRedact.getUser()); redactionLogEntry.setReason(manualOverrideReason); - redactionLogEntry.setManual(true); - redactionLogEntry.setManualRedactionType(ManualRedactionType.RESIZE); - + redactionLogEntry.getManualChanges().add(ManualChange.from(manualResizeRedact) + .withManualRedactionType(ManualRedactionType.RESIZE) + .withChange("value", manualResizeRedact.getValue())); } }); @@ -319,14 +305,17 @@ public class RedactionLogMergeService { private boolean approvedAndShouldBeInDictionary(ManualRedactionEntry manualRedactionEntry) { - return manualRedactionEntry.getStatus() - .equals(AnnotationStatus.APPROVED) && (manualRedactionEntry.isAddToDictionary() || manualRedactionEntry.isAddToDossierDictionary()); + return manualRedactionEntry.getStatus().equals(AnnotationStatus.APPROVED) && + manualRedactionEntry.getProcessedDate() != null && + (manualRedactionEntry.isAddToDictionary() || manualRedactionEntry.isAddToDossierDictionary()); } private RedactionLogEntry createRedactionLogEntry(ManualRedactionEntry manualRedactionEntry, String id, String dossierTemplateId) { + var addToDictionary = manualRedactionEntry.isAddToDictionary() || manualRedactionEntry.isAddToDossierDictionary(); + return RedactionLogEntry.builder() .id(id) .color(getColorForManualAdd(manualRedactionEntry.getType(), dossierTemplateId, manualRedactionEntry.getStatus())) @@ -340,11 +329,9 @@ public class RedactionLogMergeService { .redacted(true) .isHint(false) .sectionNumber(-1) - .manual(true) - .status(manualRedactionEntry.getStatus()) - .manualRedactionType(ManualRedactionType.ADD) - .manualRedactionUserId(manualRedactionEntry.getUser()) .rectangle(manualRedactionEntry.isRectangle()) + .manualChanges(List.of(ManualChange.from(manualRedactionEntry) + .withManualRedactionType(addToDictionary ? ManualRedactionType.ADD_TO_DICTIONARY : ManualRedactionType.ADD_LOCALLY))) .build(); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/SectionGridCreatorService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/SectionGridCreatorService.java index 0994b03a..7ef14ffd 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/SectionGridCreatorService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/SectionGridCreatorService.java @@ -1,10 +1,5 @@ package com.iqser.red.service.redaction.v1.server.redaction.service; -import java.util.ArrayList; -import java.util.List; - -import org.springframework.stereotype.Service; - import com.iqser.red.service.redaction.v1.model.CellRectangle; import com.iqser.red.service.redaction.v1.model.Point; import com.iqser.red.service.redaction.v1.model.SectionRectangle; @@ -14,8 +9,11 @@ import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock; import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer; import com.iqser.red.service.redaction.v1.server.tableextraction.model.Cell; import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table; - import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Service; + +import java.util.ArrayList; +import java.util.List; @Service @RequiredArgsConstructor diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/SectionTextBuilderService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/SectionTextBuilderService.java index b0ae577c..91be5740 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/SectionTextBuilderService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/SectionTextBuilderService.java @@ -1,35 +1,19 @@ package com.iqser.red.service.redaction.v1.server.redaction.service; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Locale; -import java.util.Map; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.stream.Collectors; - -import org.apache.commons.collections4.CollectionUtils; -import org.springframework.stereotype.Service; - import com.iqser.red.service.redaction.v1.model.Point; import com.iqser.red.service.redaction.v1.model.SectionArea; -import com.iqser.red.service.redaction.v1.server.classification.model.Document; -import com.iqser.red.service.redaction.v1.server.classification.model.Footer; -import com.iqser.red.service.redaction.v1.server.classification.model.Header; -import com.iqser.red.service.redaction.v1.server.classification.model.Paragraph; -import com.iqser.red.service.redaction.v1.server.classification.model.SectionText; -import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock; -import com.iqser.red.service.redaction.v1.server.classification.model.UnclassifiedText; -import com.iqser.red.service.redaction.v1.server.redaction.model.CellValue; -import com.iqser.red.service.redaction.v1.server.redaction.model.Image; -import com.iqser.red.service.redaction.v1.server.redaction.model.ImageType; -import com.iqser.red.service.redaction.v1.server.redaction.model.PdfImage; -import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText; +import com.iqser.red.service.redaction.v1.server.classification.model.*; +import com.iqser.red.service.redaction.v1.server.redaction.model.*; import com.iqser.red.service.redaction.v1.server.tableextraction.model.Cell; import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table; - import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.CollectionUtils; +import org.springframework.stereotype.Service; + +import java.util.*; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Collectors; @Slf4j @Service diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java index 024d5b4a..16e357d5 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java @@ -23,6 +23,7 @@ import java.util.Set; import java.util.UUID; import java.util.stream.Collectors; +import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.*; import org.apache.commons.io.IOUtils; import org.junit.After; import org.junit.Before; @@ -51,13 +52,7 @@ import com.amazonaws.services.s3.AmazonS3; import com.fasterxml.jackson.databind.ObjectMapper; import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus; import com.iqser.red.service.persistence.service.v1.api.model.annotations.Comment; -import com.iqser.red.service.persistence.service.v1.api.model.annotations.IdRemoval; -import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualForceRedaction; -import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualImageRecategorization; -import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualLegalBasisChange; -import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactionEntry; import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions; -import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualResizeRedaction; import com.iqser.red.service.persistence.service.v1.api.model.annotations.Rectangle; import com.iqser.red.service.persistence.service.v1.api.model.common.JSONPrimitive; import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.configuration.Colors; @@ -1561,4 +1556,4 @@ public class RedactionIntegrationTest { } -} \ No newline at end of file +}