From 88736855b4b6a391ac9185ae7440c19a98190e20 Mon Sep 17 00:00:00 2001 From: Andrei Isvoran Date: Wed, 29 Nov 2023 11:03:51 +0100 Subject: [PATCH] RED-7785 - Fix false positives not getting removed for unprocessed changes & refactor merging unprocessed changes into entitylog logic. --- .../service/EntityLogMergeService.java | 316 ++++++++++++++++++ .../processor/service/EntityLogService.java | 256 +------------- ...tyLogTest.java => EntityLogMergeTest.java} | 20 +- .../entitymapped/ManualRedactionEntry.java | 2 + 4 files changed, 335 insertions(+), 259 deletions(-) create mode 100644 persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/EntityLogMergeService.java rename persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/{EntityLogTest.java => EntityLogMergeTest.java} (93%) diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/EntityLogMergeService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/EntityLogMergeService.java new file mode 100644 index 000000000..e53bb73e0 --- /dev/null +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/EntityLogMergeService.java @@ -0,0 +1,316 @@ +package com.iqser.red.service.persistence.management.v1.processor.service; + +import static com.iqser.red.service.persistence.management.v1.processor.utils.TypeIdUtils.toTypeId; + +import java.time.OffsetDateTime; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Comparator; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.stream.Stream; + +import org.springframework.stereotype.Service; + +import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.TypeEntity; +import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierEntity; +import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException; +import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DictionaryPersistenceService; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Change; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ChangeType; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryType; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualChange; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position; +import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions; +import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle; +import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.BaseAnnotation; +import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval; +import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction; +import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange; +import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization; +import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry; +import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntryType; +import com.iqser.red.service.redaction.v1.model.UnprocessedManualEntity; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +@Service +@Slf4j +@RequiredArgsConstructor +public class EntityLogMergeService { + + private final DictionaryPersistenceService dictionaryPersistenceService; + + + public EntityLog mergeEntityLog(ManualRedactions manualRedactions, List unprocessedManualEntities, EntityLog entityLog, DossierEntity dossier) { + + log.info("Merging EntityLog"); + List allManualChanges = allManualChanges(manualRedactions); + + // Sort manual changes by date so we process them in order of when they were requested + allManualChanges = allManualChanges.stream().sorted(Comparator.comparing(BaseAnnotation::getRequestDate)).toList(); + allManualChanges.forEach(manualChange -> { + // this is ugly and should be replaced with switch pattern matching https://openjdk.org/jeps/406 -> requires Java 17 (preview) or higher + if (manualChange instanceof ManualRedactionEntry) { + mergeManualRedactionEntries((ManualRedactionEntry) manualChange, unprocessedManualEntities, entityLog, dossier); + } else if (manualChange instanceof IdRemoval) { + mergeIdsToRemove((IdRemoval) manualChange, entityLog); + } else if (manualChange instanceof ManualResizeRedaction) { + mergeResizeRedactions((ManualResizeRedaction) manualChange, entityLog); + } else if (manualChange instanceof ManualLegalBasisChange) { + mergeLegalBasisChanges((ManualLegalBasisChange) manualChange, entityLog); + } else if (manualChange instanceof ManualRecategorization) { + mergeRecategorizations((ManualRecategorization) manualChange, entityLog, dossier); + } else if (manualChange instanceof ManualForceRedaction) { + mergeForceRedactions((ManualForceRedaction) manualChange, entityLog); + } + }); + + log.info("EntityLog merged successfully!"); + return entityLog; + } + + + private void mergeManualRedactionEntries(ManualRedactionEntry manualRedactionEntry, List unprocessedManualEntities, EntityLog entityLog, DossierEntity dossier) { + + UnprocessedManualEntity unprocessedManualEntity = unprocessedManualEntities.stream() + .filter(manualEntity -> manualEntity.getAnnotationId().equals(manualRedactionEntry.getAnnotationId())) + .findFirst().orElseThrow(() -> new NotFoundException("Entry with annotationId " + manualRedactionEntry.getAnnotationId() + " not found")); + List changes = new ArrayList<>(); + + changes.add(Change.builder() + .analysisNumber(entityLog.getAnalysisNumber()) + .dateTime(OffsetDateTime.now()) + .type(ChangeType.ADDED) + .build()); + + boolean isHint = isHint(manualRedactionEntry.getType(), dossier); + + if (manualRedactionEntry.getDictionaryEntryType().equals(DictionaryEntryType.FALSE_POSITIVE)) { + var matchingEntities = entityLog.getEntityLogEntry().stream() + .filter(entityLogEntry -> equalPosition(manualRedactionEntry.getPositions().get(0), entityLogEntry.getPositions().get(0))) + .toList(); + matchingEntities.forEach(matchingEntity -> { + mergeFalsePositive(entityLog, matchingEntity); + }); + return; + } + + var entityLogEntryBuilder = EntityLogEntry.builder() + .id(manualRedactionEntry.getAnnotationId()) + .type(manualRedactionEntry.getType()) + .value(manualRedactionEntry.getValue()) + .legalBasis(manualRedactionEntry.getLegalBasis() == null || manualRedactionEntry.getLegalBasis().isEmpty() ? unprocessedManualEntity.getLegalBasis() : manualRedactionEntry.getLegalBasis()) + .reason(manualRedactionEntry.getReason()) + .entryType(isHint ? EntryType.HINT : EntryType.ENTITY) + .state(isHint ? EntryState.SKIPPED : EntryState.APPLIED) + .imported(false) + .matchedRule("") + .section(manualRedactionEntry.getSection()) + .color(unprocessedManualEntity.getColor()) + .positions(unprocessedManualEntity.getPositions()) + .textAfter(unprocessedManualEntity.getTextAfter()) + .textBefore(unprocessedManualEntity.getTextBefore()) + .startOffset(unprocessedManualEntity.getStartOffset()) + .endOffset(unprocessedManualEntity.getEndOffset()) + .containingNodeId(unprocessedManualEntity.getContainingNodeId()) + .closestHeadline(unprocessedManualEntity.getClosestHeadline()) + .imageHasTransparency(false) + .dictionaryEntry(manualRedactionEntry.isAddToDictionary()) + .dossierDictionaryEntry(manualRedactionEntry.isAddToDossierDictionary()) + .excluded(false) + .changes(changes) + .manualChanges(List.of(ManualChange.builder() + .manualRedactionType(ManualRedactionType.ADD_LOCALLY) + .requestedDate(manualRedactionEntry.getRequestDate()) + .processedDate(null) + .userId(manualRedactionEntry.getUser()) + .propertyChanges(Map.of("value", manualRedactionEntry.getValue())) + .build())) + .engines(new HashSet<>()) + .reference(new HashSet<>()) + .importedRedactionIntersections(new HashSet<>()); + + entityLog.getEntityLogEntry().add(entityLogEntryBuilder + .build()); + } + + + private void mergeFalsePositive(EntityLog entityLog, EntityLogEntry existingEntry) { + + existingEntry.setState(EntryState.REMOVED); + List falsePositiveChanges = new ArrayList<>(); + falsePositiveChanges.add(Change.builder() + .analysisNumber(entityLog.getAnalysisNumber()) + .dateTime(OffsetDateTime.now()) + .type(ChangeType.REMOVED) + .build()); + if (existingEntry.getChanges() != null && !existingEntry.getChanges().isEmpty()) { + existingEntry.getChanges().addAll(falsePositiveChanges); + } else { + existingEntry.setChanges(falsePositiveChanges); + } + } + + + + private void mergeIdsToRemove(IdRemoval idRemoval, EntityLog entityLog) { + + var entity = entityLog.getEntityLogEntry().stream().filter(entityLogEntry -> entityLogEntry.getId().equals(idRemoval.getAnnotationId())).findAny(); + if (entity.isPresent()) { + entity.get().setState(EntryState.IGNORED); + addChanges(entity.get().getChanges(), ChangeType.REMOVED, entityLog.getAnalysisNumber()); + entity.get().getManualChanges().add(ManualChange.builder() + .manualRedactionType(ManualRedactionType.REMOVE_LOCALLY) + .requestedDate(idRemoval.getRequestDate()) + .processedDate(null) + .userId(idRemoval.getUser()).build()); + } + } + + private void mergeResizeRedactions(ManualResizeRedaction manualResizeRedaction, EntityLog entityLog) { + + var entity = entityLog.getEntityLogEntry().stream().filter(entityLogEntry -> entityLogEntry.getId().equals(manualResizeRedaction.getAnnotationId())).findAny(); + if (entity.isPresent()) { + var newPosition = manualResizeRedaction.getPositions().get(0); + entity.get().setPositions(List.of(new Position( + newPosition.getTopLeftX(), + newPosition.getTopLeftY(), + newPosition.getWidth(), + newPosition.getHeight(), + newPosition.getPage()))); + addChanges(entity.get().getChanges(), ChangeType.CHANGED, entityLog.getAnalysisNumber()); + entity.get().getManualChanges().add(ManualChange.builder() + .manualRedactionType(ManualRedactionType.RESIZE) + .requestedDate(manualResizeRedaction.getRequestDate()) + .processedDate(null) + .propertyChanges(Map.of("value", manualResizeRedaction.getValue())) + .userId(manualResizeRedaction.getUser()).build()); + } + } + + private void mergeLegalBasisChanges(ManualLegalBasisChange manualLegalBasisChange, EntityLog entityLog) { + + var entity = entityLog.getEntityLogEntry().stream().filter(entityLogEntry -> entityLogEntry.getId().equals(manualLegalBasisChange.getAnnotationId())).findAny(); + if (entity.isPresent()) { + entity.get().setLegalBasis(manualLegalBasisChange.getLegalBasis()); + entity.get().setSection(manualLegalBasisChange.getSection()); + entity.get().setValue(manualLegalBasisChange.getValue()); + addChanges(entity.get().getChanges(), ChangeType.CHANGED, entityLog.getAnalysisNumber()); + entity.get().getManualChanges().add(ManualChange.builder() + .manualRedactionType(ManualRedactionType.LEGAL_BASIS_CHANGE) + .requestedDate(manualLegalBasisChange.getRequestDate()) + .processedDate(null) + .propertyChanges(Map.of("value", manualLegalBasisChange.getValue(), + "section", manualLegalBasisChange.getSection(), + "legalBasis", manualLegalBasisChange.getLegalBasis())) + .userId(manualLegalBasisChange.getUser()) + .build()); + } + } + + private void mergeRecategorizations(ManualRecategorization recategorization, EntityLog entityLog, DossierEntity dossier) { + + boolean isHint = isHint(recategorization.getType(), dossier); + var entity = entityLog.getEntityLogEntry().stream().filter(entityLogEntry -> entityLogEntry.getId().equals(recategorization.getAnnotationId())).findAny(); + if (entity.isPresent()) { + entity.get().setType(recategorization.getType()); + entity.get().setEntryType(getEntryType(isHint, recategorization.getType())); + entity.get().setState(isHint ? EntryState.SKIPPED : EntryState.APPLIED); + addChanges(entity.get().getChanges(), ChangeType.CHANGED, entityLog.getAnalysisNumber()); + entity.get().getManualChanges().add(ManualChange.builder() + .manualRedactionType(ManualRedactionType.RECATEGORIZE) + .requestedDate(recategorization.getRequestDate()) + .processedDate(recategorization.getProcessedDate()) + .userId(recategorization.getUser()) + .propertyChanges(Map.of("type", recategorization.getType())) + .build()); + } + } + + + private static EntryType getEntryType(boolean isHint, String type) { + + if (type.equals("image") || type.equals("logo") || type.equals("signature") || type.equals("formula")) { + return isHint ? EntryType.IMAGE_HINT : EntryType.IMAGE; + } else { + return isHint ? EntryType.HINT : EntryType.ENTITY; + } + } + + + private void mergeForceRedactions(ManualForceRedaction forceRedaction, EntityLog entityLog) { + + var entity = entityLog.getEntityLogEntry().stream().filter(entityLogEntry -> entityLogEntry.getId().equals(forceRedaction.getAnnotationId())).findAny(); + if (entity.isPresent()) { + entity.get().setLegalBasis(forceRedaction.getLegalBasis()); + entity.get().setState(EntryState.APPLIED); + addChanges(entity.get().getChanges(), ChangeType.CHANGED, entityLog.getAnalysisNumber()); + var forceRedactManualChange = ManualChange.builder() + .manualRedactionType(ManualRedactionType.FORCE_REDACT) + .requestedDate(forceRedaction.getRequestDate()) + .processedDate(forceRedaction.getProcessedDate()) + .userId(forceRedaction.getUser()); + if (forceRedaction.getLegalBasis() != null && !forceRedaction.getLegalBasis().isEmpty()) { + forceRedactManualChange.propertyChanges(Map.of("legalBasis", forceRedaction.getLegalBasis())); + } + entity.get().getManualChanges().add(forceRedactManualChange.build()); + } + } + + private void addChanges(List changes, ChangeType changeType, int analysisNumber) { + + if (!changes.isEmpty()) { + changes.add(Change.builder() + .analysisNumber(analysisNumber + 1) + .dateTime(OffsetDateTime.now()) + .type(changeType) + .build()); + } else { + changes.add(Change.builder().analysisNumber(analysisNumber).dateTime(OffsetDateTime.now()).type(changeType).build()); + } + } + + private boolean isHint(String type, DossierEntity dossier) { + + String typeId = toTypeId(type, dossier.getDossierTemplateId()); + TypeEntity typeEntity = dictionaryPersistenceService.getType(typeId); + + if (typeEntity == null) { + var optionalType = dictionaryPersistenceService.getAllTypes(false).stream().filter(typeEntity1 -> typeEntity1.getType().equals(type)).findFirst(); + if (optionalType.isPresent()) { + typeEntity = optionalType.get(); + } else { + throw new NotFoundException("TypeEntity could not be found for typeId: " + typeId); + } + } + + return typeEntity.isHint(); + } + + private boolean equalPosition(Rectangle position1, Position position2) { + + return position1.getTopLeftX() == position2.x() + && position1.getTopLeftY() == position2.y() + && position1.getWidth() == position2.w() + && position1.getHeight() == position2.h(); + } + + private List allManualChanges(ManualRedactions manualRedactions) { + + return Stream.of(manualRedactions.getEntriesToAdd(), + manualRedactions.getForceRedactions(), + manualRedactions.getResizeRedactions(), + manualRedactions.getRecategorizations(), + manualRedactions.getIdsToRemove(), + manualRedactions.getLegalBasisChanges()).flatMap(Collection::stream).map(baseAnnotation -> (BaseAnnotation) baseAnnotation).toList(); + } + +} diff --git a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/EntityLogService.java b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/EntityLogService.java index 42ad26ba7..85c023eb6 100644 --- a/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/EntityLogService.java +++ b/persistence-service-v1/persistence-service-processor-v1/src/main/java/com/iqser/red/service/persistence/management/v1/processor/service/EntityLogService.java @@ -1,45 +1,22 @@ package com.iqser.red.service.persistence.management.v1.processor.service; -import static com.iqser.red.service.persistence.management.v1.processor.utils.TypeIdUtils.toTypeId; - import java.time.OffsetDateTime; import java.util.ArrayList; import java.util.Collections; -import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Map; -import java.util.Objects; -import java.util.Set; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; import com.iqser.red.service.persistence.management.v1.processor.client.redactionservice.UnprocessedManualEntityClient; -import com.iqser.red.service.persistence.management.v1.processor.entity.configuration.TypeEntity; import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierEntity; -import com.iqser.red.service.persistence.management.v1.processor.exception.NotFoundException; import com.iqser.red.service.persistence.management.v1.processor.service.manualredactions.ManualRedactionProviderService; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DictionaryPersistenceService; -import com.iqser.red.service.persistence.management.v1.processor.service.persistence.DossierTemplatePersistenceService; -import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Change; -import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ChangeType; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry; -import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState; -import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryType; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.FilteredEntityLogRequest; -import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualChange; -import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType; -import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions; -import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval; -import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction; -import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange; -import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization; -import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry; -import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction; -import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileModel; import com.iqser.red.service.redaction.v1.model.UnprocessedManualEntity; import lombok.AccessLevel; @@ -60,8 +37,7 @@ public class EntityLogService { UnprocessedManualEntityClient unprocessedManualEntityClient; DossierService dossierService; CommentService commentService; - DictionaryPersistenceService dictionaryPersistenceService; - DossierTemplatePersistenceService dossierTemplatePersistenceService; + EntityLogMergeService entityLogMergeService; public EntityLog getEntityLog(String dossierId, String fileId) { @@ -90,7 +66,7 @@ public class EntityLogService { DossierEntity dossier = dossierService.getDossierById(dossierId); ManualRedactions manualRedactions = manualRedactionProviderService.getManualRedactions(fileId, true); List unprocessedManualEntities = getUnprocessedManualEntities(dossierId, fileId, dossier.getDossierTemplateId(), manualRedactions); - mergeEntityLog(manualRedactions, unprocessedManualEntities, entityLog, dossier, fileStatus); + entityLogMergeService.mergeEntityLog(manualRedactions, unprocessedManualEntities, entityLog, dossier); } Map commentCountPerAnnotationId = commentService.getCommentCounts(fileId); @@ -142,232 +118,4 @@ public class EntityLogService { } - public void mergeEntityLog(ManualRedactions manualRedactions, List unprocessedManualEntities, EntityLog entityLog, DossierEntity dossier, FileModel fileStatus) { - - log.info("Merging EntityLog"); - mergeManualRedactionEntries(manualRedactions.getEntriesToAdd(), unprocessedManualEntities, entityLog, dossier, fileStatus); - mergeIdsToRemove(manualRedactions.getIdsToRemove(), entityLog); - mergeResizeRedactions(manualRedactions.getResizeRedactions(), entityLog); - mergeLegalBasisChanges(manualRedactions.getLegalBasisChanges(), entityLog); - mergeRecategorizations(manualRedactions.getRecategorizations(), entityLog, dossier, fileStatus); - mergeForceRedactions(manualRedactions.getForceRedactions(), entityLog); - log.info("EntityLog merged successfully!"); - } - - - private void mergeManualRedactionEntries(Set manualRedactionEntries, List unprocessedManualEntities, EntityLog entityLog, DossierEntity dossier, FileModel fileStatus) { - - manualRedactionEntries.forEach(manualRedactionEntry -> { - UnprocessedManualEntity unprocessedManualEntity = unprocessedManualEntities.stream() - .filter(manualEntity -> manualEntity.getAnnotationId().equals(manualRedactionEntry.getAnnotationId())) - .findFirst().orElseThrow(() -> new NotFoundException("Entry with annotationId " + manualRedactionEntry.getAnnotationId() + " not found")); - List changes = new ArrayList<>(); - changes.add(Change.builder() - .analysisNumber(entityLog.getAnalysisNumber()) - .dateTime(OffsetDateTime.now()) - .type(ChangeType.ADDED) - .build()); - boolean isHint = isHint(manualRedactionEntry.getType(), dossier, fileStatus); - entityLog.getEntityLogEntry().add(EntityLogEntry.builder() - .id(manualRedactionEntry.getAnnotationId()) - .type(manualRedactionEntry.getType()) - .value(manualRedactionEntry.getValue()) - .legalBasis(manualRedactionEntry.getLegalBasis() == null || manualRedactionEntry.getLegalBasis().isEmpty() ? unprocessedManualEntity.getLegalBasis() : manualRedactionEntry.getLegalBasis()) - .reason(manualRedactionEntry.getReason()) - .entryType(isHint ? EntryType.HINT : EntryType.ENTITY) - .state(isHint ? EntryState.SKIPPED : EntryState.APPLIED) - .imported(false) - .matchedRule("") - .section(manualRedactionEntry.getSection()) - .color(unprocessedManualEntity.getColor()) - .positions(unprocessedManualEntity.getPositions()) - .textAfter(unprocessedManualEntity.getTextAfter()) - .textBefore(unprocessedManualEntity.getTextBefore()) - .startOffset(unprocessedManualEntity.getStartOffset()) - .endOffset(unprocessedManualEntity.getEndOffset()) - .containingNodeId(unprocessedManualEntity.getContainingNodeId()) - .closestHeadline(unprocessedManualEntity.getClosestHeadline()) - .imageHasTransparency(false) - .dictionaryEntry(manualRedactionEntry.isAddToDictionary()) - .dossierDictionaryEntry(manualRedactionEntry.isAddToDossierDictionary()) - .excluded(false) - .changes(changes) - .manualChanges(List.of(ManualChange.builder() - .manualRedactionType(ManualRedactionType.ADD_LOCALLY) - .requestedDate(manualRedactionEntry.getRequestDate()) - .processedDate(null) - .userId(manualRedactionEntry.getUser()) - .propertyChanges(Map.of("value", manualRedactionEntry.getValue())) - .build())) - .engines(new HashSet<>()) - .reference(new HashSet<>()) - .importedRedactionIntersections(new HashSet<>()) - .build()); - }); - } - - - private void mergeIdsToRemove(Set idRemovals, EntityLog entityLog) { - - idRemovals.forEach(idRemoval -> { - var entity = entityLog.getEntityLogEntry().stream().filter(entityLogEntry -> entityLogEntry.getId().equals(idRemoval.getAnnotationId())).findAny(); - if (entity.isPresent()) { - entity.get().setState(EntryState.IGNORED); - addChanges(entity.get().getChanges(), ChangeType.REMOVED, entityLog.getAnalysisNumber()); - entity.get().getManualChanges().add(ManualChange.builder() - .manualRedactionType(ManualRedactionType.REMOVE_LOCALLY) - .requestedDate(idRemoval.getRequestDate()) - .processedDate(null) - .userId(idRemoval.getUser()).build()); - } - }); - } - - private void mergeResizeRedactions(Set manualResizeRedactions, EntityLog entityLog) { - - manualResizeRedactions.forEach(manualResizeRedaction -> { - var entity = entityLog.getEntityLogEntry().stream().filter(entityLogEntry -> entityLogEntry.getId().equals(manualResizeRedaction.getAnnotationId())).findAny(); - if (entity.isPresent()) { - var newPosition = manualResizeRedaction.getPositions().get(0); - entity.get().setPositions(List.of(new Position( - newPosition.getTopLeftX(), - newPosition.getTopLeftY(), - newPosition.getWidth(), - newPosition.getHeight(), - newPosition.getPage()))); - addChanges(entity.get().getChanges(), ChangeType.CHANGED, entityLog.getAnalysisNumber()); - entity.get().getManualChanges().add(ManualChange.builder() - .manualRedactionType(ManualRedactionType.RESIZE) - .requestedDate(manualResizeRedaction.getRequestDate()) - .processedDate(null) - .propertyChanges(Map.of("value", manualResizeRedaction.getValue())) - .userId(manualResizeRedaction.getUser()).build()); - } - }); - } - - private void mergeLegalBasisChanges(Set manualLegalBasisChanges, EntityLog entityLog) { - - manualLegalBasisChanges.forEach(manualLegalBasisChange -> { - var entity = entityLog.getEntityLogEntry().stream().filter(entityLogEntry -> entityLogEntry.getId().equals(manualLegalBasisChange.getAnnotationId())).findAny(); - if (entity.isPresent()) { - entity.get().setLegalBasis(manualLegalBasisChange.getLegalBasis()); - entity.get().setSection(manualLegalBasisChange.getSection()); - entity.get().setValue(manualLegalBasisChange.getValue()); - addChanges(entity.get().getChanges(), ChangeType.CHANGED, entityLog.getAnalysisNumber()); - entity.get().getManualChanges().add(ManualChange.builder() - .manualRedactionType(ManualRedactionType.LEGAL_BASIS_CHANGE) - .requestedDate(manualLegalBasisChange.getRequestDate()) - .processedDate(null) - .propertyChanges(Map.of("value", manualLegalBasisChange.getValue(), - "section", manualLegalBasisChange.getSection(), - "legalBasis", manualLegalBasisChange.getLegalBasis())) - .userId(manualLegalBasisChange.getUser()) - .build()); - } - }); - } - - private void mergeRecategorizations(Set recategorizations, EntityLog entityLog, DossierEntity dossier, FileModel fileStatus) { - - recategorizations.forEach(recategorization -> { - boolean isHint = isHint(recategorization.getType(), dossier, fileStatus); - var entity = entityLog.getEntityLogEntry().stream().filter(entityLogEntry -> entityLogEntry.getId().equals(recategorization.getAnnotationId())).findAny(); - if (entity.isPresent()) { - entity.get().setType(recategorization.getType()); - entity.get().setEntryType(getEntryType(isHint, recategorization.getType())); - entity.get().setState(isHint ? EntryState.SKIPPED : EntryState.APPLIED); - addChanges(entity.get().getChanges(), ChangeType.CHANGED, entityLog.getAnalysisNumber()); - entity.get().getManualChanges().add(ManualChange.builder() - .manualRedactionType(ManualRedactionType.RECATEGORIZE) - .requestedDate(recategorization.getRequestDate()) - .processedDate(recategorization.getProcessedDate()) - .userId(recategorization.getUser()) - .propertyChanges(Map.of("type", recategorization.getType())) - .build()); - } - }); - } - - - private static EntryType getEntryType(boolean isHint, String type) { - - if (type.equals("image") || type.equals("logo") || type.equals("signature") || type.equals("formula")) { - return isHint ? EntryType.IMAGE_HINT : EntryType.IMAGE; - } else { - return isHint ? EntryType.HINT : EntryType.ENTITY; - } - } - - - private void mergeForceRedactions(Set forceRedactions, EntityLog entityLog) { - - forceRedactions.forEach(forceRedaction -> { - var entity = entityLog.getEntityLogEntry().stream().filter(entityLogEntry -> entityLogEntry.getId().equals(forceRedaction.getAnnotationId())).findAny(); - if (entity.isPresent()) { - entity.get().setLegalBasis(forceRedaction.getLegalBasis()); - entity.get().setState(EntryState.APPLIED); - addChanges(entity.get().getChanges(), ChangeType.CHANGED, entityLog.getAnalysisNumber()); - var forceRedactManualChange = ManualChange.builder() - .manualRedactionType(ManualRedactionType.FORCE_REDACT) - .requestedDate(forceRedaction.getRequestDate()) - .processedDate(forceRedaction.getProcessedDate()) - .userId(forceRedaction.getUser()); - if (forceRedaction.getLegalBasis() != null && !forceRedaction.getLegalBasis().isEmpty()) { - forceRedactManualChange.propertyChanges(Map.of("legalBasis", forceRedaction.getLegalBasis())); - } - entity.get().getManualChanges().add(forceRedactManualChange.build()); - } - }); - } - - private void addChanges(List changes, ChangeType changeType, int analysisNumber) { - - if (!changes.isEmpty()) { - changes.add(Change.builder() - .analysisNumber(analysisNumber + 1) - .dateTime(OffsetDateTime.now()) - .type(changeType) - .build()); - } else { - changes.add(Change.builder().analysisNumber(analysisNumber).dateTime(OffsetDateTime.now()).type(changeType).build()); - } - } - - private boolean isHint(String type, DossierEntity dossier, FileModel fileStatus) { - - String typeId = toTypeId(type, dossier.getDossierTemplateId()); - TypeEntity typeEntity = dictionaryPersistenceService.getType(typeId); - - if (typeEntity == null) { - var optionalType = dictionaryPersistenceService.getAllTypes(false).stream().filter(typeEntity1 -> typeEntity1.getType().equals(type)).findFirst(); - if (optionalType.isPresent()) { - typeEntity = optionalType.get(); - } else { - throw new NotFoundException("TypeEntity could not be found for typeId: " + typeId); - } - } - - if (Objects.equals(type, "CBI_address")) { - - String vertebrateStudyId = ""; - var fileAttributesMap = fileStatus.getFileAttributes(); - var dossierTemplate = dossierTemplatePersistenceService.getDossierTemplate(dossier.getDossierTemplateId()); - var fileAttributeConfigs = dossierTemplate.getFileAttributeConfigs(); - var vertebrateStudy = fileAttributeConfigs.stream().filter(fileAttributeConfigEntity -> fileAttributeConfigEntity.getLabel().equals("Vertebrate Study")).findFirst(); - - if (vertebrateStudy.isPresent()) { - vertebrateStudyId = vertebrateStudy.get().getId(); - } - - if (!Objects.equals(vertebrateStudyId, "") - && fileAttributesMap.containsKey(vertebrateStudyId) - && (fileAttributesMap.get(vertebrateStudyId) == null || !fileAttributesMap.get(vertebrateStudyId).equals("Yes"))) { - return true; - } - } - - return typeEntity.isHint(); - } - } diff --git a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/EntityLogTest.java b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/EntityLogMergeTest.java similarity index 93% rename from persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/EntityLogTest.java rename to persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/EntityLogMergeTest.java index fb54fbd7f..3edb27b6c 100644 --- a/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/EntityLogTest.java +++ b/persistence-service-v1/persistence-service-server-v1/src/test/java/com/iqser/red/service/peristence/v1/server/integration/tests/EntityLogMergeTest.java @@ -18,7 +18,7 @@ import java.util.UUID; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.Mock; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.mock.mockito.MockBean; import org.springframework.test.context.junit.jupiter.SpringExtension; import org.testcontainers.shaded.com.google.common.collect.Lists; @@ -28,6 +28,7 @@ import com.iqser.red.service.persistence.management.v1.processor.entity.configur import com.iqser.red.service.persistence.management.v1.processor.entity.dossier.DossierEntity; import com.iqser.red.service.persistence.management.v1.processor.service.CommentService; import com.iqser.red.service.persistence.management.v1.processor.service.DossierService; +import com.iqser.red.service.persistence.management.v1.processor.service.EntityLogMergeService; import com.iqser.red.service.persistence.management.v1.processor.service.EntityLogService; import com.iqser.red.service.persistence.management.v1.processor.service.FileManagementStorageService; import com.iqser.red.service.persistence.management.v1.processor.service.FileStatusService; @@ -49,10 +50,11 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileModel; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntryType; import com.iqser.red.service.redaction.v1.model.UnprocessedManualEntity; @ExtendWith(SpringExtension.class) -public class EntityLogTest { +public class EntityLogMergeTest { @MockBean private FileStatusService fileStatusService; @@ -78,12 +80,12 @@ public class EntityLogTest { @MockBean private DossierTemplatePersistenceService dossierTemplatePersistenceService; - private EntityLogService entityLogService; + private EntityLogMergeService entityLogMergeService; @BeforeEach public void setUp() { - entityLogService = new EntityLogService(fileManagementStorageService, fileStatusService, manualRedactionProviderService, unprocessedManualEntityClient, dossierService, commentService, dictionaryPersistenceService, dossierTemplatePersistenceService); + entityLogMergeService = new EntityLogMergeService(dictionaryPersistenceService); } @@ -127,7 +129,9 @@ public class EntityLogTest { .build()); when(dictionaryPersistenceService.getType(anyString())).thenReturn(TypeEntity.builder().isHint(false).build()); - EntityLog response = entityLogService.getEntityLog(dossierId, fileId, null, true); + EntityLog response = entityLogMergeService.mergeEntityLog(manualRedactions, List.of(unprocessedManualEntity), entityLog, DossierEntity.builder() + .dossierTemplateId(dossierTemplateId) + .build()); assertNotNull(response); assertFalse(response.getEntityLogEntry().isEmpty()); @@ -250,11 +254,13 @@ public class EntityLogTest { .addToDossierDictionary(false) .fileId(fileId) .requestDate(OffsetDateTime.now()) + .dictionaryEntryType(DictionaryEntryType.ENTRY) .type("manual") .build())) .idsToRemove(Set.of( IdRemoval.builder() .annotationId(entryToRemoveId) + .requestDate(OffsetDateTime.now()) .build() )) .resizeRedactions(Set.of( @@ -263,6 +269,7 @@ public class EntityLogTest { .value("Random") .annotationId(entryToResizeId) .positions(positions) + .requestDate(OffsetDateTime.now()) .build() )) .legalBasisChanges(Set.of( @@ -271,6 +278,7 @@ public class EntityLogTest { .value("Random") .legalBasis("New legal basis") .section("Section") + .requestDate(OffsetDateTime.now()) .build() )) .forceRedactions(Set.of( @@ -278,8 +286,10 @@ public class EntityLogTest { .annotationId(forceRedactionId) .fileId(fileId) .legalBasis("Force") + .requestDate(OffsetDateTime.now()) .build() )) .build(); } } + diff --git a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/annotations/entitymapped/ManualRedactionEntry.java b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/annotations/entitymapped/ManualRedactionEntry.java index 7ae2833e3..55c2f71d7 100644 --- a/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/annotations/entitymapped/ManualRedactionEntry.java +++ b/persistence-service-v1/persistence-service-shared-api-v1/src/main/java/com/iqser/red/service/persistence/service/v1/api/shared/model/annotations/entitymapped/ManualRedactionEntry.java @@ -4,6 +4,7 @@ import java.util.ArrayList; import java.util.List; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntryType; import lombok.AllArgsConstructor; import lombok.Data; @@ -30,5 +31,6 @@ public class ManualRedactionEntry extends BaseAnnotation { private String textBefore; private String textAfter; private String sourceId; + private DictionaryEntryType dictionaryEntryType; }