From a58bcedccfd6cbcc4e74eea83eee2c2d5ffaf420 Mon Sep 17 00:00:00 2001 From: Andrei Isvoran Date: Thu, 27 Jun 2024 13:49:26 +0200 Subject: [PATCH] RED-9140 - Add more information to changes --- .../redaction-service-api-v1/build.gradle.kts | 2 +- .../build.gradle.kts | 2 +- .../v1/server/migration/MigrationMapper.java | 6 +- .../v1/server/model/MigrationEntity.java | 2 +- .../service/EntityChangeLogService.java | 79 ++++- .../service/EntityLogCreatorService.java | 62 +++- .../NotFoundImportedEntitiesService.java | 3 +- .../utils/EntityLogEntryDiffChecker.java | 45 +++ .../v1/server/utils/ManualChangesUtils.java | 54 ++++ .../v1/server/RedactionIntegrationTest.java | 279 +++++++++++++++++- .../ManualChangesIntegrationTest.java | 6 + 11 files changed, 507 insertions(+), 33 deletions(-) create mode 100644 redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/utils/EntityLogEntryDiffChecker.java create mode 100644 redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/utils/ManualChangesUtils.java diff --git a/redaction-service-v1/redaction-service-api-v1/build.gradle.kts b/redaction-service-v1/redaction-service-api-v1/build.gradle.kts index 97713b61..f649d8ae 100644 --- a/redaction-service-v1/redaction-service-api-v1/build.gradle.kts +++ b/redaction-service-v1/redaction-service-api-v1/build.gradle.kts @@ -4,7 +4,7 @@ plugins { } description = "redaction-service-api-v1" -val persistenceServiceVersion = "2.439.0" +val persistenceServiceVersion = "2.467.0" dependencies { implementation("org.springframework:spring-web:6.0.12") diff --git a/redaction-service-v1/redaction-service-server-v1/build.gradle.kts b/redaction-service-v1/redaction-service-server-v1/build.gradle.kts index 342414bd..ea2f708c 100644 --- a/redaction-service-v1/redaction-service-server-v1/build.gradle.kts +++ b/redaction-service-v1/redaction-service-server-v1/build.gradle.kts @@ -16,7 +16,7 @@ val layoutParserVersion = "0.141.0" val jacksonVersion = "2.15.2" val droolsVersion = "9.44.0.Final" val pdfBoxVersion = "3.0.0" -val persistenceServiceVersion = "2.444.0" +val persistenceServiceVersion = "2.467.0" val springBootStarterVersion = "3.1.5" val springCloudVersion = "4.0.4" val testContainersVersion = "1.19.7" diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/migration/MigrationMapper.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/migration/MigrationMapper.java index 0b4f034b..72edc146 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/migration/MigrationMapper.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/migration/MigrationMapper.java @@ -18,7 +18,8 @@ public class MigrationMapper { return new com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Change(change.getAnalysisNumber(), toEntityLogType(change.getType()), - change.getDateTime()); + change.getDateTime(), + Collections.emptyMap()); } @@ -28,7 +29,8 @@ public class MigrationMapper { manualChange.getProcessedDate(), manualChange.getRequestedDate(), manualChange.getUserId(), - manualChange.getPropertyChanges()); + manualChange.getPropertyChanges(), + 0); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/MigrationEntity.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/MigrationEntity.java index 37857744..a8e91fba 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/MigrationEntity.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/MigrationEntity.java @@ -198,7 +198,7 @@ public final class MigrationEntity { throw new UnsupportedOperationException("Unknown subclass " + migratedEntity.getClass()); } - entityLogEntry.setManualChanges(ManualChangeFactory.toLocalManualChangeList(migratedEntity.getManualOverwrite().getManualChangeLog(), true)); + entityLogEntry.setManualChanges(ManualChangeFactory.toLocalManualChangeList(migratedEntity.getManualOverwrite().getManualChangeLog(), true, 0)); entityLogEntry.setColor(redactionLogEntry.getColor()); entityLogEntry.setChanges(redactionLogEntry.getChanges() .stream() diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityChangeLogService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityChangeLogService.java index cdb44c36..156856b6 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityChangeLogService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityChangeLogService.java @@ -2,7 +2,9 @@ package com.iqser.red.service.redaction.v1.server.service; import java.time.OffsetDateTime; import java.util.ArrayList; +import java.util.Collections; import java.util.List; +import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; @@ -13,6 +15,11 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ChangeType; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualChange; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.PropertyChange; +import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ChangeFactory; +import com.iqser.red.service.redaction.v1.server.utils.EntityLogEntryDiffChecker; import io.micrometer.core.annotation.Timed; import lombok.AccessLevel; @@ -31,7 +38,7 @@ public class EntityChangeLogService { var now = OffsetDateTime.now(); if (previousEntityLogEntries.isEmpty()) { - newEntityLogEntries.forEach(entry -> entry.getChanges().add(new Change(analysisNumber, ChangeType.ADDED, now))); + newEntityLogEntries.forEach(entry -> entry.getChanges().add(new Change(analysisNumber, ChangeType.ADDED, now, Collections.emptyMap()))); return new EntryChanges(newEntityLogEntries, new ArrayList<>()); } @@ -42,7 +49,7 @@ public class EntityChangeLogService { .filter(entry -> entry.getId().equals(entityLogEntry.getId())) .findAny(); if (optionalPreviousEntity.isEmpty()) { - entityLogEntry.getChanges().add(new Change(analysisNumber, ChangeType.ADDED, now)); + entityLogEntry.getChanges().add(new Change(analysisNumber, ChangeType.ADDED, now, Collections.emptyMap())); toInsert.add(entityLogEntry); continue; } @@ -51,10 +58,9 @@ public class EntityChangeLogService { entityLogEntry.getChanges().addAll(previousEntity.getChanges()); if (!previousEntity.equals(entityLogEntry)) { - if(!previousEntity.getState().equals(entityLogEntry.getState())) { - ChangeType changeType = calculateChangeType(entityLogEntry.getState(), previousEntity.getState()); - entityLogEntry.getChanges().add(new Change(analysisNumber, changeType, now)); - } + List propertyChanges = EntityLogEntryDiffChecker.compareEntityLogEntries(previousEntity, entityLogEntry); + Change change = calculateChange(previousEntity, entityLogEntry, propertyChanges, analysisNumber); + entityLogEntry.getChanges().add(change); toUpdate.add(entityLogEntry); } } @@ -77,18 +83,43 @@ public class EntityChangeLogService { .toList(); removedEntries.stream() .filter(entry -> !entry.getState().equals(EntryState.REMOVED)) - .peek(entry -> entry.getChanges().add(new Change(analysisNumber, ChangeType.REMOVED, now))) + .peek(entry -> entry.getChanges().add(new Change(analysisNumber, ChangeType.REMOVED, now, Collections.emptyMap()))) .forEach(entry -> entry.setState(EntryState.REMOVED)); newEntityLogEntries.addAll(removedEntries); return removedEntries; } + private Change calculateChange(EntityLogEntry previousEntry, EntityLogEntry currentEntry, List propertyChanges, int analysisNumber) { + + ManualChange previousLastManualChange; + ManualChange currentLastManualChange; + + if (previousEntry.getManualChanges().isEmpty() && currentEntry.getManualChanges().isEmpty()) { + return systemChange(propertyChanges, analysisNumber, previousEntry.getState(), currentEntry.getState()); + } + + if (previousEntry.getManualChanges().isEmpty() && !currentEntry.getManualChanges().isEmpty()) { + currentLastManualChange = currentEntry.getManualChanges() + .get(currentEntry.getManualChanges().size() - 1); + return manualChange(propertyChanges, analysisNumber, currentLastManualChange); + } + + previousLastManualChange = previousEntry.getManualChanges() + .get(previousEntry.getManualChanges().size() - 1); + currentLastManualChange = currentEntry.getManualChanges() + .get(currentEntry.getManualChanges().size() - 1); + + if (Objects.equals(previousLastManualChange, currentLastManualChange)) { + return systemChange(propertyChanges, analysisNumber, previousEntry.getState(), currentEntry.getState()); + } else { + return manualChange(propertyChanges, analysisNumber, currentLastManualChange); + } + } + + private ChangeType calculateChangeType(EntryState state, EntryState previousState) { - if (state.equals(previousState)) { - throw new IllegalArgumentException("States are equal, can't calculate ChangeType."); - } if (!isRemoved(previousState) && isRemoved(state)) { return ChangeType.REMOVED; } @@ -99,12 +130,40 @@ public class EntityChangeLogService { } + private Change systemChange(List propertyChanges, int analysisNumber, EntryState previousState, EntryState currentState) { + + return ChangeFactory.toChange(calculateChangeType(currentState, previousState), OffsetDateTime.now(), analysisNumber, propertyChanges.toArray(new PropertyChange[0])); + } + + + private Change manualChange(List propertyChanges, int analysisNumber, ManualChange manualChange) { + + return ChangeFactory.toChange(convertToChangeType(manualChange.getManualRedactionType()), + manualChange.getRequestedDate(), + analysisNumber, + propertyChanges.toArray(new PropertyChange[0])); + } + + private static boolean isRemoved(EntryState state) { return (state.equals(EntryState.REMOVED) || state.equals(EntryState.IGNORED)); } + public ChangeType convertToChangeType(ManualRedactionType manualRedactionType) { + + return switch (manualRedactionType) { + case ADD_LOCALLY, ADD, ADD_TO_DICTIONARY -> ChangeType.ADDED; + case FORCE_REDACT, FORCE_HINT, FORCE -> ChangeType.FORCE_REDACT; + case REMOVE_LOCALLY, REMOVE, REMOVE_FROM_DICTIONARY -> ChangeType.REMOVED; + case RECATEGORIZE, RECATEGORIZE_IN_DICTIONARY -> ChangeType.RECATEGORIZE; + case LEGAL_BASIS_CHANGE -> ChangeType.LEGAL_BASIS_CHANGE; + case RESIZE, RESIZE_IN_DICTIONARY -> ChangeType.RESIZED; + }; + } + + public record EntryChanges(List inserted, List updated) { } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java index b237c2ad..58684c34 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java @@ -20,9 +20,11 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogLegalBasis; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryType; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualChange; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualChangeFactory; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.legalbasis.LegalBasis; +import com.iqser.red.service.persistence.service.v1.api.shared.mongo.service.EntityLogMongoService; import com.iqser.red.service.redaction.v1.server.RedactionServiceSettings; import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient; import com.iqser.red.service.redaction.v1.server.model.PrecursorEntity; @@ -39,6 +41,7 @@ import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNo import com.iqser.red.service.redaction.v1.server.model.document.textblock.AtomicTextBlock; import com.iqser.red.service.redaction.v1.server.service.EntityChangeLogService.EntryChanges; import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService; +import com.iqser.red.service.redaction.v1.server.utils.ManualChangesUtils; import lombok.AccessLevel; import lombok.RequiredArgsConstructor; @@ -56,6 +59,7 @@ public class EntityLogCreatorService { LegalBasisClient legalBasisClient; EntityChangeLogService entityChangeLogService; RedactionStorageService redactionStorageService; + EntityLogMongoService entityLogMongoService; private static boolean notFalsePositiveOrFalseRecommendationOrRemoval(TextEntity textEntity) { @@ -72,7 +76,7 @@ public class EntityLogCreatorService { DictionaryVersion dictionaryVersion, long rulesVersion) { - List entityLogEntries = createEntityLogEntries(document, analyzeRequest, notFoundEntities); + List entityLogEntries = createEntityLogEntries(document, analyzeRequest, notFoundEntities, analyzeRequest.getAnalysisNumber()); List legalBasis = legalBasisClient.getLegalBasisMapping(analyzeRequest.getDossierTemplateId()); @@ -129,7 +133,7 @@ public class EntityLogCreatorService { Set sectionsToReanalyseIds, DictionaryVersion dictionaryVersion) { - List newEntityLogEntries = createEntityLogEntries(document, analyzeRequest, notFoundEntries).stream() + List newEntityLogEntries = createEntityLogEntries(document, analyzeRequest, notFoundEntries, analyzeRequest.getAnalysisNumber()).stream() .filter(entry -> entry.getContainingNodeId().isEmpty() || sectionsToReanalyseIds.contains(entry.getContainingNodeId() .get(0))) .collect(Collectors.toList()); @@ -144,7 +148,7 @@ public class EntityLogCreatorService { } - private List createEntityLogEntries(Document document, AnalyzeRequest analyzeRequest, List notFoundPrecursorEntries) { + private List createEntityLogEntries(Document document, AnalyzeRequest analyzeRequest, List notFoundPrecursorEntries, int analysisNumber) { String dossierTemplateId = analyzeRequest.getDossierTemplateId(); @@ -155,31 +159,34 @@ public class EntityLogCreatorService { .filter(entity -> !entity.getValue().isEmpty()) .filter(EntityLogCreatorService::notFalsePositiveOrFalseRecommendationOrRemoval) .filter(entity -> !entity.removed()) - .forEach(entityNode -> entries.addAll(toEntityLogEntries(entityNode))); + .forEach(entityNode -> entries.addAll(toEntityLogEntries(entityNode, analysisNumber, analyzeRequest.getDossierId(), analyzeRequest.getFileId()))); + document.streamAllImages() .filter(entity -> !entity.removed()) - .forEach(imageNode -> entries.add(createEntityLogEntry(imageNode, dossierTemplateId))); + .forEach(imageNode -> entries.add(createEntityLogEntry(imageNode, dossierTemplateId, analysisNumber, analyzeRequest.getDossierId(), analyzeRequest.getFileId()))); + notFoundPrecursorEntries.stream() .filter(entity -> !entity.removed()) - .forEach(precursorEntity -> entries.add(createEntityLogEntry(precursorEntity))); + .forEach(precursorEntity -> entries.add(createEntityLogEntry(precursorEntity, analysisNumber, analyzeRequest.getDossierId(), analyzeRequest.getFileId()))); + return entries; } - private List toEntityLogEntries(TextEntity textEntity) { + private List toEntityLogEntries(TextEntity textEntity, int analysisNumber, String dossierId, String fileId) { List entityLogEntries = new ArrayList<>(); // split entity into multiple entries if it occurs on multiple pages, since FE can't handle multi page entities for (PositionOnPage positionOnPage : textEntity.getPositionsOnPagePerPage()) { - EntityLogEntry entityLogEntry = createEntityLogEntry(textEntity); - List rectanglesPerLine = positionOnPage.getRectanglePerLine() .stream() .map(rectangle2D -> new Position(rectangle2D, positionOnPage.getPage().getNumber())) .toList(); + EntityLogEntry entityLogEntry = createEntityLogEntry(textEntity, analysisNumber, positionOnPage.getId(), dossierId, fileId); + // set the ID from the positions, since it might contain a "-" with the page number if the entity is split across multiple pages entityLogEntry.setId(positionOnPage.getId()); entityLogEntry.setPositions(rectanglesPerLine); @@ -190,10 +197,14 @@ public class EntityLogCreatorService { } - private EntityLogEntry createEntityLogEntry(Image image, String dossierTemplateId) { + private EntityLogEntry createEntityLogEntry(Image image, String dossierTemplateId, int analysisNumber, String dossierId, String fileId) { String imageType = image.getImageType().equals(ImageType.OTHER) ? "image" : image.getImageType().toString().toLowerCase(Locale.ENGLISH); boolean isHint = dictionaryService.isHint(imageType, dossierTemplateId); + + List existingManualChanges = getManualChangesByEntityLogId(dossierId, fileId, image.getId()); + List allManualChanges = ManualChangeFactory.toLocalManualChangeList(image.getManualOverwrite().getManualChangeLog(), true, analysisNumber); + return EntityLogEntry.builder() .id(image.getId()) .value(image.getValue()) @@ -209,7 +220,7 @@ public class EntityLogCreatorService { // .orElse(image.getParent().toString())) .orElse(this.buildSectionString(image.getParent()))) .imageHasTransparency(image.isTransparent()) - .manualChanges(ManualChangeFactory.toLocalManualChangeList(image.getManualOverwrite().getManualChangeLog(), true)) + .manualChanges(ManualChangesUtils.mergeManualChanges(existingManualChanges, allManualChanges)) .state(buildEntryState(image)) .entryType(isHint ? EntryType.IMAGE_HINT : EntryType.IMAGE) .engines(getEngines(null, image.getManualOverwrite())) @@ -219,11 +230,14 @@ public class EntityLogCreatorService { } - private EntityLogEntry createEntityLogEntry(PrecursorEntity precursorEntity) { + private EntityLogEntry createEntityLogEntry(PrecursorEntity precursorEntity, int analysisNumber, String dossierId, String fileId) { String type = precursorEntity.getManualOverwrite().getType() .orElse(precursorEntity.getType()); - boolean isHint = isHint(precursorEntity.getEntityType()); + + List existingManualChanges = getManualChangesByEntityLogId(dossierId, fileId, precursorEntity.getId()); + List allManualChanges = ManualChangeFactory.toLocalManualChangeList(precursorEntity.getManualOverwrite().getManualChangeLog(), true, analysisNumber); + return EntityLogEntry.builder() .id(precursorEntity.getId()) .reason(precursorEntity.buildReasonWithManualChangeDescriptions()) @@ -253,13 +267,13 @@ public class EntityLogCreatorService { //(was .imported(precursorEntity.getEngines() != null && precursorEntity.getEngines().contains(Engine.IMPORTED))) .imported(false) .reference(Collections.emptySet()) - .manualChanges(ManualChangeFactory.toLocalManualChangeList(precursorEntity.getManualOverwrite().getManualChangeLog(), true)) + .manualChanges(ManualChangesUtils.mergeManualChanges(existingManualChanges, allManualChanges)) .paragraphPageIdx(-1) .build(); } - private EntityLogEntry createEntityLogEntry(TextEntity entity) { + private EntityLogEntry createEntityLogEntry(TextEntity entity, int analysisNumber, String id, String dossierId, String fileId) { Set referenceIds = new HashSet<>(); entity.references() @@ -270,6 +284,9 @@ public class EntityLogCreatorService { EntryType entryType = buildEntryType(entity); + List existingManualChanges = getManualChangesByEntityLogId(dossierId, fileId, id); + List allManualChanges = ManualChangeFactory.toLocalManualChangeList(entity.getManualOverwrite().getManualChangeLog(), true, analysisNumber); + return EntityLogEntry.builder() .reason(entity.buildReasonWithManualChangeDescriptions()) .legalBasis(entity.legalBasis()) @@ -297,7 +314,7 @@ public class EntityLogCreatorService { //(was .imported(entity.getEngines() != null && entity.getEngines().contains(Engine.IMPORTED))) .imported(false) .reference(referenceIds) - .manualChanges(ManualChangeFactory.toLocalManualChangeList(entity.getManualOverwrite().getManualChangeLog(), true)) + .manualChanges(ManualChangesUtils.mergeManualChanges(existingManualChanges, allManualChanges)) .state(buildEntryState(entity)) .entryType(entryType) .paragraphPageIdx(determinePageParagraphIndex(entity, entryType)) @@ -403,4 +420,17 @@ public class EntityLogCreatorService { return node.getType().toString() + ": " + node.getTextBlock().buildSummary(); } + + public List getManualChangesByEntityLogId(String dossierId, String fileId, String id) { + + List manualChanges = new ArrayList<>(); + List entityLogEntries = entityLogMongoService.findEntityLogEntriesByIds(dossierId, fileId, List.of(id)); + + for (EntityLogEntry entry : entityLogEntries) { + manualChanges.addAll(entry.getManualChanges()); + } + + return manualChanges; + } + } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/NotFoundImportedEntitiesService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/NotFoundImportedEntitiesService.java index 04e687e6..e4888dc2 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/NotFoundImportedEntitiesService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/NotFoundImportedEntitiesService.java @@ -17,6 +17,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position; +import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ChangeFactory; import com.iqser.red.service.redaction.v1.server.model.PrecursorEntity; import com.iqser.red.service.redaction.v1.server.model.RectangleWithPage; import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService; @@ -96,7 +97,7 @@ public class NotFoundImportedEntitiesService { if (entityLogEntry.getState() != EntryState.REMOVED) { entityLogEntry.setState(EntryState.REMOVED); - entityLogEntry.getChanges().add(new Change(analysisNumber, ChangeType.REMOVED, OffsetDateTime.now())); + entityLogEntry.getChanges().add(ChangeFactory.toChange(ChangeType.REMOVED, OffsetDateTime.now(), analysisNumber)); } } } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/utils/EntityLogEntryDiffChecker.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/utils/EntityLogEntryDiffChecker.java new file mode 100644 index 00000000..468ec8bf --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/utils/EntityLogEntryDiffChecker.java @@ -0,0 +1,45 @@ +package com.iqser.red.service.redaction.v1.server.utils; + +import java.lang.reflect.Field; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; + +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.PropertyChange; + +import lombok.experimental.UtilityClass; +import lombok.extern.slf4j.Slf4j; + +@UtilityClass +@Slf4j +@SuppressWarnings("PMD") // Needed in order to be able to do setAccessible(true) for field comparison +public class EntityLogEntryDiffChecker { + + public List compareEntityLogEntries(EntityLogEntry previousEntityLogEntry, EntityLogEntry currentEntityLogEntry) { + + List changes = new ArrayList<>(); + Field[] fields = EntityLogEntry.class.getDeclaredFields(); + + for (Field field : fields) { + field.setAccessible(true); + try { + Object oldValue = field.get(previousEntityLogEntry); + Object newValue = field.get(currentEntityLogEntry); + + if (!Objects.equals(oldValue, newValue)) { + changes.add(PropertyChange.builder() + .property(field.getName()) + .oldValue(oldValue != null ? oldValue.toString() : "") + .newValue(newValue != null ? newValue.toString() : "") + .build()); + } + } catch (IllegalAccessException e) { + log.warn("Failed to access field: {}", field.getName()); + } + } + + return changes; + } + +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/utils/ManualChangesUtils.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/utils/ManualChangesUtils.java new file mode 100644 index 00000000..840aa38e --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/utils/ManualChangesUtils.java @@ -0,0 +1,54 @@ +package com.iqser.red.service.redaction.v1.server.utils; + +import java.time.Duration; +import java.time.OffsetDateTime; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; + +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualChange; + +import lombok.experimental.UtilityClass; +import lombok.extern.slf4j.Slf4j; + +@UtilityClass +@Slf4j +public class ManualChangesUtils { + + // there is a very slight variation of time when comparing the manual changes request Date, under 1ms. + public static final Duration TOLERANCE = Duration.ofMillis(1); + + + public List mergeManualChanges(List existingChanges, List allChanges) { + + List mergedChanges = new ArrayList<>(existingChanges); + + for (ManualChange manualChange : allChanges) { + + var existingChangeOpt = mergedChanges.stream() + .filter(existingChange -> areChangesEquivalent(existingChange, manualChange)) + .findFirst(); + + if (existingChangeOpt.isEmpty()) { + mergedChanges.add(manualChange); + } + } + + return mergedChanges; + } + + + private boolean areChangesEquivalent(ManualChange change1, ManualChange change2) { + + return change1.getManualRedactionType() == change2.getManualRedactionType() && areDatesEquivalent(change1.getRequestedDate(), change2.getRequestedDate()) && Objects.equals( + change1.getUserId(), + change2.getUserId()); + } + + + private boolean areDatesEquivalent(OffsetDateTime date1, OffsetDateTime date2) { + + return Duration.between(date1, date2).abs().compareTo(TOLERANCE) <= 0; + } + +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java index 1593a25c..ed9e61f8 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java @@ -47,6 +47,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequ import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeResult; import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute; import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ChangeType; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryType; @@ -1317,7 +1318,6 @@ public class RedactionIntegrationTest extends RulesIntegrationTest { .findFirst() .get(); - request.setManualRedactions(ManualRedactions.builder() .legalBasisChanges(Set.of(ManualLegalBasisChange.builder() .annotationId("3029651d0842a625f2d23f8375c23600") @@ -1840,6 +1840,283 @@ public class RedactionIntegrationTest extends RulesIntegrationTest { } + @Test + public void testResizeWithChanges() { + + AnalyzeRequest request = uploadFileToStorage("files/new/crafted document.pdf"); + analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request); + analyzeService.analyze(request); + + var entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID); + + var libraryOutlook = entityLog.getEntityLogEntry() + .stream() + .filter(e -> e.getValue().equals("library@outlook.com")) + .findFirst(); + assertTrue(libraryOutlook.isPresent()); + + request.setManualRedactions(ManualRedactions.builder() + .resizeRedactions(Set.of(ManualResizeRedaction.builder() + .addToAllDossiers(false) + .updateDictionary(false) + .user("user") + .fileId(TEST_FILE_ID) + .requestDate(OffsetDateTime.now()) + .annotationId(libraryOutlook.get().getId()) + .positions(List.of(Rectangle.builder() + .topLeftX(159.364f) + .topLeftY(303.364f) + .width(115.68f) + .height(15.408f) + .page(4) + .build())) + .value("in library@outlook.com") + .build())) + .build()); + + analyzeService.reanalyze(request); + entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID); + var inLibraryOutlookOpt = entityLog.getEntityLogEntry() + .stream() + .filter(e -> e.getValue().equals("in library@outlook.com")) + .findFirst(); + assertTrue(inLibraryOutlookOpt.isPresent()); + var inLibraryOutlook = inLibraryOutlookOpt.get(); + assertEquals(inLibraryOutlook.getChanges().size(), 2); + assertEquals(inLibraryOutlook.getChanges() + .get(0).getType(), ChangeType.ADDED); + assertEquals(inLibraryOutlook.getChanges() + .get(1).getType(), ChangeType.RESIZED); + assertEquals(inLibraryOutlook.getChanges() + .get(1).getPropertyChanges() + .get("reason"), "Found by Email Regex -> Found by Email Regex, resized by manual override"); + assertEquals(inLibraryOutlook.getChanges() + .get(1).getPropertyChanges() + .get("startOffset"), "3793 -> 3790"); + assertEquals(inLibraryOutlook.getChanges() + .get(1).getPropertyChanges() + .get("legalBasis"), "Article 4(1)(b), Regulation (EC) No 1049/2001 (Personal data) -> Reg (EC) No 1107/2009 Art. 63 (2e)"); + assertEquals(inLibraryOutlook.getChanges() + .get(1).getPropertyChanges() + .get("positions"), "[[171.748, 305.0, 103.296036, 12.642, 4]] -> [[159.364, 303.364, 115.68, 15.408, 4]]"); + assertEquals(inLibraryOutlook.getChanges() + .get(1).getPropertyChanges() + .get("textBefore"), "irure dolor in -> aute irure dolor "); + assertEquals(inLibraryOutlook.getChanges() + .get(1).getPropertyChanges() + .get("value"), "library@outlook.com -> in library@outlook.com"); + } + + + @Test + public void testAddWithChanges() { + + AnalyzeRequest request = uploadFileToStorage("files/new/crafted document.pdf"); + analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request); + analyzeService.analyze(request); + + String manualAddId = UUID.randomUUID().toString(); + + request.setManualRedactions(ManualRedactions.builder() + .entriesToAdd(Set.of(ManualRedactionEntry.builder() + .annotationId(manualAddId) + .requestDate(OffsetDateTime.now()) + .fileId(TEST_FILE_ID) + .user("user") + .value("not in Dictionary") + .section(null) + .reason("(Regulations (EU) 2016/679 and (EU) 2018/1725 shall apply to the processing of personal data carried out pursuant to this Regulation. Any personal data made public pursuant to Article 38 of this Regulation and this Article shall only be used to ensure the transparency of the risk assessment under this Regulation and shall not be further processed in a manner that is incompatible with these purposes, in accordance with point (b) of Article 5(1) of Regulation (EU) 2016/679 and point (b) of Article 4(1) of Regulation (EU) 2018/1725, as the case may be)") + .addToDossierDictionary(false) + .addToDictionary(false) + .legalBasis("Article 39(e)(3) of Regulation (EC) No 178/2002") + .rectangle(false) + .positions(List.of(Rectangle.builder() + .topLeftX(270.844f) + .topLeftY(238.364f) + .width(81.876f) + .height(15.408f) + .page(1) + .build())) + .type("manual") + .build())) + .build()); + + analyzeService.reanalyze(request); + var entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID); + var notInDictionaryOpt = entityLog.getEntityLogEntry() + .stream() + .filter(e -> e.getId().equals(manualAddId)) + .findFirst(); + assertTrue(notInDictionaryOpt.isPresent()); + var notInDictionary = notInDictionaryOpt.get(); + assertEquals(notInDictionary.getChanges().size(), 1); + assertEquals(notInDictionary.getChanges() + .get(0).getType(), ChangeType.ADDED); + } + + + @Test + public void testRemovalWithChanges() { + + AnalyzeRequest request = uploadFileToStorage("files/new/crafted document.pdf"); + analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request); + analyzeService.analyze(request); + + var entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID); + + var davidKsenia = entityLog.getEntityLogEntry() + .stream() + .filter(e -> e.getValue().equals("David Ksenia")) + .findFirst(); + assertTrue(davidKsenia.isPresent()); + + request.setManualRedactions(ManualRedactions.builder() + .idsToRemove(Set.of(IdRemoval.builder() + .fileId(TEST_FILE_ID) + .user("user") + .requestDate(OffsetDateTime.now()) + .annotationId(davidKsenia.get().getId()) + .removeFromDictionary(false) + .removeFromAllDossiers(false) + .build())) + .build()); + + analyzeService.reanalyze(request); + entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID); + var davidKseniaOpt = entityLog.getEntityLogEntry() + .stream() + .filter(e -> e.getId().equals(davidKsenia.get().getId())) + .findFirst(); + assertTrue(davidKseniaOpt.isPresent()); + var responseDavidKsenia = davidKseniaOpt.get(); + assertEquals(responseDavidKsenia.getChanges().size(), 2); + assertEquals(responseDavidKsenia.getState(), EntryState.IGNORED); + assertEquals(responseDavidKsenia.getChanges() + .get(0).getType(), ChangeType.ADDED); + assertEquals(responseDavidKsenia.getChanges() + .get(1).getType(), ChangeType.REMOVED); + assertEquals(responseDavidKsenia.getChanges() + .get(1).getPropertyChanges() + .get("reason"), "No vertebrate found -> removed by manual override"); + assertEquals(responseDavidKsenia.getChanges() + .get(1).getPropertyChanges() + .get("matchedRule"), "CBI.3.2 -> "); + assertEquals(responseDavidKsenia.getChanges() + .get(1).getPropertyChanges() + .get("state"), "SKIPPED -> IGNORED"); + } + + + @Test + public void testRecatgeorizeWithChanges() { + + AnalyzeRequest request = uploadFileToStorage("files/new/crafted document.pdf"); + analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request); + analyzeService.analyze(request); + + var entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID); + + var davidKsenia = entityLog.getEntityLogEntry() + .stream() + .filter(e -> e.getValue().equals("David Ksenia")) + .findFirst(); + assertTrue(davidKsenia.isPresent()); + + request.setManualRedactions(ManualRedactions.builder() + .recategorizations(Set.of(ManualRecategorization.builder() + .fileId(TEST_FILE_ID) + .user("user") + .requestDate(OffsetDateTime.now()) + .annotationId(davidKsenia.get().getId()) + .legalBasis("new legal basis") + .addToAllDossiers(false) + .addToDictionary(false) + .type("PII") + .section(null) + .build())) + .build()); + + analyzeService.reanalyze(request); + entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID); + var davidKseniaOpt = entityLog.getEntityLogEntry() + .stream() + .filter(e -> e.getId().equals(davidKsenia.get().getId())) + .findFirst(); + assertTrue(davidKseniaOpt.isPresent()); + var responseDavidKsenia = davidKseniaOpt.get(); + assertEquals(responseDavidKsenia.getChanges().size(), 2); + assertEquals(responseDavidKsenia.getChanges() + .get(0).getType(), ChangeType.ADDED); + assertEquals(responseDavidKsenia.getChanges() + .get(1).getType(), ChangeType.RECATEGORIZE); + assertEquals(responseDavidKsenia.getChanges() + .get(1).getPropertyChanges() + .get("reason"), "No vertebrate found -> Recategorized entities are applied by default., recategorized by manual override"); + assertEquals(responseDavidKsenia.getChanges() + .get(1).getPropertyChanges() + .get("matchedRule"), "CBI.3.2 -> MAN.3.3"); + assertEquals(responseDavidKsenia.getChanges() + .get(1).getPropertyChanges() + .get("legalBasis"), " -> new legal basis"); + assertEquals(responseDavidKsenia.getChanges() + .get(1).getPropertyChanges() + .get("state"), "SKIPPED -> APPLIED"); + assertEquals(responseDavidKsenia.getChanges() + .get(1).getPropertyChanges() + .get("type"), "CBI_author -> PII"); + } + + + @Test + public void testForceWithChanges() { + + AnalyzeRequest request = uploadFileToStorage("files/new/crafted document.pdf"); + analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request); + analyzeService.analyze(request); + + var entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID); + + var davidKsenia = entityLog.getEntityLogEntry() + .stream() + .filter(e -> e.getValue().equals("David Ksenia")) + .findFirst(); + assertTrue(davidKsenia.isPresent()); + + request.setManualRedactions(ManualRedactions.builder() + .forceRedactions(Set.of(ManualForceRedaction.builder() + .fileId(TEST_FILE_ID) + .user("user") + .requestDate(OffsetDateTime.now()) + .annotationId(davidKsenia.get().getId()) + .legalBasis("new legal basis") + .build())) + .build()); + + analyzeService.reanalyze(request); + entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID); + var davidKseniaOpt = entityLog.getEntityLogEntry() + .stream() + .filter(e -> e.getId().equals(davidKsenia.get().getId())) + .findFirst(); + assertTrue(davidKseniaOpt.isPresent()); + var responseDavidKsenia = davidKseniaOpt.get(); + assertEquals(responseDavidKsenia.getChanges().size(), 2); + assertEquals(responseDavidKsenia.getChanges() + .get(0).getType(), ChangeType.ADDED); + assertEquals(responseDavidKsenia.getChanges() + .get(1).getType(), ChangeType.FORCE_REDACT); + assertEquals(responseDavidKsenia.getChanges() + .get(1).getPropertyChanges() + .get("reason"), "No vertebrate found -> No vertebrate found, forced by manual override"); + assertEquals(responseDavidKsenia.getChanges() + .get(1).getPropertyChanges() + .get("legalBasis"), " -> new legal basis"); + assertEquals(responseDavidKsenia.getChanges() + .get(1).getPropertyChanges() + .get("state"), "SKIPPED -> APPLIED"); + } + + private IdRemoval getIdRemoval(String id) { return IdRemoval.builder() diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesIntegrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesIntegrationTest.java index ce91c661..7353acb7 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesIntegrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesIntegrationTest.java @@ -10,14 +10,19 @@ import java.util.Collection; import java.util.HashSet; import java.util.List; import java.util.Set; +import java.util.UUID; import java.util.stream.Collectors; import org.junit.jupiter.api.Test; import com.google.common.collect.Sets; +import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ChangeType; +import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction; +import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction; import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType; import com.iqser.red.service.redaction.v1.server.model.document.entity.PositionOnPage; @@ -25,6 +30,7 @@ import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntit import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Paragraph; import com.iqser.red.service.redaction.v1.server.rules.RulesIntegrationTest; +import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType; public class ManualChangesIntegrationTest extends RulesIntegrationTest {