From 5ffef504e400c15a0507915526b444289b9e081b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kilian=20Sch=C3=BCttler?= Date: Tue, 27 Feb 2024 10:04:27 +0100 Subject: [PATCH] More migration fixes --- .../build.gradle.kts | 2 +- .../LegacyRedactionLogMergeService.java | 6 +- .../v1/server/migration/MigrationMapper.java | 94 +++ .../migration/MigrationMessageReceiver.java | 21 +- ...dactionLogToEntityLogMigrationService.java | 135 ++--- .../v1/server/model/MigratedEntityLog.java | 4 +- .../v1/server/model/MigrationEntity.java | 333 ++++++----- .../v1/server/model/PrecursorEntity.java | 6 +- .../model/document/nodes/ImageType.java | 14 +- .../service/EntityLogCreatorService.java | 33 +- .../service/UnprocessedChangesService.java | 138 +++-- .../EntityFromPrecursorCreationService.java | 61 +- .../v1/server/utils/MigratedIdsCollector.java | 3 +- .../v1/server/MigrationIntegrationTest.java | 34 +- .../v1/server/RedactionIntegrationTest.java | 539 +++++++++++------- .../utils/LayoutParsingRequestProvider.java | 3 +- 16 files changed, 854 insertions(+), 572 deletions(-) create mode 100644 redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/migration/MigrationMapper.java diff --git a/redaction-service-v1/redaction-service-server-v1/build.gradle.kts b/redaction-service-v1/redaction-service-server-v1/build.gradle.kts index c8582faa..7eefb5f1 100644 --- a/redaction-service-v1/redaction-service-server-v1/build.gradle.kts +++ b/redaction-service-v1/redaction-service-server-v1/build.gradle.kts @@ -16,7 +16,7 @@ val layoutParserVersion = "0.91.0" val jacksonVersion = "2.15.2" val droolsVersion = "9.44.0.Final" val pdfBoxVersion = "3.0.0" -val persistenceServiceVersion = "2.351.0" +val persistenceServiceVersion = "2.359.0" val springBootStarterVersion = "3.1.5" configurations { diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/migration/LegacyRedactionLogMergeService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/migration/LegacyRedactionLogMergeService.java index aaa8a1be..0d141c87 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/migration/LegacyRedactionLogMergeService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/migration/LegacyRedactionLogMergeService.java @@ -45,7 +45,7 @@ public class LegacyRedactionLogMergeService { public RedactionLog addManualAddEntriesAndRemoveSkippedImported(RedactionLog redactionLog, ManualRedactions manualRedactions, String dossierTemplateId) { Set skippedImportedRedactions = new HashSet<>(); - log.info("Merging Redaction log with manual redactions"); + log.info("Adding manual add Entries and removing skipped or imported entries"); if (manualRedactions != null) { var manualRedactionLogEntries = addManualAddEntries(manualRedactions.getEntriesToAdd(), redactionLog.getAnalysisNumber()); @@ -92,6 +92,10 @@ public class LegacyRedactionLogMergeService { return redactionLog; } + public long getNumberOfAffectedAnnotations(ManualRedactions manualRedactions) { + + return createManualRedactionWrappers(manualRedactions).stream().map(ManualRedactionWrapper::getId).distinct().count(); + } private List createManualRedactionWrappers(ManualRedactions manualRedactions) { diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/migration/MigrationMapper.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/migration/MigrationMapper.java new file mode 100644 index 00000000..5b049564 --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/migration/MigrationMapper.java @@ -0,0 +1,94 @@ +package com.iqser.red.service.redaction.v1.server.migration; + +import java.util.Collections; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; + +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ChangeType; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualChange; +import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Change; +import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine; +import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ManualRedactionType; +import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry; +import com.iqser.red.service.redaction.v1.server.model.MigrationEntity; + +public class MigrationMapper { + + public static com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Change toEntityLogChanges(Change change) { + + return new com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Change(change.getAnalysisNumber(), + toEntityLogType(change.getType()), + change.getDateTime()); + } + + + public static com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualChange toEntityLogManualChanges(com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ManualChange manualChange) { + + return new com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualChange(toManualRedactionType(manualChange.getManualRedactionType()), + manualChange.getProcessedDate(), + manualChange.getRequestedDate(), + manualChange.getUserId(), + manualChange.getPropertyChanges()); + } + + + public static ChangeType toEntityLogType(com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ChangeType type) { + + return switch (type) { + case ADDED -> ChangeType.ADDED; + case REMOVED -> ChangeType.REMOVED; + case CHANGED -> ChangeType.CHANGED; + }; + } + + + public static com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType toManualRedactionType(ManualRedactionType manualRedactionType) { + + return switch (manualRedactionType) { + case ADD_LOCALLY -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.ADD_LOCALLY; + case ADD_TO_DICTIONARY -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.ADD_TO_DICTIONARY; + case REMOVE_LOCALLY -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.REMOVE_LOCALLY; + case REMOVE_FROM_DICTIONARY -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.REMOVE_FROM_DICTIONARY; + case FORCE_REDACT -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.FORCE_REDACT; + case FORCE_HINT -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.FORCE_HINT; + case RECATEGORIZE -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.RECATEGORIZE; + case LEGAL_BASIS_CHANGE -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.LEGAL_BASIS_CHANGE; + case RESIZE -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.RESIZE; + }; + } + + + public static com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine toEntityLogEngine(Engine engine) { + + return switch (engine) { + case DICTIONARY -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine.DICTIONARY; + case NER -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine.NER; + case RULE -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine.RULE; + }; + } + + + public static Set getMigratedEngines(RedactionLogEntry entry) { + + if (entry.getEngines() == null) { + return Collections.emptySet(); + } + return entry.getEngines() + .stream() + .map(MigrationMapper::toEntityLogEngine) + .collect(Collectors.toSet()); + } + + + public List migrateManualChanges(List manualChanges) { + + if (manualChanges == null) { + return Collections.emptyList(); + } + return manualChanges.stream() + .map(MigrationMapper::toEntityLogManualChanges) + .toList(); + } + +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/migration/MigrationMessageReceiver.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/migration/MigrationMessageReceiver.java index 66e8db83..740d4116 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/migration/MigrationMessageReceiver.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/migration/MigrationMessageReceiver.java @@ -58,19 +58,28 @@ public class MigrationMessageReceiver { if (redactionLog.getAnalysisVersion() == 0) { redactionLog = legacyVersion0MigrationService.mergeDuplicateAnnotationIds(redactionLog); } else if (migrationRequest.getManualRedactions() != null) { - redactionLog = legacyRedactionLogMergeService.addManualAddEntriesAndRemoveSkippedImported(redactionLog, migrationRequest.getManualRedactions(), migrationRequest.getDossierTemplateId()); + redactionLog = legacyRedactionLogMergeService.addManualAddEntriesAndRemoveSkippedImported(redactionLog, + migrationRequest.getManualRedactions(), + migrationRequest.getDossierTemplateId()); } - MigratedEntityLog migratedEntityLog = redactionLogToEntityLogMigrationService.migrate(redactionLog, document, migrationRequest.getDossierTemplateId(), migrationRequest.getManualRedactions()); + MigratedEntityLog migratedEntityLog = redactionLogToEntityLogMigrationService.migrate(redactionLog, + document, + migrationRequest.getDossierTemplateId(), + migrationRequest.getManualRedactions(), + migrationRequest.getFileId()); + log.info("Storing migrated entityLog and ids to migrate in DB for file {}", migrationRequest.getFileId()); redactionStorageService.storeObject(migrationRequest.getDossierId(), migrationRequest.getFileId(), FileType.ENTITY_LOG, migratedEntityLog.getEntityLog()); redactionStorageService.storeObject(migrationRequest.getDossierId(), migrationRequest.getFileId(), FileType.MIGRATED_IDS, migratedEntityLog.getMigratedIds()); sendFinished(MigrationResponse.builder().dossierId(migrationRequest.getDossierId()).fileId(migrationRequest.getFileId()).build()); - log.info("Migrated {} redactionLog entries for dossierId {} and fileId {}", - migratedEntityLog.getEntityLog().getEntityLogEntry().size(), - migrationRequest.getDossierId(), - migrationRequest.getFileId()); + log.info("Migrated {} redactionLog entries, found {} annotation ids for migration in the db, {} new manual entries, for dossierId {} and fileId {}", + migratedEntityLog.getEntityLog().getEntityLogEntry().size(), + migratedEntityLog.getMigratedIds().getMappings().size(), + migratedEntityLog.getMigratedIds().getManualRedactionEntriesToAdd().size(), + migrationRequest.getDossierId(), + migrationRequest.getFileId()); log.info(""); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/migration/RedactionLogToEntityLogMigrationService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/migration/RedactionLogToEntityLogMigrationService.java index 938109d4..a83069a1 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/migration/RedactionLogToEntityLogMigrationService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/migration/RedactionLogToEntityLogMigrationService.java @@ -19,29 +19,24 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.migration.MigratedIds; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.BaseAnnotation; -import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction; -import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ManualRedactionType; +import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogLegalBasis; -import com.iqser.red.service.redaction.v1.model.MigrationRequest; -import com.iqser.red.service.redaction.v1.server.model.PrecursorEntity; import com.iqser.red.service.redaction.v1.server.model.MigratedEntityLog; import com.iqser.red.service.redaction.v1.server.model.MigrationEntity; +import com.iqser.red.service.redaction.v1.server.model.PrecursorEntity; import com.iqser.red.service.redaction.v1.server.model.RectangleWithPage; -import com.iqser.red.service.redaction.v1.server.model.document.TextRange; -import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType; import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Image; import com.iqser.red.service.redaction.v1.server.model.document.nodes.ImageType; -import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode; import com.iqser.red.service.redaction.v1.server.service.DictionaryService; import com.iqser.red.service.redaction.v1.server.service.ManualChangesApplicationService; -import com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService; import com.iqser.red.service.redaction.v1.server.service.document.EntityEnrichmentService; import com.iqser.red.service.redaction.v1.server.service.document.EntityFindingUtility; +import com.iqser.red.service.redaction.v1.server.service.document.EntityFromPrecursorCreationService; import com.iqser.red.service.redaction.v1.server.utils.IdBuilder; import com.iqser.red.service.redaction.v1.server.utils.MigratedIdsCollector; @@ -64,13 +59,16 @@ public class RedactionLogToEntityLogMigrationService { ManualChangesApplicationService manualChangesApplicationService; - public MigratedEntityLog migrate(RedactionLog redactionLog, Document document, String dossierTemplateId, ManualRedactions manualRedactions) { + public MigratedEntityLog migrate(RedactionLog redactionLog, Document document, String dossierTemplateId, ManualRedactions manualRedactions, String fileId) { + + log.info("Migrating entities for file {}", fileId); + List entitiesToMigrate = calculateMigrationEntitiesFromRedactionLog(redactionLog, document, dossierTemplateId, fileId); - List entitiesToMigrate = calculateMigrationEntitiesFromRedactionLog(redactionLog, document, dossierTemplateId); MigratedIds migratedIds = entitiesToMigrate.stream() .collect(new MigratedIdsCollector()); applyManualChanges(entitiesToMigrate, manualRedactions); + log.info("applying manual changes to migrated entities for file {}", fileId); EntityLog entityLog = new EntityLog(); entityLog.setAnalysisNumber(redactionLog.getAnalysisNumber()); @@ -85,6 +83,8 @@ public class RedactionLogToEntityLogMigrationService { .toList()); Map oldToNewIDMapping = migratedIds.buildOldToNewMapping(); + + log.info("Writing migrated entities to entityLog for file {}", fileId); entityLog.setEntityLogEntry(entitiesToMigrate.stream() .map(migrationEntity -> migrationEntity.toEntityLogEntry(oldToNewIDMapping)) .toList()); @@ -102,6 +102,13 @@ public class RedactionLogToEntityLogMigrationService { .filter(m -> !m.getOldId().equals(m.getNewId())) .collect(new MigratedIdsCollector()); + List manualRedactionEntriesToAdd = entitiesToMigrate.stream() + .filter(MigrationEntity::needsManualEntry) + .map(MigrationEntity::buildManualRedactionEntry) + .toList(); + + idsToMigrateInDb.setManualRedactionEntriesToAdd(manualRedactionEntriesToAdd); + return new MigratedEntityLog(idsToMigrateInDb, entityLog); } @@ -117,27 +124,14 @@ public class RedactionLogToEntityLogMigrationService { manualRedactions.getForceRedactions(), manualRedactions.getResizeRedactions(), manualRedactions.getLegalBasisChanges(), - manualRedactions.getRecategorizations(), - manualRedactions.getLegalBasisChanges()) + manualRedactions.getRecategorizations()) .flatMap(Collection::stream) .collect(Collectors.groupingBy(BaseAnnotation::getAnnotationId)); - entitiesToMigrate.forEach(migrationEntity -> manualChangesPerAnnotationId.getOrDefault(migrationEntity.getOldId(), Collections.emptyList()) - .forEach(manualChange -> { - if (manualChange instanceof ManualResizeRedaction manualResizeRedaction && migrationEntity.getMigratedEntity() instanceof TextEntity textEntity) { - ManualResizeRedaction migratedManualResizeRedaction = ManualResizeRedaction.builder() - .positions(manualResizeRedaction.getPositions()) - .annotationId(migrationEntity.getNewId()) - .updateDictionary(manualResizeRedaction.getUpdateDictionary()) - .addToAllDossiers(manualResizeRedaction.isAddToAllDossiers()) - .textAfter(manualResizeRedaction.getTextAfter()) - .textBefore(manualResizeRedaction.getTextBefore()) - .build(); - manualChangesApplicationService.resize(textEntity, migratedManualResizeRedaction); - } else { - migrationEntity.getMigratedEntity().getManualOverwrite().addChange(manualChange); - } - })); + entitiesToMigrate.forEach(migrationEntity -> migrationEntity.applyManualChanges(manualChangesPerAnnotationId.getOrDefault(migrationEntity.getOldId(), + Collections.emptyList()), + manualChangesApplicationService)); + } @@ -147,10 +141,10 @@ public class RedactionLogToEntityLogMigrationService { } - private List calculateMigrationEntitiesFromRedactionLog(RedactionLog redactionLog, Document document, String dossierTemplateId) { + private List calculateMigrationEntitiesFromRedactionLog(RedactionLog redactionLog, Document document, String dossierTemplateId, String fileId) { - List images = getImageBasedMigrationEntities(redactionLog, document, dossierTemplateId); - List textMigrationEntities = getTextBasedMigrationEntities(redactionLog, document, dossierTemplateId); + List images = getImageBasedMigrationEntities(redactionLog, document, fileId); + List textMigrationEntities = getTextBasedMigrationEntities(redactionLog, document, dossierTemplateId, fileId); return Stream.of(textMigrationEntities.stream(), images.stream()) .flatMap(Function.identity()) .toList(); @@ -163,7 +157,7 @@ public class RedactionLogToEntityLogMigrationService { } - private List getImageBasedMigrationEntities(RedactionLog redactionLog, Document document, String dossierTemplateId) { + private List getImageBasedMigrationEntities(RedactionLog redactionLog, Document document, String fileId) { List images = document.streamAllImages() .collect(Collectors.toList()); @@ -195,7 +189,8 @@ public class RedactionLogToEntityLogMigrationService { } String ruleIdentifier; - String reason = Optional.ofNullable(redactionLogImage.getReason()).orElse(""); + String reason = Optional.ofNullable(redactionLogImage.getReason()) + .orElse(""); if (redactionLogImage.getMatchedRule().isBlank() || redactionLogImage.getMatchedRule() == null) { ruleIdentifier = "OLDIMG.0.0"; } else { @@ -209,7 +204,7 @@ public class RedactionLogToEntityLogMigrationService { } else { closestImage.skip(ruleIdentifier, reason); } - migrationEntities.add(new MigrationEntity(null, redactionLogImage, closestImage, redactionLogImage.getId(), closestImage.getId())); + migrationEntities.add(MigrationEntity.fromRedactionLogImage(redactionLogImage, closestImage, fileId)); } return migrationEntities; } @@ -250,40 +245,20 @@ public class RedactionLogToEntityLogMigrationService { } - private List getTextBasedMigrationEntities(RedactionLog redactionLog, Document document, String dossierTemplateId) { + private List getTextBasedMigrationEntities(RedactionLog redactionLog, Document document, String dossierTemplateId, String fileId) { List entitiesToMigrate = redactionLog.getRedactionLogEntry() .stream() .filter(redactionLogEntry -> !redactionLogEntry.isImage()) - .map(entry -> MigrationEntity.fromRedactionLogEntry(entry, dictionaryService.isHint(entry.getType(), dossierTemplateId))) - .peek(migrationEntity -> { - if (migrationEntity.getPrecursorEntity().getEntityType().equals(EntityType.HINT) &&// - !migrationEntity.getRedactionLogEntry().isHint() &&// - !migrationEntity.getRedactionLogEntry().isRedacted()) { - migrationEntity.getPrecursorEntity().ignore(migrationEntity.getPrecursorEntity().getRuleIdentifier(), migrationEntity.getPrecursorEntity().getReason()); - } else if (migrationEntity.getRedactionLogEntry().lastChangeIsRemoved()) { - migrationEntity.getPrecursorEntity().remove(migrationEntity.getPrecursorEntity().getRuleIdentifier(), migrationEntity.getPrecursorEntity().getReason()); - } else if (lastManualChangeIsRemove(migrationEntity)) { - migrationEntity.getPrecursorEntity().ignore(migrationEntity.getPrecursorEntity().getRuleIdentifier(), migrationEntity.getPrecursorEntity().getReason()); - } else if (migrationEntity.getPrecursorEntity().isApplied() && migrationEntity.getRedactionLogEntry().isRecommendation()) { - migrationEntity.getPrecursorEntity() - .skip(migrationEntity.getPrecursorEntity().getRuleIdentifier(), migrationEntity.getPrecursorEntity().getReason()); - } else if (migrationEntity.getPrecursorEntity().isApplied()) { - migrationEntity.getPrecursorEntity() - .apply(migrationEntity.getPrecursorEntity().getRuleIdentifier(), - migrationEntity.getPrecursorEntity().getReason(), - migrationEntity.getPrecursorEntity().getLegalBasis()); - } else { - migrationEntity.getPrecursorEntity() - .skip(migrationEntity.getPrecursorEntity().getRuleIdentifier(), migrationEntity.getPrecursorEntity().getReason()); - } - }) + .map(entry -> MigrationEntity.fromRedactionLogEntry(entry, dictionaryService.isHint(entry.getType(), dossierTemplateId), fileId)) .toList(); - Map> tempEntitiesByValue = entityFindingUtility.findAllPossibleEntitiesAndGroupByValue(document, - entitiesToMigrate.stream() - .map(MigrationEntity::getPrecursorEntity) - .toList()); + List precursorEntities = entitiesToMigrate.stream() + .map(MigrationEntity::getPrecursorEntity) + .toList(); + + log.info("Finding all possible entities"); + Map> tempEntitiesByValue = entityFindingUtility.findAllPossibleEntitiesAndGroupByValue(document, precursorEntities); for (MigrationEntity migrationEntity : entitiesToMigrate) { Optional optionalTextEntity = entityFindingUtility.findClosestEntityAndReturnEmptyIfNotFound(migrationEntity.getPrecursorEntity(), @@ -297,45 +272,19 @@ public class RedactionLogToEntityLogMigrationService { continue; } - TextEntity entity = createCorrectEntity(migrationEntity.getPrecursorEntity(), document, optionalTextEntity.get().getTextRange()); - migrationEntity.setMigratedEntity(entity); - migrationEntity.setOldId(migrationEntity.getPrecursorEntity().getId()); - migrationEntity.setNewId(entity.getId()); // Can only be on one page, since redactionLogEntries can only be on one page + TextEntity migratedEntity = EntityFromPrecursorCreationService.createCorrectEntity(migrationEntity.getPrecursorEntity(), optionalTextEntity.get(), true); + migrationEntity.setMigratedEntity(migratedEntity); + migrationEntity.setOldId(migrationEntity.getPrecursorEntity().getId()); + migrationEntity.setNewId(migratedEntity.getId()); } tempEntitiesByValue.values() .stream() .flatMap(Collection::stream) .forEach(TextEntity::removeFromGraph); + return entitiesToMigrate; } - - private static boolean lastManualChangeIsRemove(MigrationEntity migrationEntity) { - - if (migrationEntity.getRedactionLogEntry().getManualChanges() == null) { - return false; - } - - return migrationEntity.getRedactionLogEntry().getManualChanges() - .stream() - .reduce((a, b) -> b) - .map(m -> m.getManualRedactionType().equals(ManualRedactionType.REMOVE_LOCALLY)) - .orElse(false); - } - - - private TextEntity createCorrectEntity(PrecursorEntity precursorEntity, SemanticNode node, TextRange closestTextRange) { - - EntityCreationService entityCreationService = new EntityCreationService(entityEnrichmentService); - TextEntity correctEntity = entityCreationService.forceByTextRange(closestTextRange, precursorEntity.getType(), precursorEntity.getEntityType(), node); - - correctEntity.addMatchedRules(precursorEntity.getMatchedRuleList()); - correctEntity.setDictionaryEntry(precursorEntity.isDictionaryEntry()); - correctEntity.setDossierDictionaryEntry(precursorEntity.isDossierDictionaryEntry()); - correctEntity.getManualOverwrite().addChanges(precursorEntity.getManualOverwrite().getManualChangeLog()); - return correctEntity; - } - } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/MigratedEntityLog.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/MigratedEntityLog.java index aff967e5..31a9e30d 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/MigratedEntityLog.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/MigratedEntityLog.java @@ -1,7 +1,10 @@ package com.iqser.red.service.redaction.v1.server.model; +import java.util.List; + import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.migration.MigratedIds; +import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry; import lombok.AllArgsConstructor; import lombok.Builder; @@ -16,5 +19,4 @@ public class MigratedEntityLog { MigratedIds migratedIds; EntityLog entityLog; - } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/MigrationEntity.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/MigrationEntity.java index bd948b08..975bcc8b 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/MigrationEntity.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/MigrationEntity.java @@ -1,34 +1,45 @@ package com.iqser.red.service.redaction.v1.server.model; +import static com.iqser.red.service.redaction.v1.server.service.EntityLogCreatorService.buildEntryState; +import static com.iqser.red.service.redaction.v1.server.service.EntityLogCreatorService.buildEntryType; + +import java.awt.geom.Rectangle2D; import java.util.Collections; +import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; -import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ChangeType; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryType; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position; -import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Change; -import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine; -import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ManualChange; +import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle; +import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.BaseAnnotation; +import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry; +import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ManualRedactionType; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry; +import com.iqser.red.service.redaction.v1.server.migration.MigrationMapper; import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType; import com.iqser.red.service.redaction.v1.server.model.document.entity.IEntity; import com.iqser.red.service.redaction.v1.server.model.document.entity.ManualChangeOverwrite; import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Image; import com.iqser.red.service.redaction.v1.server.service.ManualChangeFactory; +import com.iqser.red.service.redaction.v1.server.service.ManualChangesApplicationService; import lombok.AllArgsConstructor; +import lombok.Builder; import lombok.Data; import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +@Slf4j @Data +@Builder @AllArgsConstructor @RequiredArgsConstructor public final class MigrationEntity { @@ -38,28 +49,73 @@ public final class MigrationEntity { private IEntity migratedEntity; private String oldId; private String newId; + private String fileId; + + @Builder.Default + List manualChanges = new LinkedList<>(); - public static MigrationEntity fromRedactionLogEntry(RedactionLogEntry redactionLogEntry, boolean hint) { + public static MigrationEntity fromRedactionLogEntry(RedactionLogEntry redactionLogEntry, boolean hint, String fileId) { - return new MigrationEntity(createPrecursorEntity(redactionLogEntry, hint), redactionLogEntry); + PrecursorEntity precursorEntity = createPrecursorEntity(redactionLogEntry, hint); + + if (precursorEntity.getEntityType().equals(EntityType.HINT) && !redactionLogEntry.isHint() && !redactionLogEntry.isRedacted()) { + precursorEntity.ignore(precursorEntity.getRuleIdentifier(), precursorEntity.getReason()); + } else if (redactionLogEntry.lastChangeIsRemoved()) { + precursorEntity.remove(precursorEntity.getRuleIdentifier(), precursorEntity.getReason()); + } else if (lastManualChangeIsRemove(redactionLogEntry)) { + precursorEntity.ignore(precursorEntity.getRuleIdentifier(), precursorEntity.getReason()); + } else if (precursorEntity.isApplied() && redactionLogEntry.isRecommendation()) { + precursorEntity.skip(precursorEntity.getRuleIdentifier(), precursorEntity.getReason()); + } else if (precursorEntity.isApplied()) { + precursorEntity.apply(precursorEntity.getRuleIdentifier(), precursorEntity.getReason(), precursorEntity.getLegalBasis()); + } else { + precursorEntity.skip(precursorEntity.getRuleIdentifier(), precursorEntity.getReason()); + } + + return MigrationEntity.builder().precursorEntity(precursorEntity).redactionLogEntry(redactionLogEntry).oldId(redactionLogEntry.getId()).fileId(fileId).build(); + } + + + public static MigrationEntity fromRedactionLogImage(RedactionLogEntry redactionLogImage, Image image, String fileId) { + + return MigrationEntity.builder().redactionLogEntry(redactionLogImage).migratedEntity(image).oldId(redactionLogImage.getId()).newId(image.getId()).fileId(fileId).build(); + } + + + private static boolean lastManualChangeIsRemove(RedactionLogEntry redactionLogEntry) { + + if (redactionLogEntry.getManualChanges() == null) { + return false; + } + + return redactionLogEntry.getManualChanges() + .stream() + .reduce((a, b) -> b) + .map(m -> m.getManualRedactionType().equals(ManualRedactionType.REMOVE_LOCALLY)) + .orElse(false); } public static PrecursorEntity createPrecursorEntity(RedactionLogEntry redactionLogEntry, boolean hint) { String ruleIdentifier = buildRuleIdentifier(redactionLogEntry); - List rectangleWithPages = redactionLogEntry.getPositions().stream().map(RectangleWithPage::fromRedactionLogRectangle).toList(); + List rectangleWithPages = redactionLogEntry.getPositions() + .stream() + .map(RectangleWithPage::fromRedactionLogRectangle) + .toList(); EntityType entityType = getEntityType(redactionLogEntry, hint); return PrecursorEntity.builder() .id(redactionLogEntry.getId()) .value(redactionLogEntry.getValue()) .entityPosition(rectangleWithPages) .ruleIdentifier(ruleIdentifier) - .reason(Optional.ofNullable(redactionLogEntry.getReason()).orElse("")) + .reason(Optional.ofNullable(redactionLogEntry.getReason()) + .orElse("")) .legalBasis(redactionLogEntry.getLegalBasis()) .type(redactionLogEntry.getType()) .section(redactionLogEntry.getSection()) + .engines(MigrationMapper.getMigratedEngines(redactionLogEntry)) .entityType(entityType) .applied(redactionLogEntry.isRedacted()) .isDictionaryEntry(redactionLogEntry.isDictionaryEntry()) @@ -100,14 +156,6 @@ public final class MigrationEntity { } - private static com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Change toEntityLogChanges(Change change) { - - return new com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Change(change.getAnalysisNumber(), - toEntityLogType(change.getType()), - change.getDateTime()); - } - - private static EntryType getEntryType(EntityType entityType) { return switch (entityType) { @@ -120,42 +168,6 @@ public final class MigrationEntity { } - private static com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualChange toEntityLogManualChanges(ManualChange manualChange) { - - return new com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualChange(toManualRedactionType(manualChange.getManualRedactionType()), - manualChange.getProcessedDate(), - manualChange.getRequestedDate(), - manualChange.getUserId(), - manualChange.getPropertyChanges()); - } - - - private static ChangeType toEntityLogType(com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ChangeType type) { - - return switch (type) { - case ADDED -> ChangeType.ADDED; - case REMOVED -> ChangeType.REMOVED; - case CHANGED -> ChangeType.CHANGED; - }; - } - - - private static com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType toManualRedactionType(ManualRedactionType manualRedactionType) { - - return switch (manualRedactionType) { - case ADD_LOCALLY -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.ADD_LOCALLY; - case ADD_TO_DICTIONARY -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.ADD_TO_DICTIONARY; - case REMOVE_LOCALLY -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.REMOVE_LOCALLY; - case REMOVE_FROM_DICTIONARY -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.REMOVE_FROM_DICTIONARY; - case FORCE_REDACT -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.FORCE_REDACT; - case FORCE_HINT -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.FORCE_HINT; - case RECATEGORIZE -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.RECATEGORIZE; - case LEGAL_BASIS_CHANGE -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.LEGAL_BASIS_CHANGE; - case RESIZE -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.RESIZE; - }; - } - - public EntityLogEntry toEntityLogEntry(Map oldToNewIdMapping) { EntityLogEntry entityLogEntry; @@ -171,10 +183,13 @@ public final class MigrationEntity { entityLogEntry.setManualChanges(ManualChangeFactory.toManualChangeList(migratedEntity.getManualOverwrite().getManualChangeLog(), redactionLogEntry.isHint())); entityLogEntry.setColor(redactionLogEntry.getColor()); - entityLogEntry.setChanges(redactionLogEntry.getChanges().stream().map(MigrationEntity::toEntityLogChanges).toList()); + entityLogEntry.setChanges(redactionLogEntry.getChanges() + .stream() + .map(MigrationMapper::toEntityLogChanges) + .toList()); entityLogEntry.setReference(migrateSetOfIds(redactionLogEntry.getReference(), oldToNewIdMapping)); entityLogEntry.setImportedRedactionIntersections(migrateSetOfIds(redactionLogEntry.getImportedRedactionIntersections(), oldToNewIdMapping)); - entityLogEntry.setEngines(getMigratedEngines(redactionLogEntry)); + entityLogEntry.setEngines(MigrationMapper.getMigratedEngines(redactionLogEntry)); if (redactionLogEntry.getLegalBasis() != null) { entityLogEntry.setLegalBasis(redactionLogEntry.getLegalBasis()); } @@ -198,47 +213,21 @@ public final class MigrationEntity { } - private List migrateManualChanges(List manualChanges) { - - if (manualChanges == null) { - return Collections.emptyList(); - } - return manualChanges.stream().map(MigrationEntity::toEntityLogManualChanges).toList(); - } - - - private static Set getMigratedEngines(RedactionLogEntry entry) { - - if (entry.getEngines() == null) { - return Collections.emptySet(); - } - return entry.getEngines().stream().map(MigrationEntity::toEntityLogEngine).collect(Collectors.toSet()); - } - - private Set migrateSetOfIds(Set ids, Map oldToNewIdMapping) { if (ids == null) { return Collections.emptySet(); } - return ids.stream().map(oldToNewIdMapping::get).collect(Collectors.toSet()); - } - - - private static com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine toEntityLogEngine(Engine engine) { - - return switch (engine) { - case DICTIONARY -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine.DICTIONARY; - case NER -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine.NER; - case RULE -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine.RULE; - }; + return ids.stream() + .map(oldToNewIdMapping::get) + .collect(Collectors.toSet()); } public EntityLogEntry createEntityLogEntry(Image image) { List positions = getPositionsFromOverride(image).orElse(List.of(new Position(image.getPosition(), image.getPage().getNumber()))); - return EntityLogEntry.builder() + return EntityLogEntry.builder() .id(image.getId()) .value(image.value()) .type(image.type()) @@ -249,7 +238,8 @@ public final class MigrationEntity { .positions(positions) .containingNodeId(image.getTreeId()) .closestHeadline(image.getHeadline().getTextBlock().getSearchText()) - .section(redactionLogEntry.getSection()) + .section(image.getManualOverwrite().getSection() + .orElse(redactionLogEntry.getSection())) .textAfter(redactionLogEntry.getTextAfter()) .textBefore(redactionLogEntry.getTextBefore()) .imageHasTransparency(image.isTransparent()) @@ -270,7 +260,8 @@ public final class MigrationEntity { .type(precursorEntity.type()) .state(buildEntryState(precursorEntity)) .entryType(buildEntryType(precursorEntity)) - .section(redactionLogEntry.getSection()) + .section(precursorEntity.getManualOverwrite().getSection() + .orElse(redactionLogEntry.getSection())) .textAfter(redactionLogEntry.getTextAfter()) .textBefore(redactionLogEntry.getTextBefore()) .containingNodeId(Collections.emptyList()) @@ -280,12 +271,11 @@ public final class MigrationEntity { .dossierDictionaryEntry(precursorEntity.isDossierDictionaryEntry()) .startOffset(-1) .endOffset(-1) - .positions(precursorEntity.getManualOverwrite() - .getPositions() - .orElse(precursorEntity.getEntityPosition()) - .stream() - .map(entityPosition -> new Position(entityPosition.rectangle2D(), entityPosition.pageNumber())) - .toList()) + .positions(precursorEntity.getManualOverwrite().getPositions() + .orElse(precursorEntity.getEntityPosition()) + .stream() + .map(entityPosition -> new Position(entityPosition.rectangle2D(), entityPosition.pageNumber())) + .toList()) .engines(Collections.emptySet()) .build(); } @@ -300,11 +290,13 @@ public final class MigrationEntity { .positions(rectanglesPerLine) .reason(entity.buildReasonWithManualChangeDescriptions()) .legalBasis(entity.legalBasis()) - .value(entity.getManualOverwrite().getValue().orElse(entity.getMatchedRule().isWriteValueWithLineBreaks() ? entity.getValueWithLineBreaks() : entity.getValue())) + .value(entity.getManualOverwrite().getValue() + .orElse(entity.getMatchedRule().isWriteValueWithLineBreaks() ? entity.getValueWithLineBreaks() : entity.getValue())) .type(entity.type()) - .section(redactionLogEntry.getSection()) - .textAfter(redactionLogEntry.getTextAfter()) - .textBefore(redactionLogEntry.getTextBefore()) + .section(entity.getManualOverwrite().getSection() + .orElse(redactionLogEntry.getSection())) + .textAfter(entity.getTextAfter()) + .textBefore(entity.getTextBefore()) .containingNodeId(entity.getDeepestFullyContainingNode().getTreeId()) .closestHeadline(entity.getDeepestFullyContainingNode().getHeadline().getTextBlock().getSearchText()) .matchedRule(entity.getMatchedRule().getRuleIdentifier().toString()) @@ -322,54 +314,129 @@ public final class MigrationEntity { private static List getRectanglesPerLine(TextEntity entity) { return getPositionsFromOverride(entity).orElse(entity.getPositionsOnPagePerPage() - .get(0) - .getRectanglePerLine() - .stream() - .map(rectangle2D -> new Position(rectangle2D, entity.getPositionsOnPagePerPage().get(0).getPage().getNumber())) - .toList()); + .get(0).getRectanglePerLine() + .stream() + .map(rectangle2D -> new Position(rectangle2D, + entity.getPositionsOnPagePerPage() + .get(0).getPage().getNumber())) + .toList()); } private static Optional> getPositionsFromOverride(IEntity entity) { - return entity.getManualOverwrite().getPositions().map(rects -> rects.stream().map(r -> new Position(r.rectangle2D(), r.pageNumber())).toList()); - } - - - private EntryState buildEntryState(IEntity entity) { - - if (entity.applied() && entity.active()) { - return EntryState.APPLIED; - } else if (entity.skipped() && entity.active()) { - return EntryState.SKIPPED; - } else if (entity.ignored()) { - return EntryState.IGNORED; - } else { - return EntryState.REMOVED; - } - } - - - private EntryType buildEntryType(IEntity entity) { - - if (entity instanceof TextEntity textEntity) { - return getEntryType(textEntity.getEntityType()); - } else if (entity instanceof PrecursorEntity precursorEntity) { - if (precursorEntity.isRectangle()) { - return EntryType.AREA; - } - return getEntryType(precursorEntity.getEntityType()); - } else if (entity instanceof Image) { - return EntryType.IMAGE; - } - throw new UnsupportedOperationException(String.format("Entity subclass %s is not implemented!", entity.getClass())); + return entity.getManualOverwrite().getPositions() + .map(rects -> rects.stream() + .map(r -> new Position(r.rectangle2D(), r.pageNumber())) + .toList()); } public boolean hasManualChangesOrComments() { return !(redactionLogEntry.getManualChanges() == null || redactionLogEntry.getManualChanges().isEmpty()) || // - !(redactionLogEntry.getComments() == null || redactionLogEntry.getComments().isEmpty()); + !(redactionLogEntry.getComments() == null || redactionLogEntry.getComments().isEmpty()) // + || hasManualChanges(); + } + + + public boolean hasManualChanges() { + + return !manualChanges.isEmpty(); + } + + + public void applyManualChanges(List manualChangesToApply, ManualChangesApplicationService manualChangesApplicationService) { + + manualChanges.addAll(manualChangesToApply); + manualChangesToApply.forEach(manualChange -> { + if (manualChange instanceof ManualResizeRedaction manualResizeRedaction && migratedEntity instanceof TextEntity textEntity) { + // Due to the value in the old redaction log already being resized, there is no way to find the original entity ID and therefore to migrate the resize annotation correctly. + // Instead, we add an add_locally change to the db. + ManualResizeRedaction migratedManualResizeRedaction = ManualResizeRedaction.builder() + .positions(manualResizeRedaction.getPositions()) + .annotationId(getNewId()) + .updateDictionary(manualResizeRedaction.getUpdateDictionary()) + .addToAllDossiers(manualResizeRedaction.isAddToAllDossiers()) + .textAfter(manualResizeRedaction.getTextAfter()) + .textBefore(manualResizeRedaction.getTextBefore()) + .build(); + manualChangesApplicationService.resize(textEntity, migratedManualResizeRedaction); + } else { + migratedEntity.getManualOverwrite().addChange(manualChange); + } + }); + } + + + public ManualRedactionEntry buildManualRedactionEntry() { + + assert hasManualChanges(); + + // currently we need to insert a manual redaction entry, whenever an entity has been resized. + String user = manualChanges.stream() + .filter(mc -> mc instanceof ManualResizeRedaction) + .findFirst() + .orElse(manualChanges.get(0)).getUser(); + + return ManualRedactionEntry.builder() + .annotationId(newId) + .fileId(fileId) + .type(redactionLogEntry.getType()) + .value(redactionLogEntry.getValue()) + .reason(redactionLogEntry.getReason()) + .legalBasis(redactionLogEntry.getLegalBasis()) + .section(redactionLogEntry.getSection()) + .addToDictionary(false) + .addToDossierDictionary(false) + .rectangle(false) + .positions(buildPositions(migratedEntity)) + .user(user) + .build(); + } + + + private List buildPositions(IEntity entity) { + + if (entity instanceof TextEntity textEntity) { + + var positionsOnPage = textEntity.getPositionsOnPagePerPage() + .get(0); + return positionsOnPage.getRectanglePerLine() + .stream() + .map(p -> new Rectangle((float) p.getX(), (float) p.getY(), (float) p.getWidth(), (float) p.getHeight(), positionsOnPage.getPage().getNumber())) + .toList(); + } + if (entity instanceof PrecursorEntity pEntity) { + + return pEntity.getManualOverwrite().getPositions() + .orElse(pEntity.getEntityPosition()) + .stream() + .map(p -> new Rectangle((float) p.rectangle2D().getX(), + (float) p.rectangle2D().getY(), + (float) p.rectangle2D().getWidth(), + (float) p.rectangle2D().getHeight(), + p.pageNumber())) + .toList(); + } + if (entity instanceof Image image) { + + Rectangle2D position = image.getManualOverwrite().getPositions() + .map(p -> p.get(0).rectangle2D()) + .orElse(image.getPosition()); + + return List.of(new Rectangle((float) position.getX(), (float) position.getY(), (float) position.getWidth(), (float) position.getHeight(), image.getPage().getNumber())); + + } else { + throw new UnsupportedOperationException(); + } + } + + + public boolean needsManualEntry() { + + return manualChanges.stream() + .anyMatch(mc -> mc instanceof ManualResizeRedaction && !((ManualResizeRedaction) mc).getUpdateDictionary()) && !(migratedEntity instanceof Image); } } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/PrecursorEntity.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/PrecursorEntity.java index 22416c0e..3058c5d7 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/PrecursorEntity.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/PrecursorEntity.java @@ -43,7 +43,6 @@ public class PrecursorEntity implements IEntity { String type; String section; EntityType entityType; - EntryType entryType; boolean applied; boolean isDictionaryEntry; boolean isDossierDictionaryEntry; @@ -61,8 +60,8 @@ public class PrecursorEntity implements IEntity { .stream() .map(RectangleWithPage::fromAnnotationRectangle) .toList(); + var entityType = hint ? EntityType.HINT : EntityType.ENTITY; - var entryType = hint ? EntryType.HINT : (manualRedactionEntry.isRectangle() ? EntryType.AREA : EntryType.ENTITY); ManualChangeOverwrite manualChangeOverwrite = new ManualChangeOverwrite(entityType); manualChangeOverwrite.addChange(manualRedactionEntry); return PrecursorEntity.builder() @@ -75,7 +74,6 @@ public class PrecursorEntity implements IEntity { .type(manualRedactionEntry.getType()) .section(manualRedactionEntry.getSection()) .entityType(entityType) - .entryType(entryType) .applied(true) .isDictionaryEntry(false) .isDossierDictionaryEntry(false) @@ -103,7 +101,6 @@ public class PrecursorEntity implements IEntity { .type(entityLogEntry.getType()) .section(entityLogEntry.getSection()) .entityType(entityType) - .entryType(entityLogEntry.getEntryType()) .isDictionaryEntry(entityLogEntry.isDictionaryEntry()) .isDossierDictionaryEntry(entityLogEntry.isDossierDictionaryEntry()) .manualOverwrite(new ManualChangeOverwrite(entityType)) @@ -134,7 +131,6 @@ public class PrecursorEntity implements IEntity { .type(Optional.ofNullable(importedRedaction.getType()) .orElse(IMPORTED_REDACTION_TYPE)) .entityType(entityType) - .entryType(entryType) .isDictionaryEntry(false) .isDossierDictionaryEntry(false) .rectangle(value.isBlank() || entryType.equals(EntryType.IMAGE) || entryType.equals(EntryType.IMAGE_HINT) || entryType.equals(EntryType.AREA)) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/ImageType.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/ImageType.java index 397d7b11..e5e025f0 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/ImageType.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/ImageType.java @@ -6,10 +6,22 @@ public enum ImageType { LOGO, FORMULA, SIGNATURE, - OTHER, + OTHER { + @Override + public String toString() { + + return "image"; + } + }, OCR; + public String toString() { + + return name().toLowerCase(Locale.ENGLISH); + } + + public static ImageType fromString(String imageType) { return switch (imageType.toLowerCase(Locale.ROOT)) { diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java index 8840888b..893015ca 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java @@ -67,20 +67,19 @@ public class EntityLogCreatorService { List entityLogEntries = createEntityLogEntries(document, analyzeRequest, notFoundEntities); List legalBasis = legalBasisClient.getLegalBasisMapping(analyzeRequest.getDossierTemplateId()); - EntityLog entityLog = new EntityLog(redactionServiceSettings.getAnalysisVersion(), - analyzeRequest.getAnalysisNumber(), - entityLogEntries, - toEntityLogLegalBasis(legalBasis), - dictionaryVersion.getDossierTemplateVersion(), - dictionaryVersion.getDossierVersion(), - rulesVersion, - legalBasisClient.getVersion(analyzeRequest.getDossierTemplateId())); List previousExistingEntityLogEntries = getPreviousEntityLogEntries(analyzeRequest.getDossierId(), analyzeRequest.getFileId()); entityChangeLogService.computeChanges(previousExistingEntityLogEntries, entityLogEntries, analyzeRequest.getManualRedactions(), analyzeRequest.getAnalysisNumber()); - return entityLog; + return new EntityLog(redactionServiceSettings.getAnalysisVersion(), + analyzeRequest.getAnalysisNumber(), + entityLogEntries, + toEntityLogLegalBasis(legalBasis), + dictionaryVersion.getDossierTemplateVersion(), + dictionaryVersion.getDossierVersion(), + rulesVersion, + legalBasisClient.getVersion(analyzeRequest.getDossierTemplateId())); } @@ -187,12 +186,11 @@ public class EntityLogCreatorService { private EntityLogEntry createEntityLogEntry(Image image, String dossierTemplateId) { - String imageType = image.getImageType().equals(ImageType.OTHER) ? "image" : image.getImageType().toString().toLowerCase(Locale.ENGLISH); - boolean isHint = dictionaryService.isHint(imageType, dossierTemplateId); + boolean isHint = dictionaryService.isHint(image.type(), dossierTemplateId); return EntityLogEntry.builder() .id(image.getId()) .value(image.value()) - .type(imageType) + .type(image.type()) .reason(image.buildReasonWithManualChangeDescriptions()) .legalBasis(image.legalBasis()) .matchedRule(image.getMatchedRule().getRuleIdentifier().toString()) @@ -306,7 +304,7 @@ public class EntityLogCreatorService { } - private EntryState buildEntryState(IEntity entity) { + public static EntryState buildEntryState(IEntity entity) { if (entity.applied() && entity.active()) { return EntryState.APPLIED; @@ -320,12 +318,17 @@ public class EntityLogCreatorService { } - private EntryType buildEntryType(IEntity entity) { + public static EntryType buildEntryType(IEntity entity) { if (entity instanceof TextEntity textEntity) { return getEntryType(textEntity.getEntityType()); } else if (entity instanceof PrecursorEntity precursorEntity) { - return precursorEntity.getEntryType(); + if (precursorEntity.isRectangle()) { + return EntryType.AREA; + } + return getEntryType(precursorEntity.getEntityType()); + } else if (entity instanceof Image) { + return EntryType.IMAGE; } throw new UnsupportedOperationException(String.format("Entity subclass %s is not implemented!", entity.getClass())); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/UnprocessedChangesService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/UnprocessedChangesService.java index b7dfd989..2a469cc1 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/UnprocessedChangesService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/UnprocessedChangesService.java @@ -76,11 +76,19 @@ public class UnprocessedChangesService { EntityLog previousEntityLog = redactionStorageService.getEntityLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId()); Document document = DocumentGraphMapper.toDocumentGraph(observedStorageService.getDocumentData(analyzeRequest.getDossierId(), analyzeRequest.getFileId())); - Set allAnnotationIds = analyzeRequest.getManualRedactions().getEntriesToAdd().stream().map(ManualRedactionEntry::getAnnotationId).collect(Collectors.toSet()); - Set resizeIds = analyzeRequest.getManualRedactions().getResizeRedactions().stream().map(ManualResizeRedaction::getAnnotationId).collect(Collectors.toSet()); + Set allAnnotationIds = analyzeRequest.getManualRedactions().getEntriesToAdd() + .stream() + .map(ManualRedactionEntry::getAnnotationId) + .collect(Collectors.toSet()); + Set resizeIds = analyzeRequest.getManualRedactions().getResizeRedactions() + .stream() + .map(ManualResizeRedaction::getAnnotationId) + .collect(Collectors.toSet()); allAnnotationIds.addAll(resizeIds); - List manualResizeRedactions = analyzeRequest.getManualRedactions().getResizeRedactions().stream().toList(); + List manualResizeRedactions = analyzeRequest.getManualRedactions().getResizeRedactions() + .stream() + .toList(); List manualEntitiesToBeResized = previousEntityLog.getEntityLogEntry() .stream() .filter(entityLogEntry -> resizeIds.contains(entityLogEntry.getId())) @@ -99,31 +107,36 @@ public class UnprocessedChangesService { notFoundManualEntities = entityFromPrecursorCreationService.toTextEntity(manualEntities, document); } - document.getEntities().forEach(textEntity -> { - Set processedIds = new HashSet<>(); - for (var positionsOnPerPage : textEntity.getPositionsOnPagePerPage()) { - if (processedIds.contains(positionsOnPerPage.getId())) { - continue; - } - processedIds.add(positionsOnPerPage.getId()); - List positions = positionsOnPerPage.getRectanglePerLine() - .stream() - .map(rectangle2D -> new Position(rectangle2D, positionsOnPerPage.getPage().getNumber())) - .collect(Collectors.toList()); - unprocessedManualEntities.add(UnprocessedManualEntity.builder() - .annotationId(allAnnotationIds.stream().filter(textEntity::matchesAnnotationId).findFirst().orElse("")) - .textBefore(textEntity.getTextBefore()) - .textAfter(textEntity.getTextAfter()) - .section(textEntity.getManualOverwrite().getSection().orElse(textEntity.getDeepestFullyContainingNode().toString())) - .positions(positions) - .build()); - } - }); + document.getEntities() + .forEach(textEntity -> { + Set processedIds = new HashSet<>(); + for (var positionsOnPerPage : textEntity.getPositionsOnPagePerPage()) { + if (processedIds.contains(positionsOnPerPage.getId())) { + continue; + } + processedIds.add(positionsOnPerPage.getId()); + List positions = positionsOnPerPage.getRectanglePerLine() + .stream() + .map(rectangle2D -> new Position(rectangle2D, positionsOnPerPage.getPage().getNumber())) + .collect(Collectors.toList()); + unprocessedManualEntities.add(UnprocessedManualEntity.builder() + .annotationId(allAnnotationIds.stream() + .filter(textEntity::matchesAnnotationId) + .findFirst() + .orElse("")) + .textBefore(textEntity.getTextBefore()) + .textAfter(textEntity.getTextAfter()) + .section(textEntity.getManualOverwrite().getSection() + .orElse(textEntity.getDeepestFullyContainingNode().toString())) + .positions(positions) + .build()); + } + }); notFoundManualEntities.forEach(manualEntity -> unprocessedManualEntities.add(builDefaultUnprocessedManualEntity(manualEntity))); rabbitTemplate.convertAndSend(QueueNames.REDACTION_ANALYSIS_RESPONSE_QUEUE, - AnalyzeResponse.builder().fileId(analyzeRequest.getFileId()).unprocessedManualEntities(unprocessedManualEntities).build()); + AnalyzeResponse.builder().fileId(analyzeRequest.getFileId()).unprocessedManualEntities(unprocessedManualEntities).build()); } @@ -143,13 +156,13 @@ public class UnprocessedChangesService { continue; } - TextEntity correctEntity = createCorrectEntity(precursorEntity, optionalTextEntity.get()); + TextEntity correctEntity = EntityFromPrecursorCreationService.createCorrectEntity(precursorEntity, optionalTextEntity.get()); Optional optionalManualResizeRedaction = manualResizeRedactions.stream() .filter(manualResizeRedaction -> manualResizeRedaction.getAnnotationId().equals(precursorEntity.getId())) .findFirst(); if (optionalManualResizeRedaction.isPresent()) { ManualResizeRedaction manualResizeRedaction = optionalManualResizeRedaction.get(); - manualChangesApplicationService.resizeEntityAndReinsert(correctEntity, manualResizeRedaction); + manualChangesApplicationService.resize(correctEntity, manualResizeRedaction); // If the entity's value is not the same as the manual resize request's value it means we didn't find it anywhere and we want to remove it // from the graph, so it does not get processed and sent back to persistence-service to update its value. @@ -160,60 +173,37 @@ public class UnprocessedChangesService { } // remove all temp entities from the graph - tempEntities.values().stream().flatMap(Collection::stream).forEach(TextEntity::removeFromGraph); + tempEntities.values() + .stream() + .flatMap(Collection::stream) + .forEach(TextEntity::removeFromGraph); } - private TextEntity createCorrectEntity(PrecursorEntity precursorEntity, TextEntity closestEntity) { + private UnprocessedManualEntity builDefaultUnprocessedManualEntity(PrecursorEntity precursorEntity) { - TextEntity correctEntity = TextEntity.initialEntityNode(closestEntity.getTextRange(), precursorEntity.type(), precursorEntity.getEntityType(), precursorEntity.getId()); - - correctEntity.setDeepestFullyContainingNode(closestEntity.getDeepestFullyContainingNode()); - correctEntity.setIntersectingNodes(new ArrayList<>(closestEntity.getIntersectingNodes())); - correctEntity.setDuplicateTextRanges(new ArrayList<>(closestEntity.getDuplicateTextRanges())); - correctEntity.setPages(new HashSet<>(closestEntity.getPages())); - - correctEntity.setValue(closestEntity.getValue()); - correctEntity.setTextAfter(closestEntity.getTextAfter()); - correctEntity.setTextBefore(closestEntity.getTextBefore()); - - correctEntity.getIntersectingNodes().forEach(n -> n.getEntities().add(correctEntity)); - correctEntity.getPages().forEach(page -> page.getEntities().add(correctEntity)); - - correctEntity.addMatchedRules(precursorEntity.getMatchedRuleList()); - correctEntity.setDictionaryEntry(precursorEntity.isDictionaryEntry()); - correctEntity.setDossierDictionaryEntry(precursorEntity.isDossierDictionaryEntry()); - correctEntity.getManualOverwrite().addChanges(precursorEntity.getManualOverwrite().getManualChangeLog()); - - return correctEntity; -} + return UnprocessedManualEntity.builder() + .annotationId(precursorEntity.getId()) + .textAfter("") + .textBefore("") + .section("") + .positions(precursorEntity.getManualOverwrite().getPositions() + .orElse(precursorEntity.getEntityPosition()) + .stream() + .map(entityPosition -> new Position(entityPosition.rectangle2D(), entityPosition.pageNumber())) + .toList()) + .build(); + } -private UnprocessedManualEntity builDefaultUnprocessedManualEntity(PrecursorEntity precursorEntity) { + private List manualEntitiesConverter(ManualRedactions manualRedactions, String dossierTemplateId) { - return UnprocessedManualEntity.builder() - .annotationId(precursorEntity.getId()) - .textAfter("") - .textBefore("") - .section("") - .positions(precursorEntity.getManualOverwrite() - .getPositions() - .orElse(precursorEntity.getEntityPosition()) - .stream() - .map(entityPosition -> new Position(entityPosition.rectangle2D(), entityPosition.pageNumber())) - .toList()) - .build(); -} - - -private List manualEntitiesConverter(ManualRedactions manualRedactions, String dossierTemplateId) { - - return manualRedactions.getEntriesToAdd() - .stream() - .filter(manualRedactionEntry -> manualRedactionEntry.getPositions() != null && !manualRedactionEntry.getPositions().isEmpty()) - .map(manualRedactionEntry -> PrecursorEntity.fromManualRedactionEntry(manualRedactionEntry, - dictionaryService.isHint(manualRedactionEntry.getType(), dossierTemplateId))) - .toList(); -} + return manualRedactions.getEntriesToAdd() + .stream() + .filter(manualRedactionEntry -> manualRedactionEntry.getPositions() != null && !manualRedactionEntry.getPositions().isEmpty()) + .map(manualRedactionEntry -> PrecursorEntity.fromManualRedactionEntry(manualRedactionEntry, + dictionaryService.isHint(manualRedactionEntry.getType(), dossierTemplateId))) + .toList(); + } } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityFromPrecursorCreationService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityFromPrecursorCreationService.java index faddf9bd..13033a6f 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityFromPrecursorCreationService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityFromPrecursorCreationService.java @@ -14,7 +14,6 @@ import org.springframework.stereotype.Service; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedactions; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions; -import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.BaseAnnotation; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval; import com.iqser.red.service.redaction.v1.server.model.PrecursorEntity; import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType; @@ -51,18 +50,14 @@ public class EntityFromPrecursorCreationService { Set idRemovals = manualRedactions.getIdsToRemove(); List manualEntities = manualRedactions.getEntriesToAdd() .stream() - .filter(manualRedactionEntry -> !(idRemovals.stream() - .map(BaseAnnotation::getAnnotationId) - .toList() - .contains(manualRedactionEntry.getAnnotationId()) && manualRedactionEntry.getRequestDate() - .isBefore(idRemovals.stream() - .filter(idRemoval -> idRemoval.getAnnotationId().equals(manualRedactionEntry.getAnnotationId())) - .findFirst() - .get() - .getRequestDate()))) + .filter(manualRedactionEntry -> idRemovals.stream() + .filter(idRemoval -> idRemoval.getAnnotationId().equals(manualRedactionEntry.getAnnotationId())) + .filter(idRemoval -> idRemoval.getRequestDate().isBefore(manualRedactionEntry.getRequestDate())) + .findAny()// + .isEmpty()) .filter(manualRedactionEntry -> !(manualRedactionEntry.isAddToDictionary() || manualRedactionEntry.isAddToDossierDictionary())) - .map(manualRedactionEntry -> PrecursorEntity.fromManualRedactionEntry(manualRedactionEntry, - dictionaryService.isHint(manualRedactionEntry.getType(), dossierTemplateId))) + .map(manualRedactionEntry -> // + PrecursorEntity.fromManualRedactionEntry(manualRedactionEntry, dictionaryService.isHint(manualRedactionEntry.getType(), dossierTemplateId))) .peek(manualEntity -> { if (manualEntity.getEntityType().equals(EntityType.HINT)) { manualEntity.skip("MAN.5.1", "manual hint is skipped by default"); @@ -90,8 +85,14 @@ public class EntityFromPrecursorCreationService { public List toTextEntity(List precursorEntities, SemanticNode node) { - var notFoundEntities = precursorEntities.stream().filter(PrecursorEntity::isRectangle).collect(Collectors.toList()); - var findableEntities = precursorEntities.stream().filter(precursorEntity -> !precursorEntity.isRectangle()).toList(); + var notFoundEntities = precursorEntities.stream() + .filter(PrecursorEntity::isRectangle) + .collect(Collectors.toList()); + + var findableEntities = precursorEntities.stream() + .filter(precursorEntity -> !precursorEntity.isRectangle()) + .toList(); + Map> tempEntitiesByValue = entityFindingUtility.findAllPossibleEntitiesAndGroupByValue(node, findableEntities); for (PrecursorEntity precursorEntity : findableEntities) { @@ -102,7 +103,12 @@ public class EntityFromPrecursorCreationService { } createCorrectEntity(precursorEntity, optionalClosestEntity.get()); } - tempEntitiesByValue.values().stream().flatMap(Collection::stream).forEach(TextEntity::removeFromGraph); + + tempEntitiesByValue.values() + .stream() + .flatMap(Collection::stream) + .forEach(TextEntity::removeFromGraph); + return notFoundEntities; } @@ -113,9 +119,23 @@ public class EntityFromPrecursorCreationService { * @param precursorEntity The entity identifier for the RedactionEntity. * @param closestEntity The closest Boundary to the RedactionEntity. */ - private void createCorrectEntity(PrecursorEntity precursorEntity, TextEntity closestEntity) { + public static TextEntity createCorrectEntity(PrecursorEntity precursorEntity, TextEntity closestEntity) { - TextEntity correctEntity = TextEntity.initialEntityNode(closestEntity.getTextRange(), precursorEntity.type(), precursorEntity.getEntityType(), precursorEntity.getId()); + return createCorrectEntity(precursorEntity, closestEntity, false); + } + + + public static TextEntity createCorrectEntity(PrecursorEntity precursorEntity, TextEntity closestEntity, boolean generateId) { + + TextEntity correctEntity; + if (generateId) { + correctEntity = TextEntity.initialEntityNode(closestEntity.getTextRange(), + precursorEntity.type(), + precursorEntity.getEntityType(), + closestEntity.getDeepestFullyContainingNode()); + } else { + correctEntity = TextEntity.initialEntityNode(closestEntity.getTextRange(), precursorEntity.type(), precursorEntity.getEntityType(), precursorEntity.getId()); + } correctEntity.setDeepestFullyContainingNode(closestEntity.getDeepestFullyContainingNode()); correctEntity.setIntersectingNodes(new ArrayList<>(closestEntity.getIntersectingNodes())); correctEntity.setDuplicateTextRanges(new ArrayList<>(closestEntity.getDuplicateTextRanges())); @@ -125,14 +145,17 @@ public class EntityFromPrecursorCreationService { correctEntity.setTextAfter(closestEntity.getTextAfter()); correctEntity.setTextBefore(closestEntity.getTextBefore()); - correctEntity.getIntersectingNodes().forEach(n -> n.getEntities().add(correctEntity)); - correctEntity.getPages().forEach(page -> page.getEntities().add(correctEntity)); + correctEntity.getIntersectingNodes() + .forEach(n -> n.getEntities().add(correctEntity)); + correctEntity.getPages() + .forEach(page -> page.getEntities().add(correctEntity)); correctEntity.addMatchedRules(precursorEntity.getMatchedRuleList()); correctEntity.setDictionaryEntry(precursorEntity.isDictionaryEntry()); correctEntity.setDossierDictionaryEntry(precursorEntity.isDossierDictionaryEntry()); correctEntity.getManualOverwrite().addChanges(precursorEntity.getManualOverwrite().getManualChangeLog()); correctEntity.addEngines(precursorEntity.getEngines()); + return correctEntity; } } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/utils/MigratedIdsCollector.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/utils/MigratedIdsCollector.java index 295b7e18..38656b66 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/utils/MigratedIdsCollector.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/utils/MigratedIdsCollector.java @@ -1,5 +1,6 @@ package com.iqser.red.service.redaction.v1.server.utils; +import java.util.Collections; import java.util.LinkedList; import java.util.Set; import java.util.function.BiConsumer; @@ -17,7 +18,7 @@ public class MigratedIdsCollector implements Collector supplier() { - return () -> new MigratedIds(new LinkedList<>()); + return () -> new MigratedIds(new LinkedList<>(), Collections.emptyList()); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/MigrationIntegrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/MigrationIntegrationTest.java index db60cd34..18892630 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/MigrationIntegrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/MigrationIntegrationTest.java @@ -31,6 +31,7 @@ import org.springframework.test.context.junit.jupiter.SpringExtension; import com.fasterxml.jackson.databind.ObjectMapper; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.migration.MigratedIds; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions; @@ -49,7 +50,6 @@ import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document; import com.iqser.red.service.redaction.v1.server.redaction.utils.OsUtils; import com.iqser.red.service.redaction.v1.server.service.DictionaryService; import com.iqser.red.service.redaction.v1.server.service.document.EntityFindingUtility; -import com.iqser.red.service.redaction.v1.server.utils.RectangleTransformations; import com.knecon.fforesight.tenantcommons.TenantContext; import lombok.SneakyThrows; @@ -107,7 +107,7 @@ public class MigrationIntegrationTest extends BuildDocumentIntegrationTest { @SneakyThrows public void testSave() { - MigratedIds ids = new MigratedIds(new LinkedList<>()); + MigratedIds ids = new MigratedIds(new LinkedList<>(), null); ids.addMapping("123", "321"); ids.addMapping("123", "321"); ids.addMapping("123", "321"); @@ -173,7 +173,11 @@ public class MigrationIntegrationTest extends BuildDocumentIntegrationTest { mergedRedactionLog = redactionLog; } - MigratedEntityLog migratedEntityLog = redactionLogToEntityLogMigrationService.migrate(mergedRedactionLog, document, TEST_DOSSIER_TEMPLATE_ID, manualRedactions); + MigratedEntityLog migratedEntityLog = redactionLogToEntityLogMigrationService.migrate(mergedRedactionLog, + document, + TEST_DOSSIER_TEMPLATE_ID, + manualRedactions, + TEST_FILE_ID); redactionStorageService.storeObject(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.ENTITY_LOG, migratedEntityLog.getEntityLog()); assertEquals(mergedRedactionLog.getRedactionLogEntry().size(), migratedEntityLog.getEntityLog().getEntityLogEntry().size()); @@ -187,10 +191,11 @@ public class MigrationIntegrationTest extends BuildDocumentIntegrationTest { assertEquals(mergedRedactionLog.getLegalBasis().size(), entityLog.getLegalBasis().size()); Map migratedIds = migratedEntityLog.getMigratedIds().buildOldToNewMapping(); +// assertEquals(legacyRedactionLogMergeService.getNumberOfAffectedAnnotations(manualRedactions), migratedIds.size()); + migratedIds.forEach((oldId, newId) -> assertEntryIsEqual(oldId, newId, mergedRedactionLog, entityLog, migratedIds)); - AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder().dossierId(TEST_DOSSIER_ID).fileId(TEST_FILE_ID) - .build()); + AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder().dossierId(TEST_DOSSIER_ID).fileId(TEST_FILE_ID).build()); File outputFile = Path.of(OsUtils.getTemporaryDirectory()).resolve(Path.of(fileName.replaceAll(".pdf", "_MIGRATED.pdf")).getFileName()).toFile(); try (FileOutputStream fileOutputStream = new FileOutputStream(outputFile)) { @@ -268,13 +273,24 @@ public class MigrationIntegrationTest extends BuildDocumentIntegrationTest { if (!redactionLogEntry.isImage()) { assertEquals(redactionLogEntry.getValue().toLowerCase(Locale.ENGLISH), entityLogEntry.getValue().toLowerCase(Locale.ENGLISH)); } + if (entityLogEntry.getManualChanges() + .stream() + .noneMatch(mc -> mc.getManualRedactionType().equals(ManualRedactionType.RECATEGORIZE))) { + assertEquals(redactionLogEntry.getType(), entityLogEntry.getType()); + } assertEquals(redactionLogEntry.getChanges().size(), entityLogEntry.getChanges().size()); assertTrue(redactionLogEntry.getManualChanges().size() <= entityLogEntry.getManualChanges().size()); assertEquals(redactionLogEntry.getPositions().size(), entityLogEntry.getPositions().size()); - assertTrue(positionsAlmostEqual(redactionLogEntry.getPositions(), entityLogEntry.getPositions())); -// assertEquals(redactionLogEntry.getColor(), entityLogEntry.getColor()); - assertEqualsNullSafe(redactionLogEntry.getLegalBasis(), entityLogEntry.getLegalBasis()); -// assertEqualsNullSafe(redactionLogEntry.getReason(), entityLogEntry.getReason()); + if (entityLogEntry.getManualChanges() + .stream() + .noneMatch(mc -> mc.getManualRedactionType().equals(ManualRedactionType.RESIZE) || mc.getManualRedactionType().equals(ManualRedactionType.RESIZE_IN_DICTIONARY))) { + assertTrue(positionsAlmostEqual(redactionLogEntry.getPositions(), entityLogEntry.getPositions())); + } + if (entityLogEntry.getManualChanges() + .stream() + .noneMatch(mc -> mc.getManualRedactionType().equals(ManualRedactionType.FORCE_REDACT) || mc.getManualRedactionType().equals(ManualRedactionType.FORCE_HINT))) { + assertEqualsNullSafe(redactionLogEntry.getLegalBasis(), entityLogEntry.getLegalBasis()); + } assertReferencesEqual(redactionLogEntry.getReference(), entityLogEntry.getReference(), oldToNewMapping); assertEquals(redactionLogEntry.isDictionaryEntry(), entityLogEntry.isDictionaryEntry()); assertEquals(redactionLogEntry.isDossierDictionaryEntry(), entityLogEntry.isDossierDictionaryEntry()); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java index 95980612..27b8f64c 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java @@ -119,15 +119,15 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest { when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L); when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, true)).thenReturn(List.of(Type.builder() - .id(DOSSIER_REDACTIONS_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID) - .type(DOSSIER_REDACTIONS_INDICATOR) - .dossierTemplateId(TEST_DOSSIER_ID) - .hexColor("#ffe187") - .isHint(hintTypeMap.get(DOSSIER_REDACTIONS_INDICATOR)) - .isCaseInsensitive(caseInSensitiveMap.get(DOSSIER_REDACTIONS_INDICATOR)) - .isRecommendation(recommendationTypeMap.get(DOSSIER_REDACTIONS_INDICATOR)) - .rank(rankTypeMap.get(DOSSIER_REDACTIONS_INDICATOR)) - .build())); + .id(DOSSIER_REDACTIONS_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID) + .type(DOSSIER_REDACTIONS_INDICATOR) + .dossierTemplateId(TEST_DOSSIER_ID) + .hexColor("#ffe187") + .isHint(hintTypeMap.get(DOSSIER_REDACTIONS_INDICATOR)) + .isCaseInsensitive(caseInSensitiveMap.get(DOSSIER_REDACTIONS_INDICATOR)) + .isRecommendation(recommendationTypeMap.get(DOSSIER_REDACTIONS_INDICATOR)) + .rank(rankTypeMap.get(DOSSIER_REDACTIONS_INDICATOR)) + .build())); mockDictionaryCalls(null); @@ -169,9 +169,10 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest { var entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID); - entityLog.getEntityLogEntry().forEach(entry -> { - duplicates.computeIfAbsent(entry.getId(), v -> new ArrayList<>()).add(entry); - }); + entityLog.getEntityLogEntry() + .forEach(entry -> { + duplicates.computeIfAbsent(entry.getId(), v -> new ArrayList<>()).add(entry); + }); duplicates.forEach((key, value) -> assertThat(value.size()).isEqualTo(1)); @@ -216,12 +217,12 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest { ManualRedactions manualRedactions = ManualRedactions.builder() .resizeRedactions(Set.of(ManualResizeRedaction.builder() - .annotationId("c6be5277f5ee60dc3d83527798b7fe02") - .value("Dr. Alan") - .positions(List.of(new Rectangle(236.8f, 182.90005f, 40.584f, 12.642f, 7))) - .requestDate(OffsetDateTime.now()) - .updateDictionary(false) - .build())) + .annotationId("c6be5277f5ee60dc3d83527798b7fe02") + .value("Dr. Alan") + .positions(List.of(new Rectangle(236.8f, 182.90005f, 40.584f, 12.642f, 7))) + .requestDate(OffsetDateTime.now()) + .updateDictionary(false) + .build())) .build(); request.setManualRedactions(manualRedactions); @@ -256,7 +257,10 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest { var redactionLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID); - var values = redactionLog.getEntityLogEntry().stream().map(EntityLogEntry::getValue).collect(Collectors.toList()); + var values = redactionLog.getEntityLogEntry() + .stream() + .map(EntityLogEntry::getValue) + .collect(Collectors.toList()); assertThat(values).containsExactlyInAnyOrder("Lastname M.", "Doe", "Doe J.", "M. Mustermann", "Mustermann M.", "F. Lastname"); } @@ -268,8 +272,8 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest { ClassPathResource importedRedactionClasspathResource = new ClassPathResource( "files/ImportedRedactions/18 Chlorothalonil RAR 08 Volume 3CA B 6a Oct 2017.IMPORTED_REDACTIONS.json"); storageService.storeObject(TenantContext.getTenantId(), - RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.IMPORTED_REDACTIONS), - importedRedactionClasspathResource.getInputStream()); + RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.IMPORTED_REDACTIONS), + importedRedactionClasspathResource.getInputStream()); AnalyzeRequest request = uploadFileToStorage("files/ImportedRedactions/18 Chlorothalonil RAR 08 Volume 3CA B 6a Oct 2017.pdf"); System.out.println("Start Full integration test"); @@ -353,10 +357,18 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest { var mergedEntityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID); - var cbiAddressBeforeHintRemoval = entityLog.getEntityLogEntry().stream().filter(re -> re.getType().equalsIgnoreCase("CBI_Address")).findAny().get(); + var cbiAddressBeforeHintRemoval = entityLog.getEntityLogEntry() + .stream() + .filter(re -> re.getType().equalsIgnoreCase("CBI_Address")) + .findAny() + .get(); assertThat(cbiAddressBeforeHintRemoval.getState().equals(EntryState.APPLIED)).isFalse(); - var cbiAddressAfterHintRemoval = mergedEntityLog.getEntityLogEntry().stream().filter(re -> re.getType().equalsIgnoreCase("CBI_Address")).findAny().get(); + var cbiAddressAfterHintRemoval = mergedEntityLog.getEntityLogEntry() + .stream() + .filter(re -> re.getType().equalsIgnoreCase("CBI_Address")) + .findAny() + .get(); assertThat(cbiAddressAfterHintRemoval.getState().equals(EntryState.APPLIED)).isTrue(); } @@ -386,9 +398,10 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest { var entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID); - entityLog.getEntityLogEntry().forEach(entry -> { - duplicates.computeIfAbsent(entry.getId(), v -> new ArrayList<>()).add(entry); - }); + entityLog.getEntityLogEntry() + .forEach(entry -> { + duplicates.computeIfAbsent(entry.getId(), v -> new ArrayList<>()).add(entry); + }); duplicates.forEach((id, redactionLogEntries) -> assertThat(redactionLogEntries.size()).isEqualTo(1)); @@ -421,11 +434,11 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest { AnalyzeRequest request = uploadFileToStorage(fileName); request.setFileAttributes(List.of(FileAttribute.builder() - .id("fileAttributeId") - .label("Vertebrate Study") - .placeholder("{fileattributes.vertebrateStudy}") - .value("true") - .build())); + .id("fileAttributeId") + .label("Vertebrate Study") + .placeholder("{fileattributes.vertebrateStudy}") + .value("true") + .build())); analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request); AnalyzeResult result = analyzeService.analyze(request); @@ -449,7 +462,10 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest { correctFound++; continue loop; } - if (Objects.equals(entityLogEntry.getContainingNodeId().get(0), section.getTreeId().get(0))) { + if (Objects.equals(entityLogEntry.getContainingNodeId() + .get(0), + section.getTreeId() + .get(0))) { String value = section.getTextBlock().subSequence(new TextRange(entityLogEntry.getStartOffset(), entityLogEntry.getEndOffset())).toString(); if (entityLogEntry.getValue().equalsIgnoreCase(value)) { correctFound++; @@ -481,12 +497,12 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest { ManualRedactions manualRedactions = new ManualRedactions(); manualRedactions.setEntriesToAdd(Set.of(ManualRedactionEntry.builder() - .value("Redact") - .addToDictionary(true) - .addToDossierDictionary(true) - .positions(List.of(new Rectangle(new Point(95.96979999999999f, 515.7984f), 19.866899999999987f, 46.953f, 2))) - .type("dossier_redaction") - .build())); + .value("Redact") + .addToDictionary(true) + .addToDossierDictionary(true) + .positions(List.of(new Rectangle(new Point(95.96979999999999f, 515.7984f), 19.866899999999987f, 46.953f, 2))) + .type("dossier_redaction") + .build())); request.setManualRedactions(manualRedactions); @@ -548,7 +564,11 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest { var entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID); - var changes = entityLog.getEntityLogEntry().stream().filter(entry -> entry.getValue() != null && entry.getValue().equals("report")).findFirst().get().getChanges(); + var changes = entityLog.getEntityLogEntry() + .stream() + .filter(entry -> entry.getValue() != null && entry.getValue().equals("report")) + .findFirst() + .get().getChanges(); assertThat(changes.size()).isEqualTo(2); @@ -568,18 +588,18 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest { ClassPathResource responseJson = new ClassPathResource("files/crafted_document.NER_ENTITIES.json"); storageService.storeObject(TenantContext.getTenantId(), - RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.NER_ENTITIES), - responseJson.getInputStream()); + RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.NER_ENTITIES), + responseJson.getInputStream()); long start = System.currentTimeMillis(); AnalyzeRequest request = uploadFileToStorage(fileName); request.setFileAttributes(List.of(FileAttribute.builder() - .id("fileAttributeId") - .label("Vertebrate Study") - .placeholder("{fileattributes.vertebrateStudy}") - .value("true") - .build())); + .id("fileAttributeId") + .label("Vertebrate Study") + .placeholder("{fileattributes.vertebrateStudy}") + .value("true") + .build())); analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request); AnalyzeResult result = analyzeService.analyze(request); @@ -601,7 +621,11 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest { .map(redactionLogEntry -> new TextRange(redactionLogEntry.getStartOffset(), redactionLogEntry.getEndOffset())) .map(boundary -> documentGraph.getTextBlock().subSequence(boundary).toString()) .toList(); - List valuesInRedactionLog = entityLog.getEntityLogEntry().stream().filter(e -> !e.getEntryType().equals(EntryType.IMAGE)).map(EntityLogEntry::getValue).toList(); + List valuesInRedactionLog = entityLog.getEntityLogEntry() + .stream() + .filter(e -> !e.getEntryType().equals(EntryType.IMAGE)) + .map(EntityLogEntry::getValue) + .toList(); assertEquals(valuesInRedactionLog, valuesInDocument); @@ -628,11 +652,11 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest { ManualRedactions manualRedactions = new ManualRedactions(); manualRedactions.setRecategorizations(Set.of(ManualRecategorization.builder() - .annotationId("37eee3e9d589a5cc529bfec38c3ba479") - .fileId("fileId") - .type("signature") - .requestDate(OffsetDateTime.now()) - .build())); + .annotationId("37eee3e9d589a5cc529bfec38c3ba479") + .fileId("fileId") + .type("signature") + .requestDate(OffsetDateTime.now()) + .build())); request.setManualRedactions(manualRedactions); @@ -683,40 +707,40 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest { ManualRedactions manualRedactions = new ManualRedactions(); manualRedactions.getIdsToRemove() .add(IdRemoval.builder() - .annotationId("308dab9015bfafd911568cffe0a7f7de") - .fileId(TEST_FILE_ID) - .requestDate(OffsetDateTime.of(2022, 05, 23, 8, 30, 07, 475479, ZoneOffset.UTC)) - .processedDate(OffsetDateTime.of(2022, 05, 23, 8, 30, 07, 483651, ZoneOffset.UTC)) - .build()); + .annotationId("308dab9015bfafd911568cffe0a7f7de") + .fileId(TEST_FILE_ID) + .requestDate(OffsetDateTime.of(2022, 05, 23, 8, 30, 07, 475479, ZoneOffset.UTC)) + .processedDate(OffsetDateTime.of(2022, 05, 23, 8, 30, 07, 483651, ZoneOffset.UTC)) + .build()); manualRedactions.getForceRedactions() .add(ManualForceRedaction.builder() - .annotationId("0b56ea1a87c83f351df177315af94f0d") - .fileId(TEST_FILE_ID) - .legalBasis("Something") - .requestDate(OffsetDateTime.of(2022, 05, 23, 9, 30, 15, 4653, ZoneOffset.UTC)) - .processedDate(OffsetDateTime.of(2022, 05, 23, 9, 30, 15, 794, ZoneOffset.UTC)) - .build()); + .annotationId("0b56ea1a87c83f351df177315af94f0d") + .fileId(TEST_FILE_ID) + .legalBasis("Something") + .requestDate(OffsetDateTime.of(2022, 05, 23, 9, 30, 15, 4653, ZoneOffset.UTC)) + .processedDate(OffsetDateTime.of(2022, 05, 23, 9, 30, 15, 794, ZoneOffset.UTC)) + .build()); manualRedactions.getIdsToRemove() .add(IdRemoval.builder() - .annotationId("0b56ea1a87c83f351df177315af94f0d") - .fileId(TEST_FILE_ID) - .requestDate(OffsetDateTime.of(2022, 05, 23, 8, 30, 23, 961721, ZoneOffset.UTC)) - .processedDate(OffsetDateTime.of(2022, 05, 23, 8, 30, 23, 96528, ZoneOffset.UTC)) - .build()); + .annotationId("0b56ea1a87c83f351df177315af94f0d") + .fileId(TEST_FILE_ID) + .requestDate(OffsetDateTime.of(2022, 05, 23, 8, 30, 23, 961721, ZoneOffset.UTC)) + .processedDate(OffsetDateTime.of(2022, 05, 23, 8, 30, 23, 96528, ZoneOffset.UTC)) + .build()); request.setManualRedactions(manualRedactions); AnalyzeResult result = analyzeService.analyze(request); AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder() - .manualRedactions(manualRedactions) - .colors(colors) - .types(types) - .dossierId(TEST_DOSSIER_ID) - .fileId(TEST_FILE_ID) - .build()); + .manualRedactions(manualRedactions) + .colors(colors) + .types(types) + .dossierId(TEST_DOSSIER_ID) + .fileId(TEST_FILE_ID) + .build()); try (FileOutputStream fileOutputStream = new FileOutputStream(OsUtils.getTemporaryDirectory() + "/Annotated.pdf")) { fileOutputStream.write(annotateResponse.getDocument()); @@ -932,12 +956,12 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest { var entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID); AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder() - .manualRedactions(manualRedactions) - .colors(colors) - .types(types) - .dossierId(TEST_DOSSIER_ID) - .fileId(TEST_FILE_ID) - .build()); + .manualRedactions(manualRedactions) + .colors(colors) + .types(types) + .dossierId(TEST_DOSSIER_ID) + .fileId(TEST_FILE_ID) + .build()); try (FileOutputStream fileOutputStream = new FileOutputStream(OsUtils.getTemporaryDirectory() + "/Annotated.pdf")) { fileOutputStream.write(annotateResponse.getDocument()); @@ -960,15 +984,16 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest { var redactionLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID); - redactionLog.getEntityLogEntry().forEach(entry -> { - if (!entry.getEntryType().equals(EntryType.HINT)) { - if (entry.getType().equals("CBI_author")) { - assertThat(entry.getReason()).isEqualTo("Not redacted because it's row does not belong to a vertebrate study"); - } else if (entry.getType().equals("CBI_address")) { - assertThat(entry.getReason()).isEqualTo("No vertebrate found"); - } - } - }); + redactionLog.getEntityLogEntry() + .forEach(entry -> { + if (!entry.getEntryType().equals(EntryType.HINT)) { + if (entry.getType().equals("CBI_author")) { + assertThat(entry.getReason()).isEqualTo("Not redacted because it's row does not belong to a vertebrate study"); + } else if (entry.getType().equals("CBI_address")) { + assertThat(entry.getReason()).isEqualTo("No vertebrate found"); + } + } + }); } @@ -1005,18 +1030,18 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest { String manualAddId = UUID.randomUUID().toString(); manualRedactions.setIdsToRemove(Set.of(IdRemoval.builder() - .annotationId("5b940b2cb401ed9f5be6fc24f6e77bcf") - .fileId("fileId") - .processedDate(OffsetDateTime.now()) - .requestDate(OffsetDateTime.now()) - .build())); + .annotationId("5b940b2cb401ed9f5be6fc24f6e77bcf") + .fileId("fileId") + .processedDate(OffsetDateTime.now()) + .requestDate(OffsetDateTime.now()) + .build())); manualRedactions.setForceRedactions(Set.of(ManualForceRedaction.builder() - .annotationId("675eba69b0c2917de55462c817adaa05") - .fileId("fileId") - .legalBasis("Something") - .requestDate(OffsetDateTime.now()) - .processedDate(OffsetDateTime.now()) - .build())); + .annotationId("675eba69b0c2917de55462c817adaa05") + .fileId("fileId") + .legalBasis("Something") + .requestDate(OffsetDateTime.now()) + .processedDate(OffsetDateTime.now()) + .build())); ManualRedactionEntry manualRedactionEntry = new ManualRedactionEntry(); manualRedactionEntry.setAnnotationId(manualAddId); @@ -1027,7 +1052,7 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest { manualRedactionEntry.setProcessedDate(OffsetDateTime.now()); manualRedactionEntry.setRequestDate(OffsetDateTime.now()); manualRedactionEntry.setPositions(List.of(Rectangle.builder().topLeftX(375.61096f).topLeftY(241.282f).width(7.648041f).height(43.72262f).page(1).build(), - Rectangle.builder().topLeftX(384.83517f).topLeftY(241.282f).width(7.648041f).height(17.043358f).page(1).build())); + Rectangle.builder().topLeftX(384.83517f).topLeftY(241.282f).width(7.648041f).height(17.043358f).page(1).build())); // manualRedactions.getEntriesToAdd().add(manualRedactionEntry); @@ -1038,39 +1063,63 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest { manualRedactions.getEntriesToAdd().add(manualRedactionEntry); manualRedactions.setIdsToRemove(Set.of(IdRemoval.builder() - .annotationId("5b940b2cb401ed9f5be6fc24f6e77bcf") - .fileId("fileId") - .requestDate(OffsetDateTime.now()) - .processedDate(OffsetDateTime.now()) - .build())); + .annotationId("5b940b2cb401ed9f5be6fc24f6e77bcf") + .fileId("fileId") + .requestDate(OffsetDateTime.now()) + .processedDate(OffsetDateTime.now()) + .build())); manualRedactions.setLegalBasisChanges((Set.of(ManualLegalBasisChange.builder() - .annotationId("675eba69b0c2917de55462c817adaa05") - .fileId("fileId") - .legalBasis("Manual Legal Basis Change") - .processedDate(OffsetDateTime.now()) - .requestDate(OffsetDateTime.now()) - .build()))); + .annotationId("675eba69b0c2917de55462c817adaa05") + .fileId("fileId") + .legalBasis("Manual Legal Basis Change") + .processedDate(OffsetDateTime.now()) + .requestDate(OffsetDateTime.now()) + .build()))); manualRedactions.setResizeRedactions(Set.of(ManualResizeRedaction.builder() - .annotationId("fc287b74be2421156ab2895c7474ccdd") - .fileId("fileId") - .processedDate(OffsetDateTime.now()) - .requestDate(OffsetDateTime.now()) - .value("Syngenta Crop Protection AG, Basel, Switzerland RCC Ltd., Itingen, Switzerland") - .positions(List.of(Rectangle.builder().topLeftX(289.44595f).topLeftY(327.567f).width(7.648041f).height(82.51475f).page(1).build(), - Rectangle.builder().topLeftX(298.67056f).topLeftY(327.567f).width(7.648041f).height(75.32377f).page(1).build(), - Rectangle.builder().topLeftX(307.89517f).topLeftY(327.567f).width(7.648041f).height(61.670967f).page(1).build(), - Rectangle.builder().topLeftX(316.99985f).topLeftY(327.567f).width(7.648041f).height(38.104286f).page(1).build())) - .updateDictionary(false) - .build())); + .annotationId("fc287b74be2421156ab2895c7474ccdd") + .fileId("fileId") + .processedDate(OffsetDateTime.now()) + .requestDate(OffsetDateTime.now()) + .value("Syngenta Crop Protection AG, Basel, Switzerland RCC Ltd., Itingen, Switzerland") + .positions(List.of(Rectangle.builder() + .topLeftX(289.44595f) + .topLeftY(327.567f) + .width(7.648041f) + .height(82.51475f) + .page(1) + .build(), + Rectangle.builder() + .topLeftX(298.67056f) + .topLeftY(327.567f) + .width(7.648041f) + .height(75.32377f) + .page(1) + .build(), + Rectangle.builder() + .topLeftX(307.89517f) + .topLeftY(327.567f) + .width(7.648041f) + .height(61.670967f) + .page(1) + .build(), + Rectangle.builder() + .topLeftX(316.99985f) + .topLeftY(327.567f) + .width(7.648041f) + .height(38.104286f) + .page(1) + .build())) + .updateDictionary(false) + .build())); analyzeService.reanalyze(request); AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder() - .dossierId(TEST_DOSSIER_ID) - .fileId(TEST_FILE_ID) - .dossierTemplateId(TEST_DOSSIER_TEMPLATE_ID) - .manualRedactions(manualRedactions) - .build()); + .dossierId(TEST_DOSSIER_ID) + .fileId(TEST_FILE_ID) + .dossierTemplateId(TEST_DOSSIER_TEMPLATE_ID) + .manualRedactions(manualRedactions) + .build()); try (FileOutputStream fileOutputStream = new FileOutputStream(OsUtils.getTemporaryDirectory() + "/Annotated.pdf")) { fileOutputStream.write(annotateResponse.getDocument()); @@ -1110,8 +1159,8 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest { AnalyzeRequest request = uploadFileToStorage("files/ImportedRedactions/RotateTestFile_without_highlights.pdf"); storageService.storeObject(TenantContext.getTenantId(), - RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.IMPORTED_REDACTIONS), - importedRedactions.getInputStream()); + RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.IMPORTED_REDACTIONS), + importedRedactions.getInputStream()); analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request); AnalyzeResult result = analyzeService.analyze(request); @@ -1124,17 +1173,18 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest { fileOutputStream.write(annotateResponse.getDocument()); } - entityLog.getEntityLogEntry().forEach(entry -> { - if (entry.getValue() == null) { - return; - } - if (entry.getValue().equals("David")) { - assertThat(entry.getImportedRedactionIntersections()).hasSize(1); - } - if (entry.getValue().equals("annotation")) { - assertThat(entry.getImportedRedactionIntersections()).isEmpty(); - } - }); + entityLog.getEntityLogEntry() + .forEach(entry -> { + if (entry.getValue() == null) { + return; + } + if (entry.getValue().equals("David")) { + assertThat(entry.getImportedRedactionIntersections()).hasSize(1); + } + if (entry.getValue().equals("annotation")) { + assertThat(entry.getImportedRedactionIntersections()).isEmpty(); + } + }); } @@ -1163,7 +1213,10 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest { } var entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID); - var values = entityLog.getEntityLogEntry().stream().map(EntityLogEntry::getValue).collect(Collectors.toList()); + var values = entityLog.getEntityLogEntry() + .stream() + .map(EntityLogEntry::getValue) + .collect(Collectors.toList()); assertThat(values).contains("Mrs. Robinson"); assertThat(values).contains("Mr. Bojangles"); @@ -1178,8 +1231,8 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest { ClassPathResource imageServiceResponseFileResource = new ClassPathResource("files/new/SYNGENTA_EFSA_sanitisation_GFL_v1 (1).IMAGE_INFO.json"); storageService.storeObject(TenantContext.getTenantId(), - RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.IMAGE_INFO), - imageServiceResponseFileResource.getInputStream()); + RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.IMAGE_INFO), + imageServiceResponseFileResource.getInputStream()); System.out.println("Start Full integration test"); analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request); @@ -1188,23 +1241,26 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest { System.out.println("Finished analysis"); request.setManualRedactions(ManualRedactions.builder() - .legalBasisChanges(Set.of(ManualLegalBasisChange.builder() - .annotationId("3029651d0842a625f2d23f8375c23600") - .section("[19, 2]: Paragraph: Contact point: LexCo Contact:") - .value("0049 331 441 551 14") - .requestDate(OffsetDateTime.now()) - .fileId(TEST_FILE_ID) - .legalBasis("Article 39(e)(2) of Regulation (EC) No 178/2002") - .build())) - .build()); + .legalBasisChanges(Set.of(ManualLegalBasisChange.builder() + .annotationId("3029651d0842a625f2d23f8375c23600") + .section("[19, 2]: Paragraph: Contact point: LexCo Contact:") + .value("0049 331 441 551 14") + .requestDate(OffsetDateTime.now()) + .fileId(TEST_FILE_ID) + .legalBasis("Article 39(e)(2) of Regulation (EC) No 178/2002") + .build())) + .build()); analyzeService.reanalyze(request); System.out.println("Finished reanalysis"); var entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID); - entityLog.getEntityLogEntry().stream().filter(entityLogEntry -> entityLogEntry.getType().equals("signature")).forEach(entityLogEntry -> { - assertThat(entityLogEntry.getState() == EntryState.APPLIED).isTrue(); - }); + entityLog.getEntityLogEntry() + .stream() + .filter(entityLogEntry -> entityLogEntry.getType().equals("signature")) + .forEach(entityLogEntry -> { + assertThat(entityLogEntry.getState() == EntryState.APPLIED).isTrue(); + }); } @@ -1215,8 +1271,8 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest { ClassPathResource imageServiceResponseFileResource = new ClassPathResource("files/new/SYNGENTA_EFSA_sanitisation_GFL_v1 (1).IMAGE_INFO.json"); storageService.storeObject(TenantContext.getTenantId(), - RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.IMAGE_INFO), - imageServiceResponseFileResource.getInputStream()); + RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.IMAGE_INFO), + imageServiceResponseFileResource.getInputStream()); System.out.println("Start Full integration test"); analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request); @@ -1225,22 +1281,22 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest { System.out.println("Finished analysis"); request.setManualRedactions(ManualRedactions.builder() - .legalBasisChanges(Set.of(ManualLegalBasisChange.builder() - .annotationId("3029651d0842a625f2d23f8375c23600") - .section("[19, 2]: Paragraph: Contact point: LexCo Contact:") - .value("0049 331 441 551 14") - .requestDate(OffsetDateTime.now()) - .fileId(TEST_FILE_ID) - .legalBasis("Article 39(e)(2) of Regulation (EC) No 178/2002") - .build())) - .recategorizations(Set.of(ManualRecategorization.builder() - .annotationId("3029651d0842a625f2d23f8375c23600") - .type("CBI_author") - .legalBasis("") - .requestDate(OffsetDateTime.now()) - .fileId(TEST_FILE_ID) - .build())) - .build()); + .legalBasisChanges(Set.of(ManualLegalBasisChange.builder() + .annotationId("3029651d0842a625f2d23f8375c23600") + .section("[19, 2]: Paragraph: Contact point: LexCo Contact:") + .value("0049 331 441 551 14") + .requestDate(OffsetDateTime.now()) + .fileId(TEST_FILE_ID) + .legalBasis("Article 39(e)(2) of Regulation (EC) No 178/2002") + .build())) + .recategorizations(Set.of(ManualRecategorization.builder() + .annotationId("3029651d0842a625f2d23f8375c23600") + .type("CBI_author") + .legalBasis("") + .requestDate(OffsetDateTime.now()) + .fileId(TEST_FILE_ID) + .build())) + .build()); analyzeService.reanalyze(request); System.out.println("Finished reanalysis"); @@ -1267,11 +1323,11 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest { String manualAddId2 = UUID.randomUUID().toString(); List positions = List.of(Rectangle.builder().topLeftX(305.35f).topLeftY(332.5033f).width(71.40744f).height(13.645125f).page(1).build()); ManualRedactionEntry manualRedactionEntry = getManualRedactionEntry(manualAddId, - positions, - "the manufacturing or production process, including the method and innovative aspects thereof, as well as other technical and industrial specifications inherent to that process or method, except for information which is relevant to the assessment of safety"); + positions, + "the manufacturing or production process, including the method and innovative aspects thereof, as well as other technical and industrial specifications inherent to that process or method, except for information which is relevant to the assessment of safety"); ManualRedactionEntry manualRedactionEntry2 = getManualRedactionEntry(manualAddId2, - positions, - "commercial information revealing sourcing, market shares or business strategy of the applicant"); + positions, + "commercial information revealing sourcing, market shares or business strategy of the applicant"); IdRemoval idRemoval = getIdRemoval(manualAddId); IdRemoval idRemoval2 = getIdRemoval(manualAddId2); @@ -1283,53 +1339,98 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest { var entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID); - assertTrue(entityLog.getEntityLogEntry().stream().anyMatch(entityLogEntry -> entityLogEntry.getId().equals(manualAddId))); - assertEquals(entityLog.getEntityLogEntry().stream().filter(entityLogEntry -> entityLogEntry.getId().equals(manualAddId)).findFirst().get().getState(), EntryState.APPLIED); + assertTrue(entityLog.getEntityLogEntry() + .stream() + .anyMatch(entityLogEntry -> entityLogEntry.getId().equals(manualAddId))); + assertEquals(entityLog.getEntityLogEntry() + .stream() + .filter(entityLogEntry -> entityLogEntry.getId().equals(manualAddId)) + .findFirst() + .get().getState(), EntryState.APPLIED); request.setManualRedactions(ManualRedactions.builder().entriesToAdd(Set.of(manualRedactionEntry)).idsToRemove(Set.of(idRemoval)).build()); analyzeService.reanalyze(request); entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID); - assertTrue(entityLog.getEntityLogEntry().stream().anyMatch(entityLogEntry -> entityLogEntry.getId().equals(manualAddId))); - assertEquals(entityLog.getEntityLogEntry().stream().filter(entityLogEntry -> entityLogEntry.getId().equals(manualAddId)).findFirst().get().getState(), EntryState.REMOVED); + assertTrue(entityLog.getEntityLogEntry() + .stream() + .anyMatch(entityLogEntry -> entityLogEntry.getId().equals(manualAddId))); + assertEquals(entityLog.getEntityLogEntry() + .stream() + .filter(entityLogEntry -> entityLogEntry.getId().equals(manualAddId)) + .findFirst() + .get().getState(), EntryState.REMOVED); request.setManualRedactions(ManualRedactions.builder().entriesToAdd(Set.of(manualRedactionEntry, manualRedactionEntry2)).idsToRemove(Set.of(idRemoval)).build()); analyzeService.reanalyze(request); entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID); - assertTrue(entityLog.getEntityLogEntry().stream().anyMatch(entityLogEntry -> entityLogEntry.getId().equals(manualAddId))); - assertEquals(entityLog.getEntityLogEntry().stream().filter(entityLogEntry -> entityLogEntry.getId().equals(manualAddId)).findFirst().get().getState(), EntryState.REMOVED); - assertTrue(entityLog.getEntityLogEntry().stream().anyMatch(entityLogEntry -> entityLogEntry.getId().equals(manualAddId2))); - assertEquals(entityLog.getEntityLogEntry().stream().filter(entityLogEntry -> entityLogEntry.getId().equals(manualAddId2)).findFirst().get().getState(), EntryState.APPLIED); + assertTrue(entityLog.getEntityLogEntry() + .stream() + .anyMatch(entityLogEntry -> entityLogEntry.getId().equals(manualAddId))); + assertEquals(entityLog.getEntityLogEntry() + .stream() + .filter(entityLogEntry -> entityLogEntry.getId().equals(manualAddId)) + .findFirst() + .get().getState(), EntryState.REMOVED); + assertTrue(entityLog.getEntityLogEntry() + .stream() + .anyMatch(entityLogEntry -> entityLogEntry.getId().equals(manualAddId2))); + assertEquals(entityLog.getEntityLogEntry() + .stream() + .filter(entityLogEntry -> entityLogEntry.getId().equals(manualAddId2)) + .findFirst() + .get().getState(), EntryState.APPLIED); request.setManualRedactions(ManualRedactions.builder() - .entriesToAdd(Set.of(manualRedactionEntry, manualRedactionEntry2)) - .idsToRemove(Set.of(idRemoval, idRemoval2)) - .build()); + .entriesToAdd(Set.of(manualRedactionEntry, manualRedactionEntry2)) + .idsToRemove(Set.of(idRemoval, idRemoval2)) + .build()); analyzeService.reanalyze(request); entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID); - assertTrue(entityLog.getEntityLogEntry().stream().anyMatch(entityLogEntry -> entityLogEntry.getId().equals(manualAddId))); - assertEquals(entityLog.getEntityLogEntry().stream().filter(entityLogEntry -> entityLogEntry.getId().equals(manualAddId)).findFirst().get().getState(), EntryState.REMOVED); - assertTrue(entityLog.getEntityLogEntry().stream().anyMatch(entityLogEntry -> entityLogEntry.getId().equals(manualAddId2))); - assertEquals(entityLog.getEntityLogEntry().stream().filter(entityLogEntry -> entityLogEntry.getId().equals(manualAddId2)).findFirst().get().getState(), EntryState.REMOVED); + assertTrue(entityLog.getEntityLogEntry() + .stream() + .anyMatch(entityLogEntry -> entityLogEntry.getId().equals(manualAddId))); + assertEquals(entityLog.getEntityLogEntry() + .stream() + .filter(entityLogEntry -> entityLogEntry.getId().equals(manualAddId)) + .findFirst() + .get().getState(), EntryState.REMOVED); + assertTrue(entityLog.getEntityLogEntry() + .stream() + .anyMatch(entityLogEntry -> entityLogEntry.getId().equals(manualAddId2))); + assertEquals(entityLog.getEntityLogEntry() + .stream() + .filter(entityLogEntry -> entityLogEntry.getId().equals(manualAddId2)) + .findFirst() + .get().getState(), EntryState.REMOVED); manualRedactionEntry.setRequestDate(OffsetDateTime.now()); - request.setManualRedactions(ManualRedactions.builder() - .entriesToAdd(Set.of(manualRedactionEntry, manualRedactionEntry2)) - .idsToRemove(Set.of(idRemoval, idRemoval2)) - .build()); + request.setManualRedactions(ManualRedactions.builder().entriesToAdd(Set.of(manualRedactionEntry, manualRedactionEntry2)).idsToRemove(Set.of(idRemoval2)).build()); analyzeService.reanalyze(request); entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID); - assertTrue(entityLog.getEntityLogEntry().stream().anyMatch(entityLogEntry -> entityLogEntry.getId().equals(manualAddId))); - assertEquals(entityLog.getEntityLogEntry().stream().filter(entityLogEntry -> entityLogEntry.getId().equals(manualAddId)).findFirst().get().getState(), EntryState.APPLIED); - assertTrue(entityLog.getEntityLogEntry().stream().anyMatch(entityLogEntry -> entityLogEntry.getId().equals(manualAddId2))); - assertEquals(entityLog.getEntityLogEntry().stream().filter(entityLogEntry -> entityLogEntry.getId().equals(manualAddId2)).findFirst().get().getState(), EntryState.REMOVED); + assertTrue(entityLog.getEntityLogEntry() + .stream() + .anyMatch(entityLogEntry -> entityLogEntry.getId().equals(manualAddId))); + assertEquals(entityLog.getEntityLogEntry() + .stream() + .filter(entityLogEntry -> entityLogEntry.getId().equals(manualAddId)) + .findFirst() + .get().getState(), EntryState.APPLIED); + assertTrue(entityLog.getEntityLogEntry() + .stream() + .anyMatch(entityLogEntry -> entityLogEntry.getId().equals(manualAddId2))); + assertEquals(entityLog.getEntityLogEntry() + .stream() + .filter(entityLogEntry -> entityLogEntry.getId().equals(manualAddId2)) + .findFirst() + .get().getState(), EntryState.REMOVED); } @@ -1344,21 +1445,35 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest { analyzeService.analyze(request); var entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID); - var david = entityLog.getEntityLogEntry().stream().filter(e -> e.getValue().equals("David")).findFirst().get(); + var david = entityLog.getEntityLogEntry() + .stream() + .filter(e -> e.getValue().equals("David")) + .findFirst() + .get(); request.setManualRedactions(ManualRedactions.builder() - .resizeRedactions(Set.of(ManualResizeRedaction.builder() - .updateDictionary(true) - .annotationId(david.getId()) - .requestDate(OffsetDateTime.now()) - .value("David Ksenia") - .positions(List.of(Rectangle.builder().topLeftX(56.8f).topLeftY(293.564f).width(65.592f).height(15.408f).page(1).build())) - .addToAllDossiers(false) - .build())) - .build()); + .resizeRedactions(Set.of(ManualResizeRedaction.builder() + .updateDictionary(true) + .annotationId(david.getId()) + .requestDate(OffsetDateTime.now()) + .value("David Ksenia") + .positions(List.of(Rectangle.builder() + .topLeftX(56.8f) + .topLeftY(293.564f) + .width(65.592f) + .height(15.408f) + .page(1) + .build())) + .addToAllDossiers(false) + .build())) + .build()); analyzeService.reanalyze(request); entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID); - var resizedEntity = entityLog.getEntityLogEntry().stream().filter(e -> e.getId().equals(david.getId())).findFirst().get(); + var resizedEntity = entityLog.getEntityLogEntry() + .stream() + .filter(e -> e.getId().equals(david.getId())) + .findFirst() + .get(); assertEquals(resizedEntity.getState(), EntryState.APPLIED); assertEquals(resizedEntity.getValue(), "David Ksenia"); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/utils/LayoutParsingRequestProvider.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/utils/LayoutParsingRequestProvider.java index f4625012..43c9f983 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/utils/LayoutParsingRequestProvider.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/utils/LayoutParsingRequestProvider.java @@ -33,7 +33,8 @@ public class LayoutParsingRequestProvider { .textBlockFileStorageId(textBlockFileStorageId) .positionBlockFileStorageId(positionBlockFileStorageId) .pageFileStorageId(pageFileStorageId) - .simplifiedTextStorageId(simplifiedTextStorageId).viewerDocumentStorageId(viewerDocumentStorageId) + .simplifiedTextStorageId(simplifiedTextStorageId) + .viewerDocumentStorageId(viewerDocumentStorageId) .visualLayoutParsingFileId(Optional.empty()) .build(); }