diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/migration/RedactionLogToEntityLogMigrationService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/migration/RedactionLogToEntityLogMigrationService.java index 80f9b21b..b69af59e 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/migration/RedactionLogToEntityLogMigrationService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/migration/RedactionLogToEntityLogMigrationService.java @@ -72,7 +72,7 @@ public class RedactionLogToEntityLogMigrationService { .collect(new MigratedIdsCollector()); log.info("applying manual changes to migrated entities for file {}", fileId); - applyProcessedManualChanges(entitiesToMigrate, manualRedactions); + applyLocalProcessedManualChanges(entitiesToMigrate, manualRedactions); EntityLog entityLog = new EntityLog(); entityLog.setAnalysisNumber(redactionLog.getAnalysisNumber()); @@ -117,7 +117,7 @@ public class RedactionLogToEntityLogMigrationService { } - private void applyProcessedManualChanges(List entitiesToMigrate, ManualRedactions manualRedactions) { + private void applyLocalProcessedManualChanges(List entitiesToMigrate, ManualRedactions manualRedactions) { if (manualRedactions == null) { return; @@ -131,6 +131,7 @@ public class RedactionLogToEntityLogMigrationService { manualRedactions.getRecategorizations()) .flatMap(Collection::stream) .filter(manualChange -> manualChange.getProcessedDate() != null) + .filter(BaseAnnotation::isLocal) .collect(Collectors.groupingBy(BaseAnnotation::getAnnotationId)); entitiesToMigrate.forEach(migrationEntity -> migrationEntity.applyManualChanges(manualChangesPerAnnotationId.getOrDefault(migrationEntity.getOldId(), @@ -151,7 +152,7 @@ public class RedactionLogToEntityLogMigrationService { private List calculateMigrationEntitiesFromRedactionLog(RedactionLog redactionLog, Document document, String dossierTemplateId, String fileId) { - List images = getImageBasedMigrationEntities(redactionLog, document, fileId); + List images = getImageBasedMigrationEntities(redactionLog, document, fileId, dossierTemplateId); List textMigrationEntities = getTextBasedMigrationEntities(redactionLog, document, dossierTemplateId, fileId); return Stream.of(textMigrationEntities.stream(), images.stream()) .flatMap(Function.identity()) @@ -165,7 +166,7 @@ public class RedactionLogToEntityLogMigrationService { } - private List getImageBasedMigrationEntities(RedactionLog redactionLog, Document document, String fileId) { + private List getImageBasedMigrationEntities(RedactionLog redactionLog, Document document, String fileId, String dossierTemplateId) { List images = document.streamAllImages() .collect(Collectors.toList()); @@ -212,7 +213,7 @@ public class RedactionLogToEntityLogMigrationService { } else { closestImage.skip(ruleIdentifier, reason); } - migrationEntities.add(MigrationEntity.fromRedactionLogImage(redactionLogImage, closestImage, fileId)); + migrationEntities.add(MigrationEntity.fromRedactionLogImage(redactionLogImage, closestImage, fileId, dictionaryService, dossierTemplateId)); } return migrationEntities; } @@ -259,7 +260,7 @@ public class RedactionLogToEntityLogMigrationService { .stream() .filter(redactionLogEntry -> !redactionLogEntry.isImage()) .filter(redactionLogEntry -> isOnExistingPage(redactionLogEntry, document.getNumberOfPages())) - .map(entry -> MigrationEntity.fromRedactionLogEntry(entry, dictionaryService.isHint(entry.getType(), dossierTemplateId), fileId)) + .map(entry -> MigrationEntity.fromRedactionLogEntry(entry, fileId, dictionaryService, dossierTemplateId)) .toList(); List precursorEntities = entitiesToMigrate.stream() diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/MigrationEntity.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/MigrationEntity.java index 1eb5bcf0..a3041e01 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/MigrationEntity.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/MigrationEntity.java @@ -20,6 +20,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.BaseAnnotation; +import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntryType; @@ -32,6 +33,7 @@ import com.iqser.red.service.redaction.v1.server.model.document.entity.ManualCha import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Image; import com.iqser.red.service.redaction.v1.server.model.document.nodes.ImageType; +import com.iqser.red.service.redaction.v1.server.service.DictionaryService; import com.iqser.red.service.redaction.v1.server.service.ManualChangeFactory; import com.iqser.red.service.redaction.v1.server.service.ManualChangesApplicationService; @@ -50,6 +52,8 @@ public final class MigrationEntity { private final PrecursorEntity precursorEntity; private final RedactionLogEntry redactionLogEntry; + private final DictionaryService dictionaryService; + private final String dossierTemplateId; private IEntity migratedEntity; private String oldId; private String newId; @@ -59,10 +63,10 @@ public final class MigrationEntity { List manualChanges = new LinkedList<>(); - public static MigrationEntity fromRedactionLogEntry(RedactionLogEntry redactionLogEntry, boolean hint, String fileId) { + public static MigrationEntity fromRedactionLogEntry(RedactionLogEntry redactionLogEntry, String fileId, DictionaryService dictionaryService, String dossierTemplateId) { + boolean hint = dictionaryService.isHint(redactionLogEntry.getType(), dossierTemplateId); PrecursorEntity precursorEntity = createPrecursorEntity(redactionLogEntry, hint); - if (precursorEntity.getEntityType().equals(EntityType.HINT) && !redactionLogEntry.isHint() && !redactionLogEntry.isRedacted()) { precursorEntity.ignore(precursorEntity.getRuleIdentifier(), precursorEntity.getReason()); } else if (redactionLogEntry.lastChangeIsRemoved()) { @@ -77,13 +81,32 @@ public final class MigrationEntity { precursorEntity.skip(precursorEntity.getRuleIdentifier(), precursorEntity.getReason()); } - return MigrationEntity.builder().precursorEntity(precursorEntity).redactionLogEntry(redactionLogEntry).oldId(redactionLogEntry.getId()).fileId(fileId).build(); + return MigrationEntity.builder() + .precursorEntity(precursorEntity) + .redactionLogEntry(redactionLogEntry) + .oldId(redactionLogEntry.getId()) + .fileId(fileId) + .dictionaryService(dictionaryService) + .dossierTemplateId(dossierTemplateId) + .build(); } - public static MigrationEntity fromRedactionLogImage(RedactionLogEntry redactionLogImage, Image image, String fileId) { + public static MigrationEntity fromRedactionLogImage(RedactionLogEntry redactionLogImage, + Image image, + String fileId, + DictionaryService dictionaryService, + String dossierTemplateId) { - return MigrationEntity.builder().redactionLogEntry(redactionLogImage).migratedEntity(image).oldId(redactionLogImage.getId()).newId(image.getId()).fileId(fileId).build(); + return MigrationEntity.builder() + .redactionLogEntry(redactionLogImage) + .migratedEntity(image) + .oldId(redactionLogImage.getId()) + .newId(image.getId()) + .fileId(fileId) + .dictionaryService(dictionaryService) + .dossierTemplateId(dossierTemplateId) + .build(); } @@ -160,18 +183,6 @@ public final class MigrationEntity { } - private static EntryType getEntryType(EntityType entityType) { - - return switch (entityType) { - case ENTITY -> EntryType.ENTITY; - case HINT -> EntryType.HINT; - case FALSE_POSITIVE -> EntryType.FALSE_POSITIVE; - case RECOMMENDATION -> EntryType.RECOMMENDATION; - case FALSE_RECOMMENDATION -> EntryType.FALSE_RECOMMENDATION; - }; - } - - public EntityLogEntry toEntityLogEntry(Map oldToNewIdMapping) { EntityLogEntry entityLogEntry; @@ -246,7 +257,7 @@ public final class MigrationEntity { .textBefore(redactionLogEntry.getTextBefore()) .imageHasTransparency(image.isTransparent()) .state(buildEntryState(image)) - .entryType(redactionLogEntry.isHint() ? EntryType.IMAGE_HINT : EntryType.IMAGE) + .entryType(dictionaryService.isHint(imageType, dossierTemplateId) ? EntryType.IMAGE_HINT : EntryType.IMAGE) .build(); } @@ -357,6 +368,9 @@ public final class MigrationEntity { if (manualChange instanceof ManualResizeRedaction manualResizeRedaction && migratedEntity instanceof TextEntity textEntity) { manualResizeRedaction.setAnnotationId(newId); manualChangesApplicationService.resize(textEntity, manualResizeRedaction); + } else if (manualChange instanceof ManualRecategorization manualRecategorization && migratedEntity instanceof Image image) { + image.setImageType(ImageType.fromString(manualRecategorization.getType())); + migratedEntity.getManualOverwrite().addChange(manualChange); } else { migratedEntity.getManualOverwrite().addChange(manualChange); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityChangeLogService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityChangeLogService.java index d7b7dc80..67b3e824 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityChangeLogService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityChangeLogService.java @@ -13,7 +13,6 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions; -import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval; import io.micrometer.core.annotation.Timed; import lombok.AccessLevel; @@ -39,9 +38,11 @@ public class EntityChangeLogService { boolean hasChanges = false; for (EntityLogEntry entityLogEntry : newEntityLogEntries) { + Optional optionalPreviousEntity = previousEntityLogEntries.stream() .filter(entry -> entry.getId().equals(entityLogEntry.getId())) .findAny(); + if (optionalPreviousEntity.isEmpty()) { hasChanges = true; entityLogEntry.getChanges().add(new Change(analysisNumber, ChangeType.ADDED, now)); @@ -49,14 +50,18 @@ public class EntityChangeLogService { } EntityLogEntry previousEntity = optionalPreviousEntity.get(); + entityLogEntry.getChanges().addAll(previousEntity.getChanges()); + if (!previousEntity.getState().equals(entityLogEntry.getState())) { hasChanges = true; ChangeType changeType = calculateChangeType(entityLogEntry.getState(), previousEntity.getState()); entityLogEntry.getChanges().add(new Change(analysisNumber, changeType, now)); } } + addRemovedEntriesAsRemoved(previousEntityLogEntries, newEntityLogEntries, manualRedactions, analysisNumber, now); + return hasChanges; } @@ -70,11 +75,16 @@ public class EntityChangeLogService { Set existingIds = newEntityLogEntries.stream() .map(EntityLogEntry::getId) .collect(Collectors.toSet()); + List removedEntries = previousEntityLogEntries.stream() .filter(entry -> !existingIds.contains(entry.getId())) .toList(); - removedEntries.forEach(entry -> entry.getChanges().add(new Change(analysisNumber, ChangeType.REMOVED, now))); - removedEntries.forEach(entry -> entry.setState(EntryState.REMOVED)); + + removedEntries.stream() + .filter(entry -> !entry.getState().equals(EntryState.REMOVED)) + .peek(entry -> entry.getChanges().add(new Change(analysisNumber, ChangeType.REMOVED, now))) + .forEach(entry -> entry.setState(EntryState.REMOVED)); + newEntityLogEntries.addAll(removedEntries); }