Merge branch 'RED-7384-bp' into 'release/4.244.x'

RED-7384: fixes for migration backport

See merge request redactmanager/redaction-service!288
This commit is contained in:
Kilian Schüttler 2024-02-27 10:04:38 +01:00
commit b99d9ed59c
15 changed files with 884 additions and 615 deletions

View File

@ -16,7 +16,7 @@ val layoutParserVersion = "0.91.0"
val jacksonVersion = "2.15.2" val jacksonVersion = "2.15.2"
val droolsVersion = "9.44.0.Final" val droolsVersion = "9.44.0.Final"
val pdfBoxVersion = "3.0.0" val pdfBoxVersion = "3.0.0"
val persistenceServiceVersion = "2.338.0" val persistenceServiceVersion = "2.359.0"
val springBootStarterVersion = "3.1.5" val springBootStarterVersion = "3.1.5"
configurations { configurations {

View File

@ -45,7 +45,7 @@ public class LegacyRedactionLogMergeService {
public RedactionLog addManualAddEntriesAndRemoveSkippedImported(RedactionLog redactionLog, ManualRedactions manualRedactions, String dossierTemplateId) { public RedactionLog addManualAddEntriesAndRemoveSkippedImported(RedactionLog redactionLog, ManualRedactions manualRedactions, String dossierTemplateId) {
Set<String> skippedImportedRedactions = new HashSet<>(); Set<String> skippedImportedRedactions = new HashSet<>();
log.info("Merging Redaction log with manual redactions"); log.info("Adding manual add Entries and removing skipped or imported entries");
if (manualRedactions != null) { if (manualRedactions != null) {
var manualRedactionLogEntries = addManualAddEntries(manualRedactions.getEntriesToAdd(), redactionLog.getAnalysisNumber()); var manualRedactionLogEntries = addManualAddEntries(manualRedactions.getEntriesToAdd(), redactionLog.getAnalysisNumber());
@ -92,6 +92,10 @@ public class LegacyRedactionLogMergeService {
return redactionLog; return redactionLog;
} }
public long getNumberOfAffectedAnnotations(ManualRedactions manualRedactions) {
return createManualRedactionWrappers(manualRedactions).stream().map(ManualRedactionWrapper::getId).distinct().count();
}
private List<ManualRedactionWrapper> createManualRedactionWrappers(ManualRedactions manualRedactions) { private List<ManualRedactionWrapper> createManualRedactionWrappers(ManualRedactions manualRedactions) {

View File

@ -0,0 +1,94 @@
package com.iqser.red.service.redaction.v1.server.migration;
import java.util.Collections;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ChangeType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualChange;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Change;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ManualRedactionType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
import com.iqser.red.service.redaction.v1.server.model.MigrationEntity;
public class MigrationMapper {
public static com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Change toEntityLogChanges(Change change) {
return new com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Change(change.getAnalysisNumber(),
toEntityLogType(change.getType()),
change.getDateTime());
}
public static com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualChange toEntityLogManualChanges(com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ManualChange manualChange) {
return new com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualChange(toManualRedactionType(manualChange.getManualRedactionType()),
manualChange.getProcessedDate(),
manualChange.getRequestedDate(),
manualChange.getUserId(),
manualChange.getPropertyChanges());
}
public static ChangeType toEntityLogType(com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ChangeType type) {
return switch (type) {
case ADDED -> ChangeType.ADDED;
case REMOVED -> ChangeType.REMOVED;
case CHANGED -> ChangeType.CHANGED;
};
}
public static com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType toManualRedactionType(ManualRedactionType manualRedactionType) {
return switch (manualRedactionType) {
case ADD_LOCALLY -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.ADD_LOCALLY;
case ADD_TO_DICTIONARY -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.ADD_TO_DICTIONARY;
case REMOVE_LOCALLY -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.REMOVE_LOCALLY;
case REMOVE_FROM_DICTIONARY -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.REMOVE_FROM_DICTIONARY;
case FORCE_REDACT -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.FORCE_REDACT;
case FORCE_HINT -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.FORCE_HINT;
case RECATEGORIZE -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.RECATEGORIZE;
case LEGAL_BASIS_CHANGE -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.LEGAL_BASIS_CHANGE;
case RESIZE -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.RESIZE;
};
}
public static com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine toEntityLogEngine(Engine engine) {
return switch (engine) {
case DICTIONARY -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine.DICTIONARY;
case NER -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine.NER;
case RULE -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine.RULE;
};
}
public static Set<com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine> getMigratedEngines(RedactionLogEntry entry) {
if (entry.getEngines() == null) {
return Collections.emptySet();
}
return entry.getEngines()
.stream()
.map(MigrationMapper::toEntityLogEngine)
.collect(Collectors.toSet());
}
public List<ManualChange> migrateManualChanges(List<com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ManualChange> manualChanges) {
if (manualChanges == null) {
return Collections.emptyList();
}
return manualChanges.stream()
.map(MigrationMapper::toEntityLogManualChanges)
.toList();
}
}

View File

@ -58,17 +58,26 @@ public class MigrationMessageReceiver {
if (redactionLog.getAnalysisVersion() == 0) { if (redactionLog.getAnalysisVersion() == 0) {
redactionLog = legacyVersion0MigrationService.mergeDuplicateAnnotationIds(redactionLog); redactionLog = legacyVersion0MigrationService.mergeDuplicateAnnotationIds(redactionLog);
} else if (migrationRequest.getManualRedactions() != null) { } else if (migrationRequest.getManualRedactions() != null) {
redactionLog = legacyRedactionLogMergeService.addManualAddEntriesAndRemoveSkippedImported(redactionLog, migrationRequest.getManualRedactions(), migrationRequest.getDossierTemplateId()); redactionLog = legacyRedactionLogMergeService.addManualAddEntriesAndRemoveSkippedImported(redactionLog,
migrationRequest.getManualRedactions(),
migrationRequest.getDossierTemplateId());
} }
MigratedEntityLog migratedEntityLog = redactionLogToEntityLogMigrationService.migrate(redactionLog, document, migrationRequest.getDossierTemplateId(), migrationRequest.getManualRedactions()); MigratedEntityLog migratedEntityLog = redactionLogToEntityLogMigrationService.migrate(redactionLog,
document,
migrationRequest.getDossierTemplateId(),
migrationRequest.getManualRedactions(),
migrationRequest.getFileId());
log.info("Storing migrated entityLog and ids to migrate in DB for file {}", migrationRequest.getFileId());
redactionStorageService.storeObject(migrationRequest.getDossierId(), migrationRequest.getFileId(), FileType.ENTITY_LOG, migratedEntityLog.getEntityLog()); redactionStorageService.storeObject(migrationRequest.getDossierId(), migrationRequest.getFileId(), FileType.ENTITY_LOG, migratedEntityLog.getEntityLog());
redactionStorageService.storeObject(migrationRequest.getDossierId(), migrationRequest.getFileId(), FileType.MIGRATED_IDS, migratedEntityLog.getMigratedIds()); redactionStorageService.storeObject(migrationRequest.getDossierId(), migrationRequest.getFileId(), FileType.MIGRATED_IDS, migratedEntityLog.getMigratedIds());
sendFinished(MigrationResponse.builder().dossierId(migrationRequest.getDossierId()).fileId(migrationRequest.getFileId()).build()); sendFinished(MigrationResponse.builder().dossierId(migrationRequest.getDossierId()).fileId(migrationRequest.getFileId()).build());
log.info("Migrated {} redactionLog entries for dossierId {} and fileId {}", log.info("Migrated {} redactionLog entries, found {} annotation ids for migration in the db, {} new manual entries, for dossierId {} and fileId {}",
migratedEntityLog.getEntityLog().getEntityLogEntry().size(), migratedEntityLog.getEntityLog().getEntityLogEntry().size(),
migratedEntityLog.getMigratedIds().getMappings().size(),
migratedEntityLog.getMigratedIds().getManualRedactionEntriesToAdd().size(),
migrationRequest.getDossierId(), migrationRequest.getDossierId(),
migrationRequest.getFileId()); migrationRequest.getFileId());
log.info(""); log.info("");

View File

@ -19,29 +19,24 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.migration.MigratedIds; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.migration.MigratedIds;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.BaseAnnotation; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.BaseAnnotation;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ManualRedactionType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogLegalBasis; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogLegalBasis;
import com.iqser.red.service.redaction.v1.model.MigrationRequest;
import com.iqser.red.service.redaction.v1.server.model.PrecursorEntity;
import com.iqser.red.service.redaction.v1.server.model.MigratedEntityLog; import com.iqser.red.service.redaction.v1.server.model.MigratedEntityLog;
import com.iqser.red.service.redaction.v1.server.model.MigrationEntity; import com.iqser.red.service.redaction.v1.server.model.MigrationEntity;
import com.iqser.red.service.redaction.v1.server.model.PrecursorEntity;
import com.iqser.red.service.redaction.v1.server.model.RectangleWithPage; import com.iqser.red.service.redaction.v1.server.model.RectangleWithPage;
import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity; import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Image; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Image;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.ImageType; import com.iqser.red.service.redaction.v1.server.model.document.nodes.ImageType;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
import com.iqser.red.service.redaction.v1.server.service.DictionaryService; import com.iqser.red.service.redaction.v1.server.service.DictionaryService;
import com.iqser.red.service.redaction.v1.server.service.ManualChangesApplicationService; import com.iqser.red.service.redaction.v1.server.service.ManualChangesApplicationService;
import com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService;
import com.iqser.red.service.redaction.v1.server.service.document.EntityEnrichmentService; import com.iqser.red.service.redaction.v1.server.service.document.EntityEnrichmentService;
import com.iqser.red.service.redaction.v1.server.service.document.EntityFindingUtility; import com.iqser.red.service.redaction.v1.server.service.document.EntityFindingUtility;
import com.iqser.red.service.redaction.v1.server.service.document.EntityFromPrecursorCreationService;
import com.iqser.red.service.redaction.v1.server.utils.IdBuilder; import com.iqser.red.service.redaction.v1.server.utils.IdBuilder;
import com.iqser.red.service.redaction.v1.server.utils.MigratedIdsCollector; import com.iqser.red.service.redaction.v1.server.utils.MigratedIdsCollector;
@ -64,13 +59,16 @@ public class RedactionLogToEntityLogMigrationService {
ManualChangesApplicationService manualChangesApplicationService; ManualChangesApplicationService manualChangesApplicationService;
public MigratedEntityLog migrate(RedactionLog redactionLog, Document document, String dossierTemplateId, ManualRedactions manualRedactions) { public MigratedEntityLog migrate(RedactionLog redactionLog, Document document, String dossierTemplateId, ManualRedactions manualRedactions, String fileId) {
log.info("Migrating entities for file {}", fileId);
List<MigrationEntity> entitiesToMigrate = calculateMigrationEntitiesFromRedactionLog(redactionLog, document, dossierTemplateId, fileId);
List<MigrationEntity> entitiesToMigrate = calculateMigrationEntitiesFromRedactionLog(redactionLog, document, dossierTemplateId);
MigratedIds migratedIds = entitiesToMigrate.stream() MigratedIds migratedIds = entitiesToMigrate.stream()
.collect(new MigratedIdsCollector()); .collect(new MigratedIdsCollector());
applyManualChanges(entitiesToMigrate, manualRedactions); applyManualChanges(entitiesToMigrate, manualRedactions);
log.info("applying manual changes to migrated entities for file {}", fileId);
EntityLog entityLog = new EntityLog(); EntityLog entityLog = new EntityLog();
entityLog.setAnalysisNumber(redactionLog.getAnalysisNumber()); entityLog.setAnalysisNumber(redactionLog.getAnalysisNumber());
@ -85,6 +83,8 @@ public class RedactionLogToEntityLogMigrationService {
.toList()); .toList());
Map<String, String> oldToNewIDMapping = migratedIds.buildOldToNewMapping(); Map<String, String> oldToNewIDMapping = migratedIds.buildOldToNewMapping();
log.info("Writing migrated entities to entityLog for file {}", fileId);
entityLog.setEntityLogEntry(entitiesToMigrate.stream() entityLog.setEntityLogEntry(entitiesToMigrate.stream()
.map(migrationEntity -> migrationEntity.toEntityLogEntry(oldToNewIDMapping)) .map(migrationEntity -> migrationEntity.toEntityLogEntry(oldToNewIDMapping))
.toList()); .toList());
@ -102,6 +102,13 @@ public class RedactionLogToEntityLogMigrationService {
.filter(m -> !m.getOldId().equals(m.getNewId())) .filter(m -> !m.getOldId().equals(m.getNewId()))
.collect(new MigratedIdsCollector()); .collect(new MigratedIdsCollector());
List<ManualRedactionEntry> manualRedactionEntriesToAdd = entitiesToMigrate.stream()
.filter(MigrationEntity::needsManualEntry)
.map(MigrationEntity::buildManualRedactionEntry)
.toList();
idsToMigrateInDb.setManualRedactionEntriesToAdd(manualRedactionEntriesToAdd);
return new MigratedEntityLog(idsToMigrateInDb, entityLog); return new MigratedEntityLog(idsToMigrateInDb, entityLog);
} }
@ -117,27 +124,14 @@ public class RedactionLogToEntityLogMigrationService {
manualRedactions.getForceRedactions(), manualRedactions.getForceRedactions(),
manualRedactions.getResizeRedactions(), manualRedactions.getResizeRedactions(),
manualRedactions.getLegalBasisChanges(), manualRedactions.getLegalBasisChanges(),
manualRedactions.getRecategorizations(), manualRedactions.getRecategorizations())
manualRedactions.getLegalBasisChanges())
.flatMap(Collection::stream) .flatMap(Collection::stream)
.collect(Collectors.groupingBy(BaseAnnotation::getAnnotationId)); .collect(Collectors.groupingBy(BaseAnnotation::getAnnotationId));
entitiesToMigrate.forEach(migrationEntity -> manualChangesPerAnnotationId.getOrDefault(migrationEntity.getOldId(), Collections.emptyList()) entitiesToMigrate.forEach(migrationEntity -> migrationEntity.applyManualChanges(manualChangesPerAnnotationId.getOrDefault(migrationEntity.getOldId(),
.forEach(manualChange -> { Collections.emptyList()),
if (manualChange instanceof ManualResizeRedaction manualResizeRedaction && migrationEntity.getMigratedEntity() instanceof TextEntity textEntity) { manualChangesApplicationService));
ManualResizeRedaction migratedManualResizeRedaction = ManualResizeRedaction.builder()
.positions(manualResizeRedaction.getPositions())
.annotationId(migrationEntity.getNewId())
.updateDictionary(manualResizeRedaction.getUpdateDictionary())
.addToAllDossiers(manualResizeRedaction.isAddToAllDossiers())
.textAfter(manualResizeRedaction.getTextAfter())
.textBefore(manualResizeRedaction.getTextBefore())
.build();
manualChangesApplicationService.resize(textEntity, migratedManualResizeRedaction);
} else {
migrationEntity.getMigratedEntity().getManualOverwrite().addChange(manualChange);
}
}));
} }
@ -147,10 +141,10 @@ public class RedactionLogToEntityLogMigrationService {
} }
private List<MigrationEntity> calculateMigrationEntitiesFromRedactionLog(RedactionLog redactionLog, Document document, String dossierTemplateId) { private List<MigrationEntity> calculateMigrationEntitiesFromRedactionLog(RedactionLog redactionLog, Document document, String dossierTemplateId, String fileId) {
List<MigrationEntity> images = getImageBasedMigrationEntities(redactionLog, document, dossierTemplateId); List<MigrationEntity> images = getImageBasedMigrationEntities(redactionLog, document, fileId);
List<MigrationEntity> textMigrationEntities = getTextBasedMigrationEntities(redactionLog, document, dossierTemplateId); List<MigrationEntity> textMigrationEntities = getTextBasedMigrationEntities(redactionLog, document, dossierTemplateId, fileId);
return Stream.of(textMigrationEntities.stream(), images.stream()) return Stream.of(textMigrationEntities.stream(), images.stream())
.flatMap(Function.identity()) .flatMap(Function.identity())
.toList(); .toList();
@ -163,7 +157,7 @@ public class RedactionLogToEntityLogMigrationService {
} }
private List<MigrationEntity> getImageBasedMigrationEntities(RedactionLog redactionLog, Document document, String dossierTemplateId) { private List<MigrationEntity> getImageBasedMigrationEntities(RedactionLog redactionLog, Document document, String fileId) {
List<Image> images = document.streamAllImages() List<Image> images = document.streamAllImages()
.collect(Collectors.toList()); .collect(Collectors.toList());
@ -195,7 +189,8 @@ public class RedactionLogToEntityLogMigrationService {
} }
String ruleIdentifier; String ruleIdentifier;
String reason = Optional.ofNullable(redactionLogImage.getReason()).orElse(""); String reason = Optional.ofNullable(redactionLogImage.getReason())
.orElse("");
if (redactionLogImage.getMatchedRule().isBlank() || redactionLogImage.getMatchedRule() == null) { if (redactionLogImage.getMatchedRule().isBlank() || redactionLogImage.getMatchedRule() == null) {
ruleIdentifier = "OLDIMG.0.0"; ruleIdentifier = "OLDIMG.0.0";
} else { } else {
@ -209,7 +204,7 @@ public class RedactionLogToEntityLogMigrationService {
} else { } else {
closestImage.skip(ruleIdentifier, reason); closestImage.skip(ruleIdentifier, reason);
} }
migrationEntities.add(new MigrationEntity(null, redactionLogImage, closestImage, redactionLogImage.getId(), closestImage.getId())); migrationEntities.add(MigrationEntity.fromRedactionLogImage(redactionLogImage, closestImage, fileId));
} }
return migrationEntities; return migrationEntities;
} }
@ -250,40 +245,20 @@ public class RedactionLogToEntityLogMigrationService {
} }
private List<MigrationEntity> getTextBasedMigrationEntities(RedactionLog redactionLog, Document document, String dossierTemplateId) { private List<MigrationEntity> getTextBasedMigrationEntities(RedactionLog redactionLog, Document document, String dossierTemplateId, String fileId) {
List<MigrationEntity> entitiesToMigrate = redactionLog.getRedactionLogEntry() List<MigrationEntity> entitiesToMigrate = redactionLog.getRedactionLogEntry()
.stream() .stream()
.filter(redactionLogEntry -> !redactionLogEntry.isImage()) .filter(redactionLogEntry -> !redactionLogEntry.isImage())
.map(entry -> MigrationEntity.fromRedactionLogEntry(entry, dictionaryService.isHint(entry.getType(), dossierTemplateId))) .map(entry -> MigrationEntity.fromRedactionLogEntry(entry, dictionaryService.isHint(entry.getType(), dossierTemplateId), fileId))
.peek(migrationEntity -> {
if (migrationEntity.getPrecursorEntity().getEntityType().equals(EntityType.HINT) &&//
!migrationEntity.getRedactionLogEntry().isHint() &&//
!migrationEntity.getRedactionLogEntry().isRedacted()) {
migrationEntity.getPrecursorEntity().ignore(migrationEntity.getPrecursorEntity().getRuleIdentifier(), migrationEntity.getPrecursorEntity().getReason());
} else if (migrationEntity.getRedactionLogEntry().lastChangeIsRemoved()) {
migrationEntity.getPrecursorEntity().remove(migrationEntity.getPrecursorEntity().getRuleIdentifier(), migrationEntity.getPrecursorEntity().getReason());
} else if (lastManualChangeIsRemove(migrationEntity)) {
migrationEntity.getPrecursorEntity().ignore(migrationEntity.getPrecursorEntity().getRuleIdentifier(), migrationEntity.getPrecursorEntity().getReason());
} else if (migrationEntity.getPrecursorEntity().isApplied() && migrationEntity.getRedactionLogEntry().isRecommendation()) {
migrationEntity.getPrecursorEntity()
.skip(migrationEntity.getPrecursorEntity().getRuleIdentifier(), migrationEntity.getPrecursorEntity().getReason());
} else if (migrationEntity.getPrecursorEntity().isApplied()) {
migrationEntity.getPrecursorEntity()
.apply(migrationEntity.getPrecursorEntity().getRuleIdentifier(),
migrationEntity.getPrecursorEntity().getReason(),
migrationEntity.getPrecursorEntity().getLegalBasis());
} else {
migrationEntity.getPrecursorEntity()
.skip(migrationEntity.getPrecursorEntity().getRuleIdentifier(), migrationEntity.getPrecursorEntity().getReason());
}
})
.toList(); .toList();
Map<String, List<TextEntity>> tempEntitiesByValue = entityFindingUtility.findAllPossibleEntitiesAndGroupByValue(document, List<PrecursorEntity> precursorEntities = entitiesToMigrate.stream()
entitiesToMigrate.stream()
.map(MigrationEntity::getPrecursorEntity) .map(MigrationEntity::getPrecursorEntity)
.toList()); .toList();
log.info("Finding all possible entities");
Map<String, List<TextEntity>> tempEntitiesByValue = entityFindingUtility.findAllPossibleEntitiesAndGroupByValue(document, precursorEntities);
for (MigrationEntity migrationEntity : entitiesToMigrate) { for (MigrationEntity migrationEntity : entitiesToMigrate) {
Optional<TextEntity> optionalTextEntity = entityFindingUtility.findClosestEntityAndReturnEmptyIfNotFound(migrationEntity.getPrecursorEntity(), Optional<TextEntity> optionalTextEntity = entityFindingUtility.findClosestEntityAndReturnEmptyIfNotFound(migrationEntity.getPrecursorEntity(),
@ -297,45 +272,19 @@ public class RedactionLogToEntityLogMigrationService {
continue; continue;
} }
TextEntity entity = createCorrectEntity(migrationEntity.getPrecursorEntity(), document, optionalTextEntity.get().getTextRange()); TextEntity migratedEntity = EntityFromPrecursorCreationService.createCorrectEntity(migrationEntity.getPrecursorEntity(), optionalTextEntity.get(), true);
migrationEntity.setMigratedEntity(entity);
migrationEntity.setOldId(migrationEntity.getPrecursorEntity().getId());
migrationEntity.setNewId(entity.getId()); // Can only be on one page, since redactionLogEntries can only be on one page
migrationEntity.setMigratedEntity(migratedEntity);
migrationEntity.setOldId(migrationEntity.getPrecursorEntity().getId());
migrationEntity.setNewId(migratedEntity.getId());
} }
tempEntitiesByValue.values() tempEntitiesByValue.values()
.stream() .stream()
.flatMap(Collection::stream) .flatMap(Collection::stream)
.forEach(TextEntity::removeFromGraph); .forEach(TextEntity::removeFromGraph);
return entitiesToMigrate; return entitiesToMigrate;
} }
private static boolean lastManualChangeIsRemove(MigrationEntity migrationEntity) {
if (migrationEntity.getRedactionLogEntry().getManualChanges() == null) {
return false;
}
return migrationEntity.getRedactionLogEntry().getManualChanges()
.stream()
.reduce((a, b) -> b)
.map(m -> m.getManualRedactionType().equals(ManualRedactionType.REMOVE_LOCALLY))
.orElse(false);
}
private TextEntity createCorrectEntity(PrecursorEntity precursorEntity, SemanticNode node, TextRange closestTextRange) {
EntityCreationService entityCreationService = new EntityCreationService(entityEnrichmentService);
TextEntity correctEntity = entityCreationService.forceByTextRange(closestTextRange, precursorEntity.getType(), precursorEntity.getEntityType(), node);
correctEntity.addMatchedRules(precursorEntity.getMatchedRuleList());
correctEntity.setDictionaryEntry(precursorEntity.isDictionaryEntry());
correctEntity.setDossierDictionaryEntry(precursorEntity.isDossierDictionaryEntry());
correctEntity.getManualOverwrite().addChanges(precursorEntity.getManualOverwrite().getManualChangeLog());
return correctEntity;
}
} }

View File

@ -1,7 +1,10 @@
package com.iqser.red.service.redaction.v1.server.model; package com.iqser.red.service.redaction.v1.server.model;
import java.util.List;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.migration.MigratedIds; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.migration.MigratedIds;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
import lombok.AllArgsConstructor; import lombok.AllArgsConstructor;
import lombok.Builder; import lombok.Builder;
@ -16,5 +19,4 @@ public class MigratedEntityLog {
MigratedIds migratedIds; MigratedIds migratedIds;
EntityLog entityLog; EntityLog entityLog;
} }

View File

@ -1,34 +1,45 @@
package com.iqser.red.service.redaction.v1.server.model; package com.iqser.red.service.redaction.v1.server.model;
import static com.iqser.red.service.redaction.v1.server.service.EntityLogCreatorService.buildEntryState;
import static com.iqser.red.service.redaction.v1.server.service.EntityLogCreatorService.buildEntryType;
import java.awt.geom.Rectangle2D;
import java.util.Collections; import java.util.Collections;
import java.util.LinkedList;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Optional; import java.util.Optional;
import java.util.Set; import java.util.Set;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ChangeType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryType; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Change; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.BaseAnnotation;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ManualChange; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ManualRedactionType; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ManualRedactionType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
import com.iqser.red.service.redaction.v1.server.migration.MigrationMapper;
import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType; import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
import com.iqser.red.service.redaction.v1.server.model.document.entity.IEntity; import com.iqser.red.service.redaction.v1.server.model.document.entity.IEntity;
import com.iqser.red.service.redaction.v1.server.model.document.entity.ManualChangeOverwrite; import com.iqser.red.service.redaction.v1.server.model.document.entity.ManualChangeOverwrite;
import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity; import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Image; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Image;
import com.iqser.red.service.redaction.v1.server.service.ManualChangeFactory; import com.iqser.red.service.redaction.v1.server.service.ManualChangeFactory;
import com.iqser.red.service.redaction.v1.server.service.ManualChangesApplicationService;
import lombok.AllArgsConstructor; import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data; import lombok.Data;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@Data @Data
@Builder
@AllArgsConstructor @AllArgsConstructor
@RequiredArgsConstructor @RequiredArgsConstructor
public final class MigrationEntity { public final class MigrationEntity {
@ -38,28 +49,73 @@ public final class MigrationEntity {
private IEntity migratedEntity; private IEntity migratedEntity;
private String oldId; private String oldId;
private String newId; private String newId;
private String fileId;
@Builder.Default
List<BaseAnnotation> manualChanges = new LinkedList<>();
public static MigrationEntity fromRedactionLogEntry(RedactionLogEntry redactionLogEntry, boolean hint) { public static MigrationEntity fromRedactionLogEntry(RedactionLogEntry redactionLogEntry, boolean hint, String fileId) {
return new MigrationEntity(createPrecursorEntity(redactionLogEntry, hint), redactionLogEntry); PrecursorEntity precursorEntity = createPrecursorEntity(redactionLogEntry, hint);
if (precursorEntity.getEntityType().equals(EntityType.HINT) && !redactionLogEntry.isHint() && !redactionLogEntry.isRedacted()) {
precursorEntity.ignore(precursorEntity.getRuleIdentifier(), precursorEntity.getReason());
} else if (redactionLogEntry.lastChangeIsRemoved()) {
precursorEntity.remove(precursorEntity.getRuleIdentifier(), precursorEntity.getReason());
} else if (lastManualChangeIsRemove(redactionLogEntry)) {
precursorEntity.ignore(precursorEntity.getRuleIdentifier(), precursorEntity.getReason());
} else if (precursorEntity.isApplied() && redactionLogEntry.isRecommendation()) {
precursorEntity.skip(precursorEntity.getRuleIdentifier(), precursorEntity.getReason());
} else if (precursorEntity.isApplied()) {
precursorEntity.apply(precursorEntity.getRuleIdentifier(), precursorEntity.getReason(), precursorEntity.getLegalBasis());
} else {
precursorEntity.skip(precursorEntity.getRuleIdentifier(), precursorEntity.getReason());
}
return MigrationEntity.builder().precursorEntity(precursorEntity).redactionLogEntry(redactionLogEntry).oldId(redactionLogEntry.getId()).fileId(fileId).build();
}
public static MigrationEntity fromRedactionLogImage(RedactionLogEntry redactionLogImage, Image image, String fileId) {
return MigrationEntity.builder().redactionLogEntry(redactionLogImage).migratedEntity(image).oldId(redactionLogImage.getId()).newId(image.getId()).fileId(fileId).build();
}
private static boolean lastManualChangeIsRemove(RedactionLogEntry redactionLogEntry) {
if (redactionLogEntry.getManualChanges() == null) {
return false;
}
return redactionLogEntry.getManualChanges()
.stream()
.reduce((a, b) -> b)
.map(m -> m.getManualRedactionType().equals(ManualRedactionType.REMOVE_LOCALLY))
.orElse(false);
} }
public static PrecursorEntity createPrecursorEntity(RedactionLogEntry redactionLogEntry, boolean hint) { public static PrecursorEntity createPrecursorEntity(RedactionLogEntry redactionLogEntry, boolean hint) {
String ruleIdentifier = buildRuleIdentifier(redactionLogEntry); String ruleIdentifier = buildRuleIdentifier(redactionLogEntry);
List<RectangleWithPage> rectangleWithPages = redactionLogEntry.getPositions().stream().map(RectangleWithPage::fromRedactionLogRectangle).toList(); List<RectangleWithPage> rectangleWithPages = redactionLogEntry.getPositions()
.stream()
.map(RectangleWithPage::fromRedactionLogRectangle)
.toList();
EntityType entityType = getEntityType(redactionLogEntry, hint); EntityType entityType = getEntityType(redactionLogEntry, hint);
return PrecursorEntity.builder() return PrecursorEntity.builder()
.id(redactionLogEntry.getId()) .id(redactionLogEntry.getId())
.value(redactionLogEntry.getValue()) .value(redactionLogEntry.getValue())
.entityPosition(rectangleWithPages) .entityPosition(rectangleWithPages)
.ruleIdentifier(ruleIdentifier) .ruleIdentifier(ruleIdentifier)
.reason(Optional.ofNullable(redactionLogEntry.getReason()).orElse("")) .reason(Optional.ofNullable(redactionLogEntry.getReason())
.orElse(""))
.legalBasis(redactionLogEntry.getLegalBasis()) .legalBasis(redactionLogEntry.getLegalBasis())
.type(redactionLogEntry.getType()) .type(redactionLogEntry.getType())
.section(redactionLogEntry.getSection()) .section(redactionLogEntry.getSection())
.engines(MigrationMapper.getMigratedEngines(redactionLogEntry))
.entityType(entityType) .entityType(entityType)
.applied(redactionLogEntry.isRedacted()) .applied(redactionLogEntry.isRedacted())
.isDictionaryEntry(redactionLogEntry.isDictionaryEntry()) .isDictionaryEntry(redactionLogEntry.isDictionaryEntry())
@ -100,14 +156,6 @@ public final class MigrationEntity {
} }
private static com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Change toEntityLogChanges(Change change) {
return new com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Change(change.getAnalysisNumber(),
toEntityLogType(change.getType()),
change.getDateTime());
}
private static EntryType getEntryType(EntityType entityType) { private static EntryType getEntryType(EntityType entityType) {
return switch (entityType) { return switch (entityType) {
@ -120,42 +168,6 @@ public final class MigrationEntity {
} }
private static com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualChange toEntityLogManualChanges(ManualChange manualChange) {
return new com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualChange(toManualRedactionType(manualChange.getManualRedactionType()),
manualChange.getProcessedDate(),
manualChange.getRequestedDate(),
manualChange.getUserId(),
manualChange.getPropertyChanges());
}
private static ChangeType toEntityLogType(com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ChangeType type) {
return switch (type) {
case ADDED -> ChangeType.ADDED;
case REMOVED -> ChangeType.REMOVED;
case CHANGED -> ChangeType.CHANGED;
};
}
private static com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType toManualRedactionType(ManualRedactionType manualRedactionType) {
return switch (manualRedactionType) {
case ADD_LOCALLY -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.ADD_LOCALLY;
case ADD_TO_DICTIONARY -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.ADD_TO_DICTIONARY;
case REMOVE_LOCALLY -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.REMOVE_LOCALLY;
case REMOVE_FROM_DICTIONARY -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.REMOVE_FROM_DICTIONARY;
case FORCE_REDACT -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.FORCE_REDACT;
case FORCE_HINT -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.FORCE_HINT;
case RECATEGORIZE -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.RECATEGORIZE;
case LEGAL_BASIS_CHANGE -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.LEGAL_BASIS_CHANGE;
case RESIZE -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.RESIZE;
};
}
public EntityLogEntry toEntityLogEntry(Map<String, String> oldToNewIdMapping) { public EntityLogEntry toEntityLogEntry(Map<String, String> oldToNewIdMapping) {
EntityLogEntry entityLogEntry; EntityLogEntry entityLogEntry;
@ -171,10 +183,13 @@ public final class MigrationEntity {
entityLogEntry.setManualChanges(ManualChangeFactory.toManualChangeList(migratedEntity.getManualOverwrite().getManualChangeLog(), redactionLogEntry.isHint())); entityLogEntry.setManualChanges(ManualChangeFactory.toManualChangeList(migratedEntity.getManualOverwrite().getManualChangeLog(), redactionLogEntry.isHint()));
entityLogEntry.setColor(redactionLogEntry.getColor()); entityLogEntry.setColor(redactionLogEntry.getColor());
entityLogEntry.setChanges(redactionLogEntry.getChanges().stream().map(MigrationEntity::toEntityLogChanges).toList()); entityLogEntry.setChanges(redactionLogEntry.getChanges()
.stream()
.map(MigrationMapper::toEntityLogChanges)
.toList());
entityLogEntry.setReference(migrateSetOfIds(redactionLogEntry.getReference(), oldToNewIdMapping)); entityLogEntry.setReference(migrateSetOfIds(redactionLogEntry.getReference(), oldToNewIdMapping));
entityLogEntry.setImportedRedactionIntersections(migrateSetOfIds(redactionLogEntry.getImportedRedactionIntersections(), oldToNewIdMapping)); entityLogEntry.setImportedRedactionIntersections(migrateSetOfIds(redactionLogEntry.getImportedRedactionIntersections(), oldToNewIdMapping));
entityLogEntry.setEngines(getMigratedEngines(redactionLogEntry)); entityLogEntry.setEngines(MigrationMapper.getMigratedEngines(redactionLogEntry));
if (redactionLogEntry.getLegalBasis() != null) { if (redactionLogEntry.getLegalBasis() != null) {
entityLogEntry.setLegalBasis(redactionLogEntry.getLegalBasis()); entityLogEntry.setLegalBasis(redactionLogEntry.getLegalBasis());
} }
@ -198,40 +213,14 @@ public final class MigrationEntity {
} }
private List<com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualChange> migrateManualChanges(List<ManualChange> manualChanges) {
if (manualChanges == null) {
return Collections.emptyList();
}
return manualChanges.stream().map(MigrationEntity::toEntityLogManualChanges).toList();
}
private static Set<com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine> getMigratedEngines(RedactionLogEntry entry) {
if (entry.getEngines() == null) {
return Collections.emptySet();
}
return entry.getEngines().stream().map(MigrationEntity::toEntityLogEngine).collect(Collectors.toSet());
}
private Set<String> migrateSetOfIds(Set<String> ids, Map<String, String> oldToNewIdMapping) { private Set<String> migrateSetOfIds(Set<String> ids, Map<String, String> oldToNewIdMapping) {
if (ids == null) { if (ids == null) {
return Collections.emptySet(); return Collections.emptySet();
} }
return ids.stream().map(oldToNewIdMapping::get).collect(Collectors.toSet()); return ids.stream()
} .map(oldToNewIdMapping::get)
.collect(Collectors.toSet());
private static com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine toEntityLogEngine(Engine engine) {
return switch (engine) {
case DICTIONARY -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine.DICTIONARY;
case NER -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine.NER;
case RULE -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine.RULE;
};
} }
@ -249,7 +238,8 @@ public final class MigrationEntity {
.positions(positions) .positions(positions)
.containingNodeId(image.getTreeId()) .containingNodeId(image.getTreeId())
.closestHeadline(image.getHeadline().getTextBlock().getSearchText()) .closestHeadline(image.getHeadline().getTextBlock().getSearchText())
.section(redactionLogEntry.getSection()) .section(image.getManualOverwrite().getSection()
.orElse(redactionLogEntry.getSection()))
.textAfter(redactionLogEntry.getTextAfter()) .textAfter(redactionLogEntry.getTextAfter())
.textBefore(redactionLogEntry.getTextBefore()) .textBefore(redactionLogEntry.getTextBefore())
.imageHasTransparency(image.isTransparent()) .imageHasTransparency(image.isTransparent())
@ -270,7 +260,8 @@ public final class MigrationEntity {
.type(precursorEntity.type()) .type(precursorEntity.type())
.state(buildEntryState(precursorEntity)) .state(buildEntryState(precursorEntity))
.entryType(buildEntryType(precursorEntity)) .entryType(buildEntryType(precursorEntity))
.section(redactionLogEntry.getSection()) .section(precursorEntity.getManualOverwrite().getSection()
.orElse(redactionLogEntry.getSection()))
.textAfter(redactionLogEntry.getTextAfter()) .textAfter(redactionLogEntry.getTextAfter())
.textBefore(redactionLogEntry.getTextBefore()) .textBefore(redactionLogEntry.getTextBefore())
.containingNodeId(Collections.emptyList()) .containingNodeId(Collections.emptyList())
@ -280,8 +271,7 @@ public final class MigrationEntity {
.dossierDictionaryEntry(precursorEntity.isDossierDictionaryEntry()) .dossierDictionaryEntry(precursorEntity.isDossierDictionaryEntry())
.startOffset(-1) .startOffset(-1)
.endOffset(-1) .endOffset(-1)
.positions(precursorEntity.getManualOverwrite() .positions(precursorEntity.getManualOverwrite().getPositions()
.getPositions()
.orElse(precursorEntity.getEntityPosition()) .orElse(precursorEntity.getEntityPosition())
.stream() .stream()
.map(entityPosition -> new Position(entityPosition.rectangle2D(), entityPosition.pageNumber())) .map(entityPosition -> new Position(entityPosition.rectangle2D(), entityPosition.pageNumber()))
@ -300,11 +290,13 @@ public final class MigrationEntity {
.positions(rectanglesPerLine) .positions(rectanglesPerLine)
.reason(entity.buildReasonWithManualChangeDescriptions()) .reason(entity.buildReasonWithManualChangeDescriptions())
.legalBasis(entity.legalBasis()) .legalBasis(entity.legalBasis())
.value(entity.getManualOverwrite().getValue().orElse(entity.getMatchedRule().isWriteValueWithLineBreaks() ? entity.getValueWithLineBreaks() : entity.getValue())) .value(entity.getManualOverwrite().getValue()
.orElse(entity.getMatchedRule().isWriteValueWithLineBreaks() ? entity.getValueWithLineBreaks() : entity.getValue()))
.type(entity.type()) .type(entity.type())
.section(redactionLogEntry.getSection()) .section(entity.getManualOverwrite().getSection()
.textAfter(redactionLogEntry.getTextAfter()) .orElse(redactionLogEntry.getSection()))
.textBefore(redactionLogEntry.getTextBefore()) .textAfter(entity.getTextAfter())
.textBefore(entity.getTextBefore())
.containingNodeId(entity.getDeepestFullyContainingNode().getTreeId()) .containingNodeId(entity.getDeepestFullyContainingNode().getTreeId())
.closestHeadline(entity.getDeepestFullyContainingNode().getHeadline().getTextBlock().getSearchText()) .closestHeadline(entity.getDeepestFullyContainingNode().getHeadline().getTextBlock().getSearchText())
.matchedRule(entity.getMatchedRule().getRuleIdentifier().toString()) .matchedRule(entity.getMatchedRule().getRuleIdentifier().toString())
@ -322,54 +314,129 @@ public final class MigrationEntity {
private static List<Position> getRectanglesPerLine(TextEntity entity) { private static List<Position> getRectanglesPerLine(TextEntity entity) {
return getPositionsFromOverride(entity).orElse(entity.getPositionsOnPagePerPage() return getPositionsFromOverride(entity).orElse(entity.getPositionsOnPagePerPage()
.get(0) .get(0).getRectanglePerLine()
.getRectanglePerLine()
.stream() .stream()
.map(rectangle2D -> new Position(rectangle2D, entity.getPositionsOnPagePerPage().get(0).getPage().getNumber())) .map(rectangle2D -> new Position(rectangle2D,
entity.getPositionsOnPagePerPage()
.get(0).getPage().getNumber()))
.toList()); .toList());
} }
private static Optional<List<Position>> getPositionsFromOverride(IEntity entity) { private static Optional<List<Position>> getPositionsFromOverride(IEntity entity) {
return entity.getManualOverwrite().getPositions().map(rects -> rects.stream().map(r -> new Position(r.rectangle2D(), r.pageNumber())).toList()); return entity.getManualOverwrite().getPositions()
} .map(rects -> rects.stream()
.map(r -> new Position(r.rectangle2D(), r.pageNumber()))
.toList());
private EntryState buildEntryState(IEntity entity) {
if (entity.applied() && entity.active()) {
return EntryState.APPLIED;
} else if (entity.skipped() && entity.active()) {
return EntryState.SKIPPED;
} else if (entity.ignored()) {
return EntryState.IGNORED;
} else {
return EntryState.REMOVED;
}
}
private EntryType buildEntryType(IEntity entity) {
if (entity instanceof TextEntity textEntity) {
return getEntryType(textEntity.getEntityType());
} else if (entity instanceof PrecursorEntity precursorEntity) {
if (precursorEntity.isRectangle()) {
return EntryType.AREA;
}
return getEntryType(precursorEntity.getEntityType());
} else if (entity instanceof Image) {
return EntryType.IMAGE;
}
throw new UnsupportedOperationException(String.format("Entity subclass %s is not implemented!", entity.getClass()));
} }
public boolean hasManualChangesOrComments() { public boolean hasManualChangesOrComments() {
return !(redactionLogEntry.getManualChanges() == null || redactionLogEntry.getManualChanges().isEmpty()) || // return !(redactionLogEntry.getManualChanges() == null || redactionLogEntry.getManualChanges().isEmpty()) || //
!(redactionLogEntry.getComments() == null || redactionLogEntry.getComments().isEmpty()); !(redactionLogEntry.getComments() == null || redactionLogEntry.getComments().isEmpty()) //
|| hasManualChanges();
}
public boolean hasManualChanges() {
return !manualChanges.isEmpty();
}
public void applyManualChanges(List<BaseAnnotation> manualChangesToApply, ManualChangesApplicationService manualChangesApplicationService) {
manualChanges.addAll(manualChangesToApply);
manualChangesToApply.forEach(manualChange -> {
if (manualChange instanceof ManualResizeRedaction manualResizeRedaction && migratedEntity instanceof TextEntity textEntity) {
// Due to the value in the old redaction log already being resized, there is no way to find the original entity ID and therefore to migrate the resize annotation correctly.
// Instead, we add an add_locally change to the db.
ManualResizeRedaction migratedManualResizeRedaction = ManualResizeRedaction.builder()
.positions(manualResizeRedaction.getPositions())
.annotationId(getNewId())
.updateDictionary(manualResizeRedaction.getUpdateDictionary())
.addToAllDossiers(manualResizeRedaction.isAddToAllDossiers())
.textAfter(manualResizeRedaction.getTextAfter())
.textBefore(manualResizeRedaction.getTextBefore())
.build();
manualChangesApplicationService.resize(textEntity, migratedManualResizeRedaction);
} else {
migratedEntity.getManualOverwrite().addChange(manualChange);
}
});
}
public ManualRedactionEntry buildManualRedactionEntry() {
assert hasManualChanges();
// currently we need to insert a manual redaction entry, whenever an entity has been resized.
String user = manualChanges.stream()
.filter(mc -> mc instanceof ManualResizeRedaction)
.findFirst()
.orElse(manualChanges.get(0)).getUser();
return ManualRedactionEntry.builder()
.annotationId(newId)
.fileId(fileId)
.type(redactionLogEntry.getType())
.value(redactionLogEntry.getValue())
.reason(redactionLogEntry.getReason())
.legalBasis(redactionLogEntry.getLegalBasis())
.section(redactionLogEntry.getSection())
.addToDictionary(false)
.addToDossierDictionary(false)
.rectangle(false)
.positions(buildPositions(migratedEntity))
.user(user)
.build();
}
private List<Rectangle> buildPositions(IEntity entity) {
if (entity instanceof TextEntity textEntity) {
var positionsOnPage = textEntity.getPositionsOnPagePerPage()
.get(0);
return positionsOnPage.getRectanglePerLine()
.stream()
.map(p -> new Rectangle((float) p.getX(), (float) p.getY(), (float) p.getWidth(), (float) p.getHeight(), positionsOnPage.getPage().getNumber()))
.toList();
}
if (entity instanceof PrecursorEntity pEntity) {
return pEntity.getManualOverwrite().getPositions()
.orElse(pEntity.getEntityPosition())
.stream()
.map(p -> new Rectangle((float) p.rectangle2D().getX(),
(float) p.rectangle2D().getY(),
(float) p.rectangle2D().getWidth(),
(float) p.rectangle2D().getHeight(),
p.pageNumber()))
.toList();
}
if (entity instanceof Image image) {
Rectangle2D position = image.getManualOverwrite().getPositions()
.map(p -> p.get(0).rectangle2D())
.orElse(image.getPosition());
return List.of(new Rectangle((float) position.getX(), (float) position.getY(), (float) position.getWidth(), (float) position.getHeight(), image.getPage().getNumber()));
} else {
throw new UnsupportedOperationException();
}
}
public boolean needsManualEntry() {
return manualChanges.stream()
.anyMatch(mc -> mc instanceof ManualResizeRedaction && !((ManualResizeRedaction) mc).getUpdateDictionary()) && !(migratedEntity instanceof Image);
} }
} }

View File

@ -43,7 +43,6 @@ public class PrecursorEntity implements IEntity {
String type; String type;
String section; String section;
EntityType entityType; EntityType entityType;
EntryType entryType;
boolean applied; boolean applied;
boolean isDictionaryEntry; boolean isDictionaryEntry;
boolean isDossierDictionaryEntry; boolean isDossierDictionaryEntry;
@ -61,8 +60,8 @@ public class PrecursorEntity implements IEntity {
.stream() .stream()
.map(RectangleWithPage::fromAnnotationRectangle) .map(RectangleWithPage::fromAnnotationRectangle)
.toList(); .toList();
var entityType = hint ? EntityType.HINT : EntityType.ENTITY; var entityType = hint ? EntityType.HINT : EntityType.ENTITY;
var entryType = hint ? EntryType.HINT : (manualRedactionEntry.isRectangle() ? EntryType.AREA : EntryType.ENTITY);
ManualChangeOverwrite manualChangeOverwrite = new ManualChangeOverwrite(entityType); ManualChangeOverwrite manualChangeOverwrite = new ManualChangeOverwrite(entityType);
manualChangeOverwrite.addChange(manualRedactionEntry); manualChangeOverwrite.addChange(manualRedactionEntry);
return PrecursorEntity.builder() return PrecursorEntity.builder()
@ -75,7 +74,6 @@ public class PrecursorEntity implements IEntity {
.type(manualRedactionEntry.getType()) .type(manualRedactionEntry.getType())
.section(manualRedactionEntry.getSection()) .section(manualRedactionEntry.getSection())
.entityType(entityType) .entityType(entityType)
.entryType(entryType)
.applied(true) .applied(true)
.isDictionaryEntry(false) .isDictionaryEntry(false)
.isDossierDictionaryEntry(false) .isDossierDictionaryEntry(false)
@ -103,7 +101,6 @@ public class PrecursorEntity implements IEntity {
.type(entityLogEntry.getType()) .type(entityLogEntry.getType())
.section(entityLogEntry.getSection()) .section(entityLogEntry.getSection())
.entityType(entityType) .entityType(entityType)
.entryType(entityLogEntry.getEntryType())
.isDictionaryEntry(entityLogEntry.isDictionaryEntry()) .isDictionaryEntry(entityLogEntry.isDictionaryEntry())
.isDossierDictionaryEntry(entityLogEntry.isDossierDictionaryEntry()) .isDossierDictionaryEntry(entityLogEntry.isDossierDictionaryEntry())
.manualOverwrite(new ManualChangeOverwrite(entityType)) .manualOverwrite(new ManualChangeOverwrite(entityType))
@ -134,7 +131,6 @@ public class PrecursorEntity implements IEntity {
.type(Optional.ofNullable(importedRedaction.getType()) .type(Optional.ofNullable(importedRedaction.getType())
.orElse(IMPORTED_REDACTION_TYPE)) .orElse(IMPORTED_REDACTION_TYPE))
.entityType(entityType) .entityType(entityType)
.entryType(entryType)
.isDictionaryEntry(false) .isDictionaryEntry(false)
.isDossierDictionaryEntry(false) .isDossierDictionaryEntry(false)
.rectangle(value.isBlank() || entryType.equals(EntryType.IMAGE) || entryType.equals(EntryType.IMAGE_HINT) || entryType.equals(EntryType.AREA)) .rectangle(value.isBlank() || entryType.equals(EntryType.IMAGE) || entryType.equals(EntryType.IMAGE_HINT) || entryType.equals(EntryType.AREA))

View File

@ -6,10 +6,22 @@ public enum ImageType {
LOGO, LOGO,
FORMULA, FORMULA,
SIGNATURE, SIGNATURE,
OTHER, OTHER {
@Override
public String toString() {
return "image";
}
},
OCR; OCR;
public String toString() {
return name().toLowerCase(Locale.ENGLISH);
}
public static ImageType fromString(String imageType) { public static ImageType fromString(String imageType) {
return switch (imageType.toLowerCase(Locale.ROOT)) { return switch (imageType.toLowerCase(Locale.ROOT)) {

View File

@ -4,7 +4,6 @@ import java.util.ArrayList;
import java.util.Collections; import java.util.Collections;
import java.util.HashSet; import java.util.HashSet;
import java.util.List; import java.util.List;
import java.util.Locale;
import java.util.Set; import java.util.Set;
import java.util.stream.Collectors; import java.util.stream.Collectors;
@ -19,7 +18,6 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryType; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.legalbasis.LegalBasis; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.legalbasis.LegalBasis;
import com.iqser.red.service.redaction.v1.server.RedactionServiceSettings; import com.iqser.red.service.redaction.v1.server.RedactionServiceSettings;
import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient; import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient;
@ -32,7 +30,6 @@ import com.iqser.red.service.redaction.v1.server.model.document.entity.PositionO
import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity; import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Image; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Image;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.ImageType;
import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService; import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
import lombok.AccessLevel; import lombok.AccessLevel;
@ -68,7 +65,12 @@ public class EntityLogCreatorService {
List<EntityLogEntry> entityLogEntries = createEntityLogEntries(document, analyzeRequest, notFoundEntities); List<EntityLogEntry> entityLogEntries = createEntityLogEntries(document, analyzeRequest, notFoundEntities);
List<LegalBasis> legalBasis = legalBasisClient.getLegalBasisMapping(analyzeRequest.getDossierTemplateId()); List<LegalBasis> legalBasis = legalBasisClient.getLegalBasisMapping(analyzeRequest.getDossierTemplateId());
EntityLog entityLog = new EntityLog(redactionServiceSettings.getAnalysisVersion(),
List<EntityLogEntry> previousExistingEntityLogEntries = getPreviousEntityLogEntries(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
entityChangeLogService.computeChanges(previousExistingEntityLogEntries, entityLogEntries, analyzeRequest.getManualRedactions(), analyzeRequest.getAnalysisNumber());
return new EntityLog(redactionServiceSettings.getAnalysisVersion(),
analyzeRequest.getAnalysisNumber(), analyzeRequest.getAnalysisNumber(),
entityLogEntries, entityLogEntries,
toEntityLogLegalBasis(legalBasis), toEntityLogLegalBasis(legalBasis),
@ -76,12 +78,6 @@ public class EntityLogCreatorService {
dictionaryVersion.getDossierVersion(), dictionaryVersion.getDossierVersion(),
rulesVersion, rulesVersion,
legalBasisClient.getVersion(analyzeRequest.getDossierTemplateId())); legalBasisClient.getVersion(analyzeRequest.getDossierTemplateId()));
List<EntityLogEntry> previousExistingEntityLogEntries = getPreviousEntityLogEntries(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
entityChangeLogService.computeChanges(previousExistingEntityLogEntries, entityLogEntries, analyzeRequest.getManualRedactions(), analyzeRequest.getAnalysisNumber());
return entityLog;
} }
@ -116,9 +112,12 @@ public class EntityLogCreatorService {
DictionaryVersion dictionaryVersion) { DictionaryVersion dictionaryVersion) {
List<EntityLogEntry> newEntityLogEntries = createEntityLogEntries(document, analyzeRequest, notFoundEntries).stream() List<EntityLogEntry> newEntityLogEntries = createEntityLogEntries(document, analyzeRequest, notFoundEntries).stream()
.filter(entry -> entry.getContainingNodeId().isEmpty() || sectionsToReanalyseIds.contains(entry.getContainingNodeId().get(0))) .filter(entry -> entry.getContainingNodeId().isEmpty() || sectionsToReanalyseIds.contains(entry.getContainingNodeId()
.get(0)))
.collect(Collectors.toList()); .collect(Collectors.toList());
Set<String> newEntityIds = newEntityLogEntries.stream().map(EntityLogEntry::getId).collect(Collectors.toSet()); Set<String> newEntityIds = newEntityLogEntries.stream()
.map(EntityLogEntry::getId)
.collect(Collectors.toSet());
List<EntityLogEntry> previousEntriesFromReAnalyzedSections = previousEntityLog.getEntityLogEntry() List<EntityLogEntry> previousEntriesFromReAnalyzedSections = previousEntityLog.getEntityLogEntry()
.stream() .stream()
@ -139,22 +138,6 @@ public class EntityLogCreatorService {
private List<EntityLogEntry> createEntityLogEntries(Document document, AnalyzeRequest analyzeRequest, List<PrecursorEntity> notFoundPrecursorEntries) { private List<EntityLogEntry> createEntityLogEntries(Document document, AnalyzeRequest analyzeRequest, List<PrecursorEntity> notFoundPrecursorEntries) {
Set<ManualRedactionEntry> dictionaryEntries;
Set<String> dictionaryEntriesValues;
if (analyzeRequest.getManualRedactions() != null && !analyzeRequest.getManualRedactions().getEntriesToAdd().isEmpty()) {
dictionaryEntries = analyzeRequest.getManualRedactions().getEntriesToAdd()
.stream()
.filter(e -> e.isAddToDictionary() || e.isAddToDossierDictionary())
.collect(Collectors.toSet());
dictionaryEntriesValues = dictionaryEntries.stream()
.map(ManualRedactionEntry::getValue)
.collect(Collectors.toSet());
} else {
dictionaryEntriesValues = new HashSet<>();
dictionaryEntries = new HashSet<>();
}
String dossierTemplateId = analyzeRequest.getDossierTemplateId(); String dossierTemplateId = analyzeRequest.getDossierTemplateId();
List<EntityLogEntry> entries = new ArrayList<>(); List<EntityLogEntry> entries = new ArrayList<>();
@ -164,22 +147,21 @@ public class EntityLogCreatorService {
.filter(entity -> !entity.getValue().isEmpty()) .filter(entity -> !entity.getValue().isEmpty())
.filter(EntityLogCreatorService::notFalsePositiveOrFalseRecommendation) .filter(EntityLogCreatorService::notFalsePositiveOrFalseRecommendation)
.filter(entity -> !entity.removed()) .filter(entity -> !entity.removed())
.forEach(entityNode -> entries.addAll(toEntityLogEntries(entityNode, dictionaryEntries, dictionaryEntriesValues))); .forEach(entityNode -> entries.addAll(toEntityLogEntries(entityNode)));
document.streamAllImages().filter(entity -> !entity.removed()).forEach(imageNode -> entries.add(createEntityLogEntry(imageNode, dossierTemplateId))); document.streamAllImages()
notFoundPrecursorEntries.stream().filter(entity -> !entity.removed()).forEach(precursorEntity -> entries.add(createEntityLogEntry(precursorEntity, dossierTemplateId))); .filter(entity -> !entity.removed())
.forEach(imageNode -> entries.add(createEntityLogEntry(imageNode, dossierTemplateId)));
notFoundPrecursorEntries.stream()
.filter(entity -> !entity.removed())
.forEach(precursorEntity -> entries.add(createEntityLogEntry(precursorEntity, dossierTemplateId)));
return entries; return entries;
} }
private List<EntityLogEntry> toEntityLogEntries(TextEntity textEntity, Set<ManualRedactionEntry> dictionaryEntries, Set<String> dictionaryEntriesValues) { private List<EntityLogEntry> toEntityLogEntries(TextEntity textEntity) {
List<EntityLogEntry> entityLogEntries = new ArrayList<>(); List<EntityLogEntry> entityLogEntries = new ArrayList<>();
// Adding ADD_TO_DICTIONARY manual change to the entity's manual overwrite
if (dictionaryEntriesValues.contains(textEntity.getValue())) {
textEntity.getManualOverwrite().addChange(dictionaryEntries.stream().filter(entry -> entry.getValue().equals(textEntity.getValue())).findFirst().get());
}
// split entity into multiple entries if it occurs on multiple pages, since FE can't handle multi page entities // split entity into multiple entries if it occurs on multiple pages, since FE can't handle multi page entities
for (PositionOnPage positionOnPage : textEntity.getPositionsOnPagePerPage()) { for (PositionOnPage positionOnPage : textEntity.getPositionsOnPagePerPage()) {
@ -202,12 +184,11 @@ public class EntityLogCreatorService {
private EntityLogEntry createEntityLogEntry(Image image, String dossierTemplateId) { private EntityLogEntry createEntityLogEntry(Image image, String dossierTemplateId) {
String imageType = image.getImageType().equals(ImageType.OTHER) ? "image" : image.getImageType().toString().toLowerCase(Locale.ENGLISH); boolean isHint = dictionaryService.isHint(image.type(), dossierTemplateId);
boolean isHint = dictionaryService.isHint(imageType, dossierTemplateId);
return EntityLogEntry.builder() return EntityLogEntry.builder()
.id(image.getId()) .id(image.getId())
.value(image.value()) .value(image.value())
.type(imageType) .type(image.type())
.reason(image.buildReasonWithManualChangeDescriptions()) .reason(image.buildReasonWithManualChangeDescriptions())
.legalBasis(image.legalBasis()) .legalBasis(image.legalBasis())
.matchedRule(image.getMatchedRule().getRuleIdentifier().toString()) .matchedRule(image.getMatchedRule().getRuleIdentifier().toString())
@ -229,7 +210,8 @@ public class EntityLogCreatorService {
private EntityLogEntry createEntityLogEntry(PrecursorEntity precursorEntity, String dossierTemplateId) { private EntityLogEntry createEntityLogEntry(PrecursorEntity precursorEntity, String dossierTemplateId) {
String type = precursorEntity.getManualOverwrite().getType().orElse(precursorEntity.getType()); String type = precursorEntity.getManualOverwrite().getType()
.orElse(precursorEntity.getType());
boolean isHint = isHint(precursorEntity.getEntityType()); boolean isHint = isHint(precursorEntity.getEntityType());
return EntityLogEntry.builder() return EntityLogEntry.builder()
.id(precursorEntity.getId()) .id(precursorEntity.getId())
@ -239,7 +221,8 @@ public class EntityLogCreatorService {
.type(type) .type(type)
.state(buildEntryState(precursorEntity)) .state(buildEntryState(precursorEntity))
.entryType(buildEntryType(precursorEntity)) .entryType(buildEntryType(precursorEntity))
.section(precursorEntity.getManualOverwrite().getSection().orElse(precursorEntity.getSection())) .section(precursorEntity.getManualOverwrite().getSection()
.orElse(precursorEntity.getSection()))
.containingNodeId(Collections.emptyList()) .containingNodeId(Collections.emptyList())
.closestHeadline("") .closestHeadline("")
.matchedRule(precursorEntity.getMatchedRule().getRuleIdentifier().toString()) .matchedRule(precursorEntity.getMatchedRule().getRuleIdentifier().toString())
@ -267,12 +250,17 @@ public class EntityLogCreatorService {
private EntityLogEntry createEntityLogEntry(TextEntity entity) { private EntityLogEntry createEntityLogEntry(TextEntity entity) {
Set<String> referenceIds = new HashSet<>(); Set<String> referenceIds = new HashSet<>();
entity.references().stream().filter(TextEntity::active).forEach(ref -> ref.getPositionsOnPagePerPage().forEach(pos -> referenceIds.add(pos.getId()))); entity.references()
.stream()
.filter(TextEntity::active)
.forEach(ref -> ref.getPositionsOnPagePerPage()
.forEach(pos -> referenceIds.add(pos.getId())));
boolean isHint = isHint(entity.getEntityType()); boolean isHint = isHint(entity.getEntityType());
return EntityLogEntry.builder() return EntityLogEntry.builder()
.reason(entity.buildReasonWithManualChangeDescriptions()) .reason(entity.buildReasonWithManualChangeDescriptions())
.legalBasis(entity.legalBasis()) .legalBasis(entity.legalBasis())
.value(entity.getManualOverwrite().getValue().orElse(entity.getMatchedRule().isWriteValueWithLineBreaks() ? entity.getValueWithLineBreaks() : entity.getValue())) .value(entity.getManualOverwrite().getValue()
.orElse(entity.getMatchedRule().isWriteValueWithLineBreaks() ? entity.getValueWithLineBreaks() : entity.getValue()))
.type(entity.type()) .type(entity.type())
.section(entity.getManualOverwrite().getSection() .section(entity.getManualOverwrite().getSection()
.orElse(entity.getDeepestFullyContainingNode().toString())) .orElse(entity.getDeepestFullyContainingNode().toString()))
@ -314,7 +302,7 @@ public class EntityLogCreatorService {
} }
private EntryState buildEntryState(IEntity entity) { public static EntryState buildEntryState(IEntity entity) {
if (entity.applied() && entity.active()) { if (entity.applied() && entity.active()) {
return EntryState.APPLIED; return EntryState.APPLIED;
@ -328,12 +316,17 @@ public class EntityLogCreatorService {
} }
private EntryType buildEntryType(IEntity entity) { public static EntryType buildEntryType(IEntity entity) {
if (entity instanceof TextEntity textEntity) { if (entity instanceof TextEntity textEntity) {
return getEntryType(textEntity.getEntityType()); return getEntryType(textEntity.getEntityType());
} else if (entity instanceof PrecursorEntity precursorEntity) { } else if (entity instanceof PrecursorEntity precursorEntity) {
return precursorEntity.getEntryType(); if (precursorEntity.isRectangle()) {
return EntryType.AREA;
}
return getEntryType(precursorEntity.getEntityType());
} else if (entity instanceof Image) {
return EntryType.IMAGE;
} }
throw new UnsupportedOperationException(String.format("Entity subclass %s is not implemented!", entity.getClass())); throw new UnsupportedOperationException(String.format("Entity subclass %s is not implemented!", entity.getClass()));
} }
@ -353,7 +346,9 @@ public class EntityLogCreatorService {
private List<EntityLogLegalBasis> toEntityLogLegalBasis(List<LegalBasis> legalBasis) { private List<EntityLogLegalBasis> toEntityLogLegalBasis(List<LegalBasis> legalBasis) {
return legalBasis.stream().map(l -> new EntityLogLegalBasis(l.getName(), l.getDescription(), l.getReason())).collect(Collectors.toList()); return legalBasis.stream()
.map(l -> new EntityLogLegalBasis(l.getName(), l.getDescription(), l.getReason()))
.collect(Collectors.toList());
} }
} }

View File

@ -76,11 +76,19 @@ public class UnprocessedChangesService {
EntityLog previousEntityLog = redactionStorageService.getEntityLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId()); EntityLog previousEntityLog = redactionStorageService.getEntityLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
Document document = DocumentGraphMapper.toDocumentGraph(observedStorageService.getDocumentData(analyzeRequest.getDossierId(), analyzeRequest.getFileId())); Document document = DocumentGraphMapper.toDocumentGraph(observedStorageService.getDocumentData(analyzeRequest.getDossierId(), analyzeRequest.getFileId()));
Set<String> allAnnotationIds = analyzeRequest.getManualRedactions().getEntriesToAdd().stream().map(ManualRedactionEntry::getAnnotationId).collect(Collectors.toSet()); Set<String> allAnnotationIds = analyzeRequest.getManualRedactions().getEntriesToAdd()
Set<String> resizeIds = analyzeRequest.getManualRedactions().getResizeRedactions().stream().map(ManualResizeRedaction::getAnnotationId).collect(Collectors.toSet()); .stream()
.map(ManualRedactionEntry::getAnnotationId)
.collect(Collectors.toSet());
Set<String> resizeIds = analyzeRequest.getManualRedactions().getResizeRedactions()
.stream()
.map(ManualResizeRedaction::getAnnotationId)
.collect(Collectors.toSet());
allAnnotationIds.addAll(resizeIds); allAnnotationIds.addAll(resizeIds);
List<ManualResizeRedaction> manualResizeRedactions = analyzeRequest.getManualRedactions().getResizeRedactions().stream().toList(); List<ManualResizeRedaction> manualResizeRedactions = analyzeRequest.getManualRedactions().getResizeRedactions()
.stream()
.toList();
List<PrecursorEntity> manualEntitiesToBeResized = previousEntityLog.getEntityLogEntry() List<PrecursorEntity> manualEntitiesToBeResized = previousEntityLog.getEntityLogEntry()
.stream() .stream()
.filter(entityLogEntry -> resizeIds.contains(entityLogEntry.getId())) .filter(entityLogEntry -> resizeIds.contains(entityLogEntry.getId()))
@ -99,7 +107,8 @@ public class UnprocessedChangesService {
notFoundManualEntities = entityFromPrecursorCreationService.toTextEntity(manualEntities, document); notFoundManualEntities = entityFromPrecursorCreationService.toTextEntity(manualEntities, document);
} }
document.getEntities().forEach(textEntity -> { document.getEntities()
.forEach(textEntity -> {
Set<String> processedIds = new HashSet<>(); Set<String> processedIds = new HashSet<>();
for (var positionsOnPerPage : textEntity.getPositionsOnPagePerPage()) { for (var positionsOnPerPage : textEntity.getPositionsOnPagePerPage()) {
if (processedIds.contains(positionsOnPerPage.getId())) { if (processedIds.contains(positionsOnPerPage.getId())) {
@ -111,10 +120,14 @@ public class UnprocessedChangesService {
.map(rectangle2D -> new Position(rectangle2D, positionsOnPerPage.getPage().getNumber())) .map(rectangle2D -> new Position(rectangle2D, positionsOnPerPage.getPage().getNumber()))
.collect(Collectors.toList()); .collect(Collectors.toList());
unprocessedManualEntities.add(UnprocessedManualEntity.builder() unprocessedManualEntities.add(UnprocessedManualEntity.builder()
.annotationId(allAnnotationIds.stream().filter(textEntity::matchesAnnotationId).findFirst().orElse("")) .annotationId(allAnnotationIds.stream()
.filter(textEntity::matchesAnnotationId)
.findFirst()
.orElse(""))
.textBefore(textEntity.getTextBefore()) .textBefore(textEntity.getTextBefore())
.textAfter(textEntity.getTextAfter()) .textAfter(textEntity.getTextAfter())
.section(textEntity.getManualOverwrite().getSection().orElse(textEntity.getDeepestFullyContainingNode().toString())) .section(textEntity.getManualOverwrite().getSection()
.orElse(textEntity.getDeepestFullyContainingNode().toString()))
.positions(positions) .positions(positions)
.build()); .build());
} }
@ -143,13 +156,13 @@ public class UnprocessedChangesService {
continue; continue;
} }
TextEntity correctEntity = createCorrectEntity(precursorEntity, optionalTextEntity.get()); TextEntity correctEntity = EntityFromPrecursorCreationService.createCorrectEntity(precursorEntity, optionalTextEntity.get());
Optional<ManualResizeRedaction> optionalManualResizeRedaction = manualResizeRedactions.stream() Optional<ManualResizeRedaction> optionalManualResizeRedaction = manualResizeRedactions.stream()
.filter(manualResizeRedaction -> manualResizeRedaction.getAnnotationId().equals(precursorEntity.getId())) .filter(manualResizeRedaction -> manualResizeRedaction.getAnnotationId().equals(precursorEntity.getId()))
.findFirst(); .findFirst();
if (optionalManualResizeRedaction.isPresent()) { if (optionalManualResizeRedaction.isPresent()) {
ManualResizeRedaction manualResizeRedaction = optionalManualResizeRedaction.get(); ManualResizeRedaction manualResizeRedaction = optionalManualResizeRedaction.get();
manualChangesApplicationService.resizeEntityAndReinsert(correctEntity, manualResizeRedaction); manualChangesApplicationService.resize(correctEntity, manualResizeRedaction);
// If the entity's value is not the same as the manual resize request's value it means we didn't find it anywhere and we want to remove it // If the entity's value is not the same as the manual resize request's value it means we didn't find it anywhere and we want to remove it
// from the graph, so it does not get processed and sent back to persistence-service to update its value. // from the graph, so it does not get processed and sent back to persistence-service to update its value.
@ -160,53 +173,30 @@ public class UnprocessedChangesService {
} }
// remove all temp entities from the graph // remove all temp entities from the graph
tempEntities.values().stream().flatMap(Collection::stream).forEach(TextEntity::removeFromGraph); tempEntities.values()
.stream()
.flatMap(Collection::stream)
.forEach(TextEntity::removeFromGraph);
} }
private TextEntity createCorrectEntity(PrecursorEntity precursorEntity, TextEntity closestEntity) { private UnprocessedManualEntity builDefaultUnprocessedManualEntity(PrecursorEntity precursorEntity) {
TextEntity correctEntity = TextEntity.initialEntityNode(closestEntity.getTextRange(), precursorEntity.type(), precursorEntity.getEntityType(), precursorEntity.getId());
correctEntity.setDeepestFullyContainingNode(closestEntity.getDeepestFullyContainingNode());
correctEntity.setIntersectingNodes(new ArrayList<>(closestEntity.getIntersectingNodes()));
correctEntity.setDuplicateTextRanges(new ArrayList<>(closestEntity.getDuplicateTextRanges()));
correctEntity.setPages(new HashSet<>(closestEntity.getPages()));
correctEntity.setValue(closestEntity.getValue());
correctEntity.setTextAfter(closestEntity.getTextAfter());
correctEntity.setTextBefore(closestEntity.getTextBefore());
correctEntity.getIntersectingNodes().forEach(n -> n.getEntities().add(correctEntity));
correctEntity.getPages().forEach(page -> page.getEntities().add(correctEntity));
correctEntity.addMatchedRules(precursorEntity.getMatchedRuleList());
correctEntity.setDictionaryEntry(precursorEntity.isDictionaryEntry());
correctEntity.setDossierDictionaryEntry(precursorEntity.isDossierDictionaryEntry());
correctEntity.getManualOverwrite().addChanges(precursorEntity.getManualOverwrite().getManualChangeLog());
return correctEntity;
}
private UnprocessedManualEntity builDefaultUnprocessedManualEntity(PrecursorEntity precursorEntity) {
return UnprocessedManualEntity.builder() return UnprocessedManualEntity.builder()
.annotationId(precursorEntity.getId()) .annotationId(precursorEntity.getId())
.textAfter("") .textAfter("")
.textBefore("") .textBefore("")
.section("") .section("")
.positions(precursorEntity.getManualOverwrite() .positions(precursorEntity.getManualOverwrite().getPositions()
.getPositions()
.orElse(precursorEntity.getEntityPosition()) .orElse(precursorEntity.getEntityPosition())
.stream() .stream()
.map(entityPosition -> new Position(entityPosition.rectangle2D(), entityPosition.pageNumber())) .map(entityPosition -> new Position(entityPosition.rectangle2D(), entityPosition.pageNumber()))
.toList()) .toList())
.build(); .build();
} }
private List<PrecursorEntity> manualEntitiesConverter(ManualRedactions manualRedactions, String dossierTemplateId) { private List<PrecursorEntity> manualEntitiesConverter(ManualRedactions manualRedactions, String dossierTemplateId) {
return manualRedactions.getEntriesToAdd() return manualRedactions.getEntriesToAdd()
.stream() .stream()
@ -214,6 +204,6 @@ private List<PrecursorEntity> manualEntitiesConverter(ManualRedactions manualRed
.map(manualRedactionEntry -> PrecursorEntity.fromManualRedactionEntry(manualRedactionEntry, .map(manualRedactionEntry -> PrecursorEntity.fromManualRedactionEntry(manualRedactionEntry,
dictionaryService.isHint(manualRedactionEntry.getType(), dossierTemplateId))) dictionaryService.isHint(manualRedactionEntry.getType(), dossierTemplateId)))
.toList(); .toList();
} }
} }

View File

@ -14,7 +14,6 @@ import org.springframework.stereotype.Service;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedactions; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedactions;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.BaseAnnotation;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;
import com.iqser.red.service.redaction.v1.server.model.PrecursorEntity; import com.iqser.red.service.redaction.v1.server.model.PrecursorEntity;
import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType; import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
@ -51,18 +50,14 @@ public class EntityFromPrecursorCreationService {
Set<IdRemoval> idRemovals = manualRedactions.getIdsToRemove(); Set<IdRemoval> idRemovals = manualRedactions.getIdsToRemove();
List<PrecursorEntity> manualEntities = manualRedactions.getEntriesToAdd() List<PrecursorEntity> manualEntities = manualRedactions.getEntriesToAdd()
.stream() .stream()
.filter(manualRedactionEntry -> !(idRemovals.stream() .filter(manualRedactionEntry -> idRemovals.stream()
.map(BaseAnnotation::getAnnotationId)
.toList()
.contains(manualRedactionEntry.getAnnotationId()) && manualRedactionEntry.getRequestDate()
.isBefore(idRemovals.stream()
.filter(idRemoval -> idRemoval.getAnnotationId().equals(manualRedactionEntry.getAnnotationId())) .filter(idRemoval -> idRemoval.getAnnotationId().equals(manualRedactionEntry.getAnnotationId()))
.findFirst() .filter(idRemoval -> idRemoval.getRequestDate().isBefore(manualRedactionEntry.getRequestDate()))
.get() .findAny()//
.getRequestDate()))) .isEmpty())
.filter(manualRedactionEntry -> !(manualRedactionEntry.isAddToDictionary() || manualRedactionEntry.isAddToDossierDictionary())) .filter(manualRedactionEntry -> !(manualRedactionEntry.isAddToDictionary() || manualRedactionEntry.isAddToDossierDictionary()))
.map(manualRedactionEntry -> PrecursorEntity.fromManualRedactionEntry(manualRedactionEntry, .map(manualRedactionEntry -> //
dictionaryService.isHint(manualRedactionEntry.getType(), dossierTemplateId))) PrecursorEntity.fromManualRedactionEntry(manualRedactionEntry, dictionaryService.isHint(manualRedactionEntry.getType(), dossierTemplateId)))
.peek(manualEntity -> { .peek(manualEntity -> {
if (manualEntity.getEntityType().equals(EntityType.HINT)) { if (manualEntity.getEntityType().equals(EntityType.HINT)) {
manualEntity.skip("MAN.5.1", "manual hint is skipped by default"); manualEntity.skip("MAN.5.1", "manual hint is skipped by default");
@ -90,8 +85,14 @@ public class EntityFromPrecursorCreationService {
public List<PrecursorEntity> toTextEntity(List<PrecursorEntity> precursorEntities, SemanticNode node) { public List<PrecursorEntity> toTextEntity(List<PrecursorEntity> precursorEntities, SemanticNode node) {
var notFoundEntities = precursorEntities.stream().filter(PrecursorEntity::isRectangle).collect(Collectors.toList()); var notFoundEntities = precursorEntities.stream()
var findableEntities = precursorEntities.stream().filter(precursorEntity -> !precursorEntity.isRectangle()).toList(); .filter(PrecursorEntity::isRectangle)
.collect(Collectors.toList());
var findableEntities = precursorEntities.stream()
.filter(precursorEntity -> !precursorEntity.isRectangle())
.toList();
Map<String, List<TextEntity>> tempEntitiesByValue = entityFindingUtility.findAllPossibleEntitiesAndGroupByValue(node, findableEntities); Map<String, List<TextEntity>> tempEntitiesByValue = entityFindingUtility.findAllPossibleEntitiesAndGroupByValue(node, findableEntities);
for (PrecursorEntity precursorEntity : findableEntities) { for (PrecursorEntity precursorEntity : findableEntities) {
@ -102,7 +103,12 @@ public class EntityFromPrecursorCreationService {
} }
createCorrectEntity(precursorEntity, optionalClosestEntity.get()); createCorrectEntity(precursorEntity, optionalClosestEntity.get());
} }
tempEntitiesByValue.values().stream().flatMap(Collection::stream).forEach(TextEntity::removeFromGraph);
tempEntitiesByValue.values()
.stream()
.flatMap(Collection::stream)
.forEach(TextEntity::removeFromGraph);
return notFoundEntities; return notFoundEntities;
} }
@ -113,9 +119,23 @@ public class EntityFromPrecursorCreationService {
* @param precursorEntity The entity identifier for the RedactionEntity. * @param precursorEntity The entity identifier for the RedactionEntity.
* @param closestEntity The closest Boundary to the RedactionEntity. * @param closestEntity The closest Boundary to the RedactionEntity.
*/ */
private void createCorrectEntity(PrecursorEntity precursorEntity, TextEntity closestEntity) { public static TextEntity createCorrectEntity(PrecursorEntity precursorEntity, TextEntity closestEntity) {
TextEntity correctEntity = TextEntity.initialEntityNode(closestEntity.getTextRange(), precursorEntity.type(), precursorEntity.getEntityType(), precursorEntity.getId()); return createCorrectEntity(precursorEntity, closestEntity, false);
}
public static TextEntity createCorrectEntity(PrecursorEntity precursorEntity, TextEntity closestEntity, boolean generateId) {
TextEntity correctEntity;
if (generateId) {
correctEntity = TextEntity.initialEntityNode(closestEntity.getTextRange(),
precursorEntity.type(),
precursorEntity.getEntityType(),
closestEntity.getDeepestFullyContainingNode());
} else {
correctEntity = TextEntity.initialEntityNode(closestEntity.getTextRange(), precursorEntity.type(), precursorEntity.getEntityType(), precursorEntity.getId());
}
correctEntity.setDeepestFullyContainingNode(closestEntity.getDeepestFullyContainingNode()); correctEntity.setDeepestFullyContainingNode(closestEntity.getDeepestFullyContainingNode());
correctEntity.setIntersectingNodes(new ArrayList<>(closestEntity.getIntersectingNodes())); correctEntity.setIntersectingNodes(new ArrayList<>(closestEntity.getIntersectingNodes()));
correctEntity.setDuplicateTextRanges(new ArrayList<>(closestEntity.getDuplicateTextRanges())); correctEntity.setDuplicateTextRanges(new ArrayList<>(closestEntity.getDuplicateTextRanges()));
@ -125,14 +145,17 @@ public class EntityFromPrecursorCreationService {
correctEntity.setTextAfter(closestEntity.getTextAfter()); correctEntity.setTextAfter(closestEntity.getTextAfter());
correctEntity.setTextBefore(closestEntity.getTextBefore()); correctEntity.setTextBefore(closestEntity.getTextBefore());
correctEntity.getIntersectingNodes().forEach(n -> n.getEntities().add(correctEntity)); correctEntity.getIntersectingNodes()
correctEntity.getPages().forEach(page -> page.getEntities().add(correctEntity)); .forEach(n -> n.getEntities().add(correctEntity));
correctEntity.getPages()
.forEach(page -> page.getEntities().add(correctEntity));
correctEntity.addMatchedRules(precursorEntity.getMatchedRuleList()); correctEntity.addMatchedRules(precursorEntity.getMatchedRuleList());
correctEntity.setDictionaryEntry(precursorEntity.isDictionaryEntry()); correctEntity.setDictionaryEntry(precursorEntity.isDictionaryEntry());
correctEntity.setDossierDictionaryEntry(precursorEntity.isDossierDictionaryEntry()); correctEntity.setDossierDictionaryEntry(precursorEntity.isDossierDictionaryEntry());
correctEntity.getManualOverwrite().addChanges(precursorEntity.getManualOverwrite().getManualChangeLog()); correctEntity.getManualOverwrite().addChanges(precursorEntity.getManualOverwrite().getManualChangeLog());
correctEntity.addEngines(precursorEntity.getEngines()); correctEntity.addEngines(precursorEntity.getEngines());
return correctEntity;
} }
} }

View File

@ -1,5 +1,6 @@
package com.iqser.red.service.redaction.v1.server.utils; package com.iqser.red.service.redaction.v1.server.utils;
import java.util.Collections;
import java.util.LinkedList; import java.util.LinkedList;
import java.util.Set; import java.util.Set;
import java.util.function.BiConsumer; import java.util.function.BiConsumer;
@ -17,7 +18,7 @@ public class MigratedIdsCollector implements Collector<MigrationEntity, Migrated
@Override @Override
public Supplier<MigratedIds> supplier() { public Supplier<MigratedIds> supplier() {
return () -> new MigratedIds(new LinkedList<>()); return () -> new MigratedIds(new LinkedList<>(), Collections.emptyList());
} }

View File

@ -31,6 +31,7 @@ import org.springframework.test.context.junit.jupiter.SpringExtension;
import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectMapper;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.migration.MigratedIds; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.migration.MigratedIds;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
@ -49,7 +50,6 @@ import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
import com.iqser.red.service.redaction.v1.server.redaction.utils.OsUtils; import com.iqser.red.service.redaction.v1.server.redaction.utils.OsUtils;
import com.iqser.red.service.redaction.v1.server.service.DictionaryService; import com.iqser.red.service.redaction.v1.server.service.DictionaryService;
import com.iqser.red.service.redaction.v1.server.service.document.EntityFindingUtility; import com.iqser.red.service.redaction.v1.server.service.document.EntityFindingUtility;
import com.iqser.red.service.redaction.v1.server.utils.RectangleTransformations;
import com.knecon.fforesight.tenantcommons.TenantContext; import com.knecon.fforesight.tenantcommons.TenantContext;
import lombok.SneakyThrows; import lombok.SneakyThrows;
@ -107,7 +107,7 @@ public class MigrationIntegrationTest extends BuildDocumentIntegrationTest {
@SneakyThrows @SneakyThrows
public void testSave() { public void testSave() {
MigratedIds ids = new MigratedIds(new LinkedList<>()); MigratedIds ids = new MigratedIds(new LinkedList<>(), null);
ids.addMapping("123", "321"); ids.addMapping("123", "321");
ids.addMapping("123", "321"); ids.addMapping("123", "321");
ids.addMapping("123", "321"); ids.addMapping("123", "321");
@ -173,7 +173,11 @@ public class MigrationIntegrationTest extends BuildDocumentIntegrationTest {
mergedRedactionLog = redactionLog; mergedRedactionLog = redactionLog;
} }
MigratedEntityLog migratedEntityLog = redactionLogToEntityLogMigrationService.migrate(mergedRedactionLog, document, TEST_DOSSIER_TEMPLATE_ID, manualRedactions); MigratedEntityLog migratedEntityLog = redactionLogToEntityLogMigrationService.migrate(mergedRedactionLog,
document,
TEST_DOSSIER_TEMPLATE_ID,
manualRedactions,
TEST_FILE_ID);
redactionStorageService.storeObject(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.ENTITY_LOG, migratedEntityLog.getEntityLog()); redactionStorageService.storeObject(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.ENTITY_LOG, migratedEntityLog.getEntityLog());
assertEquals(mergedRedactionLog.getRedactionLogEntry().size(), migratedEntityLog.getEntityLog().getEntityLogEntry().size()); assertEquals(mergedRedactionLog.getRedactionLogEntry().size(), migratedEntityLog.getEntityLog().getEntityLogEntry().size());
@ -187,10 +191,11 @@ public class MigrationIntegrationTest extends BuildDocumentIntegrationTest {
assertEquals(mergedRedactionLog.getLegalBasis().size(), entityLog.getLegalBasis().size()); assertEquals(mergedRedactionLog.getLegalBasis().size(), entityLog.getLegalBasis().size());
Map<String, String> migratedIds = migratedEntityLog.getMigratedIds().buildOldToNewMapping(); Map<String, String> migratedIds = migratedEntityLog.getMigratedIds().buildOldToNewMapping();
// assertEquals(legacyRedactionLogMergeService.getNumberOfAffectedAnnotations(manualRedactions), migratedIds.size());
migratedIds.forEach((oldId, newId) -> assertEntryIsEqual(oldId, newId, mergedRedactionLog, entityLog, migratedIds)); migratedIds.forEach((oldId, newId) -> assertEntryIsEqual(oldId, newId, mergedRedactionLog, entityLog, migratedIds));
AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder().dossierId(TEST_DOSSIER_ID).fileId(TEST_FILE_ID) AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder().dossierId(TEST_DOSSIER_ID).fileId(TEST_FILE_ID).build());
.build());
File outputFile = Path.of(OsUtils.getTemporaryDirectory()).resolve(Path.of(fileName.replaceAll(".pdf", "_MIGRATED.pdf")).getFileName()).toFile(); File outputFile = Path.of(OsUtils.getTemporaryDirectory()).resolve(Path.of(fileName.replaceAll(".pdf", "_MIGRATED.pdf")).getFileName()).toFile();
try (FileOutputStream fileOutputStream = new FileOutputStream(outputFile)) { try (FileOutputStream fileOutputStream = new FileOutputStream(outputFile)) {
@ -268,13 +273,24 @@ public class MigrationIntegrationTest extends BuildDocumentIntegrationTest {
if (!redactionLogEntry.isImage()) { if (!redactionLogEntry.isImage()) {
assertEquals(redactionLogEntry.getValue().toLowerCase(Locale.ENGLISH), entityLogEntry.getValue().toLowerCase(Locale.ENGLISH)); assertEquals(redactionLogEntry.getValue().toLowerCase(Locale.ENGLISH), entityLogEntry.getValue().toLowerCase(Locale.ENGLISH));
} }
if (entityLogEntry.getManualChanges()
.stream()
.noneMatch(mc -> mc.getManualRedactionType().equals(ManualRedactionType.RECATEGORIZE))) {
assertEquals(redactionLogEntry.getType(), entityLogEntry.getType());
}
assertEquals(redactionLogEntry.getChanges().size(), entityLogEntry.getChanges().size()); assertEquals(redactionLogEntry.getChanges().size(), entityLogEntry.getChanges().size());
assertTrue(redactionLogEntry.getManualChanges().size() <= entityLogEntry.getManualChanges().size()); assertTrue(redactionLogEntry.getManualChanges().size() <= entityLogEntry.getManualChanges().size());
assertEquals(redactionLogEntry.getPositions().size(), entityLogEntry.getPositions().size()); assertEquals(redactionLogEntry.getPositions().size(), entityLogEntry.getPositions().size());
if (entityLogEntry.getManualChanges()
.stream()
.noneMatch(mc -> mc.getManualRedactionType().equals(ManualRedactionType.RESIZE) || mc.getManualRedactionType().equals(ManualRedactionType.RESIZE_IN_DICTIONARY))) {
assertTrue(positionsAlmostEqual(redactionLogEntry.getPositions(), entityLogEntry.getPositions())); assertTrue(positionsAlmostEqual(redactionLogEntry.getPositions(), entityLogEntry.getPositions()));
// assertEquals(redactionLogEntry.getColor(), entityLogEntry.getColor()); }
if (entityLogEntry.getManualChanges()
.stream()
.noneMatch(mc -> mc.getManualRedactionType().equals(ManualRedactionType.FORCE_REDACT) || mc.getManualRedactionType().equals(ManualRedactionType.FORCE_HINT))) {
assertEqualsNullSafe(redactionLogEntry.getLegalBasis(), entityLogEntry.getLegalBasis()); assertEqualsNullSafe(redactionLogEntry.getLegalBasis(), entityLogEntry.getLegalBasis());
// assertEqualsNullSafe(redactionLogEntry.getReason(), entityLogEntry.getReason()); }
assertReferencesEqual(redactionLogEntry.getReference(), entityLogEntry.getReference(), oldToNewMapping); assertReferencesEqual(redactionLogEntry.getReference(), entityLogEntry.getReference(), oldToNewMapping);
assertEquals(redactionLogEntry.isDictionaryEntry(), entityLogEntry.isDictionaryEntry()); assertEquals(redactionLogEntry.isDictionaryEntry(), entityLogEntry.isDictionaryEntry());
assertEquals(redactionLogEntry.isDossierDictionaryEntry(), entityLogEntry.isDossierDictionaryEntry()); assertEquals(redactionLogEntry.isDossierDictionaryEntry(), entityLogEntry.isDossierDictionaryEntry());

View File

@ -169,7 +169,8 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
var entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID); var entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
entityLog.getEntityLogEntry().forEach(entry -> { entityLog.getEntityLogEntry()
.forEach(entry -> {
duplicates.computeIfAbsent(entry.getId(), v -> new ArrayList<>()).add(entry); duplicates.computeIfAbsent(entry.getId(), v -> new ArrayList<>()).add(entry);
}); });
@ -256,7 +257,10 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
var redactionLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID); var redactionLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
var values = redactionLog.getEntityLogEntry().stream().map(EntityLogEntry::getValue).collect(Collectors.toList()); var values = redactionLog.getEntityLogEntry()
.stream()
.map(EntityLogEntry::getValue)
.collect(Collectors.toList());
assertThat(values).containsExactlyInAnyOrder("Lastname M.", "Doe", "Doe J.", "M. Mustermann", "Mustermann M.", "F. Lastname"); assertThat(values).containsExactlyInAnyOrder("Lastname M.", "Doe", "Doe J.", "M. Mustermann", "Mustermann M.", "F. Lastname");
} }
@ -353,10 +357,18 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
var mergedEntityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID); var mergedEntityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
var cbiAddressBeforeHintRemoval = entityLog.getEntityLogEntry().stream().filter(re -> re.getType().equalsIgnoreCase("CBI_Address")).findAny().get(); var cbiAddressBeforeHintRemoval = entityLog.getEntityLogEntry()
.stream()
.filter(re -> re.getType().equalsIgnoreCase("CBI_Address"))
.findAny()
.get();
assertThat(cbiAddressBeforeHintRemoval.getState().equals(EntryState.APPLIED)).isFalse(); assertThat(cbiAddressBeforeHintRemoval.getState().equals(EntryState.APPLIED)).isFalse();
var cbiAddressAfterHintRemoval = mergedEntityLog.getEntityLogEntry().stream().filter(re -> re.getType().equalsIgnoreCase("CBI_Address")).findAny().get(); var cbiAddressAfterHintRemoval = mergedEntityLog.getEntityLogEntry()
.stream()
.filter(re -> re.getType().equalsIgnoreCase("CBI_Address"))
.findAny()
.get();
assertThat(cbiAddressAfterHintRemoval.getState().equals(EntryState.APPLIED)).isTrue(); assertThat(cbiAddressAfterHintRemoval.getState().equals(EntryState.APPLIED)).isTrue();
} }
@ -386,7 +398,8 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
var entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID); var entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
entityLog.getEntityLogEntry().forEach(entry -> { entityLog.getEntityLogEntry()
.forEach(entry -> {
duplicates.computeIfAbsent(entry.getId(), v -> new ArrayList<>()).add(entry); duplicates.computeIfAbsent(entry.getId(), v -> new ArrayList<>()).add(entry);
}); });
@ -449,7 +462,10 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
correctFound++; correctFound++;
continue loop; continue loop;
} }
if (Objects.equals(entityLogEntry.getContainingNodeId().get(0), section.getTreeId().get(0))) { if (Objects.equals(entityLogEntry.getContainingNodeId()
.get(0),
section.getTreeId()
.get(0))) {
String value = section.getTextBlock().subSequence(new TextRange(entityLogEntry.getStartOffset(), entityLogEntry.getEndOffset())).toString(); String value = section.getTextBlock().subSequence(new TextRange(entityLogEntry.getStartOffset(), entityLogEntry.getEndOffset())).toString();
if (entityLogEntry.getValue().equalsIgnoreCase(value)) { if (entityLogEntry.getValue().equalsIgnoreCase(value)) {
correctFound++; correctFound++;
@ -548,7 +564,11 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
var entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID); var entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
var changes = entityLog.getEntityLogEntry().stream().filter(entry -> entry.getValue() != null && entry.getValue().equals("report")).findFirst().get().getChanges(); var changes = entityLog.getEntityLogEntry()
.stream()
.filter(entry -> entry.getValue() != null && entry.getValue().equals("report"))
.findFirst()
.get().getChanges();
assertThat(changes.size()).isEqualTo(2); assertThat(changes.size()).isEqualTo(2);
@ -601,7 +621,11 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
.map(redactionLogEntry -> new TextRange(redactionLogEntry.getStartOffset(), redactionLogEntry.getEndOffset())) .map(redactionLogEntry -> new TextRange(redactionLogEntry.getStartOffset(), redactionLogEntry.getEndOffset()))
.map(boundary -> documentGraph.getTextBlock().subSequence(boundary).toString()) .map(boundary -> documentGraph.getTextBlock().subSequence(boundary).toString())
.toList(); .toList();
List<String> valuesInRedactionLog = entityLog.getEntityLogEntry().stream().filter(e -> !e.getEntryType().equals(EntryType.IMAGE)).map(EntityLogEntry::getValue).toList(); List<String> valuesInRedactionLog = entityLog.getEntityLogEntry()
.stream()
.filter(e -> !e.getEntryType().equals(EntryType.IMAGE))
.map(EntityLogEntry::getValue)
.toList();
assertEquals(valuesInRedactionLog, valuesInDocument); assertEquals(valuesInRedactionLog, valuesInDocument);
@ -960,7 +984,8 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
var redactionLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID); var redactionLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
redactionLog.getEntityLogEntry().forEach(entry -> { redactionLog.getEntityLogEntry()
.forEach(entry -> {
if (!entry.getEntryType().equals(EntryType.HINT)) { if (!entry.getEntryType().equals(EntryType.HINT)) {
if (entry.getType().equals("CBI_author")) { if (entry.getType().equals("CBI_author")) {
assertThat(entry.getReason()).isEqualTo("Not redacted because it's row does not belong to a vertebrate study"); assertThat(entry.getReason()).isEqualTo("Not redacted because it's row does not belong to a vertebrate study");
@ -1056,10 +1081,34 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
.processedDate(OffsetDateTime.now()) .processedDate(OffsetDateTime.now())
.requestDate(OffsetDateTime.now()) .requestDate(OffsetDateTime.now())
.value("Syngenta Crop Protection AG, Basel, Switzerland RCC Ltd., Itingen, Switzerland") .value("Syngenta Crop Protection AG, Basel, Switzerland RCC Ltd., Itingen, Switzerland")
.positions(List.of(Rectangle.builder().topLeftX(289.44595f).topLeftY(327.567f).width(7.648041f).height(82.51475f).page(1).build(), .positions(List.of(Rectangle.builder()
Rectangle.builder().topLeftX(298.67056f).topLeftY(327.567f).width(7.648041f).height(75.32377f).page(1).build(), .topLeftX(289.44595f)
Rectangle.builder().topLeftX(307.89517f).topLeftY(327.567f).width(7.648041f).height(61.670967f).page(1).build(), .topLeftY(327.567f)
Rectangle.builder().topLeftX(316.99985f).topLeftY(327.567f).width(7.648041f).height(38.104286f).page(1).build())) .width(7.648041f)
.height(82.51475f)
.page(1)
.build(),
Rectangle.builder()
.topLeftX(298.67056f)
.topLeftY(327.567f)
.width(7.648041f)
.height(75.32377f)
.page(1)
.build(),
Rectangle.builder()
.topLeftX(307.89517f)
.topLeftY(327.567f)
.width(7.648041f)
.height(61.670967f)
.page(1)
.build(),
Rectangle.builder()
.topLeftX(316.99985f)
.topLeftY(327.567f)
.width(7.648041f)
.height(38.104286f)
.page(1)
.build()))
.updateDictionary(false) .updateDictionary(false)
.build())); .build()));
@ -1124,7 +1173,8 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
fileOutputStream.write(annotateResponse.getDocument()); fileOutputStream.write(annotateResponse.getDocument());
} }
entityLog.getEntityLogEntry().forEach(entry -> { entityLog.getEntityLogEntry()
.forEach(entry -> {
if (entry.getValue() == null) { if (entry.getValue() == null) {
return; return;
} }
@ -1163,7 +1213,10 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
} }
var entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID); var entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
var values = entityLog.getEntityLogEntry().stream().map(EntityLogEntry::getValue).collect(Collectors.toList()); var values = entityLog.getEntityLogEntry()
.stream()
.map(EntityLogEntry::getValue)
.collect(Collectors.toList());
assertThat(values).contains("Mrs. Robinson"); assertThat(values).contains("Mrs. Robinson");
assertThat(values).contains("Mr. Bojangles"); assertThat(values).contains("Mr. Bojangles");
@ -1202,7 +1255,10 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
System.out.println("Finished reanalysis"); System.out.println("Finished reanalysis");
var entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID); var entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
entityLog.getEntityLogEntry().stream().filter(entityLogEntry -> entityLogEntry.getType().equals("signature")).forEach(entityLogEntry -> { entityLog.getEntityLogEntry()
.stream()
.filter(entityLogEntry -> entityLogEntry.getType().equals("signature"))
.forEach(entityLogEntry -> {
assertThat(entityLogEntry.getState() == EntryState.APPLIED).isTrue(); assertThat(entityLogEntry.getState() == EntryState.APPLIED).isTrue();
}); });
} }
@ -1282,26 +1338,50 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
var entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID); var entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
assertTrue(entityLog.getEntityLogEntry().stream().anyMatch(entityLogEntry -> entityLogEntry.getId().equals(manualAddId))); assertTrue(entityLog.getEntityLogEntry()
assertEquals(entityLog.getEntityLogEntry().stream().filter(entityLogEntry -> entityLogEntry.getId().equals(manualAddId)).findFirst().get().getState(), EntryState.APPLIED); .stream()
.anyMatch(entityLogEntry -> entityLogEntry.getId().equals(manualAddId)));
assertEquals(entityLog.getEntityLogEntry()
.stream()
.filter(entityLogEntry -> entityLogEntry.getId().equals(manualAddId))
.findFirst()
.get().getState(), EntryState.APPLIED);
request.setManualRedactions(ManualRedactions.builder().entriesToAdd(Set.of(manualRedactionEntry)).idsToRemove(Set.of(idRemoval)).build()); request.setManualRedactions(ManualRedactions.builder().entriesToAdd(Set.of(manualRedactionEntry)).idsToRemove(Set.of(idRemoval)).build());
analyzeService.reanalyze(request); analyzeService.reanalyze(request);
entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID); entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
assertTrue(entityLog.getEntityLogEntry().stream().anyMatch(entityLogEntry -> entityLogEntry.getId().equals(manualAddId))); assertTrue(entityLog.getEntityLogEntry()
assertEquals(entityLog.getEntityLogEntry().stream().filter(entityLogEntry -> entityLogEntry.getId().equals(manualAddId)).findFirst().get().getState(), EntryState.REMOVED); .stream()
.anyMatch(entityLogEntry -> entityLogEntry.getId().equals(manualAddId)));
assertEquals(entityLog.getEntityLogEntry()
.stream()
.filter(entityLogEntry -> entityLogEntry.getId().equals(manualAddId))
.findFirst()
.get().getState(), EntryState.REMOVED);
request.setManualRedactions(ManualRedactions.builder().entriesToAdd(Set.of(manualRedactionEntry, manualRedactionEntry2)).idsToRemove(Set.of(idRemoval)).build()); request.setManualRedactions(ManualRedactions.builder().entriesToAdd(Set.of(manualRedactionEntry, manualRedactionEntry2)).idsToRemove(Set.of(idRemoval)).build());
analyzeService.reanalyze(request); analyzeService.reanalyze(request);
entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID); entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
assertTrue(entityLog.getEntityLogEntry().stream().anyMatch(entityLogEntry -> entityLogEntry.getId().equals(manualAddId))); assertTrue(entityLog.getEntityLogEntry()
assertEquals(entityLog.getEntityLogEntry().stream().filter(entityLogEntry -> entityLogEntry.getId().equals(manualAddId)).findFirst().get().getState(), EntryState.REMOVED); .stream()
assertTrue(entityLog.getEntityLogEntry().stream().anyMatch(entityLogEntry -> entityLogEntry.getId().equals(manualAddId2))); .anyMatch(entityLogEntry -> entityLogEntry.getId().equals(manualAddId)));
assertEquals(entityLog.getEntityLogEntry().stream().filter(entityLogEntry -> entityLogEntry.getId().equals(manualAddId2)).findFirst().get().getState(), EntryState.APPLIED); assertEquals(entityLog.getEntityLogEntry()
.stream()
.filter(entityLogEntry -> entityLogEntry.getId().equals(manualAddId))
.findFirst()
.get().getState(), EntryState.REMOVED);
assertTrue(entityLog.getEntityLogEntry()
.stream()
.anyMatch(entityLogEntry -> entityLogEntry.getId().equals(manualAddId2)));
assertEquals(entityLog.getEntityLogEntry()
.stream()
.filter(entityLogEntry -> entityLogEntry.getId().equals(manualAddId2))
.findFirst()
.get().getState(), EntryState.APPLIED);
request.setManualRedactions(ManualRedactions.builder() request.setManualRedactions(ManualRedactions.builder()
.entriesToAdd(Set.of(manualRedactionEntry, manualRedactionEntry2)) .entriesToAdd(Set.of(manualRedactionEntry, manualRedactionEntry2))
@ -1311,26 +1391,48 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID); entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
assertTrue(entityLog.getEntityLogEntry().stream().anyMatch(entityLogEntry -> entityLogEntry.getId().equals(manualAddId))); assertTrue(entityLog.getEntityLogEntry()
assertEquals(entityLog.getEntityLogEntry().stream().filter(entityLogEntry -> entityLogEntry.getId().equals(manualAddId)).findFirst().get().getState(), EntryState.REMOVED); .stream()
assertTrue(entityLog.getEntityLogEntry().stream().anyMatch(entityLogEntry -> entityLogEntry.getId().equals(manualAddId2))); .anyMatch(entityLogEntry -> entityLogEntry.getId().equals(manualAddId)));
assertEquals(entityLog.getEntityLogEntry().stream().filter(entityLogEntry -> entityLogEntry.getId().equals(manualAddId2)).findFirst().get().getState(), EntryState.REMOVED); assertEquals(entityLog.getEntityLogEntry()
.stream()
.filter(entityLogEntry -> entityLogEntry.getId().equals(manualAddId))
.findFirst()
.get().getState(), EntryState.REMOVED);
assertTrue(entityLog.getEntityLogEntry()
.stream()
.anyMatch(entityLogEntry -> entityLogEntry.getId().equals(manualAddId2)));
assertEquals(entityLog.getEntityLogEntry()
.stream()
.filter(entityLogEntry -> entityLogEntry.getId().equals(manualAddId2))
.findFirst()
.get().getState(), EntryState.REMOVED);
manualRedactionEntry.setRequestDate(OffsetDateTime.now()); manualRedactionEntry.setRequestDate(OffsetDateTime.now());
request.setManualRedactions(ManualRedactions.builder() request.setManualRedactions(ManualRedactions.builder().entriesToAdd(Set.of(manualRedactionEntry, manualRedactionEntry2)).idsToRemove(Set.of(idRemoval2)).build());
.entriesToAdd(Set.of(manualRedactionEntry, manualRedactionEntry2))
.idsToRemove(Set.of(idRemoval, idRemoval2))
.build());
analyzeService.reanalyze(request); analyzeService.reanalyze(request);
entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID); entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
assertTrue(entityLog.getEntityLogEntry().stream().anyMatch(entityLogEntry -> entityLogEntry.getId().equals(manualAddId))); assertTrue(entityLog.getEntityLogEntry()
assertEquals(entityLog.getEntityLogEntry().stream().filter(entityLogEntry -> entityLogEntry.getId().equals(manualAddId)).findFirst().get().getState(), EntryState.APPLIED); .stream()
assertTrue(entityLog.getEntityLogEntry().stream().anyMatch(entityLogEntry -> entityLogEntry.getId().equals(manualAddId2))); .anyMatch(entityLogEntry -> entityLogEntry.getId().equals(manualAddId)));
assertEquals(entityLog.getEntityLogEntry().stream().filter(entityLogEntry -> entityLogEntry.getId().equals(manualAddId2)).findFirst().get().getState(), EntryState.REMOVED); assertEquals(entityLog.getEntityLogEntry()
.stream()
.filter(entityLogEntry -> entityLogEntry.getId().equals(manualAddId))
.findFirst()
.get().getState(), EntryState.APPLIED);
assertTrue(entityLog.getEntityLogEntry()
.stream()
.anyMatch(entityLogEntry -> entityLogEntry.getId().equals(manualAddId2)));
assertEquals(entityLog.getEntityLogEntry()
.stream()
.filter(entityLogEntry -> entityLogEntry.getId().equals(manualAddId2))
.findFirst()
.get().getState(), EntryState.REMOVED);
} }
@Test @Test
@SneakyThrows @SneakyThrows
public void testResizeWithUpdateDictionaryTrue() { public void testResizeWithUpdateDictionaryTrue() {
@ -1342,7 +1444,11 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
analyzeService.analyze(request); analyzeService.analyze(request);
var entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID); var entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
var david = entityLog.getEntityLogEntry().stream().filter(e -> e.getValue().equals("David")).findFirst().get(); var david = entityLog.getEntityLogEntry()
.stream()
.filter(e -> e.getValue().equals("David"))
.findFirst()
.get();
request.setManualRedactions(ManualRedactions.builder() request.setManualRedactions(ManualRedactions.builder()
.resizeRedactions(Set.of(ManualResizeRedaction.builder() .resizeRedactions(Set.of(ManualResizeRedaction.builder()
@ -1350,13 +1456,23 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
.annotationId(david.getId()) .annotationId(david.getId())
.requestDate(OffsetDateTime.now()) .requestDate(OffsetDateTime.now())
.value("David Ksenia") .value("David Ksenia")
.positions(List.of(Rectangle.builder().topLeftX(56.8f).topLeftY(293.564f).width(65.592f).height(15.408f).page(1).build())) .positions(List.of(Rectangle.builder()
.topLeftX(56.8f)
.topLeftY(293.564f)
.width(65.592f)
.height(15.408f)
.page(1)
.build()))
.addToAllDossiers(false) .addToAllDossiers(false)
.build())) .build()))
.build()); .build());
analyzeService.reanalyze(request); analyzeService.reanalyze(request);
entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID); entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
var resizedEntity = entityLog.getEntityLogEntry().stream().filter(e -> e.getId().equals(david.getId())).findFirst().get(); var resizedEntity = entityLog.getEntityLogEntry()
.stream()
.filter(e -> e.getId().equals(david.getId()))
.findFirst()
.get();
assertEquals(resizedEntity.getState(), EntryState.APPLIED); assertEquals(resizedEntity.getState(), EntryState.APPLIED);
assertEquals(resizedEntity.getValue(), "David Ksenia"); assertEquals(resizedEntity.getValue(), "David Ksenia");
} }
@ -1364,12 +1480,7 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
private IdRemoval getIdRemoval(String id) { private IdRemoval getIdRemoval(String id) {
return IdRemoval.builder() return IdRemoval.builder().annotationId(id).removeFromAllDossiers(false).removeFromDictionary(false).requestDate(OffsetDateTime.now()).build();
.annotationId(id)
.removeFromAllDossiers(false)
.removeFromDictionary(false)
.requestDate(OffsetDateTime.now())
.build();
} }