Merge branch 'RED-7384-bp' into 'release/4.244.x'
RED-7384: fixes for migration backport See merge request redactmanager/redaction-service!288
This commit is contained in:
commit
b99d9ed59c
@ -16,7 +16,7 @@ val layoutParserVersion = "0.91.0"
|
|||||||
val jacksonVersion = "2.15.2"
|
val jacksonVersion = "2.15.2"
|
||||||
val droolsVersion = "9.44.0.Final"
|
val droolsVersion = "9.44.0.Final"
|
||||||
val pdfBoxVersion = "3.0.0"
|
val pdfBoxVersion = "3.0.0"
|
||||||
val persistenceServiceVersion = "2.338.0"
|
val persistenceServiceVersion = "2.359.0"
|
||||||
val springBootStarterVersion = "3.1.5"
|
val springBootStarterVersion = "3.1.5"
|
||||||
|
|
||||||
configurations {
|
configurations {
|
||||||
|
|||||||
@ -45,7 +45,7 @@ public class LegacyRedactionLogMergeService {
|
|||||||
public RedactionLog addManualAddEntriesAndRemoveSkippedImported(RedactionLog redactionLog, ManualRedactions manualRedactions, String dossierTemplateId) {
|
public RedactionLog addManualAddEntriesAndRemoveSkippedImported(RedactionLog redactionLog, ManualRedactions manualRedactions, String dossierTemplateId) {
|
||||||
|
|
||||||
Set<String> skippedImportedRedactions = new HashSet<>();
|
Set<String> skippedImportedRedactions = new HashSet<>();
|
||||||
log.info("Merging Redaction log with manual redactions");
|
log.info("Adding manual add Entries and removing skipped or imported entries");
|
||||||
if (manualRedactions != null) {
|
if (manualRedactions != null) {
|
||||||
|
|
||||||
var manualRedactionLogEntries = addManualAddEntries(manualRedactions.getEntriesToAdd(), redactionLog.getAnalysisNumber());
|
var manualRedactionLogEntries = addManualAddEntries(manualRedactions.getEntriesToAdd(), redactionLog.getAnalysisNumber());
|
||||||
@ -92,6 +92,10 @@ public class LegacyRedactionLogMergeService {
|
|||||||
return redactionLog;
|
return redactionLog;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public long getNumberOfAffectedAnnotations(ManualRedactions manualRedactions) {
|
||||||
|
|
||||||
|
return createManualRedactionWrappers(manualRedactions).stream().map(ManualRedactionWrapper::getId).distinct().count();
|
||||||
|
}
|
||||||
|
|
||||||
private List<ManualRedactionWrapper> createManualRedactionWrappers(ManualRedactions manualRedactions) {
|
private List<ManualRedactionWrapper> createManualRedactionWrappers(ManualRedactions manualRedactions) {
|
||||||
|
|
||||||
|
|||||||
@ -0,0 +1,94 @@
|
|||||||
|
package com.iqser.red.service.redaction.v1.server.migration;
|
||||||
|
|
||||||
|
import java.util.Collections;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Set;
|
||||||
|
import java.util.stream.Collectors;
|
||||||
|
|
||||||
|
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ChangeType;
|
||||||
|
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualChange;
|
||||||
|
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Change;
|
||||||
|
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine;
|
||||||
|
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ManualRedactionType;
|
||||||
|
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
|
||||||
|
import com.iqser.red.service.redaction.v1.server.model.MigrationEntity;
|
||||||
|
|
||||||
|
public class MigrationMapper {
|
||||||
|
|
||||||
|
public static com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Change toEntityLogChanges(Change change) {
|
||||||
|
|
||||||
|
return new com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Change(change.getAnalysisNumber(),
|
||||||
|
toEntityLogType(change.getType()),
|
||||||
|
change.getDateTime());
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
public static com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualChange toEntityLogManualChanges(com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ManualChange manualChange) {
|
||||||
|
|
||||||
|
return new com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualChange(toManualRedactionType(manualChange.getManualRedactionType()),
|
||||||
|
manualChange.getProcessedDate(),
|
||||||
|
manualChange.getRequestedDate(),
|
||||||
|
manualChange.getUserId(),
|
||||||
|
manualChange.getPropertyChanges());
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
public static ChangeType toEntityLogType(com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ChangeType type) {
|
||||||
|
|
||||||
|
return switch (type) {
|
||||||
|
case ADDED -> ChangeType.ADDED;
|
||||||
|
case REMOVED -> ChangeType.REMOVED;
|
||||||
|
case CHANGED -> ChangeType.CHANGED;
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
public static com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType toManualRedactionType(ManualRedactionType manualRedactionType) {
|
||||||
|
|
||||||
|
return switch (manualRedactionType) {
|
||||||
|
case ADD_LOCALLY -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.ADD_LOCALLY;
|
||||||
|
case ADD_TO_DICTIONARY -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.ADD_TO_DICTIONARY;
|
||||||
|
case REMOVE_LOCALLY -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.REMOVE_LOCALLY;
|
||||||
|
case REMOVE_FROM_DICTIONARY -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.REMOVE_FROM_DICTIONARY;
|
||||||
|
case FORCE_REDACT -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.FORCE_REDACT;
|
||||||
|
case FORCE_HINT -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.FORCE_HINT;
|
||||||
|
case RECATEGORIZE -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.RECATEGORIZE;
|
||||||
|
case LEGAL_BASIS_CHANGE -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.LEGAL_BASIS_CHANGE;
|
||||||
|
case RESIZE -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.RESIZE;
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
public static com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine toEntityLogEngine(Engine engine) {
|
||||||
|
|
||||||
|
return switch (engine) {
|
||||||
|
case DICTIONARY -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine.DICTIONARY;
|
||||||
|
case NER -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine.NER;
|
||||||
|
case RULE -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine.RULE;
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
public static Set<com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine> getMigratedEngines(RedactionLogEntry entry) {
|
||||||
|
|
||||||
|
if (entry.getEngines() == null) {
|
||||||
|
return Collections.emptySet();
|
||||||
|
}
|
||||||
|
return entry.getEngines()
|
||||||
|
.stream()
|
||||||
|
.map(MigrationMapper::toEntityLogEngine)
|
||||||
|
.collect(Collectors.toSet());
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
public List<ManualChange> migrateManualChanges(List<com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ManualChange> manualChanges) {
|
||||||
|
|
||||||
|
if (manualChanges == null) {
|
||||||
|
return Collections.emptyList();
|
||||||
|
}
|
||||||
|
return manualChanges.stream()
|
||||||
|
.map(MigrationMapper::toEntityLogManualChanges)
|
||||||
|
.toList();
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
@ -58,17 +58,26 @@ public class MigrationMessageReceiver {
|
|||||||
if (redactionLog.getAnalysisVersion() == 0) {
|
if (redactionLog.getAnalysisVersion() == 0) {
|
||||||
redactionLog = legacyVersion0MigrationService.mergeDuplicateAnnotationIds(redactionLog);
|
redactionLog = legacyVersion0MigrationService.mergeDuplicateAnnotationIds(redactionLog);
|
||||||
} else if (migrationRequest.getManualRedactions() != null) {
|
} else if (migrationRequest.getManualRedactions() != null) {
|
||||||
redactionLog = legacyRedactionLogMergeService.addManualAddEntriesAndRemoveSkippedImported(redactionLog, migrationRequest.getManualRedactions(), migrationRequest.getDossierTemplateId());
|
redactionLog = legacyRedactionLogMergeService.addManualAddEntriesAndRemoveSkippedImported(redactionLog,
|
||||||
|
migrationRequest.getManualRedactions(),
|
||||||
|
migrationRequest.getDossierTemplateId());
|
||||||
}
|
}
|
||||||
|
|
||||||
MigratedEntityLog migratedEntityLog = redactionLogToEntityLogMigrationService.migrate(redactionLog, document, migrationRequest.getDossierTemplateId(), migrationRequest.getManualRedactions());
|
MigratedEntityLog migratedEntityLog = redactionLogToEntityLogMigrationService.migrate(redactionLog,
|
||||||
|
document,
|
||||||
|
migrationRequest.getDossierTemplateId(),
|
||||||
|
migrationRequest.getManualRedactions(),
|
||||||
|
migrationRequest.getFileId());
|
||||||
|
|
||||||
|
log.info("Storing migrated entityLog and ids to migrate in DB for file {}", migrationRequest.getFileId());
|
||||||
redactionStorageService.storeObject(migrationRequest.getDossierId(), migrationRequest.getFileId(), FileType.ENTITY_LOG, migratedEntityLog.getEntityLog());
|
redactionStorageService.storeObject(migrationRequest.getDossierId(), migrationRequest.getFileId(), FileType.ENTITY_LOG, migratedEntityLog.getEntityLog());
|
||||||
redactionStorageService.storeObject(migrationRequest.getDossierId(), migrationRequest.getFileId(), FileType.MIGRATED_IDS, migratedEntityLog.getMigratedIds());
|
redactionStorageService.storeObject(migrationRequest.getDossierId(), migrationRequest.getFileId(), FileType.MIGRATED_IDS, migratedEntityLog.getMigratedIds());
|
||||||
|
|
||||||
sendFinished(MigrationResponse.builder().dossierId(migrationRequest.getDossierId()).fileId(migrationRequest.getFileId()).build());
|
sendFinished(MigrationResponse.builder().dossierId(migrationRequest.getDossierId()).fileId(migrationRequest.getFileId()).build());
|
||||||
log.info("Migrated {} redactionLog entries for dossierId {} and fileId {}",
|
log.info("Migrated {} redactionLog entries, found {} annotation ids for migration in the db, {} new manual entries, for dossierId {} and fileId {}",
|
||||||
migratedEntityLog.getEntityLog().getEntityLogEntry().size(),
|
migratedEntityLog.getEntityLog().getEntityLogEntry().size(),
|
||||||
|
migratedEntityLog.getMigratedIds().getMappings().size(),
|
||||||
|
migratedEntityLog.getMigratedIds().getManualRedactionEntriesToAdd().size(),
|
||||||
migrationRequest.getDossierId(),
|
migrationRequest.getDossierId(),
|
||||||
migrationRequest.getFileId());
|
migrationRequest.getFileId());
|
||||||
log.info("");
|
log.info("");
|
||||||
|
|||||||
@ -19,29 +19,24 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog
|
|||||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.migration.MigratedIds;
|
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.migration.MigratedIds;
|
||||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
|
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
|
||||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.BaseAnnotation;
|
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.BaseAnnotation;
|
||||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
|
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
|
||||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ManualRedactionType;
|
|
||||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
|
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
|
||||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog;
|
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog;
|
||||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
|
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
|
||||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogLegalBasis;
|
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogLegalBasis;
|
||||||
import com.iqser.red.service.redaction.v1.model.MigrationRequest;
|
|
||||||
import com.iqser.red.service.redaction.v1.server.model.PrecursorEntity;
|
|
||||||
import com.iqser.red.service.redaction.v1.server.model.MigratedEntityLog;
|
import com.iqser.red.service.redaction.v1.server.model.MigratedEntityLog;
|
||||||
import com.iqser.red.service.redaction.v1.server.model.MigrationEntity;
|
import com.iqser.red.service.redaction.v1.server.model.MigrationEntity;
|
||||||
|
import com.iqser.red.service.redaction.v1.server.model.PrecursorEntity;
|
||||||
import com.iqser.red.service.redaction.v1.server.model.RectangleWithPage;
|
import com.iqser.red.service.redaction.v1.server.model.RectangleWithPage;
|
||||||
import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
|
|
||||||
import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
|
|
||||||
import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
|
import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
|
||||||
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
|
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
|
||||||
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Image;
|
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Image;
|
||||||
import com.iqser.red.service.redaction.v1.server.model.document.nodes.ImageType;
|
import com.iqser.red.service.redaction.v1.server.model.document.nodes.ImageType;
|
||||||
import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
|
|
||||||
import com.iqser.red.service.redaction.v1.server.service.DictionaryService;
|
import com.iqser.red.service.redaction.v1.server.service.DictionaryService;
|
||||||
import com.iqser.red.service.redaction.v1.server.service.ManualChangesApplicationService;
|
import com.iqser.red.service.redaction.v1.server.service.ManualChangesApplicationService;
|
||||||
import com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService;
|
|
||||||
import com.iqser.red.service.redaction.v1.server.service.document.EntityEnrichmentService;
|
import com.iqser.red.service.redaction.v1.server.service.document.EntityEnrichmentService;
|
||||||
import com.iqser.red.service.redaction.v1.server.service.document.EntityFindingUtility;
|
import com.iqser.red.service.redaction.v1.server.service.document.EntityFindingUtility;
|
||||||
|
import com.iqser.red.service.redaction.v1.server.service.document.EntityFromPrecursorCreationService;
|
||||||
import com.iqser.red.service.redaction.v1.server.utils.IdBuilder;
|
import com.iqser.red.service.redaction.v1.server.utils.IdBuilder;
|
||||||
import com.iqser.red.service.redaction.v1.server.utils.MigratedIdsCollector;
|
import com.iqser.red.service.redaction.v1.server.utils.MigratedIdsCollector;
|
||||||
|
|
||||||
@ -64,13 +59,16 @@ public class RedactionLogToEntityLogMigrationService {
|
|||||||
ManualChangesApplicationService manualChangesApplicationService;
|
ManualChangesApplicationService manualChangesApplicationService;
|
||||||
|
|
||||||
|
|
||||||
public MigratedEntityLog migrate(RedactionLog redactionLog, Document document, String dossierTemplateId, ManualRedactions manualRedactions) {
|
public MigratedEntityLog migrate(RedactionLog redactionLog, Document document, String dossierTemplateId, ManualRedactions manualRedactions, String fileId) {
|
||||||
|
|
||||||
|
log.info("Migrating entities for file {}", fileId);
|
||||||
|
List<MigrationEntity> entitiesToMigrate = calculateMigrationEntitiesFromRedactionLog(redactionLog, document, dossierTemplateId, fileId);
|
||||||
|
|
||||||
List<MigrationEntity> entitiesToMigrate = calculateMigrationEntitiesFromRedactionLog(redactionLog, document, dossierTemplateId);
|
|
||||||
MigratedIds migratedIds = entitiesToMigrate.stream()
|
MigratedIds migratedIds = entitiesToMigrate.stream()
|
||||||
.collect(new MigratedIdsCollector());
|
.collect(new MigratedIdsCollector());
|
||||||
|
|
||||||
applyManualChanges(entitiesToMigrate, manualRedactions);
|
applyManualChanges(entitiesToMigrate, manualRedactions);
|
||||||
|
log.info("applying manual changes to migrated entities for file {}", fileId);
|
||||||
|
|
||||||
EntityLog entityLog = new EntityLog();
|
EntityLog entityLog = new EntityLog();
|
||||||
entityLog.setAnalysisNumber(redactionLog.getAnalysisNumber());
|
entityLog.setAnalysisNumber(redactionLog.getAnalysisNumber());
|
||||||
@ -85,6 +83,8 @@ public class RedactionLogToEntityLogMigrationService {
|
|||||||
.toList());
|
.toList());
|
||||||
|
|
||||||
Map<String, String> oldToNewIDMapping = migratedIds.buildOldToNewMapping();
|
Map<String, String> oldToNewIDMapping = migratedIds.buildOldToNewMapping();
|
||||||
|
|
||||||
|
log.info("Writing migrated entities to entityLog for file {}", fileId);
|
||||||
entityLog.setEntityLogEntry(entitiesToMigrate.stream()
|
entityLog.setEntityLogEntry(entitiesToMigrate.stream()
|
||||||
.map(migrationEntity -> migrationEntity.toEntityLogEntry(oldToNewIDMapping))
|
.map(migrationEntity -> migrationEntity.toEntityLogEntry(oldToNewIDMapping))
|
||||||
.toList());
|
.toList());
|
||||||
@ -102,6 +102,13 @@ public class RedactionLogToEntityLogMigrationService {
|
|||||||
.filter(m -> !m.getOldId().equals(m.getNewId()))
|
.filter(m -> !m.getOldId().equals(m.getNewId()))
|
||||||
.collect(new MigratedIdsCollector());
|
.collect(new MigratedIdsCollector());
|
||||||
|
|
||||||
|
List<ManualRedactionEntry> manualRedactionEntriesToAdd = entitiesToMigrate.stream()
|
||||||
|
.filter(MigrationEntity::needsManualEntry)
|
||||||
|
.map(MigrationEntity::buildManualRedactionEntry)
|
||||||
|
.toList();
|
||||||
|
|
||||||
|
idsToMigrateInDb.setManualRedactionEntriesToAdd(manualRedactionEntriesToAdd);
|
||||||
|
|
||||||
return new MigratedEntityLog(idsToMigrateInDb, entityLog);
|
return new MigratedEntityLog(idsToMigrateInDb, entityLog);
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -117,27 +124,14 @@ public class RedactionLogToEntityLogMigrationService {
|
|||||||
manualRedactions.getForceRedactions(),
|
manualRedactions.getForceRedactions(),
|
||||||
manualRedactions.getResizeRedactions(),
|
manualRedactions.getResizeRedactions(),
|
||||||
manualRedactions.getLegalBasisChanges(),
|
manualRedactions.getLegalBasisChanges(),
|
||||||
manualRedactions.getRecategorizations(),
|
manualRedactions.getRecategorizations())
|
||||||
manualRedactions.getLegalBasisChanges())
|
|
||||||
.flatMap(Collection::stream)
|
.flatMap(Collection::stream)
|
||||||
.collect(Collectors.groupingBy(BaseAnnotation::getAnnotationId));
|
.collect(Collectors.groupingBy(BaseAnnotation::getAnnotationId));
|
||||||
|
|
||||||
entitiesToMigrate.forEach(migrationEntity -> manualChangesPerAnnotationId.getOrDefault(migrationEntity.getOldId(), Collections.emptyList())
|
entitiesToMigrate.forEach(migrationEntity -> migrationEntity.applyManualChanges(manualChangesPerAnnotationId.getOrDefault(migrationEntity.getOldId(),
|
||||||
.forEach(manualChange -> {
|
Collections.emptyList()),
|
||||||
if (manualChange instanceof ManualResizeRedaction manualResizeRedaction && migrationEntity.getMigratedEntity() instanceof TextEntity textEntity) {
|
manualChangesApplicationService));
|
||||||
ManualResizeRedaction migratedManualResizeRedaction = ManualResizeRedaction.builder()
|
|
||||||
.positions(manualResizeRedaction.getPositions())
|
|
||||||
.annotationId(migrationEntity.getNewId())
|
|
||||||
.updateDictionary(manualResizeRedaction.getUpdateDictionary())
|
|
||||||
.addToAllDossiers(manualResizeRedaction.isAddToAllDossiers())
|
|
||||||
.textAfter(manualResizeRedaction.getTextAfter())
|
|
||||||
.textBefore(manualResizeRedaction.getTextBefore())
|
|
||||||
.build();
|
|
||||||
manualChangesApplicationService.resize(textEntity, migratedManualResizeRedaction);
|
|
||||||
} else {
|
|
||||||
migrationEntity.getMigratedEntity().getManualOverwrite().addChange(manualChange);
|
|
||||||
}
|
|
||||||
}));
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@ -147,10 +141,10 @@ public class RedactionLogToEntityLogMigrationService {
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
private List<MigrationEntity> calculateMigrationEntitiesFromRedactionLog(RedactionLog redactionLog, Document document, String dossierTemplateId) {
|
private List<MigrationEntity> calculateMigrationEntitiesFromRedactionLog(RedactionLog redactionLog, Document document, String dossierTemplateId, String fileId) {
|
||||||
|
|
||||||
List<MigrationEntity> images = getImageBasedMigrationEntities(redactionLog, document, dossierTemplateId);
|
List<MigrationEntity> images = getImageBasedMigrationEntities(redactionLog, document, fileId);
|
||||||
List<MigrationEntity> textMigrationEntities = getTextBasedMigrationEntities(redactionLog, document, dossierTemplateId);
|
List<MigrationEntity> textMigrationEntities = getTextBasedMigrationEntities(redactionLog, document, dossierTemplateId, fileId);
|
||||||
return Stream.of(textMigrationEntities.stream(), images.stream())
|
return Stream.of(textMigrationEntities.stream(), images.stream())
|
||||||
.flatMap(Function.identity())
|
.flatMap(Function.identity())
|
||||||
.toList();
|
.toList();
|
||||||
@ -163,7 +157,7 @@ public class RedactionLogToEntityLogMigrationService {
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
private List<MigrationEntity> getImageBasedMigrationEntities(RedactionLog redactionLog, Document document, String dossierTemplateId) {
|
private List<MigrationEntity> getImageBasedMigrationEntities(RedactionLog redactionLog, Document document, String fileId) {
|
||||||
|
|
||||||
List<Image> images = document.streamAllImages()
|
List<Image> images = document.streamAllImages()
|
||||||
.collect(Collectors.toList());
|
.collect(Collectors.toList());
|
||||||
@ -195,7 +189,8 @@ public class RedactionLogToEntityLogMigrationService {
|
|||||||
}
|
}
|
||||||
|
|
||||||
String ruleIdentifier;
|
String ruleIdentifier;
|
||||||
String reason = Optional.ofNullable(redactionLogImage.getReason()).orElse("");
|
String reason = Optional.ofNullable(redactionLogImage.getReason())
|
||||||
|
.orElse("");
|
||||||
if (redactionLogImage.getMatchedRule().isBlank() || redactionLogImage.getMatchedRule() == null) {
|
if (redactionLogImage.getMatchedRule().isBlank() || redactionLogImage.getMatchedRule() == null) {
|
||||||
ruleIdentifier = "OLDIMG.0.0";
|
ruleIdentifier = "OLDIMG.0.0";
|
||||||
} else {
|
} else {
|
||||||
@ -209,7 +204,7 @@ public class RedactionLogToEntityLogMigrationService {
|
|||||||
} else {
|
} else {
|
||||||
closestImage.skip(ruleIdentifier, reason);
|
closestImage.skip(ruleIdentifier, reason);
|
||||||
}
|
}
|
||||||
migrationEntities.add(new MigrationEntity(null, redactionLogImage, closestImage, redactionLogImage.getId(), closestImage.getId()));
|
migrationEntities.add(MigrationEntity.fromRedactionLogImage(redactionLogImage, closestImage, fileId));
|
||||||
}
|
}
|
||||||
return migrationEntities;
|
return migrationEntities;
|
||||||
}
|
}
|
||||||
@ -250,40 +245,20 @@ public class RedactionLogToEntityLogMigrationService {
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
private List<MigrationEntity> getTextBasedMigrationEntities(RedactionLog redactionLog, Document document, String dossierTemplateId) {
|
private List<MigrationEntity> getTextBasedMigrationEntities(RedactionLog redactionLog, Document document, String dossierTemplateId, String fileId) {
|
||||||
|
|
||||||
List<MigrationEntity> entitiesToMigrate = redactionLog.getRedactionLogEntry()
|
List<MigrationEntity> entitiesToMigrate = redactionLog.getRedactionLogEntry()
|
||||||
.stream()
|
.stream()
|
||||||
.filter(redactionLogEntry -> !redactionLogEntry.isImage())
|
.filter(redactionLogEntry -> !redactionLogEntry.isImage())
|
||||||
.map(entry -> MigrationEntity.fromRedactionLogEntry(entry, dictionaryService.isHint(entry.getType(), dossierTemplateId)))
|
.map(entry -> MigrationEntity.fromRedactionLogEntry(entry, dictionaryService.isHint(entry.getType(), dossierTemplateId), fileId))
|
||||||
.peek(migrationEntity -> {
|
|
||||||
if (migrationEntity.getPrecursorEntity().getEntityType().equals(EntityType.HINT) &&//
|
|
||||||
!migrationEntity.getRedactionLogEntry().isHint() &&//
|
|
||||||
!migrationEntity.getRedactionLogEntry().isRedacted()) {
|
|
||||||
migrationEntity.getPrecursorEntity().ignore(migrationEntity.getPrecursorEntity().getRuleIdentifier(), migrationEntity.getPrecursorEntity().getReason());
|
|
||||||
} else if (migrationEntity.getRedactionLogEntry().lastChangeIsRemoved()) {
|
|
||||||
migrationEntity.getPrecursorEntity().remove(migrationEntity.getPrecursorEntity().getRuleIdentifier(), migrationEntity.getPrecursorEntity().getReason());
|
|
||||||
} else if (lastManualChangeIsRemove(migrationEntity)) {
|
|
||||||
migrationEntity.getPrecursorEntity().ignore(migrationEntity.getPrecursorEntity().getRuleIdentifier(), migrationEntity.getPrecursorEntity().getReason());
|
|
||||||
} else if (migrationEntity.getPrecursorEntity().isApplied() && migrationEntity.getRedactionLogEntry().isRecommendation()) {
|
|
||||||
migrationEntity.getPrecursorEntity()
|
|
||||||
.skip(migrationEntity.getPrecursorEntity().getRuleIdentifier(), migrationEntity.getPrecursorEntity().getReason());
|
|
||||||
} else if (migrationEntity.getPrecursorEntity().isApplied()) {
|
|
||||||
migrationEntity.getPrecursorEntity()
|
|
||||||
.apply(migrationEntity.getPrecursorEntity().getRuleIdentifier(),
|
|
||||||
migrationEntity.getPrecursorEntity().getReason(),
|
|
||||||
migrationEntity.getPrecursorEntity().getLegalBasis());
|
|
||||||
} else {
|
|
||||||
migrationEntity.getPrecursorEntity()
|
|
||||||
.skip(migrationEntity.getPrecursorEntity().getRuleIdentifier(), migrationEntity.getPrecursorEntity().getReason());
|
|
||||||
}
|
|
||||||
})
|
|
||||||
.toList();
|
.toList();
|
||||||
|
|
||||||
Map<String, List<TextEntity>> tempEntitiesByValue = entityFindingUtility.findAllPossibleEntitiesAndGroupByValue(document,
|
List<PrecursorEntity> precursorEntities = entitiesToMigrate.stream()
|
||||||
entitiesToMigrate.stream()
|
|
||||||
.map(MigrationEntity::getPrecursorEntity)
|
.map(MigrationEntity::getPrecursorEntity)
|
||||||
.toList());
|
.toList();
|
||||||
|
|
||||||
|
log.info("Finding all possible entities");
|
||||||
|
Map<String, List<TextEntity>> tempEntitiesByValue = entityFindingUtility.findAllPossibleEntitiesAndGroupByValue(document, precursorEntities);
|
||||||
|
|
||||||
for (MigrationEntity migrationEntity : entitiesToMigrate) {
|
for (MigrationEntity migrationEntity : entitiesToMigrate) {
|
||||||
Optional<TextEntity> optionalTextEntity = entityFindingUtility.findClosestEntityAndReturnEmptyIfNotFound(migrationEntity.getPrecursorEntity(),
|
Optional<TextEntity> optionalTextEntity = entityFindingUtility.findClosestEntityAndReturnEmptyIfNotFound(migrationEntity.getPrecursorEntity(),
|
||||||
@ -297,45 +272,19 @@ public class RedactionLogToEntityLogMigrationService {
|
|||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
TextEntity entity = createCorrectEntity(migrationEntity.getPrecursorEntity(), document, optionalTextEntity.get().getTextRange());
|
TextEntity migratedEntity = EntityFromPrecursorCreationService.createCorrectEntity(migrationEntity.getPrecursorEntity(), optionalTextEntity.get(), true);
|
||||||
migrationEntity.setMigratedEntity(entity);
|
|
||||||
migrationEntity.setOldId(migrationEntity.getPrecursorEntity().getId());
|
|
||||||
migrationEntity.setNewId(entity.getId()); // Can only be on one page, since redactionLogEntries can only be on one page
|
|
||||||
|
|
||||||
|
migrationEntity.setMigratedEntity(migratedEntity);
|
||||||
|
migrationEntity.setOldId(migrationEntity.getPrecursorEntity().getId());
|
||||||
|
migrationEntity.setNewId(migratedEntity.getId());
|
||||||
}
|
}
|
||||||
|
|
||||||
tempEntitiesByValue.values()
|
tempEntitiesByValue.values()
|
||||||
.stream()
|
.stream()
|
||||||
.flatMap(Collection::stream)
|
.flatMap(Collection::stream)
|
||||||
.forEach(TextEntity::removeFromGraph);
|
.forEach(TextEntity::removeFromGraph);
|
||||||
|
|
||||||
return entitiesToMigrate;
|
return entitiesToMigrate;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
private static boolean lastManualChangeIsRemove(MigrationEntity migrationEntity) {
|
|
||||||
|
|
||||||
if (migrationEntity.getRedactionLogEntry().getManualChanges() == null) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
return migrationEntity.getRedactionLogEntry().getManualChanges()
|
|
||||||
.stream()
|
|
||||||
.reduce((a, b) -> b)
|
|
||||||
.map(m -> m.getManualRedactionType().equals(ManualRedactionType.REMOVE_LOCALLY))
|
|
||||||
.orElse(false);
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
private TextEntity createCorrectEntity(PrecursorEntity precursorEntity, SemanticNode node, TextRange closestTextRange) {
|
|
||||||
|
|
||||||
EntityCreationService entityCreationService = new EntityCreationService(entityEnrichmentService);
|
|
||||||
TextEntity correctEntity = entityCreationService.forceByTextRange(closestTextRange, precursorEntity.getType(), precursorEntity.getEntityType(), node);
|
|
||||||
|
|
||||||
correctEntity.addMatchedRules(precursorEntity.getMatchedRuleList());
|
|
||||||
correctEntity.setDictionaryEntry(precursorEntity.isDictionaryEntry());
|
|
||||||
correctEntity.setDossierDictionaryEntry(precursorEntity.isDossierDictionaryEntry());
|
|
||||||
correctEntity.getManualOverwrite().addChanges(precursorEntity.getManualOverwrite().getManualChangeLog());
|
|
||||||
return correctEntity;
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|||||||
@ -1,7 +1,10 @@
|
|||||||
package com.iqser.red.service.redaction.v1.server.model;
|
package com.iqser.red.service.redaction.v1.server.model;
|
||||||
|
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
|
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
|
||||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.migration.MigratedIds;
|
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.migration.MigratedIds;
|
||||||
|
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
|
||||||
|
|
||||||
import lombok.AllArgsConstructor;
|
import lombok.AllArgsConstructor;
|
||||||
import lombok.Builder;
|
import lombok.Builder;
|
||||||
@ -16,5 +19,4 @@ public class MigratedEntityLog {
|
|||||||
|
|
||||||
MigratedIds migratedIds;
|
MigratedIds migratedIds;
|
||||||
EntityLog entityLog;
|
EntityLog entityLog;
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|||||||
@ -1,34 +1,45 @@
|
|||||||
package com.iqser.red.service.redaction.v1.server.model;
|
package com.iqser.red.service.redaction.v1.server.model;
|
||||||
|
|
||||||
|
import static com.iqser.red.service.redaction.v1.server.service.EntityLogCreatorService.buildEntryState;
|
||||||
|
import static com.iqser.red.service.redaction.v1.server.service.EntityLogCreatorService.buildEntryType;
|
||||||
|
|
||||||
|
import java.awt.geom.Rectangle2D;
|
||||||
import java.util.Collections;
|
import java.util.Collections;
|
||||||
|
import java.util.LinkedList;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
import java.util.Optional;
|
import java.util.Optional;
|
||||||
import java.util.Set;
|
import java.util.Set;
|
||||||
import java.util.stream.Collectors;
|
import java.util.stream.Collectors;
|
||||||
|
|
||||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ChangeType;
|
|
||||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
|
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
|
||||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState;
|
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState;
|
||||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryType;
|
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryType;
|
||||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position;
|
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position;
|
||||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Change;
|
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle;
|
||||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine;
|
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.BaseAnnotation;
|
||||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ManualChange;
|
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
|
||||||
|
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
|
||||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ManualRedactionType;
|
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ManualRedactionType;
|
||||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
|
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
|
||||||
|
import com.iqser.red.service.redaction.v1.server.migration.MigrationMapper;
|
||||||
import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
|
import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
|
||||||
import com.iqser.red.service.redaction.v1.server.model.document.entity.IEntity;
|
import com.iqser.red.service.redaction.v1.server.model.document.entity.IEntity;
|
||||||
import com.iqser.red.service.redaction.v1.server.model.document.entity.ManualChangeOverwrite;
|
import com.iqser.red.service.redaction.v1.server.model.document.entity.ManualChangeOverwrite;
|
||||||
import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
|
import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
|
||||||
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Image;
|
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Image;
|
||||||
import com.iqser.red.service.redaction.v1.server.service.ManualChangeFactory;
|
import com.iqser.red.service.redaction.v1.server.service.ManualChangeFactory;
|
||||||
|
import com.iqser.red.service.redaction.v1.server.service.ManualChangesApplicationService;
|
||||||
|
|
||||||
import lombok.AllArgsConstructor;
|
import lombok.AllArgsConstructor;
|
||||||
|
import lombok.Builder;
|
||||||
import lombok.Data;
|
import lombok.Data;
|
||||||
import lombok.RequiredArgsConstructor;
|
import lombok.RequiredArgsConstructor;
|
||||||
|
import lombok.extern.slf4j.Slf4j;
|
||||||
|
|
||||||
|
@Slf4j
|
||||||
@Data
|
@Data
|
||||||
|
@Builder
|
||||||
@AllArgsConstructor
|
@AllArgsConstructor
|
||||||
@RequiredArgsConstructor
|
@RequiredArgsConstructor
|
||||||
public final class MigrationEntity {
|
public final class MigrationEntity {
|
||||||
@ -38,28 +49,73 @@ public final class MigrationEntity {
|
|||||||
private IEntity migratedEntity;
|
private IEntity migratedEntity;
|
||||||
private String oldId;
|
private String oldId;
|
||||||
private String newId;
|
private String newId;
|
||||||
|
private String fileId;
|
||||||
|
|
||||||
|
@Builder.Default
|
||||||
|
List<BaseAnnotation> manualChanges = new LinkedList<>();
|
||||||
|
|
||||||
|
|
||||||
public static MigrationEntity fromRedactionLogEntry(RedactionLogEntry redactionLogEntry, boolean hint) {
|
public static MigrationEntity fromRedactionLogEntry(RedactionLogEntry redactionLogEntry, boolean hint, String fileId) {
|
||||||
|
|
||||||
return new MigrationEntity(createPrecursorEntity(redactionLogEntry, hint), redactionLogEntry);
|
PrecursorEntity precursorEntity = createPrecursorEntity(redactionLogEntry, hint);
|
||||||
|
|
||||||
|
if (precursorEntity.getEntityType().equals(EntityType.HINT) && !redactionLogEntry.isHint() && !redactionLogEntry.isRedacted()) {
|
||||||
|
precursorEntity.ignore(precursorEntity.getRuleIdentifier(), precursorEntity.getReason());
|
||||||
|
} else if (redactionLogEntry.lastChangeIsRemoved()) {
|
||||||
|
precursorEntity.remove(precursorEntity.getRuleIdentifier(), precursorEntity.getReason());
|
||||||
|
} else if (lastManualChangeIsRemove(redactionLogEntry)) {
|
||||||
|
precursorEntity.ignore(precursorEntity.getRuleIdentifier(), precursorEntity.getReason());
|
||||||
|
} else if (precursorEntity.isApplied() && redactionLogEntry.isRecommendation()) {
|
||||||
|
precursorEntity.skip(precursorEntity.getRuleIdentifier(), precursorEntity.getReason());
|
||||||
|
} else if (precursorEntity.isApplied()) {
|
||||||
|
precursorEntity.apply(precursorEntity.getRuleIdentifier(), precursorEntity.getReason(), precursorEntity.getLegalBasis());
|
||||||
|
} else {
|
||||||
|
precursorEntity.skip(precursorEntity.getRuleIdentifier(), precursorEntity.getReason());
|
||||||
|
}
|
||||||
|
|
||||||
|
return MigrationEntity.builder().precursorEntity(precursorEntity).redactionLogEntry(redactionLogEntry).oldId(redactionLogEntry.getId()).fileId(fileId).build();
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
public static MigrationEntity fromRedactionLogImage(RedactionLogEntry redactionLogImage, Image image, String fileId) {
|
||||||
|
|
||||||
|
return MigrationEntity.builder().redactionLogEntry(redactionLogImage).migratedEntity(image).oldId(redactionLogImage.getId()).newId(image.getId()).fileId(fileId).build();
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
private static boolean lastManualChangeIsRemove(RedactionLogEntry redactionLogEntry) {
|
||||||
|
|
||||||
|
if (redactionLogEntry.getManualChanges() == null) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
return redactionLogEntry.getManualChanges()
|
||||||
|
.stream()
|
||||||
|
.reduce((a, b) -> b)
|
||||||
|
.map(m -> m.getManualRedactionType().equals(ManualRedactionType.REMOVE_LOCALLY))
|
||||||
|
.orElse(false);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
public static PrecursorEntity createPrecursorEntity(RedactionLogEntry redactionLogEntry, boolean hint) {
|
public static PrecursorEntity createPrecursorEntity(RedactionLogEntry redactionLogEntry, boolean hint) {
|
||||||
|
|
||||||
String ruleIdentifier = buildRuleIdentifier(redactionLogEntry);
|
String ruleIdentifier = buildRuleIdentifier(redactionLogEntry);
|
||||||
List<RectangleWithPage> rectangleWithPages = redactionLogEntry.getPositions().stream().map(RectangleWithPage::fromRedactionLogRectangle).toList();
|
List<RectangleWithPage> rectangleWithPages = redactionLogEntry.getPositions()
|
||||||
|
.stream()
|
||||||
|
.map(RectangleWithPage::fromRedactionLogRectangle)
|
||||||
|
.toList();
|
||||||
EntityType entityType = getEntityType(redactionLogEntry, hint);
|
EntityType entityType = getEntityType(redactionLogEntry, hint);
|
||||||
return PrecursorEntity.builder()
|
return PrecursorEntity.builder()
|
||||||
.id(redactionLogEntry.getId())
|
.id(redactionLogEntry.getId())
|
||||||
.value(redactionLogEntry.getValue())
|
.value(redactionLogEntry.getValue())
|
||||||
.entityPosition(rectangleWithPages)
|
.entityPosition(rectangleWithPages)
|
||||||
.ruleIdentifier(ruleIdentifier)
|
.ruleIdentifier(ruleIdentifier)
|
||||||
.reason(Optional.ofNullable(redactionLogEntry.getReason()).orElse(""))
|
.reason(Optional.ofNullable(redactionLogEntry.getReason())
|
||||||
|
.orElse(""))
|
||||||
.legalBasis(redactionLogEntry.getLegalBasis())
|
.legalBasis(redactionLogEntry.getLegalBasis())
|
||||||
.type(redactionLogEntry.getType())
|
.type(redactionLogEntry.getType())
|
||||||
.section(redactionLogEntry.getSection())
|
.section(redactionLogEntry.getSection())
|
||||||
|
.engines(MigrationMapper.getMigratedEngines(redactionLogEntry))
|
||||||
.entityType(entityType)
|
.entityType(entityType)
|
||||||
.applied(redactionLogEntry.isRedacted())
|
.applied(redactionLogEntry.isRedacted())
|
||||||
.isDictionaryEntry(redactionLogEntry.isDictionaryEntry())
|
.isDictionaryEntry(redactionLogEntry.isDictionaryEntry())
|
||||||
@ -100,14 +156,6 @@ public final class MigrationEntity {
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
private static com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Change toEntityLogChanges(Change change) {
|
|
||||||
|
|
||||||
return new com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Change(change.getAnalysisNumber(),
|
|
||||||
toEntityLogType(change.getType()),
|
|
||||||
change.getDateTime());
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
private static EntryType getEntryType(EntityType entityType) {
|
private static EntryType getEntryType(EntityType entityType) {
|
||||||
|
|
||||||
return switch (entityType) {
|
return switch (entityType) {
|
||||||
@ -120,42 +168,6 @@ public final class MigrationEntity {
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
private static com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualChange toEntityLogManualChanges(ManualChange manualChange) {
|
|
||||||
|
|
||||||
return new com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualChange(toManualRedactionType(manualChange.getManualRedactionType()),
|
|
||||||
manualChange.getProcessedDate(),
|
|
||||||
manualChange.getRequestedDate(),
|
|
||||||
manualChange.getUserId(),
|
|
||||||
manualChange.getPropertyChanges());
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
private static ChangeType toEntityLogType(com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ChangeType type) {
|
|
||||||
|
|
||||||
return switch (type) {
|
|
||||||
case ADDED -> ChangeType.ADDED;
|
|
||||||
case REMOVED -> ChangeType.REMOVED;
|
|
||||||
case CHANGED -> ChangeType.CHANGED;
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
private static com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType toManualRedactionType(ManualRedactionType manualRedactionType) {
|
|
||||||
|
|
||||||
return switch (manualRedactionType) {
|
|
||||||
case ADD_LOCALLY -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.ADD_LOCALLY;
|
|
||||||
case ADD_TO_DICTIONARY -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.ADD_TO_DICTIONARY;
|
|
||||||
case REMOVE_LOCALLY -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.REMOVE_LOCALLY;
|
|
||||||
case REMOVE_FROM_DICTIONARY -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.REMOVE_FROM_DICTIONARY;
|
|
||||||
case FORCE_REDACT -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.FORCE_REDACT;
|
|
||||||
case FORCE_HINT -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.FORCE_HINT;
|
|
||||||
case RECATEGORIZE -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.RECATEGORIZE;
|
|
||||||
case LEGAL_BASIS_CHANGE -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.LEGAL_BASIS_CHANGE;
|
|
||||||
case RESIZE -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.RESIZE;
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
public EntityLogEntry toEntityLogEntry(Map<String, String> oldToNewIdMapping) {
|
public EntityLogEntry toEntityLogEntry(Map<String, String> oldToNewIdMapping) {
|
||||||
|
|
||||||
EntityLogEntry entityLogEntry;
|
EntityLogEntry entityLogEntry;
|
||||||
@ -171,10 +183,13 @@ public final class MigrationEntity {
|
|||||||
|
|
||||||
entityLogEntry.setManualChanges(ManualChangeFactory.toManualChangeList(migratedEntity.getManualOverwrite().getManualChangeLog(), redactionLogEntry.isHint()));
|
entityLogEntry.setManualChanges(ManualChangeFactory.toManualChangeList(migratedEntity.getManualOverwrite().getManualChangeLog(), redactionLogEntry.isHint()));
|
||||||
entityLogEntry.setColor(redactionLogEntry.getColor());
|
entityLogEntry.setColor(redactionLogEntry.getColor());
|
||||||
entityLogEntry.setChanges(redactionLogEntry.getChanges().stream().map(MigrationEntity::toEntityLogChanges).toList());
|
entityLogEntry.setChanges(redactionLogEntry.getChanges()
|
||||||
|
.stream()
|
||||||
|
.map(MigrationMapper::toEntityLogChanges)
|
||||||
|
.toList());
|
||||||
entityLogEntry.setReference(migrateSetOfIds(redactionLogEntry.getReference(), oldToNewIdMapping));
|
entityLogEntry.setReference(migrateSetOfIds(redactionLogEntry.getReference(), oldToNewIdMapping));
|
||||||
entityLogEntry.setImportedRedactionIntersections(migrateSetOfIds(redactionLogEntry.getImportedRedactionIntersections(), oldToNewIdMapping));
|
entityLogEntry.setImportedRedactionIntersections(migrateSetOfIds(redactionLogEntry.getImportedRedactionIntersections(), oldToNewIdMapping));
|
||||||
entityLogEntry.setEngines(getMigratedEngines(redactionLogEntry));
|
entityLogEntry.setEngines(MigrationMapper.getMigratedEngines(redactionLogEntry));
|
||||||
if (redactionLogEntry.getLegalBasis() != null) {
|
if (redactionLogEntry.getLegalBasis() != null) {
|
||||||
entityLogEntry.setLegalBasis(redactionLogEntry.getLegalBasis());
|
entityLogEntry.setLegalBasis(redactionLogEntry.getLegalBasis());
|
||||||
}
|
}
|
||||||
@ -198,40 +213,14 @@ public final class MigrationEntity {
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
private List<com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualChange> migrateManualChanges(List<ManualChange> manualChanges) {
|
|
||||||
|
|
||||||
if (manualChanges == null) {
|
|
||||||
return Collections.emptyList();
|
|
||||||
}
|
|
||||||
return manualChanges.stream().map(MigrationEntity::toEntityLogManualChanges).toList();
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
private static Set<com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine> getMigratedEngines(RedactionLogEntry entry) {
|
|
||||||
|
|
||||||
if (entry.getEngines() == null) {
|
|
||||||
return Collections.emptySet();
|
|
||||||
}
|
|
||||||
return entry.getEngines().stream().map(MigrationEntity::toEntityLogEngine).collect(Collectors.toSet());
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
private Set<String> migrateSetOfIds(Set<String> ids, Map<String, String> oldToNewIdMapping) {
|
private Set<String> migrateSetOfIds(Set<String> ids, Map<String, String> oldToNewIdMapping) {
|
||||||
|
|
||||||
if (ids == null) {
|
if (ids == null) {
|
||||||
return Collections.emptySet();
|
return Collections.emptySet();
|
||||||
}
|
}
|
||||||
return ids.stream().map(oldToNewIdMapping::get).collect(Collectors.toSet());
|
return ids.stream()
|
||||||
}
|
.map(oldToNewIdMapping::get)
|
||||||
|
.collect(Collectors.toSet());
|
||||||
|
|
||||||
private static com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine toEntityLogEngine(Engine engine) {
|
|
||||||
|
|
||||||
return switch (engine) {
|
|
||||||
case DICTIONARY -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine.DICTIONARY;
|
|
||||||
case NER -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine.NER;
|
|
||||||
case RULE -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine.RULE;
|
|
||||||
};
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@ -249,7 +238,8 @@ public final class MigrationEntity {
|
|||||||
.positions(positions)
|
.positions(positions)
|
||||||
.containingNodeId(image.getTreeId())
|
.containingNodeId(image.getTreeId())
|
||||||
.closestHeadline(image.getHeadline().getTextBlock().getSearchText())
|
.closestHeadline(image.getHeadline().getTextBlock().getSearchText())
|
||||||
.section(redactionLogEntry.getSection())
|
.section(image.getManualOverwrite().getSection()
|
||||||
|
.orElse(redactionLogEntry.getSection()))
|
||||||
.textAfter(redactionLogEntry.getTextAfter())
|
.textAfter(redactionLogEntry.getTextAfter())
|
||||||
.textBefore(redactionLogEntry.getTextBefore())
|
.textBefore(redactionLogEntry.getTextBefore())
|
||||||
.imageHasTransparency(image.isTransparent())
|
.imageHasTransparency(image.isTransparent())
|
||||||
@ -270,7 +260,8 @@ public final class MigrationEntity {
|
|||||||
.type(precursorEntity.type())
|
.type(precursorEntity.type())
|
||||||
.state(buildEntryState(precursorEntity))
|
.state(buildEntryState(precursorEntity))
|
||||||
.entryType(buildEntryType(precursorEntity))
|
.entryType(buildEntryType(precursorEntity))
|
||||||
.section(redactionLogEntry.getSection())
|
.section(precursorEntity.getManualOverwrite().getSection()
|
||||||
|
.orElse(redactionLogEntry.getSection()))
|
||||||
.textAfter(redactionLogEntry.getTextAfter())
|
.textAfter(redactionLogEntry.getTextAfter())
|
||||||
.textBefore(redactionLogEntry.getTextBefore())
|
.textBefore(redactionLogEntry.getTextBefore())
|
||||||
.containingNodeId(Collections.emptyList())
|
.containingNodeId(Collections.emptyList())
|
||||||
@ -280,8 +271,7 @@ public final class MigrationEntity {
|
|||||||
.dossierDictionaryEntry(precursorEntity.isDossierDictionaryEntry())
|
.dossierDictionaryEntry(precursorEntity.isDossierDictionaryEntry())
|
||||||
.startOffset(-1)
|
.startOffset(-1)
|
||||||
.endOffset(-1)
|
.endOffset(-1)
|
||||||
.positions(precursorEntity.getManualOverwrite()
|
.positions(precursorEntity.getManualOverwrite().getPositions()
|
||||||
.getPositions()
|
|
||||||
.orElse(precursorEntity.getEntityPosition())
|
.orElse(precursorEntity.getEntityPosition())
|
||||||
.stream()
|
.stream()
|
||||||
.map(entityPosition -> new Position(entityPosition.rectangle2D(), entityPosition.pageNumber()))
|
.map(entityPosition -> new Position(entityPosition.rectangle2D(), entityPosition.pageNumber()))
|
||||||
@ -300,11 +290,13 @@ public final class MigrationEntity {
|
|||||||
.positions(rectanglesPerLine)
|
.positions(rectanglesPerLine)
|
||||||
.reason(entity.buildReasonWithManualChangeDescriptions())
|
.reason(entity.buildReasonWithManualChangeDescriptions())
|
||||||
.legalBasis(entity.legalBasis())
|
.legalBasis(entity.legalBasis())
|
||||||
.value(entity.getManualOverwrite().getValue().orElse(entity.getMatchedRule().isWriteValueWithLineBreaks() ? entity.getValueWithLineBreaks() : entity.getValue()))
|
.value(entity.getManualOverwrite().getValue()
|
||||||
|
.orElse(entity.getMatchedRule().isWriteValueWithLineBreaks() ? entity.getValueWithLineBreaks() : entity.getValue()))
|
||||||
.type(entity.type())
|
.type(entity.type())
|
||||||
.section(redactionLogEntry.getSection())
|
.section(entity.getManualOverwrite().getSection()
|
||||||
.textAfter(redactionLogEntry.getTextAfter())
|
.orElse(redactionLogEntry.getSection()))
|
||||||
.textBefore(redactionLogEntry.getTextBefore())
|
.textAfter(entity.getTextAfter())
|
||||||
|
.textBefore(entity.getTextBefore())
|
||||||
.containingNodeId(entity.getDeepestFullyContainingNode().getTreeId())
|
.containingNodeId(entity.getDeepestFullyContainingNode().getTreeId())
|
||||||
.closestHeadline(entity.getDeepestFullyContainingNode().getHeadline().getTextBlock().getSearchText())
|
.closestHeadline(entity.getDeepestFullyContainingNode().getHeadline().getTextBlock().getSearchText())
|
||||||
.matchedRule(entity.getMatchedRule().getRuleIdentifier().toString())
|
.matchedRule(entity.getMatchedRule().getRuleIdentifier().toString())
|
||||||
@ -322,54 +314,129 @@ public final class MigrationEntity {
|
|||||||
private static List<Position> getRectanglesPerLine(TextEntity entity) {
|
private static List<Position> getRectanglesPerLine(TextEntity entity) {
|
||||||
|
|
||||||
return getPositionsFromOverride(entity).orElse(entity.getPositionsOnPagePerPage()
|
return getPositionsFromOverride(entity).orElse(entity.getPositionsOnPagePerPage()
|
||||||
.get(0)
|
.get(0).getRectanglePerLine()
|
||||||
.getRectanglePerLine()
|
|
||||||
.stream()
|
.stream()
|
||||||
.map(rectangle2D -> new Position(rectangle2D, entity.getPositionsOnPagePerPage().get(0).getPage().getNumber()))
|
.map(rectangle2D -> new Position(rectangle2D,
|
||||||
|
entity.getPositionsOnPagePerPage()
|
||||||
|
.get(0).getPage().getNumber()))
|
||||||
.toList());
|
.toList());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
private static Optional<List<Position>> getPositionsFromOverride(IEntity entity) {
|
private static Optional<List<Position>> getPositionsFromOverride(IEntity entity) {
|
||||||
|
|
||||||
return entity.getManualOverwrite().getPositions().map(rects -> rects.stream().map(r -> new Position(r.rectangle2D(), r.pageNumber())).toList());
|
return entity.getManualOverwrite().getPositions()
|
||||||
}
|
.map(rects -> rects.stream()
|
||||||
|
.map(r -> new Position(r.rectangle2D(), r.pageNumber()))
|
||||||
|
.toList());
|
||||||
private EntryState buildEntryState(IEntity entity) {
|
|
||||||
|
|
||||||
if (entity.applied() && entity.active()) {
|
|
||||||
return EntryState.APPLIED;
|
|
||||||
} else if (entity.skipped() && entity.active()) {
|
|
||||||
return EntryState.SKIPPED;
|
|
||||||
} else if (entity.ignored()) {
|
|
||||||
return EntryState.IGNORED;
|
|
||||||
} else {
|
|
||||||
return EntryState.REMOVED;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
private EntryType buildEntryType(IEntity entity) {
|
|
||||||
|
|
||||||
if (entity instanceof TextEntity textEntity) {
|
|
||||||
return getEntryType(textEntity.getEntityType());
|
|
||||||
} else if (entity instanceof PrecursorEntity precursorEntity) {
|
|
||||||
if (precursorEntity.isRectangle()) {
|
|
||||||
return EntryType.AREA;
|
|
||||||
}
|
|
||||||
return getEntryType(precursorEntity.getEntityType());
|
|
||||||
} else if (entity instanceof Image) {
|
|
||||||
return EntryType.IMAGE;
|
|
||||||
}
|
|
||||||
throw new UnsupportedOperationException(String.format("Entity subclass %s is not implemented!", entity.getClass()));
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
public boolean hasManualChangesOrComments() {
|
public boolean hasManualChangesOrComments() {
|
||||||
|
|
||||||
return !(redactionLogEntry.getManualChanges() == null || redactionLogEntry.getManualChanges().isEmpty()) || //
|
return !(redactionLogEntry.getManualChanges() == null || redactionLogEntry.getManualChanges().isEmpty()) || //
|
||||||
!(redactionLogEntry.getComments() == null || redactionLogEntry.getComments().isEmpty());
|
!(redactionLogEntry.getComments() == null || redactionLogEntry.getComments().isEmpty()) //
|
||||||
|
|| hasManualChanges();
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
public boolean hasManualChanges() {
|
||||||
|
|
||||||
|
return !manualChanges.isEmpty();
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
public void applyManualChanges(List<BaseAnnotation> manualChangesToApply, ManualChangesApplicationService manualChangesApplicationService) {
|
||||||
|
|
||||||
|
manualChanges.addAll(manualChangesToApply);
|
||||||
|
manualChangesToApply.forEach(manualChange -> {
|
||||||
|
if (manualChange instanceof ManualResizeRedaction manualResizeRedaction && migratedEntity instanceof TextEntity textEntity) {
|
||||||
|
// Due to the value in the old redaction log already being resized, there is no way to find the original entity ID and therefore to migrate the resize annotation correctly.
|
||||||
|
// Instead, we add an add_locally change to the db.
|
||||||
|
ManualResizeRedaction migratedManualResizeRedaction = ManualResizeRedaction.builder()
|
||||||
|
.positions(manualResizeRedaction.getPositions())
|
||||||
|
.annotationId(getNewId())
|
||||||
|
.updateDictionary(manualResizeRedaction.getUpdateDictionary())
|
||||||
|
.addToAllDossiers(manualResizeRedaction.isAddToAllDossiers())
|
||||||
|
.textAfter(manualResizeRedaction.getTextAfter())
|
||||||
|
.textBefore(manualResizeRedaction.getTextBefore())
|
||||||
|
.build();
|
||||||
|
manualChangesApplicationService.resize(textEntity, migratedManualResizeRedaction);
|
||||||
|
} else {
|
||||||
|
migratedEntity.getManualOverwrite().addChange(manualChange);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
public ManualRedactionEntry buildManualRedactionEntry() {
|
||||||
|
|
||||||
|
assert hasManualChanges();
|
||||||
|
|
||||||
|
// currently we need to insert a manual redaction entry, whenever an entity has been resized.
|
||||||
|
String user = manualChanges.stream()
|
||||||
|
.filter(mc -> mc instanceof ManualResizeRedaction)
|
||||||
|
.findFirst()
|
||||||
|
.orElse(manualChanges.get(0)).getUser();
|
||||||
|
|
||||||
|
return ManualRedactionEntry.builder()
|
||||||
|
.annotationId(newId)
|
||||||
|
.fileId(fileId)
|
||||||
|
.type(redactionLogEntry.getType())
|
||||||
|
.value(redactionLogEntry.getValue())
|
||||||
|
.reason(redactionLogEntry.getReason())
|
||||||
|
.legalBasis(redactionLogEntry.getLegalBasis())
|
||||||
|
.section(redactionLogEntry.getSection())
|
||||||
|
.addToDictionary(false)
|
||||||
|
.addToDossierDictionary(false)
|
||||||
|
.rectangle(false)
|
||||||
|
.positions(buildPositions(migratedEntity))
|
||||||
|
.user(user)
|
||||||
|
.build();
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
private List<Rectangle> buildPositions(IEntity entity) {
|
||||||
|
|
||||||
|
if (entity instanceof TextEntity textEntity) {
|
||||||
|
|
||||||
|
var positionsOnPage = textEntity.getPositionsOnPagePerPage()
|
||||||
|
.get(0);
|
||||||
|
return positionsOnPage.getRectanglePerLine()
|
||||||
|
.stream()
|
||||||
|
.map(p -> new Rectangle((float) p.getX(), (float) p.getY(), (float) p.getWidth(), (float) p.getHeight(), positionsOnPage.getPage().getNumber()))
|
||||||
|
.toList();
|
||||||
|
}
|
||||||
|
if (entity instanceof PrecursorEntity pEntity) {
|
||||||
|
|
||||||
|
return pEntity.getManualOverwrite().getPositions()
|
||||||
|
.orElse(pEntity.getEntityPosition())
|
||||||
|
.stream()
|
||||||
|
.map(p -> new Rectangle((float) p.rectangle2D().getX(),
|
||||||
|
(float) p.rectangle2D().getY(),
|
||||||
|
(float) p.rectangle2D().getWidth(),
|
||||||
|
(float) p.rectangle2D().getHeight(),
|
||||||
|
p.pageNumber()))
|
||||||
|
.toList();
|
||||||
|
}
|
||||||
|
if (entity instanceof Image image) {
|
||||||
|
|
||||||
|
Rectangle2D position = image.getManualOverwrite().getPositions()
|
||||||
|
.map(p -> p.get(0).rectangle2D())
|
||||||
|
.orElse(image.getPosition());
|
||||||
|
|
||||||
|
return List.of(new Rectangle((float) position.getX(), (float) position.getY(), (float) position.getWidth(), (float) position.getHeight(), image.getPage().getNumber()));
|
||||||
|
|
||||||
|
} else {
|
||||||
|
throw new UnsupportedOperationException();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
public boolean needsManualEntry() {
|
||||||
|
|
||||||
|
return manualChanges.stream()
|
||||||
|
.anyMatch(mc -> mc instanceof ManualResizeRedaction && !((ManualResizeRedaction) mc).getUpdateDictionary()) && !(migratedEntity instanceof Image);
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|||||||
@ -43,7 +43,6 @@ public class PrecursorEntity implements IEntity {
|
|||||||
String type;
|
String type;
|
||||||
String section;
|
String section;
|
||||||
EntityType entityType;
|
EntityType entityType;
|
||||||
EntryType entryType;
|
|
||||||
boolean applied;
|
boolean applied;
|
||||||
boolean isDictionaryEntry;
|
boolean isDictionaryEntry;
|
||||||
boolean isDossierDictionaryEntry;
|
boolean isDossierDictionaryEntry;
|
||||||
@ -61,8 +60,8 @@ public class PrecursorEntity implements IEntity {
|
|||||||
.stream()
|
.stream()
|
||||||
.map(RectangleWithPage::fromAnnotationRectangle)
|
.map(RectangleWithPage::fromAnnotationRectangle)
|
||||||
.toList();
|
.toList();
|
||||||
|
|
||||||
var entityType = hint ? EntityType.HINT : EntityType.ENTITY;
|
var entityType = hint ? EntityType.HINT : EntityType.ENTITY;
|
||||||
var entryType = hint ? EntryType.HINT : (manualRedactionEntry.isRectangle() ? EntryType.AREA : EntryType.ENTITY);
|
|
||||||
ManualChangeOverwrite manualChangeOverwrite = new ManualChangeOverwrite(entityType);
|
ManualChangeOverwrite manualChangeOverwrite = new ManualChangeOverwrite(entityType);
|
||||||
manualChangeOverwrite.addChange(manualRedactionEntry);
|
manualChangeOverwrite.addChange(manualRedactionEntry);
|
||||||
return PrecursorEntity.builder()
|
return PrecursorEntity.builder()
|
||||||
@ -75,7 +74,6 @@ public class PrecursorEntity implements IEntity {
|
|||||||
.type(manualRedactionEntry.getType())
|
.type(manualRedactionEntry.getType())
|
||||||
.section(manualRedactionEntry.getSection())
|
.section(manualRedactionEntry.getSection())
|
||||||
.entityType(entityType)
|
.entityType(entityType)
|
||||||
.entryType(entryType)
|
|
||||||
.applied(true)
|
.applied(true)
|
||||||
.isDictionaryEntry(false)
|
.isDictionaryEntry(false)
|
||||||
.isDossierDictionaryEntry(false)
|
.isDossierDictionaryEntry(false)
|
||||||
@ -103,7 +101,6 @@ public class PrecursorEntity implements IEntity {
|
|||||||
.type(entityLogEntry.getType())
|
.type(entityLogEntry.getType())
|
||||||
.section(entityLogEntry.getSection())
|
.section(entityLogEntry.getSection())
|
||||||
.entityType(entityType)
|
.entityType(entityType)
|
||||||
.entryType(entityLogEntry.getEntryType())
|
|
||||||
.isDictionaryEntry(entityLogEntry.isDictionaryEntry())
|
.isDictionaryEntry(entityLogEntry.isDictionaryEntry())
|
||||||
.isDossierDictionaryEntry(entityLogEntry.isDossierDictionaryEntry())
|
.isDossierDictionaryEntry(entityLogEntry.isDossierDictionaryEntry())
|
||||||
.manualOverwrite(new ManualChangeOverwrite(entityType))
|
.manualOverwrite(new ManualChangeOverwrite(entityType))
|
||||||
@ -134,7 +131,6 @@ public class PrecursorEntity implements IEntity {
|
|||||||
.type(Optional.ofNullable(importedRedaction.getType())
|
.type(Optional.ofNullable(importedRedaction.getType())
|
||||||
.orElse(IMPORTED_REDACTION_TYPE))
|
.orElse(IMPORTED_REDACTION_TYPE))
|
||||||
.entityType(entityType)
|
.entityType(entityType)
|
||||||
.entryType(entryType)
|
|
||||||
.isDictionaryEntry(false)
|
.isDictionaryEntry(false)
|
||||||
.isDossierDictionaryEntry(false)
|
.isDossierDictionaryEntry(false)
|
||||||
.rectangle(value.isBlank() || entryType.equals(EntryType.IMAGE) || entryType.equals(EntryType.IMAGE_HINT) || entryType.equals(EntryType.AREA))
|
.rectangle(value.isBlank() || entryType.equals(EntryType.IMAGE) || entryType.equals(EntryType.IMAGE_HINT) || entryType.equals(EntryType.AREA))
|
||||||
|
|||||||
@ -6,10 +6,22 @@ public enum ImageType {
|
|||||||
LOGO,
|
LOGO,
|
||||||
FORMULA,
|
FORMULA,
|
||||||
SIGNATURE,
|
SIGNATURE,
|
||||||
OTHER,
|
OTHER {
|
||||||
|
@Override
|
||||||
|
public String toString() {
|
||||||
|
|
||||||
|
return "image";
|
||||||
|
}
|
||||||
|
},
|
||||||
OCR;
|
OCR;
|
||||||
|
|
||||||
|
|
||||||
|
public String toString() {
|
||||||
|
|
||||||
|
return name().toLowerCase(Locale.ENGLISH);
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
public static ImageType fromString(String imageType) {
|
public static ImageType fromString(String imageType) {
|
||||||
|
|
||||||
return switch (imageType.toLowerCase(Locale.ROOT)) {
|
return switch (imageType.toLowerCase(Locale.ROOT)) {
|
||||||
|
|||||||
@ -4,7 +4,6 @@ import java.util.ArrayList;
|
|||||||
import java.util.Collections;
|
import java.util.Collections;
|
||||||
import java.util.HashSet;
|
import java.util.HashSet;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Locale;
|
|
||||||
import java.util.Set;
|
import java.util.Set;
|
||||||
import java.util.stream.Collectors;
|
import java.util.stream.Collectors;
|
||||||
|
|
||||||
@ -19,7 +18,6 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog
|
|||||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState;
|
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState;
|
||||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryType;
|
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryType;
|
||||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position;
|
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position;
|
||||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
|
|
||||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.legalbasis.LegalBasis;
|
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.legalbasis.LegalBasis;
|
||||||
import com.iqser.red.service.redaction.v1.server.RedactionServiceSettings;
|
import com.iqser.red.service.redaction.v1.server.RedactionServiceSettings;
|
||||||
import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient;
|
import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient;
|
||||||
@ -32,7 +30,6 @@ import com.iqser.red.service.redaction.v1.server.model.document.entity.PositionO
|
|||||||
import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
|
import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
|
||||||
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
|
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
|
||||||
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Image;
|
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Image;
|
||||||
import com.iqser.red.service.redaction.v1.server.model.document.nodes.ImageType;
|
|
||||||
import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
|
import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
|
||||||
|
|
||||||
import lombok.AccessLevel;
|
import lombok.AccessLevel;
|
||||||
@ -68,7 +65,12 @@ public class EntityLogCreatorService {
|
|||||||
List<EntityLogEntry> entityLogEntries = createEntityLogEntries(document, analyzeRequest, notFoundEntities);
|
List<EntityLogEntry> entityLogEntries = createEntityLogEntries(document, analyzeRequest, notFoundEntities);
|
||||||
|
|
||||||
List<LegalBasis> legalBasis = legalBasisClient.getLegalBasisMapping(analyzeRequest.getDossierTemplateId());
|
List<LegalBasis> legalBasis = legalBasisClient.getLegalBasisMapping(analyzeRequest.getDossierTemplateId());
|
||||||
EntityLog entityLog = new EntityLog(redactionServiceSettings.getAnalysisVersion(),
|
|
||||||
|
List<EntityLogEntry> previousExistingEntityLogEntries = getPreviousEntityLogEntries(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
|
||||||
|
|
||||||
|
entityChangeLogService.computeChanges(previousExistingEntityLogEntries, entityLogEntries, analyzeRequest.getManualRedactions(), analyzeRequest.getAnalysisNumber());
|
||||||
|
|
||||||
|
return new EntityLog(redactionServiceSettings.getAnalysisVersion(),
|
||||||
analyzeRequest.getAnalysisNumber(),
|
analyzeRequest.getAnalysisNumber(),
|
||||||
entityLogEntries,
|
entityLogEntries,
|
||||||
toEntityLogLegalBasis(legalBasis),
|
toEntityLogLegalBasis(legalBasis),
|
||||||
@ -76,12 +78,6 @@ public class EntityLogCreatorService {
|
|||||||
dictionaryVersion.getDossierVersion(),
|
dictionaryVersion.getDossierVersion(),
|
||||||
rulesVersion,
|
rulesVersion,
|
||||||
legalBasisClient.getVersion(analyzeRequest.getDossierTemplateId()));
|
legalBasisClient.getVersion(analyzeRequest.getDossierTemplateId()));
|
||||||
|
|
||||||
List<EntityLogEntry> previousExistingEntityLogEntries = getPreviousEntityLogEntries(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
|
|
||||||
|
|
||||||
entityChangeLogService.computeChanges(previousExistingEntityLogEntries, entityLogEntries, analyzeRequest.getManualRedactions(), analyzeRequest.getAnalysisNumber());
|
|
||||||
|
|
||||||
return entityLog;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@ -116,9 +112,12 @@ public class EntityLogCreatorService {
|
|||||||
DictionaryVersion dictionaryVersion) {
|
DictionaryVersion dictionaryVersion) {
|
||||||
|
|
||||||
List<EntityLogEntry> newEntityLogEntries = createEntityLogEntries(document, analyzeRequest, notFoundEntries).stream()
|
List<EntityLogEntry> newEntityLogEntries = createEntityLogEntries(document, analyzeRequest, notFoundEntries).stream()
|
||||||
.filter(entry -> entry.getContainingNodeId().isEmpty() || sectionsToReanalyseIds.contains(entry.getContainingNodeId().get(0)))
|
.filter(entry -> entry.getContainingNodeId().isEmpty() || sectionsToReanalyseIds.contains(entry.getContainingNodeId()
|
||||||
|
.get(0)))
|
||||||
.collect(Collectors.toList());
|
.collect(Collectors.toList());
|
||||||
Set<String> newEntityIds = newEntityLogEntries.stream().map(EntityLogEntry::getId).collect(Collectors.toSet());
|
Set<String> newEntityIds = newEntityLogEntries.stream()
|
||||||
|
.map(EntityLogEntry::getId)
|
||||||
|
.collect(Collectors.toSet());
|
||||||
|
|
||||||
List<EntityLogEntry> previousEntriesFromReAnalyzedSections = previousEntityLog.getEntityLogEntry()
|
List<EntityLogEntry> previousEntriesFromReAnalyzedSections = previousEntityLog.getEntityLogEntry()
|
||||||
.stream()
|
.stream()
|
||||||
@ -139,22 +138,6 @@ public class EntityLogCreatorService {
|
|||||||
|
|
||||||
private List<EntityLogEntry> createEntityLogEntries(Document document, AnalyzeRequest analyzeRequest, List<PrecursorEntity> notFoundPrecursorEntries) {
|
private List<EntityLogEntry> createEntityLogEntries(Document document, AnalyzeRequest analyzeRequest, List<PrecursorEntity> notFoundPrecursorEntries) {
|
||||||
|
|
||||||
Set<ManualRedactionEntry> dictionaryEntries;
|
|
||||||
Set<String> dictionaryEntriesValues;
|
|
||||||
|
|
||||||
if (analyzeRequest.getManualRedactions() != null && !analyzeRequest.getManualRedactions().getEntriesToAdd().isEmpty()) {
|
|
||||||
dictionaryEntries = analyzeRequest.getManualRedactions().getEntriesToAdd()
|
|
||||||
.stream()
|
|
||||||
.filter(e -> e.isAddToDictionary() || e.isAddToDossierDictionary())
|
|
||||||
.collect(Collectors.toSet());
|
|
||||||
dictionaryEntriesValues = dictionaryEntries.stream()
|
|
||||||
.map(ManualRedactionEntry::getValue)
|
|
||||||
.collect(Collectors.toSet());
|
|
||||||
} else {
|
|
||||||
dictionaryEntriesValues = new HashSet<>();
|
|
||||||
dictionaryEntries = new HashSet<>();
|
|
||||||
}
|
|
||||||
|
|
||||||
String dossierTemplateId = analyzeRequest.getDossierTemplateId();
|
String dossierTemplateId = analyzeRequest.getDossierTemplateId();
|
||||||
|
|
||||||
List<EntityLogEntry> entries = new ArrayList<>();
|
List<EntityLogEntry> entries = new ArrayList<>();
|
||||||
@ -164,22 +147,21 @@ public class EntityLogCreatorService {
|
|||||||
.filter(entity -> !entity.getValue().isEmpty())
|
.filter(entity -> !entity.getValue().isEmpty())
|
||||||
.filter(EntityLogCreatorService::notFalsePositiveOrFalseRecommendation)
|
.filter(EntityLogCreatorService::notFalsePositiveOrFalseRecommendation)
|
||||||
.filter(entity -> !entity.removed())
|
.filter(entity -> !entity.removed())
|
||||||
.forEach(entityNode -> entries.addAll(toEntityLogEntries(entityNode, dictionaryEntries, dictionaryEntriesValues)));
|
.forEach(entityNode -> entries.addAll(toEntityLogEntries(entityNode)));
|
||||||
document.streamAllImages().filter(entity -> !entity.removed()).forEach(imageNode -> entries.add(createEntityLogEntry(imageNode, dossierTemplateId)));
|
document.streamAllImages()
|
||||||
notFoundPrecursorEntries.stream().filter(entity -> !entity.removed()).forEach(precursorEntity -> entries.add(createEntityLogEntry(precursorEntity, dossierTemplateId)));
|
.filter(entity -> !entity.removed())
|
||||||
|
.forEach(imageNode -> entries.add(createEntityLogEntry(imageNode, dossierTemplateId)));
|
||||||
|
notFoundPrecursorEntries.stream()
|
||||||
|
.filter(entity -> !entity.removed())
|
||||||
|
.forEach(precursorEntity -> entries.add(createEntityLogEntry(precursorEntity, dossierTemplateId)));
|
||||||
return entries;
|
return entries;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
private List<EntityLogEntry> toEntityLogEntries(TextEntity textEntity, Set<ManualRedactionEntry> dictionaryEntries, Set<String> dictionaryEntriesValues) {
|
private List<EntityLogEntry> toEntityLogEntries(TextEntity textEntity) {
|
||||||
|
|
||||||
List<EntityLogEntry> entityLogEntries = new ArrayList<>();
|
List<EntityLogEntry> entityLogEntries = new ArrayList<>();
|
||||||
|
|
||||||
// Adding ADD_TO_DICTIONARY manual change to the entity's manual overwrite
|
|
||||||
if (dictionaryEntriesValues.contains(textEntity.getValue())) {
|
|
||||||
textEntity.getManualOverwrite().addChange(dictionaryEntries.stream().filter(entry -> entry.getValue().equals(textEntity.getValue())).findFirst().get());
|
|
||||||
}
|
|
||||||
|
|
||||||
// split entity into multiple entries if it occurs on multiple pages, since FE can't handle multi page entities
|
// split entity into multiple entries if it occurs on multiple pages, since FE can't handle multi page entities
|
||||||
for (PositionOnPage positionOnPage : textEntity.getPositionsOnPagePerPage()) {
|
for (PositionOnPage positionOnPage : textEntity.getPositionsOnPagePerPage()) {
|
||||||
|
|
||||||
@ -202,12 +184,11 @@ public class EntityLogCreatorService {
|
|||||||
|
|
||||||
private EntityLogEntry createEntityLogEntry(Image image, String dossierTemplateId) {
|
private EntityLogEntry createEntityLogEntry(Image image, String dossierTemplateId) {
|
||||||
|
|
||||||
String imageType = image.getImageType().equals(ImageType.OTHER) ? "image" : image.getImageType().toString().toLowerCase(Locale.ENGLISH);
|
boolean isHint = dictionaryService.isHint(image.type(), dossierTemplateId);
|
||||||
boolean isHint = dictionaryService.isHint(imageType, dossierTemplateId);
|
|
||||||
return EntityLogEntry.builder()
|
return EntityLogEntry.builder()
|
||||||
.id(image.getId())
|
.id(image.getId())
|
||||||
.value(image.value())
|
.value(image.value())
|
||||||
.type(imageType)
|
.type(image.type())
|
||||||
.reason(image.buildReasonWithManualChangeDescriptions())
|
.reason(image.buildReasonWithManualChangeDescriptions())
|
||||||
.legalBasis(image.legalBasis())
|
.legalBasis(image.legalBasis())
|
||||||
.matchedRule(image.getMatchedRule().getRuleIdentifier().toString())
|
.matchedRule(image.getMatchedRule().getRuleIdentifier().toString())
|
||||||
@ -229,7 +210,8 @@ public class EntityLogCreatorService {
|
|||||||
|
|
||||||
private EntityLogEntry createEntityLogEntry(PrecursorEntity precursorEntity, String dossierTemplateId) {
|
private EntityLogEntry createEntityLogEntry(PrecursorEntity precursorEntity, String dossierTemplateId) {
|
||||||
|
|
||||||
String type = precursorEntity.getManualOverwrite().getType().orElse(precursorEntity.getType());
|
String type = precursorEntity.getManualOverwrite().getType()
|
||||||
|
.orElse(precursorEntity.getType());
|
||||||
boolean isHint = isHint(precursorEntity.getEntityType());
|
boolean isHint = isHint(precursorEntity.getEntityType());
|
||||||
return EntityLogEntry.builder()
|
return EntityLogEntry.builder()
|
||||||
.id(precursorEntity.getId())
|
.id(precursorEntity.getId())
|
||||||
@ -239,7 +221,8 @@ public class EntityLogCreatorService {
|
|||||||
.type(type)
|
.type(type)
|
||||||
.state(buildEntryState(precursorEntity))
|
.state(buildEntryState(precursorEntity))
|
||||||
.entryType(buildEntryType(precursorEntity))
|
.entryType(buildEntryType(precursorEntity))
|
||||||
.section(precursorEntity.getManualOverwrite().getSection().orElse(precursorEntity.getSection()))
|
.section(precursorEntity.getManualOverwrite().getSection()
|
||||||
|
.orElse(precursorEntity.getSection()))
|
||||||
.containingNodeId(Collections.emptyList())
|
.containingNodeId(Collections.emptyList())
|
||||||
.closestHeadline("")
|
.closestHeadline("")
|
||||||
.matchedRule(precursorEntity.getMatchedRule().getRuleIdentifier().toString())
|
.matchedRule(precursorEntity.getMatchedRule().getRuleIdentifier().toString())
|
||||||
@ -267,12 +250,17 @@ public class EntityLogCreatorService {
|
|||||||
private EntityLogEntry createEntityLogEntry(TextEntity entity) {
|
private EntityLogEntry createEntityLogEntry(TextEntity entity) {
|
||||||
|
|
||||||
Set<String> referenceIds = new HashSet<>();
|
Set<String> referenceIds = new HashSet<>();
|
||||||
entity.references().stream().filter(TextEntity::active).forEach(ref -> ref.getPositionsOnPagePerPage().forEach(pos -> referenceIds.add(pos.getId())));
|
entity.references()
|
||||||
|
.stream()
|
||||||
|
.filter(TextEntity::active)
|
||||||
|
.forEach(ref -> ref.getPositionsOnPagePerPage()
|
||||||
|
.forEach(pos -> referenceIds.add(pos.getId())));
|
||||||
boolean isHint = isHint(entity.getEntityType());
|
boolean isHint = isHint(entity.getEntityType());
|
||||||
return EntityLogEntry.builder()
|
return EntityLogEntry.builder()
|
||||||
.reason(entity.buildReasonWithManualChangeDescriptions())
|
.reason(entity.buildReasonWithManualChangeDescriptions())
|
||||||
.legalBasis(entity.legalBasis())
|
.legalBasis(entity.legalBasis())
|
||||||
.value(entity.getManualOverwrite().getValue().orElse(entity.getMatchedRule().isWriteValueWithLineBreaks() ? entity.getValueWithLineBreaks() : entity.getValue()))
|
.value(entity.getManualOverwrite().getValue()
|
||||||
|
.orElse(entity.getMatchedRule().isWriteValueWithLineBreaks() ? entity.getValueWithLineBreaks() : entity.getValue()))
|
||||||
.type(entity.type())
|
.type(entity.type())
|
||||||
.section(entity.getManualOverwrite().getSection()
|
.section(entity.getManualOverwrite().getSection()
|
||||||
.orElse(entity.getDeepestFullyContainingNode().toString()))
|
.orElse(entity.getDeepestFullyContainingNode().toString()))
|
||||||
@ -314,7 +302,7 @@ public class EntityLogCreatorService {
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
private EntryState buildEntryState(IEntity entity) {
|
public static EntryState buildEntryState(IEntity entity) {
|
||||||
|
|
||||||
if (entity.applied() && entity.active()) {
|
if (entity.applied() && entity.active()) {
|
||||||
return EntryState.APPLIED;
|
return EntryState.APPLIED;
|
||||||
@ -328,12 +316,17 @@ public class EntityLogCreatorService {
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
private EntryType buildEntryType(IEntity entity) {
|
public static EntryType buildEntryType(IEntity entity) {
|
||||||
|
|
||||||
if (entity instanceof TextEntity textEntity) {
|
if (entity instanceof TextEntity textEntity) {
|
||||||
return getEntryType(textEntity.getEntityType());
|
return getEntryType(textEntity.getEntityType());
|
||||||
} else if (entity instanceof PrecursorEntity precursorEntity) {
|
} else if (entity instanceof PrecursorEntity precursorEntity) {
|
||||||
return precursorEntity.getEntryType();
|
if (precursorEntity.isRectangle()) {
|
||||||
|
return EntryType.AREA;
|
||||||
|
}
|
||||||
|
return getEntryType(precursorEntity.getEntityType());
|
||||||
|
} else if (entity instanceof Image) {
|
||||||
|
return EntryType.IMAGE;
|
||||||
}
|
}
|
||||||
throw new UnsupportedOperationException(String.format("Entity subclass %s is not implemented!", entity.getClass()));
|
throw new UnsupportedOperationException(String.format("Entity subclass %s is not implemented!", entity.getClass()));
|
||||||
}
|
}
|
||||||
@ -353,7 +346,9 @@ public class EntityLogCreatorService {
|
|||||||
|
|
||||||
private List<EntityLogLegalBasis> toEntityLogLegalBasis(List<LegalBasis> legalBasis) {
|
private List<EntityLogLegalBasis> toEntityLogLegalBasis(List<LegalBasis> legalBasis) {
|
||||||
|
|
||||||
return legalBasis.stream().map(l -> new EntityLogLegalBasis(l.getName(), l.getDescription(), l.getReason())).collect(Collectors.toList());
|
return legalBasis.stream()
|
||||||
|
.map(l -> new EntityLogLegalBasis(l.getName(), l.getDescription(), l.getReason()))
|
||||||
|
.collect(Collectors.toList());
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|||||||
@ -76,11 +76,19 @@ public class UnprocessedChangesService {
|
|||||||
EntityLog previousEntityLog = redactionStorageService.getEntityLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
|
EntityLog previousEntityLog = redactionStorageService.getEntityLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
|
||||||
Document document = DocumentGraphMapper.toDocumentGraph(observedStorageService.getDocumentData(analyzeRequest.getDossierId(), analyzeRequest.getFileId()));
|
Document document = DocumentGraphMapper.toDocumentGraph(observedStorageService.getDocumentData(analyzeRequest.getDossierId(), analyzeRequest.getFileId()));
|
||||||
|
|
||||||
Set<String> allAnnotationIds = analyzeRequest.getManualRedactions().getEntriesToAdd().stream().map(ManualRedactionEntry::getAnnotationId).collect(Collectors.toSet());
|
Set<String> allAnnotationIds = analyzeRequest.getManualRedactions().getEntriesToAdd()
|
||||||
Set<String> resizeIds = analyzeRequest.getManualRedactions().getResizeRedactions().stream().map(ManualResizeRedaction::getAnnotationId).collect(Collectors.toSet());
|
.stream()
|
||||||
|
.map(ManualRedactionEntry::getAnnotationId)
|
||||||
|
.collect(Collectors.toSet());
|
||||||
|
Set<String> resizeIds = analyzeRequest.getManualRedactions().getResizeRedactions()
|
||||||
|
.stream()
|
||||||
|
.map(ManualResizeRedaction::getAnnotationId)
|
||||||
|
.collect(Collectors.toSet());
|
||||||
allAnnotationIds.addAll(resizeIds);
|
allAnnotationIds.addAll(resizeIds);
|
||||||
|
|
||||||
List<ManualResizeRedaction> manualResizeRedactions = analyzeRequest.getManualRedactions().getResizeRedactions().stream().toList();
|
List<ManualResizeRedaction> manualResizeRedactions = analyzeRequest.getManualRedactions().getResizeRedactions()
|
||||||
|
.stream()
|
||||||
|
.toList();
|
||||||
List<PrecursorEntity> manualEntitiesToBeResized = previousEntityLog.getEntityLogEntry()
|
List<PrecursorEntity> manualEntitiesToBeResized = previousEntityLog.getEntityLogEntry()
|
||||||
.stream()
|
.stream()
|
||||||
.filter(entityLogEntry -> resizeIds.contains(entityLogEntry.getId()))
|
.filter(entityLogEntry -> resizeIds.contains(entityLogEntry.getId()))
|
||||||
@ -99,7 +107,8 @@ public class UnprocessedChangesService {
|
|||||||
notFoundManualEntities = entityFromPrecursorCreationService.toTextEntity(manualEntities, document);
|
notFoundManualEntities = entityFromPrecursorCreationService.toTextEntity(manualEntities, document);
|
||||||
}
|
}
|
||||||
|
|
||||||
document.getEntities().forEach(textEntity -> {
|
document.getEntities()
|
||||||
|
.forEach(textEntity -> {
|
||||||
Set<String> processedIds = new HashSet<>();
|
Set<String> processedIds = new HashSet<>();
|
||||||
for (var positionsOnPerPage : textEntity.getPositionsOnPagePerPage()) {
|
for (var positionsOnPerPage : textEntity.getPositionsOnPagePerPage()) {
|
||||||
if (processedIds.contains(positionsOnPerPage.getId())) {
|
if (processedIds.contains(positionsOnPerPage.getId())) {
|
||||||
@ -111,10 +120,14 @@ public class UnprocessedChangesService {
|
|||||||
.map(rectangle2D -> new Position(rectangle2D, positionsOnPerPage.getPage().getNumber()))
|
.map(rectangle2D -> new Position(rectangle2D, positionsOnPerPage.getPage().getNumber()))
|
||||||
.collect(Collectors.toList());
|
.collect(Collectors.toList());
|
||||||
unprocessedManualEntities.add(UnprocessedManualEntity.builder()
|
unprocessedManualEntities.add(UnprocessedManualEntity.builder()
|
||||||
.annotationId(allAnnotationIds.stream().filter(textEntity::matchesAnnotationId).findFirst().orElse(""))
|
.annotationId(allAnnotationIds.stream()
|
||||||
|
.filter(textEntity::matchesAnnotationId)
|
||||||
|
.findFirst()
|
||||||
|
.orElse(""))
|
||||||
.textBefore(textEntity.getTextBefore())
|
.textBefore(textEntity.getTextBefore())
|
||||||
.textAfter(textEntity.getTextAfter())
|
.textAfter(textEntity.getTextAfter())
|
||||||
.section(textEntity.getManualOverwrite().getSection().orElse(textEntity.getDeepestFullyContainingNode().toString()))
|
.section(textEntity.getManualOverwrite().getSection()
|
||||||
|
.orElse(textEntity.getDeepestFullyContainingNode().toString()))
|
||||||
.positions(positions)
|
.positions(positions)
|
||||||
.build());
|
.build());
|
||||||
}
|
}
|
||||||
@ -143,13 +156,13 @@ public class UnprocessedChangesService {
|
|||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
TextEntity correctEntity = createCorrectEntity(precursorEntity, optionalTextEntity.get());
|
TextEntity correctEntity = EntityFromPrecursorCreationService.createCorrectEntity(precursorEntity, optionalTextEntity.get());
|
||||||
Optional<ManualResizeRedaction> optionalManualResizeRedaction = manualResizeRedactions.stream()
|
Optional<ManualResizeRedaction> optionalManualResizeRedaction = manualResizeRedactions.stream()
|
||||||
.filter(manualResizeRedaction -> manualResizeRedaction.getAnnotationId().equals(precursorEntity.getId()))
|
.filter(manualResizeRedaction -> manualResizeRedaction.getAnnotationId().equals(precursorEntity.getId()))
|
||||||
.findFirst();
|
.findFirst();
|
||||||
if (optionalManualResizeRedaction.isPresent()) {
|
if (optionalManualResizeRedaction.isPresent()) {
|
||||||
ManualResizeRedaction manualResizeRedaction = optionalManualResizeRedaction.get();
|
ManualResizeRedaction manualResizeRedaction = optionalManualResizeRedaction.get();
|
||||||
manualChangesApplicationService.resizeEntityAndReinsert(correctEntity, manualResizeRedaction);
|
manualChangesApplicationService.resize(correctEntity, manualResizeRedaction);
|
||||||
|
|
||||||
// If the entity's value is not the same as the manual resize request's value it means we didn't find it anywhere and we want to remove it
|
// If the entity's value is not the same as the manual resize request's value it means we didn't find it anywhere and we want to remove it
|
||||||
// from the graph, so it does not get processed and sent back to persistence-service to update its value.
|
// from the graph, so it does not get processed and sent back to persistence-service to update its value.
|
||||||
@ -160,53 +173,30 @@ public class UnprocessedChangesService {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// remove all temp entities from the graph
|
// remove all temp entities from the graph
|
||||||
tempEntities.values().stream().flatMap(Collection::stream).forEach(TextEntity::removeFromGraph);
|
tempEntities.values()
|
||||||
|
.stream()
|
||||||
|
.flatMap(Collection::stream)
|
||||||
|
.forEach(TextEntity::removeFromGraph);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
private TextEntity createCorrectEntity(PrecursorEntity precursorEntity, TextEntity closestEntity) {
|
private UnprocessedManualEntity builDefaultUnprocessedManualEntity(PrecursorEntity precursorEntity) {
|
||||||
|
|
||||||
TextEntity correctEntity = TextEntity.initialEntityNode(closestEntity.getTextRange(), precursorEntity.type(), precursorEntity.getEntityType(), precursorEntity.getId());
|
|
||||||
|
|
||||||
correctEntity.setDeepestFullyContainingNode(closestEntity.getDeepestFullyContainingNode());
|
|
||||||
correctEntity.setIntersectingNodes(new ArrayList<>(closestEntity.getIntersectingNodes()));
|
|
||||||
correctEntity.setDuplicateTextRanges(new ArrayList<>(closestEntity.getDuplicateTextRanges()));
|
|
||||||
correctEntity.setPages(new HashSet<>(closestEntity.getPages()));
|
|
||||||
|
|
||||||
correctEntity.setValue(closestEntity.getValue());
|
|
||||||
correctEntity.setTextAfter(closestEntity.getTextAfter());
|
|
||||||
correctEntity.setTextBefore(closestEntity.getTextBefore());
|
|
||||||
|
|
||||||
correctEntity.getIntersectingNodes().forEach(n -> n.getEntities().add(correctEntity));
|
|
||||||
correctEntity.getPages().forEach(page -> page.getEntities().add(correctEntity));
|
|
||||||
|
|
||||||
correctEntity.addMatchedRules(precursorEntity.getMatchedRuleList());
|
|
||||||
correctEntity.setDictionaryEntry(precursorEntity.isDictionaryEntry());
|
|
||||||
correctEntity.setDossierDictionaryEntry(precursorEntity.isDossierDictionaryEntry());
|
|
||||||
correctEntity.getManualOverwrite().addChanges(precursorEntity.getManualOverwrite().getManualChangeLog());
|
|
||||||
|
|
||||||
return correctEntity;
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
private UnprocessedManualEntity builDefaultUnprocessedManualEntity(PrecursorEntity precursorEntity) {
|
|
||||||
|
|
||||||
return UnprocessedManualEntity.builder()
|
return UnprocessedManualEntity.builder()
|
||||||
.annotationId(precursorEntity.getId())
|
.annotationId(precursorEntity.getId())
|
||||||
.textAfter("")
|
.textAfter("")
|
||||||
.textBefore("")
|
.textBefore("")
|
||||||
.section("")
|
.section("")
|
||||||
.positions(precursorEntity.getManualOverwrite()
|
.positions(precursorEntity.getManualOverwrite().getPositions()
|
||||||
.getPositions()
|
|
||||||
.orElse(precursorEntity.getEntityPosition())
|
.orElse(precursorEntity.getEntityPosition())
|
||||||
.stream()
|
.stream()
|
||||||
.map(entityPosition -> new Position(entityPosition.rectangle2D(), entityPosition.pageNumber()))
|
.map(entityPosition -> new Position(entityPosition.rectangle2D(), entityPosition.pageNumber()))
|
||||||
.toList())
|
.toList())
|
||||||
.build();
|
.build();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
private List<PrecursorEntity> manualEntitiesConverter(ManualRedactions manualRedactions, String dossierTemplateId) {
|
private List<PrecursorEntity> manualEntitiesConverter(ManualRedactions manualRedactions, String dossierTemplateId) {
|
||||||
|
|
||||||
return manualRedactions.getEntriesToAdd()
|
return manualRedactions.getEntriesToAdd()
|
||||||
.stream()
|
.stream()
|
||||||
@ -214,6 +204,6 @@ private List<PrecursorEntity> manualEntitiesConverter(ManualRedactions manualRed
|
|||||||
.map(manualRedactionEntry -> PrecursorEntity.fromManualRedactionEntry(manualRedactionEntry,
|
.map(manualRedactionEntry -> PrecursorEntity.fromManualRedactionEntry(manualRedactionEntry,
|
||||||
dictionaryService.isHint(manualRedactionEntry.getType(), dossierTemplateId)))
|
dictionaryService.isHint(manualRedactionEntry.getType(), dossierTemplateId)))
|
||||||
.toList();
|
.toList();
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|||||||
@ -14,7 +14,6 @@ import org.springframework.stereotype.Service;
|
|||||||
|
|
||||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedactions;
|
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedactions;
|
||||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
|
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
|
||||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.BaseAnnotation;
|
|
||||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;
|
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;
|
||||||
import com.iqser.red.service.redaction.v1.server.model.PrecursorEntity;
|
import com.iqser.red.service.redaction.v1.server.model.PrecursorEntity;
|
||||||
import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
|
import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
|
||||||
@ -51,18 +50,14 @@ public class EntityFromPrecursorCreationService {
|
|||||||
Set<IdRemoval> idRemovals = manualRedactions.getIdsToRemove();
|
Set<IdRemoval> idRemovals = manualRedactions.getIdsToRemove();
|
||||||
List<PrecursorEntity> manualEntities = manualRedactions.getEntriesToAdd()
|
List<PrecursorEntity> manualEntities = manualRedactions.getEntriesToAdd()
|
||||||
.stream()
|
.stream()
|
||||||
.filter(manualRedactionEntry -> !(idRemovals.stream()
|
.filter(manualRedactionEntry -> idRemovals.stream()
|
||||||
.map(BaseAnnotation::getAnnotationId)
|
|
||||||
.toList()
|
|
||||||
.contains(manualRedactionEntry.getAnnotationId()) && manualRedactionEntry.getRequestDate()
|
|
||||||
.isBefore(idRemovals.stream()
|
|
||||||
.filter(idRemoval -> idRemoval.getAnnotationId().equals(manualRedactionEntry.getAnnotationId()))
|
.filter(idRemoval -> idRemoval.getAnnotationId().equals(manualRedactionEntry.getAnnotationId()))
|
||||||
.findFirst()
|
.filter(idRemoval -> idRemoval.getRequestDate().isBefore(manualRedactionEntry.getRequestDate()))
|
||||||
.get()
|
.findAny()//
|
||||||
.getRequestDate())))
|
.isEmpty())
|
||||||
.filter(manualRedactionEntry -> !(manualRedactionEntry.isAddToDictionary() || manualRedactionEntry.isAddToDossierDictionary()))
|
.filter(manualRedactionEntry -> !(manualRedactionEntry.isAddToDictionary() || manualRedactionEntry.isAddToDossierDictionary()))
|
||||||
.map(manualRedactionEntry -> PrecursorEntity.fromManualRedactionEntry(manualRedactionEntry,
|
.map(manualRedactionEntry -> //
|
||||||
dictionaryService.isHint(manualRedactionEntry.getType(), dossierTemplateId)))
|
PrecursorEntity.fromManualRedactionEntry(manualRedactionEntry, dictionaryService.isHint(manualRedactionEntry.getType(), dossierTemplateId)))
|
||||||
.peek(manualEntity -> {
|
.peek(manualEntity -> {
|
||||||
if (manualEntity.getEntityType().equals(EntityType.HINT)) {
|
if (manualEntity.getEntityType().equals(EntityType.HINT)) {
|
||||||
manualEntity.skip("MAN.5.1", "manual hint is skipped by default");
|
manualEntity.skip("MAN.5.1", "manual hint is skipped by default");
|
||||||
@ -90,8 +85,14 @@ public class EntityFromPrecursorCreationService {
|
|||||||
|
|
||||||
public List<PrecursorEntity> toTextEntity(List<PrecursorEntity> precursorEntities, SemanticNode node) {
|
public List<PrecursorEntity> toTextEntity(List<PrecursorEntity> precursorEntities, SemanticNode node) {
|
||||||
|
|
||||||
var notFoundEntities = precursorEntities.stream().filter(PrecursorEntity::isRectangle).collect(Collectors.toList());
|
var notFoundEntities = precursorEntities.stream()
|
||||||
var findableEntities = precursorEntities.stream().filter(precursorEntity -> !precursorEntity.isRectangle()).toList();
|
.filter(PrecursorEntity::isRectangle)
|
||||||
|
.collect(Collectors.toList());
|
||||||
|
|
||||||
|
var findableEntities = precursorEntities.stream()
|
||||||
|
.filter(precursorEntity -> !precursorEntity.isRectangle())
|
||||||
|
.toList();
|
||||||
|
|
||||||
Map<String, List<TextEntity>> tempEntitiesByValue = entityFindingUtility.findAllPossibleEntitiesAndGroupByValue(node, findableEntities);
|
Map<String, List<TextEntity>> tempEntitiesByValue = entityFindingUtility.findAllPossibleEntitiesAndGroupByValue(node, findableEntities);
|
||||||
|
|
||||||
for (PrecursorEntity precursorEntity : findableEntities) {
|
for (PrecursorEntity precursorEntity : findableEntities) {
|
||||||
@ -102,7 +103,12 @@ public class EntityFromPrecursorCreationService {
|
|||||||
}
|
}
|
||||||
createCorrectEntity(precursorEntity, optionalClosestEntity.get());
|
createCorrectEntity(precursorEntity, optionalClosestEntity.get());
|
||||||
}
|
}
|
||||||
tempEntitiesByValue.values().stream().flatMap(Collection::stream).forEach(TextEntity::removeFromGraph);
|
|
||||||
|
tempEntitiesByValue.values()
|
||||||
|
.stream()
|
||||||
|
.flatMap(Collection::stream)
|
||||||
|
.forEach(TextEntity::removeFromGraph);
|
||||||
|
|
||||||
return notFoundEntities;
|
return notFoundEntities;
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -113,9 +119,23 @@ public class EntityFromPrecursorCreationService {
|
|||||||
* @param precursorEntity The entity identifier for the RedactionEntity.
|
* @param precursorEntity The entity identifier for the RedactionEntity.
|
||||||
* @param closestEntity The closest Boundary to the RedactionEntity.
|
* @param closestEntity The closest Boundary to the RedactionEntity.
|
||||||
*/
|
*/
|
||||||
private void createCorrectEntity(PrecursorEntity precursorEntity, TextEntity closestEntity) {
|
public static TextEntity createCorrectEntity(PrecursorEntity precursorEntity, TextEntity closestEntity) {
|
||||||
|
|
||||||
TextEntity correctEntity = TextEntity.initialEntityNode(closestEntity.getTextRange(), precursorEntity.type(), precursorEntity.getEntityType(), precursorEntity.getId());
|
return createCorrectEntity(precursorEntity, closestEntity, false);
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
public static TextEntity createCorrectEntity(PrecursorEntity precursorEntity, TextEntity closestEntity, boolean generateId) {
|
||||||
|
|
||||||
|
TextEntity correctEntity;
|
||||||
|
if (generateId) {
|
||||||
|
correctEntity = TextEntity.initialEntityNode(closestEntity.getTextRange(),
|
||||||
|
precursorEntity.type(),
|
||||||
|
precursorEntity.getEntityType(),
|
||||||
|
closestEntity.getDeepestFullyContainingNode());
|
||||||
|
} else {
|
||||||
|
correctEntity = TextEntity.initialEntityNode(closestEntity.getTextRange(), precursorEntity.type(), precursorEntity.getEntityType(), precursorEntity.getId());
|
||||||
|
}
|
||||||
correctEntity.setDeepestFullyContainingNode(closestEntity.getDeepestFullyContainingNode());
|
correctEntity.setDeepestFullyContainingNode(closestEntity.getDeepestFullyContainingNode());
|
||||||
correctEntity.setIntersectingNodes(new ArrayList<>(closestEntity.getIntersectingNodes()));
|
correctEntity.setIntersectingNodes(new ArrayList<>(closestEntity.getIntersectingNodes()));
|
||||||
correctEntity.setDuplicateTextRanges(new ArrayList<>(closestEntity.getDuplicateTextRanges()));
|
correctEntity.setDuplicateTextRanges(new ArrayList<>(closestEntity.getDuplicateTextRanges()));
|
||||||
@ -125,14 +145,17 @@ public class EntityFromPrecursorCreationService {
|
|||||||
correctEntity.setTextAfter(closestEntity.getTextAfter());
|
correctEntity.setTextAfter(closestEntity.getTextAfter());
|
||||||
correctEntity.setTextBefore(closestEntity.getTextBefore());
|
correctEntity.setTextBefore(closestEntity.getTextBefore());
|
||||||
|
|
||||||
correctEntity.getIntersectingNodes().forEach(n -> n.getEntities().add(correctEntity));
|
correctEntity.getIntersectingNodes()
|
||||||
correctEntity.getPages().forEach(page -> page.getEntities().add(correctEntity));
|
.forEach(n -> n.getEntities().add(correctEntity));
|
||||||
|
correctEntity.getPages()
|
||||||
|
.forEach(page -> page.getEntities().add(correctEntity));
|
||||||
|
|
||||||
correctEntity.addMatchedRules(precursorEntity.getMatchedRuleList());
|
correctEntity.addMatchedRules(precursorEntity.getMatchedRuleList());
|
||||||
correctEntity.setDictionaryEntry(precursorEntity.isDictionaryEntry());
|
correctEntity.setDictionaryEntry(precursorEntity.isDictionaryEntry());
|
||||||
correctEntity.setDossierDictionaryEntry(precursorEntity.isDossierDictionaryEntry());
|
correctEntity.setDossierDictionaryEntry(precursorEntity.isDossierDictionaryEntry());
|
||||||
correctEntity.getManualOverwrite().addChanges(precursorEntity.getManualOverwrite().getManualChangeLog());
|
correctEntity.getManualOverwrite().addChanges(precursorEntity.getManualOverwrite().getManualChangeLog());
|
||||||
correctEntity.addEngines(precursorEntity.getEngines());
|
correctEntity.addEngines(precursorEntity.getEngines());
|
||||||
|
return correctEntity;
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|||||||
@ -1,5 +1,6 @@
|
|||||||
package com.iqser.red.service.redaction.v1.server.utils;
|
package com.iqser.red.service.redaction.v1.server.utils;
|
||||||
|
|
||||||
|
import java.util.Collections;
|
||||||
import java.util.LinkedList;
|
import java.util.LinkedList;
|
||||||
import java.util.Set;
|
import java.util.Set;
|
||||||
import java.util.function.BiConsumer;
|
import java.util.function.BiConsumer;
|
||||||
@ -17,7 +18,7 @@ public class MigratedIdsCollector implements Collector<MigrationEntity, Migrated
|
|||||||
@Override
|
@Override
|
||||||
public Supplier<MigratedIds> supplier() {
|
public Supplier<MigratedIds> supplier() {
|
||||||
|
|
||||||
return () -> new MigratedIds(new LinkedList<>());
|
return () -> new MigratedIds(new LinkedList<>(), Collections.emptyList());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@ -31,6 +31,7 @@ import org.springframework.test.context.junit.jupiter.SpringExtension;
|
|||||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
|
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
|
||||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
|
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
|
||||||
|
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType;
|
||||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position;
|
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position;
|
||||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.migration.MigratedIds;
|
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.migration.MigratedIds;
|
||||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
|
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
|
||||||
@ -49,7 +50,6 @@ import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
|
|||||||
import com.iqser.red.service.redaction.v1.server.redaction.utils.OsUtils;
|
import com.iqser.red.service.redaction.v1.server.redaction.utils.OsUtils;
|
||||||
import com.iqser.red.service.redaction.v1.server.service.DictionaryService;
|
import com.iqser.red.service.redaction.v1.server.service.DictionaryService;
|
||||||
import com.iqser.red.service.redaction.v1.server.service.document.EntityFindingUtility;
|
import com.iqser.red.service.redaction.v1.server.service.document.EntityFindingUtility;
|
||||||
import com.iqser.red.service.redaction.v1.server.utils.RectangleTransformations;
|
|
||||||
import com.knecon.fforesight.tenantcommons.TenantContext;
|
import com.knecon.fforesight.tenantcommons.TenantContext;
|
||||||
|
|
||||||
import lombok.SneakyThrows;
|
import lombok.SneakyThrows;
|
||||||
@ -107,7 +107,7 @@ public class MigrationIntegrationTest extends BuildDocumentIntegrationTest {
|
|||||||
@SneakyThrows
|
@SneakyThrows
|
||||||
public void testSave() {
|
public void testSave() {
|
||||||
|
|
||||||
MigratedIds ids = new MigratedIds(new LinkedList<>());
|
MigratedIds ids = new MigratedIds(new LinkedList<>(), null);
|
||||||
ids.addMapping("123", "321");
|
ids.addMapping("123", "321");
|
||||||
ids.addMapping("123", "321");
|
ids.addMapping("123", "321");
|
||||||
ids.addMapping("123", "321");
|
ids.addMapping("123", "321");
|
||||||
@ -173,7 +173,11 @@ public class MigrationIntegrationTest extends BuildDocumentIntegrationTest {
|
|||||||
mergedRedactionLog = redactionLog;
|
mergedRedactionLog = redactionLog;
|
||||||
}
|
}
|
||||||
|
|
||||||
MigratedEntityLog migratedEntityLog = redactionLogToEntityLogMigrationService.migrate(mergedRedactionLog, document, TEST_DOSSIER_TEMPLATE_ID, manualRedactions);
|
MigratedEntityLog migratedEntityLog = redactionLogToEntityLogMigrationService.migrate(mergedRedactionLog,
|
||||||
|
document,
|
||||||
|
TEST_DOSSIER_TEMPLATE_ID,
|
||||||
|
manualRedactions,
|
||||||
|
TEST_FILE_ID);
|
||||||
|
|
||||||
redactionStorageService.storeObject(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.ENTITY_LOG, migratedEntityLog.getEntityLog());
|
redactionStorageService.storeObject(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.ENTITY_LOG, migratedEntityLog.getEntityLog());
|
||||||
assertEquals(mergedRedactionLog.getRedactionLogEntry().size(), migratedEntityLog.getEntityLog().getEntityLogEntry().size());
|
assertEquals(mergedRedactionLog.getRedactionLogEntry().size(), migratedEntityLog.getEntityLog().getEntityLogEntry().size());
|
||||||
@ -187,10 +191,11 @@ public class MigrationIntegrationTest extends BuildDocumentIntegrationTest {
|
|||||||
assertEquals(mergedRedactionLog.getLegalBasis().size(), entityLog.getLegalBasis().size());
|
assertEquals(mergedRedactionLog.getLegalBasis().size(), entityLog.getLegalBasis().size());
|
||||||
|
|
||||||
Map<String, String> migratedIds = migratedEntityLog.getMigratedIds().buildOldToNewMapping();
|
Map<String, String> migratedIds = migratedEntityLog.getMigratedIds().buildOldToNewMapping();
|
||||||
|
// assertEquals(legacyRedactionLogMergeService.getNumberOfAffectedAnnotations(manualRedactions), migratedIds.size());
|
||||||
|
|
||||||
migratedIds.forEach((oldId, newId) -> assertEntryIsEqual(oldId, newId, mergedRedactionLog, entityLog, migratedIds));
|
migratedIds.forEach((oldId, newId) -> assertEntryIsEqual(oldId, newId, mergedRedactionLog, entityLog, migratedIds));
|
||||||
|
|
||||||
AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder().dossierId(TEST_DOSSIER_ID).fileId(TEST_FILE_ID)
|
AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder().dossierId(TEST_DOSSIER_ID).fileId(TEST_FILE_ID).build());
|
||||||
.build());
|
|
||||||
File outputFile = Path.of(OsUtils.getTemporaryDirectory()).resolve(Path.of(fileName.replaceAll(".pdf", "_MIGRATED.pdf")).getFileName()).toFile();
|
File outputFile = Path.of(OsUtils.getTemporaryDirectory()).resolve(Path.of(fileName.replaceAll(".pdf", "_MIGRATED.pdf")).getFileName()).toFile();
|
||||||
|
|
||||||
try (FileOutputStream fileOutputStream = new FileOutputStream(outputFile)) {
|
try (FileOutputStream fileOutputStream = new FileOutputStream(outputFile)) {
|
||||||
@ -268,13 +273,24 @@ public class MigrationIntegrationTest extends BuildDocumentIntegrationTest {
|
|||||||
if (!redactionLogEntry.isImage()) {
|
if (!redactionLogEntry.isImage()) {
|
||||||
assertEquals(redactionLogEntry.getValue().toLowerCase(Locale.ENGLISH), entityLogEntry.getValue().toLowerCase(Locale.ENGLISH));
|
assertEquals(redactionLogEntry.getValue().toLowerCase(Locale.ENGLISH), entityLogEntry.getValue().toLowerCase(Locale.ENGLISH));
|
||||||
}
|
}
|
||||||
|
if (entityLogEntry.getManualChanges()
|
||||||
|
.stream()
|
||||||
|
.noneMatch(mc -> mc.getManualRedactionType().equals(ManualRedactionType.RECATEGORIZE))) {
|
||||||
|
assertEquals(redactionLogEntry.getType(), entityLogEntry.getType());
|
||||||
|
}
|
||||||
assertEquals(redactionLogEntry.getChanges().size(), entityLogEntry.getChanges().size());
|
assertEquals(redactionLogEntry.getChanges().size(), entityLogEntry.getChanges().size());
|
||||||
assertTrue(redactionLogEntry.getManualChanges().size() <= entityLogEntry.getManualChanges().size());
|
assertTrue(redactionLogEntry.getManualChanges().size() <= entityLogEntry.getManualChanges().size());
|
||||||
assertEquals(redactionLogEntry.getPositions().size(), entityLogEntry.getPositions().size());
|
assertEquals(redactionLogEntry.getPositions().size(), entityLogEntry.getPositions().size());
|
||||||
|
if (entityLogEntry.getManualChanges()
|
||||||
|
.stream()
|
||||||
|
.noneMatch(mc -> mc.getManualRedactionType().equals(ManualRedactionType.RESIZE) || mc.getManualRedactionType().equals(ManualRedactionType.RESIZE_IN_DICTIONARY))) {
|
||||||
assertTrue(positionsAlmostEqual(redactionLogEntry.getPositions(), entityLogEntry.getPositions()));
|
assertTrue(positionsAlmostEqual(redactionLogEntry.getPositions(), entityLogEntry.getPositions()));
|
||||||
// assertEquals(redactionLogEntry.getColor(), entityLogEntry.getColor());
|
}
|
||||||
|
if (entityLogEntry.getManualChanges()
|
||||||
|
.stream()
|
||||||
|
.noneMatch(mc -> mc.getManualRedactionType().equals(ManualRedactionType.FORCE_REDACT) || mc.getManualRedactionType().equals(ManualRedactionType.FORCE_HINT))) {
|
||||||
assertEqualsNullSafe(redactionLogEntry.getLegalBasis(), entityLogEntry.getLegalBasis());
|
assertEqualsNullSafe(redactionLogEntry.getLegalBasis(), entityLogEntry.getLegalBasis());
|
||||||
// assertEqualsNullSafe(redactionLogEntry.getReason(), entityLogEntry.getReason());
|
}
|
||||||
assertReferencesEqual(redactionLogEntry.getReference(), entityLogEntry.getReference(), oldToNewMapping);
|
assertReferencesEqual(redactionLogEntry.getReference(), entityLogEntry.getReference(), oldToNewMapping);
|
||||||
assertEquals(redactionLogEntry.isDictionaryEntry(), entityLogEntry.isDictionaryEntry());
|
assertEquals(redactionLogEntry.isDictionaryEntry(), entityLogEntry.isDictionaryEntry());
|
||||||
assertEquals(redactionLogEntry.isDossierDictionaryEntry(), entityLogEntry.isDossierDictionaryEntry());
|
assertEquals(redactionLogEntry.isDossierDictionaryEntry(), entityLogEntry.isDossierDictionaryEntry());
|
||||||
|
|||||||
@ -169,7 +169,8 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
|
|||||||
|
|
||||||
var entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
|
var entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
|
||||||
|
|
||||||
entityLog.getEntityLogEntry().forEach(entry -> {
|
entityLog.getEntityLogEntry()
|
||||||
|
.forEach(entry -> {
|
||||||
duplicates.computeIfAbsent(entry.getId(), v -> new ArrayList<>()).add(entry);
|
duplicates.computeIfAbsent(entry.getId(), v -> new ArrayList<>()).add(entry);
|
||||||
});
|
});
|
||||||
|
|
||||||
@ -256,7 +257,10 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
|
|||||||
|
|
||||||
var redactionLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
|
var redactionLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
|
||||||
|
|
||||||
var values = redactionLog.getEntityLogEntry().stream().map(EntityLogEntry::getValue).collect(Collectors.toList());
|
var values = redactionLog.getEntityLogEntry()
|
||||||
|
.stream()
|
||||||
|
.map(EntityLogEntry::getValue)
|
||||||
|
.collect(Collectors.toList());
|
||||||
|
|
||||||
assertThat(values).containsExactlyInAnyOrder("Lastname M.", "Doe", "Doe J.", "M. Mustermann", "Mustermann M.", "F. Lastname");
|
assertThat(values).containsExactlyInAnyOrder("Lastname M.", "Doe", "Doe J.", "M. Mustermann", "Mustermann M.", "F. Lastname");
|
||||||
}
|
}
|
||||||
@ -353,10 +357,18 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
|
|||||||
|
|
||||||
var mergedEntityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
|
var mergedEntityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
|
||||||
|
|
||||||
var cbiAddressBeforeHintRemoval = entityLog.getEntityLogEntry().stream().filter(re -> re.getType().equalsIgnoreCase("CBI_Address")).findAny().get();
|
var cbiAddressBeforeHintRemoval = entityLog.getEntityLogEntry()
|
||||||
|
.stream()
|
||||||
|
.filter(re -> re.getType().equalsIgnoreCase("CBI_Address"))
|
||||||
|
.findAny()
|
||||||
|
.get();
|
||||||
assertThat(cbiAddressBeforeHintRemoval.getState().equals(EntryState.APPLIED)).isFalse();
|
assertThat(cbiAddressBeforeHintRemoval.getState().equals(EntryState.APPLIED)).isFalse();
|
||||||
|
|
||||||
var cbiAddressAfterHintRemoval = mergedEntityLog.getEntityLogEntry().stream().filter(re -> re.getType().equalsIgnoreCase("CBI_Address")).findAny().get();
|
var cbiAddressAfterHintRemoval = mergedEntityLog.getEntityLogEntry()
|
||||||
|
.stream()
|
||||||
|
.filter(re -> re.getType().equalsIgnoreCase("CBI_Address"))
|
||||||
|
.findAny()
|
||||||
|
.get();
|
||||||
assertThat(cbiAddressAfterHintRemoval.getState().equals(EntryState.APPLIED)).isTrue();
|
assertThat(cbiAddressAfterHintRemoval.getState().equals(EntryState.APPLIED)).isTrue();
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -386,7 +398,8 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
|
|||||||
|
|
||||||
var entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
|
var entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
|
||||||
|
|
||||||
entityLog.getEntityLogEntry().forEach(entry -> {
|
entityLog.getEntityLogEntry()
|
||||||
|
.forEach(entry -> {
|
||||||
duplicates.computeIfAbsent(entry.getId(), v -> new ArrayList<>()).add(entry);
|
duplicates.computeIfAbsent(entry.getId(), v -> new ArrayList<>()).add(entry);
|
||||||
});
|
});
|
||||||
|
|
||||||
@ -449,7 +462,10 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
|
|||||||
correctFound++;
|
correctFound++;
|
||||||
continue loop;
|
continue loop;
|
||||||
}
|
}
|
||||||
if (Objects.equals(entityLogEntry.getContainingNodeId().get(0), section.getTreeId().get(0))) {
|
if (Objects.equals(entityLogEntry.getContainingNodeId()
|
||||||
|
.get(0),
|
||||||
|
section.getTreeId()
|
||||||
|
.get(0))) {
|
||||||
String value = section.getTextBlock().subSequence(new TextRange(entityLogEntry.getStartOffset(), entityLogEntry.getEndOffset())).toString();
|
String value = section.getTextBlock().subSequence(new TextRange(entityLogEntry.getStartOffset(), entityLogEntry.getEndOffset())).toString();
|
||||||
if (entityLogEntry.getValue().equalsIgnoreCase(value)) {
|
if (entityLogEntry.getValue().equalsIgnoreCase(value)) {
|
||||||
correctFound++;
|
correctFound++;
|
||||||
@ -548,7 +564,11 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
|
|||||||
|
|
||||||
var entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
|
var entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
|
||||||
|
|
||||||
var changes = entityLog.getEntityLogEntry().stream().filter(entry -> entry.getValue() != null && entry.getValue().equals("report")).findFirst().get().getChanges();
|
var changes = entityLog.getEntityLogEntry()
|
||||||
|
.stream()
|
||||||
|
.filter(entry -> entry.getValue() != null && entry.getValue().equals("report"))
|
||||||
|
.findFirst()
|
||||||
|
.get().getChanges();
|
||||||
|
|
||||||
assertThat(changes.size()).isEqualTo(2);
|
assertThat(changes.size()).isEqualTo(2);
|
||||||
|
|
||||||
@ -601,7 +621,11 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
|
|||||||
.map(redactionLogEntry -> new TextRange(redactionLogEntry.getStartOffset(), redactionLogEntry.getEndOffset()))
|
.map(redactionLogEntry -> new TextRange(redactionLogEntry.getStartOffset(), redactionLogEntry.getEndOffset()))
|
||||||
.map(boundary -> documentGraph.getTextBlock().subSequence(boundary).toString())
|
.map(boundary -> documentGraph.getTextBlock().subSequence(boundary).toString())
|
||||||
.toList();
|
.toList();
|
||||||
List<String> valuesInRedactionLog = entityLog.getEntityLogEntry().stream().filter(e -> !e.getEntryType().equals(EntryType.IMAGE)).map(EntityLogEntry::getValue).toList();
|
List<String> valuesInRedactionLog = entityLog.getEntityLogEntry()
|
||||||
|
.stream()
|
||||||
|
.filter(e -> !e.getEntryType().equals(EntryType.IMAGE))
|
||||||
|
.map(EntityLogEntry::getValue)
|
||||||
|
.toList();
|
||||||
|
|
||||||
assertEquals(valuesInRedactionLog, valuesInDocument);
|
assertEquals(valuesInRedactionLog, valuesInDocument);
|
||||||
|
|
||||||
@ -960,7 +984,8 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
|
|||||||
|
|
||||||
var redactionLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
|
var redactionLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
|
||||||
|
|
||||||
redactionLog.getEntityLogEntry().forEach(entry -> {
|
redactionLog.getEntityLogEntry()
|
||||||
|
.forEach(entry -> {
|
||||||
if (!entry.getEntryType().equals(EntryType.HINT)) {
|
if (!entry.getEntryType().equals(EntryType.HINT)) {
|
||||||
if (entry.getType().equals("CBI_author")) {
|
if (entry.getType().equals("CBI_author")) {
|
||||||
assertThat(entry.getReason()).isEqualTo("Not redacted because it's row does not belong to a vertebrate study");
|
assertThat(entry.getReason()).isEqualTo("Not redacted because it's row does not belong to a vertebrate study");
|
||||||
@ -1056,10 +1081,34 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
|
|||||||
.processedDate(OffsetDateTime.now())
|
.processedDate(OffsetDateTime.now())
|
||||||
.requestDate(OffsetDateTime.now())
|
.requestDate(OffsetDateTime.now())
|
||||||
.value("Syngenta Crop Protection AG, Basel, Switzerland RCC Ltd., Itingen, Switzerland")
|
.value("Syngenta Crop Protection AG, Basel, Switzerland RCC Ltd., Itingen, Switzerland")
|
||||||
.positions(List.of(Rectangle.builder().topLeftX(289.44595f).topLeftY(327.567f).width(7.648041f).height(82.51475f).page(1).build(),
|
.positions(List.of(Rectangle.builder()
|
||||||
Rectangle.builder().topLeftX(298.67056f).topLeftY(327.567f).width(7.648041f).height(75.32377f).page(1).build(),
|
.topLeftX(289.44595f)
|
||||||
Rectangle.builder().topLeftX(307.89517f).topLeftY(327.567f).width(7.648041f).height(61.670967f).page(1).build(),
|
.topLeftY(327.567f)
|
||||||
Rectangle.builder().topLeftX(316.99985f).topLeftY(327.567f).width(7.648041f).height(38.104286f).page(1).build()))
|
.width(7.648041f)
|
||||||
|
.height(82.51475f)
|
||||||
|
.page(1)
|
||||||
|
.build(),
|
||||||
|
Rectangle.builder()
|
||||||
|
.topLeftX(298.67056f)
|
||||||
|
.topLeftY(327.567f)
|
||||||
|
.width(7.648041f)
|
||||||
|
.height(75.32377f)
|
||||||
|
.page(1)
|
||||||
|
.build(),
|
||||||
|
Rectangle.builder()
|
||||||
|
.topLeftX(307.89517f)
|
||||||
|
.topLeftY(327.567f)
|
||||||
|
.width(7.648041f)
|
||||||
|
.height(61.670967f)
|
||||||
|
.page(1)
|
||||||
|
.build(),
|
||||||
|
Rectangle.builder()
|
||||||
|
.topLeftX(316.99985f)
|
||||||
|
.topLeftY(327.567f)
|
||||||
|
.width(7.648041f)
|
||||||
|
.height(38.104286f)
|
||||||
|
.page(1)
|
||||||
|
.build()))
|
||||||
.updateDictionary(false)
|
.updateDictionary(false)
|
||||||
.build()));
|
.build()));
|
||||||
|
|
||||||
@ -1124,7 +1173,8 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
|
|||||||
fileOutputStream.write(annotateResponse.getDocument());
|
fileOutputStream.write(annotateResponse.getDocument());
|
||||||
}
|
}
|
||||||
|
|
||||||
entityLog.getEntityLogEntry().forEach(entry -> {
|
entityLog.getEntityLogEntry()
|
||||||
|
.forEach(entry -> {
|
||||||
if (entry.getValue() == null) {
|
if (entry.getValue() == null) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
@ -1163,7 +1213,10 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
|
|||||||
}
|
}
|
||||||
|
|
||||||
var entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
|
var entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
|
||||||
var values = entityLog.getEntityLogEntry().stream().map(EntityLogEntry::getValue).collect(Collectors.toList());
|
var values = entityLog.getEntityLogEntry()
|
||||||
|
.stream()
|
||||||
|
.map(EntityLogEntry::getValue)
|
||||||
|
.collect(Collectors.toList());
|
||||||
|
|
||||||
assertThat(values).contains("Mrs. Robinson");
|
assertThat(values).contains("Mrs. Robinson");
|
||||||
assertThat(values).contains("Mr. Bojangles");
|
assertThat(values).contains("Mr. Bojangles");
|
||||||
@ -1202,7 +1255,10 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
|
|||||||
System.out.println("Finished reanalysis");
|
System.out.println("Finished reanalysis");
|
||||||
|
|
||||||
var entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
|
var entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
|
||||||
entityLog.getEntityLogEntry().stream().filter(entityLogEntry -> entityLogEntry.getType().equals("signature")).forEach(entityLogEntry -> {
|
entityLog.getEntityLogEntry()
|
||||||
|
.stream()
|
||||||
|
.filter(entityLogEntry -> entityLogEntry.getType().equals("signature"))
|
||||||
|
.forEach(entityLogEntry -> {
|
||||||
assertThat(entityLogEntry.getState() == EntryState.APPLIED).isTrue();
|
assertThat(entityLogEntry.getState() == EntryState.APPLIED).isTrue();
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
@ -1282,26 +1338,50 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
|
|||||||
|
|
||||||
var entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
|
var entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
|
||||||
|
|
||||||
assertTrue(entityLog.getEntityLogEntry().stream().anyMatch(entityLogEntry -> entityLogEntry.getId().equals(manualAddId)));
|
assertTrue(entityLog.getEntityLogEntry()
|
||||||
assertEquals(entityLog.getEntityLogEntry().stream().filter(entityLogEntry -> entityLogEntry.getId().equals(manualAddId)).findFirst().get().getState(), EntryState.APPLIED);
|
.stream()
|
||||||
|
.anyMatch(entityLogEntry -> entityLogEntry.getId().equals(manualAddId)));
|
||||||
|
assertEquals(entityLog.getEntityLogEntry()
|
||||||
|
.stream()
|
||||||
|
.filter(entityLogEntry -> entityLogEntry.getId().equals(manualAddId))
|
||||||
|
.findFirst()
|
||||||
|
.get().getState(), EntryState.APPLIED);
|
||||||
|
|
||||||
request.setManualRedactions(ManualRedactions.builder().entriesToAdd(Set.of(manualRedactionEntry)).idsToRemove(Set.of(idRemoval)).build());
|
request.setManualRedactions(ManualRedactions.builder().entriesToAdd(Set.of(manualRedactionEntry)).idsToRemove(Set.of(idRemoval)).build());
|
||||||
analyzeService.reanalyze(request);
|
analyzeService.reanalyze(request);
|
||||||
|
|
||||||
entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
|
entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
|
||||||
|
|
||||||
assertTrue(entityLog.getEntityLogEntry().stream().anyMatch(entityLogEntry -> entityLogEntry.getId().equals(manualAddId)));
|
assertTrue(entityLog.getEntityLogEntry()
|
||||||
assertEquals(entityLog.getEntityLogEntry().stream().filter(entityLogEntry -> entityLogEntry.getId().equals(manualAddId)).findFirst().get().getState(), EntryState.REMOVED);
|
.stream()
|
||||||
|
.anyMatch(entityLogEntry -> entityLogEntry.getId().equals(manualAddId)));
|
||||||
|
assertEquals(entityLog.getEntityLogEntry()
|
||||||
|
.stream()
|
||||||
|
.filter(entityLogEntry -> entityLogEntry.getId().equals(manualAddId))
|
||||||
|
.findFirst()
|
||||||
|
.get().getState(), EntryState.REMOVED);
|
||||||
|
|
||||||
request.setManualRedactions(ManualRedactions.builder().entriesToAdd(Set.of(manualRedactionEntry, manualRedactionEntry2)).idsToRemove(Set.of(idRemoval)).build());
|
request.setManualRedactions(ManualRedactions.builder().entriesToAdd(Set.of(manualRedactionEntry, manualRedactionEntry2)).idsToRemove(Set.of(idRemoval)).build());
|
||||||
analyzeService.reanalyze(request);
|
analyzeService.reanalyze(request);
|
||||||
|
|
||||||
entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
|
entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
|
||||||
|
|
||||||
assertTrue(entityLog.getEntityLogEntry().stream().anyMatch(entityLogEntry -> entityLogEntry.getId().equals(manualAddId)));
|
assertTrue(entityLog.getEntityLogEntry()
|
||||||
assertEquals(entityLog.getEntityLogEntry().stream().filter(entityLogEntry -> entityLogEntry.getId().equals(manualAddId)).findFirst().get().getState(), EntryState.REMOVED);
|
.stream()
|
||||||
assertTrue(entityLog.getEntityLogEntry().stream().anyMatch(entityLogEntry -> entityLogEntry.getId().equals(manualAddId2)));
|
.anyMatch(entityLogEntry -> entityLogEntry.getId().equals(manualAddId)));
|
||||||
assertEquals(entityLog.getEntityLogEntry().stream().filter(entityLogEntry -> entityLogEntry.getId().equals(manualAddId2)).findFirst().get().getState(), EntryState.APPLIED);
|
assertEquals(entityLog.getEntityLogEntry()
|
||||||
|
.stream()
|
||||||
|
.filter(entityLogEntry -> entityLogEntry.getId().equals(manualAddId))
|
||||||
|
.findFirst()
|
||||||
|
.get().getState(), EntryState.REMOVED);
|
||||||
|
assertTrue(entityLog.getEntityLogEntry()
|
||||||
|
.stream()
|
||||||
|
.anyMatch(entityLogEntry -> entityLogEntry.getId().equals(manualAddId2)));
|
||||||
|
assertEquals(entityLog.getEntityLogEntry()
|
||||||
|
.stream()
|
||||||
|
.filter(entityLogEntry -> entityLogEntry.getId().equals(manualAddId2))
|
||||||
|
.findFirst()
|
||||||
|
.get().getState(), EntryState.APPLIED);
|
||||||
|
|
||||||
request.setManualRedactions(ManualRedactions.builder()
|
request.setManualRedactions(ManualRedactions.builder()
|
||||||
.entriesToAdd(Set.of(manualRedactionEntry, manualRedactionEntry2))
|
.entriesToAdd(Set.of(manualRedactionEntry, manualRedactionEntry2))
|
||||||
@ -1311,26 +1391,48 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
|
|||||||
|
|
||||||
entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
|
entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
|
||||||
|
|
||||||
assertTrue(entityLog.getEntityLogEntry().stream().anyMatch(entityLogEntry -> entityLogEntry.getId().equals(manualAddId)));
|
assertTrue(entityLog.getEntityLogEntry()
|
||||||
assertEquals(entityLog.getEntityLogEntry().stream().filter(entityLogEntry -> entityLogEntry.getId().equals(manualAddId)).findFirst().get().getState(), EntryState.REMOVED);
|
.stream()
|
||||||
assertTrue(entityLog.getEntityLogEntry().stream().anyMatch(entityLogEntry -> entityLogEntry.getId().equals(manualAddId2)));
|
.anyMatch(entityLogEntry -> entityLogEntry.getId().equals(manualAddId)));
|
||||||
assertEquals(entityLog.getEntityLogEntry().stream().filter(entityLogEntry -> entityLogEntry.getId().equals(manualAddId2)).findFirst().get().getState(), EntryState.REMOVED);
|
assertEquals(entityLog.getEntityLogEntry()
|
||||||
|
.stream()
|
||||||
|
.filter(entityLogEntry -> entityLogEntry.getId().equals(manualAddId))
|
||||||
|
.findFirst()
|
||||||
|
.get().getState(), EntryState.REMOVED);
|
||||||
|
assertTrue(entityLog.getEntityLogEntry()
|
||||||
|
.stream()
|
||||||
|
.anyMatch(entityLogEntry -> entityLogEntry.getId().equals(manualAddId2)));
|
||||||
|
assertEquals(entityLog.getEntityLogEntry()
|
||||||
|
.stream()
|
||||||
|
.filter(entityLogEntry -> entityLogEntry.getId().equals(manualAddId2))
|
||||||
|
.findFirst()
|
||||||
|
.get().getState(), EntryState.REMOVED);
|
||||||
|
|
||||||
manualRedactionEntry.setRequestDate(OffsetDateTime.now());
|
manualRedactionEntry.setRequestDate(OffsetDateTime.now());
|
||||||
request.setManualRedactions(ManualRedactions.builder()
|
request.setManualRedactions(ManualRedactions.builder().entriesToAdd(Set.of(manualRedactionEntry, manualRedactionEntry2)).idsToRemove(Set.of(idRemoval2)).build());
|
||||||
.entriesToAdd(Set.of(manualRedactionEntry, manualRedactionEntry2))
|
|
||||||
.idsToRemove(Set.of(idRemoval, idRemoval2))
|
|
||||||
.build());
|
|
||||||
analyzeService.reanalyze(request);
|
analyzeService.reanalyze(request);
|
||||||
|
|
||||||
entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
|
entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
|
||||||
|
|
||||||
assertTrue(entityLog.getEntityLogEntry().stream().anyMatch(entityLogEntry -> entityLogEntry.getId().equals(manualAddId)));
|
assertTrue(entityLog.getEntityLogEntry()
|
||||||
assertEquals(entityLog.getEntityLogEntry().stream().filter(entityLogEntry -> entityLogEntry.getId().equals(manualAddId)).findFirst().get().getState(), EntryState.APPLIED);
|
.stream()
|
||||||
assertTrue(entityLog.getEntityLogEntry().stream().anyMatch(entityLogEntry -> entityLogEntry.getId().equals(manualAddId2)));
|
.anyMatch(entityLogEntry -> entityLogEntry.getId().equals(manualAddId)));
|
||||||
assertEquals(entityLog.getEntityLogEntry().stream().filter(entityLogEntry -> entityLogEntry.getId().equals(manualAddId2)).findFirst().get().getState(), EntryState.REMOVED);
|
assertEquals(entityLog.getEntityLogEntry()
|
||||||
|
.stream()
|
||||||
|
.filter(entityLogEntry -> entityLogEntry.getId().equals(manualAddId))
|
||||||
|
.findFirst()
|
||||||
|
.get().getState(), EntryState.APPLIED);
|
||||||
|
assertTrue(entityLog.getEntityLogEntry()
|
||||||
|
.stream()
|
||||||
|
.anyMatch(entityLogEntry -> entityLogEntry.getId().equals(manualAddId2)));
|
||||||
|
assertEquals(entityLog.getEntityLogEntry()
|
||||||
|
.stream()
|
||||||
|
.filter(entityLogEntry -> entityLogEntry.getId().equals(manualAddId2))
|
||||||
|
.findFirst()
|
||||||
|
.get().getState(), EntryState.REMOVED);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
@SneakyThrows
|
@SneakyThrows
|
||||||
public void testResizeWithUpdateDictionaryTrue() {
|
public void testResizeWithUpdateDictionaryTrue() {
|
||||||
@ -1342,7 +1444,11 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
|
|||||||
analyzeService.analyze(request);
|
analyzeService.analyze(request);
|
||||||
|
|
||||||
var entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
|
var entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
|
||||||
var david = entityLog.getEntityLogEntry().stream().filter(e -> e.getValue().equals("David")).findFirst().get();
|
var david = entityLog.getEntityLogEntry()
|
||||||
|
.stream()
|
||||||
|
.filter(e -> e.getValue().equals("David"))
|
||||||
|
.findFirst()
|
||||||
|
.get();
|
||||||
|
|
||||||
request.setManualRedactions(ManualRedactions.builder()
|
request.setManualRedactions(ManualRedactions.builder()
|
||||||
.resizeRedactions(Set.of(ManualResizeRedaction.builder()
|
.resizeRedactions(Set.of(ManualResizeRedaction.builder()
|
||||||
@ -1350,13 +1456,23 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
|
|||||||
.annotationId(david.getId())
|
.annotationId(david.getId())
|
||||||
.requestDate(OffsetDateTime.now())
|
.requestDate(OffsetDateTime.now())
|
||||||
.value("David Ksenia")
|
.value("David Ksenia")
|
||||||
.positions(List.of(Rectangle.builder().topLeftX(56.8f).topLeftY(293.564f).width(65.592f).height(15.408f).page(1).build()))
|
.positions(List.of(Rectangle.builder()
|
||||||
|
.topLeftX(56.8f)
|
||||||
|
.topLeftY(293.564f)
|
||||||
|
.width(65.592f)
|
||||||
|
.height(15.408f)
|
||||||
|
.page(1)
|
||||||
|
.build()))
|
||||||
.addToAllDossiers(false)
|
.addToAllDossiers(false)
|
||||||
.build()))
|
.build()))
|
||||||
.build());
|
.build());
|
||||||
analyzeService.reanalyze(request);
|
analyzeService.reanalyze(request);
|
||||||
entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
|
entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
|
||||||
var resizedEntity = entityLog.getEntityLogEntry().stream().filter(e -> e.getId().equals(david.getId())).findFirst().get();
|
var resizedEntity = entityLog.getEntityLogEntry()
|
||||||
|
.stream()
|
||||||
|
.filter(e -> e.getId().equals(david.getId()))
|
||||||
|
.findFirst()
|
||||||
|
.get();
|
||||||
assertEquals(resizedEntity.getState(), EntryState.APPLIED);
|
assertEquals(resizedEntity.getState(), EntryState.APPLIED);
|
||||||
assertEquals(resizedEntity.getValue(), "David Ksenia");
|
assertEquals(resizedEntity.getValue(), "David Ksenia");
|
||||||
}
|
}
|
||||||
@ -1364,12 +1480,7 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
|
|||||||
|
|
||||||
private IdRemoval getIdRemoval(String id) {
|
private IdRemoval getIdRemoval(String id) {
|
||||||
|
|
||||||
return IdRemoval.builder()
|
return IdRemoval.builder().annotationId(id).removeFromAllDossiers(false).removeFromDictionary(false).requestDate(OffsetDateTime.now()).build();
|
||||||
.annotationId(id)
|
|
||||||
.removeFromAllDossiers(false)
|
|
||||||
.removeFromDictionary(false)
|
|
||||||
.requestDate(OffsetDateTime.now())
|
|
||||||
.build();
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user