RED-7384: handle pending dict application in redaction-service instead of persistence
This commit is contained in:
parent
90ba3239fc
commit
df694a90a8
@ -1,11 +1,15 @@
|
|||||||
package com.iqser.red.service.redaction.v1.model;
|
package com.iqser.red.service.redaction.v1.model;
|
||||||
|
|
||||||
|
import java.util.Collections;
|
||||||
|
import java.util.Set;
|
||||||
|
|
||||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
|
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
|
||||||
|
|
||||||
import lombok.AllArgsConstructor;
|
import lombok.AllArgsConstructor;
|
||||||
import lombok.Builder;
|
import lombok.Builder;
|
||||||
import lombok.Data;
|
import lombok.Data;
|
||||||
import lombok.NoArgsConstructor;
|
import lombok.NoArgsConstructor;
|
||||||
|
import lombok.NonNull;
|
||||||
|
|
||||||
@Data
|
@Data
|
||||||
@Builder
|
@Builder
|
||||||
@ -13,9 +17,18 @@ import lombok.NoArgsConstructor;
|
|||||||
@AllArgsConstructor
|
@AllArgsConstructor
|
||||||
public class MigrationRequest {
|
public class MigrationRequest {
|
||||||
|
|
||||||
|
@NonNull
|
||||||
String dossierTemplateId;
|
String dossierTemplateId;
|
||||||
|
@NonNull
|
||||||
String dossierId;
|
String dossierId;
|
||||||
|
@NonNull
|
||||||
String fileId;
|
String fileId;
|
||||||
|
|
||||||
|
boolean fileIsApproved;
|
||||||
|
@NonNull
|
||||||
ManualRedactions manualRedactions;
|
ManualRedactions manualRedactions;
|
||||||
|
@NonNull
|
||||||
|
@Builder.Default
|
||||||
|
Set<String> entitiesWithComments = Collections.emptySet();
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|||||||
@ -57,7 +57,7 @@ public class MigrationMessageReceiver {
|
|||||||
|
|
||||||
if (redactionLog.getAnalysisVersion() == 0) {
|
if (redactionLog.getAnalysisVersion() == 0) {
|
||||||
redactionLog = legacyVersion0MigrationService.mergeDuplicateAnnotationIds(redactionLog);
|
redactionLog = legacyVersion0MigrationService.mergeDuplicateAnnotationIds(redactionLog);
|
||||||
} else if (migrationRequest.getManualRedactions() != null) {
|
} else {
|
||||||
redactionLog = legacyRedactionLogMergeService.addManualAddEntriesAndRemoveSkippedImported(redactionLog,
|
redactionLog = legacyRedactionLogMergeService.addManualAddEntriesAndRemoveSkippedImported(redactionLog,
|
||||||
migrationRequest.getManualRedactions(),
|
migrationRequest.getManualRedactions(),
|
||||||
migrationRequest.getDossierTemplateId());
|
migrationRequest.getDossierTemplateId());
|
||||||
@ -67,7 +67,9 @@ public class MigrationMessageReceiver {
|
|||||||
document,
|
document,
|
||||||
migrationRequest.getDossierTemplateId(),
|
migrationRequest.getDossierTemplateId(),
|
||||||
migrationRequest.getManualRedactions(),
|
migrationRequest.getManualRedactions(),
|
||||||
migrationRequest.getFileId());
|
migrationRequest.getFileId(),
|
||||||
|
migrationRequest.getEntitiesWithComments(),
|
||||||
|
migrationRequest.isFileIsApproved());
|
||||||
|
|
||||||
log.info("Storing migrated entityLog and ids to migrate in DB for file {}", migrationRequest.getFileId());
|
log.info("Storing migrated entityLog and ids to migrate in DB for file {}", migrationRequest.getFileId());
|
||||||
redactionStorageService.storeObject(migrationRequest.getDossierId(), migrationRequest.getFileId(), FileType.ENTITY_LOG, migratedEntityLog.getEntityLog());
|
redactionStorageService.storeObject(migrationRequest.getDossierId(), migrationRequest.getFileId(), FileType.ENTITY_LOG, migratedEntityLog.getEntityLog());
|
||||||
|
|||||||
@ -8,6 +8,7 @@ import java.util.LinkedList;
|
|||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
import java.util.Optional;
|
import java.util.Optional;
|
||||||
|
import java.util.Set;
|
||||||
import java.util.function.Function;
|
import java.util.function.Function;
|
||||||
import java.util.stream.Collectors;
|
import java.util.stream.Collectors;
|
||||||
import java.util.stream.Stream;
|
import java.util.stream.Stream;
|
||||||
@ -19,7 +20,9 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog
|
|||||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.migration.MigratedIds;
|
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.migration.MigratedIds;
|
||||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
|
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
|
||||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.BaseAnnotation;
|
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.BaseAnnotation;
|
||||||
|
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;
|
||||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
|
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
|
||||||
|
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
|
||||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
|
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
|
||||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog;
|
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog;
|
||||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
|
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
|
||||||
@ -57,7 +60,13 @@ public class RedactionLogToEntityLogMigrationService {
|
|||||||
ManualChangesApplicationService manualChangesApplicationService;
|
ManualChangesApplicationService manualChangesApplicationService;
|
||||||
|
|
||||||
|
|
||||||
public MigratedEntityLog migrate(RedactionLog redactionLog, Document document, String dossierTemplateId, ManualRedactions manualRedactions, String fileId) {
|
public MigratedEntityLog migrate(RedactionLog redactionLog,
|
||||||
|
Document document,
|
||||||
|
String dossierTemplateId,
|
||||||
|
ManualRedactions manualRedactions,
|
||||||
|
String fileId,
|
||||||
|
Set<String> entitiesWithComments,
|
||||||
|
boolean fileIsApproved) {
|
||||||
|
|
||||||
log.info("Migrating entities for file {}", fileId);
|
log.info("Migrating entities for file {}", fileId);
|
||||||
List<MigrationEntity> entitiesToMigrate = calculateMigrationEntitiesFromRedactionLog(redactionLog, document, dossierTemplateId, fileId);
|
List<MigrationEntity> entitiesToMigrate = calculateMigrationEntitiesFromRedactionLog(redactionLog, document, dossierTemplateId, fileId);
|
||||||
@ -66,7 +75,7 @@ public class RedactionLogToEntityLogMigrationService {
|
|||||||
.collect(new MigratedIdsCollector());
|
.collect(new MigratedIdsCollector());
|
||||||
|
|
||||||
log.info("applying manual changes to migrated entities for file {}", fileId);
|
log.info("applying manual changes to migrated entities for file {}", fileId);
|
||||||
applyManualChanges(entitiesToMigrate, manualRedactions);
|
applyLocalProcessedManualChanges(entitiesToMigrate, manualRedactions, fileIsApproved);
|
||||||
|
|
||||||
EntityLog entityLog = new EntityLog();
|
EntityLog entityLog = new EntityLog();
|
||||||
entityLog.setAnalysisNumber(redactionLog.getAnalysisNumber());
|
entityLog.setAnalysisNumber(redactionLog.getAnalysisNumber());
|
||||||
@ -87,7 +96,7 @@ public class RedactionLogToEntityLogMigrationService {
|
|||||||
.map(migrationEntity -> migrationEntity.toEntityLogEntry(oldToNewIDMapping))
|
.map(migrationEntity -> migrationEntity.toEntityLogEntry(oldToNewIDMapping))
|
||||||
.toList());
|
.toList());
|
||||||
|
|
||||||
if (getNumberOfApprovedEntries(redactionLog) != entityLog.getEntityLogEntry().size()) {
|
if (getNumberOfApprovedEntries(redactionLog, document.getNumberOfPages()) != entityLog.getEntityLogEntry().size()) {
|
||||||
String message = String.format("Not all entities have been found during the migration redactionLog has %d entries and new entityLog %d",
|
String message = String.format("Not all entities have been found during the migration redactionLog has %d entries and new entityLog %d",
|
||||||
redactionLog.getRedactionLogEntry().size(),
|
redactionLog.getRedactionLogEntry().size(),
|
||||||
entityLog.getEntityLogEntry().size());
|
entityLog.getEntityLogEntry().size());
|
||||||
@ -95,8 +104,14 @@ public class RedactionLogToEntityLogMigrationService {
|
|||||||
throw new AssertionError(message);
|
throw new AssertionError(message);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
Set<String> entitiesWithUnprocessedChanges = manualRedactions.buildAll()
|
||||||
|
.stream()
|
||||||
|
.filter(manualRedaction -> manualRedaction.getProcessedDate() == null)
|
||||||
|
.map(BaseAnnotation::getAnnotationId)
|
||||||
|
.collect(Collectors.toSet());
|
||||||
|
|
||||||
MigratedIds idsToMigrateInDb = entitiesToMigrate.stream()
|
MigratedIds idsToMigrateInDb = entitiesToMigrate.stream()
|
||||||
.filter(MigrationEntity::hasManualChangesOrComments)
|
.filter(migrationEntity -> migrationEntity.hasManualChangesOrComments(entitiesWithComments, entitiesWithUnprocessedChanges))
|
||||||
.filter(m -> !m.getOldId().equals(m.getNewId()))
|
.filter(m -> !m.getOldId().equals(m.getNewId()))
|
||||||
.collect(new MigratedIdsCollector());
|
.collect(new MigratedIdsCollector());
|
||||||
|
|
||||||
@ -111,20 +126,29 @@ public class RedactionLogToEntityLogMigrationService {
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
private void applyManualChanges(List<MigrationEntity> entitiesToMigrate, ManualRedactions manualRedactions) {
|
private void applyLocalProcessedManualChanges(List<MigrationEntity> entitiesToMigrate, ManualRedactions manualRedactions, boolean fileIsApproved) {
|
||||||
|
|
||||||
if (manualRedactions == null) {
|
if (manualRedactions == null) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
Map<String, List<BaseAnnotation>> manualChangesPerAnnotationId;
|
||||||
|
|
||||||
Map<String, List<BaseAnnotation>> manualChangesPerAnnotationId = Stream.of(manualRedactions.getIdsToRemove(),
|
if (fileIsApproved) {
|
||||||
manualRedactions.getEntriesToAdd(),
|
manualChangesPerAnnotationId = manualRedactions.buildAll()
|
||||||
manualRedactions.getForceRedactions(),
|
.stream()
|
||||||
manualRedactions.getResizeRedactions(),
|
.filter(manualChange -> (manualChange.getProcessedDate() != null && manualChange.isLocal()) //
|
||||||
manualRedactions.getLegalBasisChanges(),
|
// unprocessed dict change of type IdRemoval or ManualResize must be applied for approved documents
|
||||||
manualRedactions.getRecategorizations())
|
|| (manualChange.getProcessedDate() == null && !manualChange.isLocal() //
|
||||||
.flatMap(Collection::stream)
|
&& (manualChange instanceof IdRemoval || manualChange instanceof ManualResizeRedaction)))
|
||||||
|
.map(this::convertPendingDictChangesToLocal)
|
||||||
.collect(Collectors.groupingBy(BaseAnnotation::getAnnotationId));
|
.collect(Collectors.groupingBy(BaseAnnotation::getAnnotationId));
|
||||||
|
} else {
|
||||||
|
manualChangesPerAnnotationId = manualRedactions.buildAll()
|
||||||
|
.stream()
|
||||||
|
.filter(manualChange -> manualChange.getProcessedDate() != null)
|
||||||
|
.filter(BaseAnnotation::isLocal)
|
||||||
|
.collect(Collectors.groupingBy(BaseAnnotation::getAnnotationId));
|
||||||
|
}
|
||||||
|
|
||||||
entitiesToMigrate.forEach(migrationEntity -> migrationEntity.applyManualChanges(manualChangesPerAnnotationId.getOrDefault(migrationEntity.getOldId(),
|
entitiesToMigrate.forEach(migrationEntity -> migrationEntity.applyManualChanges(manualChangesPerAnnotationId.getOrDefault(migrationEntity.getOldId(),
|
||||||
Collections.emptyList()),
|
Collections.emptyList()),
|
||||||
@ -133,15 +157,40 @@ public class RedactionLogToEntityLogMigrationService {
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
private static long getNumberOfApprovedEntries(RedactionLog redactionLog) {
|
private BaseAnnotation convertPendingDictChangesToLocal(BaseAnnotation baseAnnotation) {
|
||||||
|
|
||||||
return redactionLog.getRedactionLogEntry().size();
|
if (baseAnnotation.getProcessedDate() != null) {
|
||||||
|
return baseAnnotation;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (baseAnnotation.isLocal()) {
|
||||||
|
return baseAnnotation;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (baseAnnotation instanceof ManualResizeRedaction manualResizeRedaction) {
|
||||||
|
manualResizeRedaction.setAddToAllDossiers(false);
|
||||||
|
manualResizeRedaction.setUpdateDictionary(false);
|
||||||
|
} else if (baseAnnotation instanceof IdRemoval idRemoval) {
|
||||||
|
idRemoval.setRemoveFromAllDossiers(false);
|
||||||
|
idRemoval.setRemoveFromDictionary(false);
|
||||||
|
}
|
||||||
|
|
||||||
|
return baseAnnotation;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
private long getNumberOfApprovedEntries(RedactionLog redactionLog, int numberOfPages) {
|
||||||
|
|
||||||
|
return redactionLog.getRedactionLogEntry()
|
||||||
|
.stream()
|
||||||
|
.filter(redactionLogEntry -> isOnExistingPage(redactionLogEntry, numberOfPages))
|
||||||
|
.count();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
private List<MigrationEntity> calculateMigrationEntitiesFromRedactionLog(RedactionLog redactionLog, Document document, String dossierTemplateId, String fileId) {
|
private List<MigrationEntity> calculateMigrationEntitiesFromRedactionLog(RedactionLog redactionLog, Document document, String dossierTemplateId, String fileId) {
|
||||||
|
|
||||||
List<MigrationEntity> images = getImageBasedMigrationEntities(redactionLog, document, fileId);
|
List<MigrationEntity> images = getImageBasedMigrationEntities(redactionLog, document, fileId, dossierTemplateId);
|
||||||
List<MigrationEntity> textMigrationEntities = getTextBasedMigrationEntities(redactionLog, document, dossierTemplateId, fileId);
|
List<MigrationEntity> textMigrationEntities = getTextBasedMigrationEntities(redactionLog, document, dossierTemplateId, fileId);
|
||||||
return Stream.of(textMigrationEntities.stream(), images.stream())
|
return Stream.of(textMigrationEntities.stream(), images.stream())
|
||||||
.flatMap(Function.identity())
|
.flatMap(Function.identity())
|
||||||
@ -155,7 +204,7 @@ public class RedactionLogToEntityLogMigrationService {
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
private List<MigrationEntity> getImageBasedMigrationEntities(RedactionLog redactionLog, Document document, String fileId) {
|
private List<MigrationEntity> getImageBasedMigrationEntities(RedactionLog redactionLog, Document document, String fileId, String dossierTemplateId) {
|
||||||
|
|
||||||
List<Image> images = document.streamAllImages()
|
List<Image> images = document.streamAllImages()
|
||||||
.collect(Collectors.toList());
|
.collect(Collectors.toList());
|
||||||
@ -202,7 +251,7 @@ public class RedactionLogToEntityLogMigrationService {
|
|||||||
} else {
|
} else {
|
||||||
closestImage.skip(ruleIdentifier, reason);
|
closestImage.skip(ruleIdentifier, reason);
|
||||||
}
|
}
|
||||||
migrationEntities.add(MigrationEntity.fromRedactionLogImage(redactionLogImage, closestImage, fileId));
|
migrationEntities.add(MigrationEntity.fromRedactionLogImage(redactionLogImage, closestImage, fileId, dictionaryService, dossierTemplateId));
|
||||||
}
|
}
|
||||||
return migrationEntities;
|
return migrationEntities;
|
||||||
}
|
}
|
||||||
@ -248,7 +297,8 @@ public class RedactionLogToEntityLogMigrationService {
|
|||||||
List<MigrationEntity> entitiesToMigrate = redactionLog.getRedactionLogEntry()
|
List<MigrationEntity> entitiesToMigrate = redactionLog.getRedactionLogEntry()
|
||||||
.stream()
|
.stream()
|
||||||
.filter(redactionLogEntry -> !redactionLogEntry.isImage())
|
.filter(redactionLogEntry -> !redactionLogEntry.isImage())
|
||||||
.map(entry -> MigrationEntity.fromRedactionLogEntry(entry, dictionaryService.isHint(entry.getType(), dossierTemplateId), fileId))
|
.filter(redactionLogEntry -> isOnExistingPage(redactionLogEntry, document.getNumberOfPages()))
|
||||||
|
.map(entry -> MigrationEntity.fromRedactionLogEntry(entry, fileId, dictionaryService, dossierTemplateId))
|
||||||
.toList();
|
.toList();
|
||||||
|
|
||||||
List<PrecursorEntity> precursorEntities = entitiesToMigrate.stream()
|
List<PrecursorEntity> precursorEntities = entitiesToMigrate.stream()
|
||||||
@ -285,4 +335,20 @@ public class RedactionLogToEntityLogMigrationService {
|
|||||||
return entitiesToMigrate;
|
return entitiesToMigrate;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
private boolean isOnExistingPage(RedactionLogEntry redactionLogEntry, int numberOfPages) {
|
||||||
|
|
||||||
|
var pages = redactionLogEntry.getPositions()
|
||||||
|
.stream()
|
||||||
|
.map(Rectangle::getPage)
|
||||||
|
.collect(Collectors.toSet());
|
||||||
|
|
||||||
|
for (int page : pages) {
|
||||||
|
if (page > numberOfPages) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|||||||
@ -8,6 +8,7 @@ import java.time.OffsetDateTime;
|
|||||||
import java.util.Collections;
|
import java.util.Collections;
|
||||||
import java.util.LinkedList;
|
import java.util.LinkedList;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
import java.util.Locale;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
import java.util.Optional;
|
import java.util.Optional;
|
||||||
import java.util.Set;
|
import java.util.Set;
|
||||||
@ -19,6 +20,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog
|
|||||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position;
|
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position;
|
||||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle;
|
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle;
|
||||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.BaseAnnotation;
|
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.BaseAnnotation;
|
||||||
|
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
|
||||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
|
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
|
||||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
|
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
|
||||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntryType;
|
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntryType;
|
||||||
@ -30,6 +32,8 @@ import com.iqser.red.service.redaction.v1.server.model.document.entity.IEntity;
|
|||||||
import com.iqser.red.service.redaction.v1.server.model.document.entity.ManualChangeOverwrite;
|
import com.iqser.red.service.redaction.v1.server.model.document.entity.ManualChangeOverwrite;
|
||||||
import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
|
import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
|
||||||
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Image;
|
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Image;
|
||||||
|
import com.iqser.red.service.redaction.v1.server.model.document.nodes.ImageType;
|
||||||
|
import com.iqser.red.service.redaction.v1.server.service.DictionaryService;
|
||||||
import com.iqser.red.service.redaction.v1.server.service.ManualChangeFactory;
|
import com.iqser.red.service.redaction.v1.server.service.ManualChangeFactory;
|
||||||
import com.iqser.red.service.redaction.v1.server.service.ManualChangesApplicationService;
|
import com.iqser.red.service.redaction.v1.server.service.ManualChangesApplicationService;
|
||||||
|
|
||||||
@ -48,6 +52,8 @@ public final class MigrationEntity {
|
|||||||
|
|
||||||
private final PrecursorEntity precursorEntity;
|
private final PrecursorEntity precursorEntity;
|
||||||
private final RedactionLogEntry redactionLogEntry;
|
private final RedactionLogEntry redactionLogEntry;
|
||||||
|
private final DictionaryService dictionaryService;
|
||||||
|
private final String dossierTemplateId;
|
||||||
private IEntity migratedEntity;
|
private IEntity migratedEntity;
|
||||||
private String oldId;
|
private String oldId;
|
||||||
private String newId;
|
private String newId;
|
||||||
@ -57,10 +63,10 @@ public final class MigrationEntity {
|
|||||||
List<BaseAnnotation> manualChanges = new LinkedList<>();
|
List<BaseAnnotation> manualChanges = new LinkedList<>();
|
||||||
|
|
||||||
|
|
||||||
public static MigrationEntity fromRedactionLogEntry(RedactionLogEntry redactionLogEntry, boolean hint, String fileId) {
|
public static MigrationEntity fromRedactionLogEntry(RedactionLogEntry redactionLogEntry, String fileId, DictionaryService dictionaryService, String dossierTemplateId) {
|
||||||
|
|
||||||
|
boolean hint = dictionaryService.isHint(redactionLogEntry.getType(), dossierTemplateId);
|
||||||
PrecursorEntity precursorEntity = createPrecursorEntity(redactionLogEntry, hint);
|
PrecursorEntity precursorEntity = createPrecursorEntity(redactionLogEntry, hint);
|
||||||
|
|
||||||
if (precursorEntity.getEntityType().equals(EntityType.HINT) && !redactionLogEntry.isHint() && !redactionLogEntry.isRedacted()) {
|
if (precursorEntity.getEntityType().equals(EntityType.HINT) && !redactionLogEntry.isHint() && !redactionLogEntry.isRedacted()) {
|
||||||
precursorEntity.ignore(precursorEntity.getRuleIdentifier(), precursorEntity.getReason());
|
precursorEntity.ignore(precursorEntity.getRuleIdentifier(), precursorEntity.getReason());
|
||||||
} else if (redactionLogEntry.lastChangeIsRemoved()) {
|
} else if (redactionLogEntry.lastChangeIsRemoved()) {
|
||||||
@ -75,13 +81,32 @@ public final class MigrationEntity {
|
|||||||
precursorEntity.skip(precursorEntity.getRuleIdentifier(), precursorEntity.getReason());
|
precursorEntity.skip(precursorEntity.getRuleIdentifier(), precursorEntity.getReason());
|
||||||
}
|
}
|
||||||
|
|
||||||
return MigrationEntity.builder().precursorEntity(precursorEntity).redactionLogEntry(redactionLogEntry).oldId(redactionLogEntry.getId()).fileId(fileId).build();
|
return MigrationEntity.builder()
|
||||||
|
.precursorEntity(precursorEntity)
|
||||||
|
.redactionLogEntry(redactionLogEntry)
|
||||||
|
.oldId(redactionLogEntry.getId())
|
||||||
|
.fileId(fileId)
|
||||||
|
.dictionaryService(dictionaryService)
|
||||||
|
.dossierTemplateId(dossierTemplateId)
|
||||||
|
.build();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
public static MigrationEntity fromRedactionLogImage(RedactionLogEntry redactionLogImage, Image image, String fileId) {
|
public static MigrationEntity fromRedactionLogImage(RedactionLogEntry redactionLogImage,
|
||||||
|
Image image,
|
||||||
|
String fileId,
|
||||||
|
DictionaryService dictionaryService,
|
||||||
|
String dossierTemplateId) {
|
||||||
|
|
||||||
return MigrationEntity.builder().redactionLogEntry(redactionLogImage).migratedEntity(image).oldId(redactionLogImage.getId()).newId(image.getId()).fileId(fileId).build();
|
return MigrationEntity.builder()
|
||||||
|
.redactionLogEntry(redactionLogImage)
|
||||||
|
.migratedEntity(image)
|
||||||
|
.oldId(redactionLogImage.getId())
|
||||||
|
.newId(image.getId())
|
||||||
|
.fileId(fileId)
|
||||||
|
.dictionaryService(dictionaryService)
|
||||||
|
.dossierTemplateId(dossierTemplateId)
|
||||||
|
.build();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@ -158,19 +183,6 @@ public final class MigrationEntity {
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
private static EntryType getEntryType(EntityType entityType) {
|
|
||||||
|
|
||||||
return switch (entityType) {
|
|
||||||
case ENTITY -> EntryType.ENTITY;
|
|
||||||
case HINT -> EntryType.HINT;
|
|
||||||
case FALSE_POSITIVE -> EntryType.FALSE_POSITIVE;
|
|
||||||
case RECOMMENDATION -> EntryType.RECOMMENDATION;
|
|
||||||
case FALSE_RECOMMENDATION -> EntryType.FALSE_RECOMMENDATION;
|
|
||||||
default -> EntryType.FALSE_POSITIVE;
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
public EntityLogEntry toEntityLogEntry(Map<String, String> oldToNewIdMapping) {
|
public EntityLogEntry toEntityLogEntry(Map<String, String> oldToNewIdMapping) {
|
||||||
|
|
||||||
EntityLogEntry entityLogEntry;
|
EntityLogEntry entityLogEntry;
|
||||||
@ -193,9 +205,6 @@ public final class MigrationEntity {
|
|||||||
entityLogEntry.setReference(migrateSetOfIds(redactionLogEntry.getReference(), oldToNewIdMapping));
|
entityLogEntry.setReference(migrateSetOfIds(redactionLogEntry.getReference(), oldToNewIdMapping));
|
||||||
entityLogEntry.setImportedRedactionIntersections(migrateSetOfIds(redactionLogEntry.getImportedRedactionIntersections(), oldToNewIdMapping));
|
entityLogEntry.setImportedRedactionIntersections(migrateSetOfIds(redactionLogEntry.getImportedRedactionIntersections(), oldToNewIdMapping));
|
||||||
entityLogEntry.setEngines(MigrationMapper.getMigratedEngines(redactionLogEntry));
|
entityLogEntry.setEngines(MigrationMapper.getMigratedEngines(redactionLogEntry));
|
||||||
if (redactionLogEntry.getLegalBasis() != null) {
|
|
||||||
entityLogEntry.setLegalBasis(redactionLogEntry.getLegalBasis());
|
|
||||||
}
|
|
||||||
|
|
||||||
if (entityLogEntry.getEntryType().equals(EntryType.HINT) && lastManualChangeIsRemoveLocally(entityLogEntry)) {
|
if (entityLogEntry.getEntryType().equals(EntryType.HINT) && lastManualChangeIsRemoveLocally(entityLogEntry)) {
|
||||||
entityLogEntry.setState(EntryState.IGNORED);
|
entityLogEntry.setState(EntryState.IGNORED);
|
||||||
@ -228,13 +237,15 @@ public final class MigrationEntity {
|
|||||||
|
|
||||||
public EntityLogEntry createEntityLogEntry(Image image) {
|
public EntityLogEntry createEntityLogEntry(Image image) {
|
||||||
|
|
||||||
|
String imageType = image.getImageType().equals(ImageType.OTHER) ? "image" : image.getImageType().toString().toLowerCase(Locale.ENGLISH);
|
||||||
List<Position> positions = getPositionsFromOverride(image).orElse(List.of(new Position(image.getPosition(), image.getPage().getNumber())));
|
List<Position> positions = getPositionsFromOverride(image).orElse(List.of(new Position(image.getPosition(), image.getPage().getNumber())));
|
||||||
return EntityLogEntry.builder()
|
return EntityLogEntry.builder()
|
||||||
.id(image.getId())
|
.id(image.getId())
|
||||||
.value(image.value())
|
.value(image.getValue())
|
||||||
.type(image.type())
|
.type(imageType)
|
||||||
.reason(image.buildReasonWithManualChangeDescriptions())
|
.reason(image.buildReasonWithManualChangeDescriptions())
|
||||||
.legalBasis(image.legalBasis())
|
.legalBasis(image.getManualOverwrite().getLegalBasis()
|
||||||
|
.orElse(redactionLogEntry.getLegalBasis()))
|
||||||
.matchedRule(image.getMatchedRule().getRuleIdentifier().toString())
|
.matchedRule(image.getMatchedRule().getRuleIdentifier().toString())
|
||||||
.dictionaryEntry(false)
|
.dictionaryEntry(false)
|
||||||
.positions(positions)
|
.positions(positions)
|
||||||
@ -246,7 +257,7 @@ public final class MigrationEntity {
|
|||||||
.textBefore(redactionLogEntry.getTextBefore())
|
.textBefore(redactionLogEntry.getTextBefore())
|
||||||
.imageHasTransparency(image.isTransparent())
|
.imageHasTransparency(image.isTransparent())
|
||||||
.state(buildEntryState(image))
|
.state(buildEntryState(image))
|
||||||
.entryType(redactionLogEntry.isHint() ? EntryType.IMAGE_HINT : EntryType.IMAGE)
|
.entryType(dictionaryService.isHint(imageType, dossierTemplateId) ? EntryType.IMAGE_HINT : EntryType.IMAGE)
|
||||||
.build();
|
.build();
|
||||||
|
|
||||||
}
|
}
|
||||||
@ -257,7 +268,8 @@ public final class MigrationEntity {
|
|||||||
return EntityLogEntry.builder()
|
return EntityLogEntry.builder()
|
||||||
.id(precursorEntity.getId())
|
.id(precursorEntity.getId())
|
||||||
.reason(precursorEntity.buildReasonWithManualChangeDescriptions())
|
.reason(precursorEntity.buildReasonWithManualChangeDescriptions())
|
||||||
.legalBasis(precursorEntity.legalBasis())
|
.legalBasis(precursorEntity.getManualOverwrite().getLegalBasis()
|
||||||
|
.orElse(redactionLogEntry.getLegalBasis()))
|
||||||
.value(precursorEntity.value())
|
.value(precursorEntity.value())
|
||||||
.type(precursorEntity.type())
|
.type(precursorEntity.type())
|
||||||
.state(buildEntryState(precursorEntity))
|
.state(buildEntryState(precursorEntity))
|
||||||
@ -291,7 +303,8 @@ public final class MigrationEntity {
|
|||||||
.id(entity.getId())
|
.id(entity.getId())
|
||||||
.positions(rectanglesPerLine)
|
.positions(rectanglesPerLine)
|
||||||
.reason(entity.buildReasonWithManualChangeDescriptions())
|
.reason(entity.buildReasonWithManualChangeDescriptions())
|
||||||
.legalBasis(entity.legalBasis())
|
.legalBasis(entity.getManualOverwrite().getLegalBasis()
|
||||||
|
.orElse(redactionLogEntry.getLegalBasis()))
|
||||||
.value(entity.getManualOverwrite().getValue()
|
.value(entity.getManualOverwrite().getValue()
|
||||||
.orElse(entity.getMatchedRule().isWriteValueWithLineBreaks() ? entity.getValueWithLineBreaks() : entity.getValue()))
|
.orElse(entity.getMatchedRule().isWriteValueWithLineBreaks() ? entity.getValueWithLineBreaks() : entity.getValue()))
|
||||||
.type(entity.type())
|
.type(entity.type())
|
||||||
@ -334,11 +347,11 @@ public final class MigrationEntity {
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
public boolean hasManualChangesOrComments() {
|
public boolean hasManualChangesOrComments(Set<String> entitiesWithComments, Set<String> entitiesWithUnprocessedChanges) {
|
||||||
|
|
||||||
return !(redactionLogEntry.getManualChanges() == null || redactionLogEntry.getManualChanges().isEmpty()) || //
|
return !(redactionLogEntry.getManualChanges() == null || redactionLogEntry.getManualChanges().isEmpty()) || //
|
||||||
!(redactionLogEntry.getComments() == null || redactionLogEntry.getComments().isEmpty()) //
|
!(redactionLogEntry.getComments() == null || redactionLogEntry.getComments().isEmpty()) //
|
||||||
|| hasManualChanges();
|
|| hasManualChanges() || entitiesWithComments.contains(oldId) || entitiesWithUnprocessedChanges.contains(oldId);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@ -355,6 +368,9 @@ public final class MigrationEntity {
|
|||||||
if (manualChange instanceof ManualResizeRedaction manualResizeRedaction && migratedEntity instanceof TextEntity textEntity) {
|
if (manualChange instanceof ManualResizeRedaction manualResizeRedaction && migratedEntity instanceof TextEntity textEntity) {
|
||||||
manualResizeRedaction.setAnnotationId(newId);
|
manualResizeRedaction.setAnnotationId(newId);
|
||||||
manualChangesApplicationService.resize(textEntity, manualResizeRedaction);
|
manualChangesApplicationService.resize(textEntity, manualResizeRedaction);
|
||||||
|
} else if (manualChange instanceof ManualRecategorization manualRecategorization && migratedEntity instanceof Image image) {
|
||||||
|
image.setImageType(ImageType.fromString(manualRecategorization.getType()));
|
||||||
|
migratedEntity.getManualOverwrite().addChange(manualChange);
|
||||||
} else {
|
} else {
|
||||||
migratedEntity.getManualOverwrite().addChange(manualChange);
|
migratedEntity.getManualOverwrite().addChange(manualChange);
|
||||||
}
|
}
|
||||||
@ -372,10 +388,7 @@ public final class MigrationEntity {
|
|||||||
.findFirst()
|
.findFirst()
|
||||||
.orElse(manualChanges.get(0)).getUser();
|
.orElse(manualChanges.get(0)).getUser();
|
||||||
|
|
||||||
OffsetDateTime requestDate = manualChanges.stream()
|
OffsetDateTime requestDate = manualChanges.get(0).getRequestDate();
|
||||||
.filter(mc -> mc instanceof ManualResizeRedaction)
|
|
||||||
.findFirst()
|
|
||||||
.orElse(manualChanges.get(0)).getRequestDate();
|
|
||||||
|
|
||||||
return ManualRedactionEntry.builder()
|
return ManualRedactionEntry.builder()
|
||||||
.annotationId(newId)
|
.annotationId(newId)
|
||||||
|
|||||||
@ -177,7 +177,9 @@ public class MigrationIntegrationTest extends BuildDocumentIntegrationTest {
|
|||||||
document,
|
document,
|
||||||
TEST_DOSSIER_TEMPLATE_ID,
|
TEST_DOSSIER_TEMPLATE_ID,
|
||||||
manualRedactions,
|
manualRedactions,
|
||||||
TEST_FILE_ID);
|
TEST_FILE_ID,
|
||||||
|
Collections.emptySet(),
|
||||||
|
false);
|
||||||
|
|
||||||
redactionStorageService.storeObject(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.ENTITY_LOG, migratedEntityLog.getEntityLog());
|
redactionStorageService.storeObject(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.ENTITY_LOG, migratedEntityLog.getEntityLog());
|
||||||
assertEquals(mergedRedactionLog.getRedactionLogEntry().size(), migratedEntityLog.getEntityLog().getEntityLogEntry().size());
|
assertEquals(mergedRedactionLog.getRedactionLogEntry().size(), migratedEntityLog.getEntityLog().getEntityLogEntry().size());
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user