Merge branch 'RED-7384' into 'release/4.244.x'
RED-7384: migration fixes, ignore non-local manual changes in migration See merge request redactmanager/redaction-service!344
This commit is contained in:
Kilian Schüttler
commit
7dbc585274
@ -72,7 +72,7 @@ public class RedactionLogToEntityLogMigrationService {
|
||||
.collect(new MigratedIdsCollector());
|
||||
|
||||
log.info("applying manual changes to migrated entities for file {}", fileId);
|
||||
applyProcessedManualChanges(entitiesToMigrate, manualRedactions);
|
||||
applyLocalProcessedManualChanges(entitiesToMigrate, manualRedactions);
|
||||
|
||||
EntityLog entityLog = new EntityLog();
|
||||
entityLog.setAnalysisNumber(redactionLog.getAnalysisNumber());
|
||||
@ -117,7 +117,7 @@ public class RedactionLogToEntityLogMigrationService {
|
||||
}
|
||||
|
||||
|
||||
private void applyProcessedManualChanges(List<MigrationEntity> entitiesToMigrate, ManualRedactions manualRedactions) {
|
||||
private void applyLocalProcessedManualChanges(List<MigrationEntity> entitiesToMigrate, ManualRedactions manualRedactions) {
|
||||
|
||||
if (manualRedactions == null) {
|
||||
return;
|
||||
@ -131,6 +131,7 @@ public class RedactionLogToEntityLogMigrationService {
|
||||
manualRedactions.getRecategorizations())
|
||||
.flatMap(Collection::stream)
|
||||
.filter(manualChange -> manualChange.getProcessedDate() != null)
|
||||
.filter(BaseAnnotation::isLocal)
|
||||
.collect(Collectors.groupingBy(BaseAnnotation::getAnnotationId));
|
||||
|
||||
entitiesToMigrate.forEach(migrationEntity -> migrationEntity.applyManualChanges(manualChangesPerAnnotationId.getOrDefault(migrationEntity.getOldId(),
|
||||
@ -151,7 +152,7 @@ public class RedactionLogToEntityLogMigrationService {
|
||||
|
||||
private List<MigrationEntity> calculateMigrationEntitiesFromRedactionLog(RedactionLog redactionLog, Document document, String dossierTemplateId, String fileId) {
|
||||
|
||||
List<MigrationEntity> images = getImageBasedMigrationEntities(redactionLog, document, fileId);
|
||||
List<MigrationEntity> images = getImageBasedMigrationEntities(redactionLog, document, fileId, dossierTemplateId);
|
||||
List<MigrationEntity> textMigrationEntities = getTextBasedMigrationEntities(redactionLog, document, dossierTemplateId, fileId);
|
||||
return Stream.of(textMigrationEntities.stream(), images.stream())
|
||||
.flatMap(Function.identity())
|
||||
@ -165,7 +166,7 @@ public class RedactionLogToEntityLogMigrationService {
|
||||
}
|
||||
|
||||
|
||||
private List<MigrationEntity> getImageBasedMigrationEntities(RedactionLog redactionLog, Document document, String fileId) {
|
||||
private List<MigrationEntity> getImageBasedMigrationEntities(RedactionLog redactionLog, Document document, String fileId, String dossierTemplateId) {
|
||||
|
||||
List<Image> images = document.streamAllImages()
|
||||
.collect(Collectors.toList());
|
||||
@ -212,7 +213,7 @@ public class RedactionLogToEntityLogMigrationService {
|
||||
} else {
|
||||
closestImage.skip(ruleIdentifier, reason);
|
||||
}
|
||||
migrationEntities.add(MigrationEntity.fromRedactionLogImage(redactionLogImage, closestImage, fileId));
|
||||
migrationEntities.add(MigrationEntity.fromRedactionLogImage(redactionLogImage, closestImage, fileId, dictionaryService, dossierTemplateId));
|
||||
}
|
||||
return migrationEntities;
|
||||
}
|
||||
@ -259,7 +260,7 @@ public class RedactionLogToEntityLogMigrationService {
|
||||
.stream()
|
||||
.filter(redactionLogEntry -> !redactionLogEntry.isImage())
|
||||
.filter(redactionLogEntry -> isOnExistingPage(redactionLogEntry, document.getNumberOfPages()))
|
||||
.map(entry -> MigrationEntity.fromRedactionLogEntry(entry, dictionaryService.isHint(entry.getType(), dossierTemplateId), fileId))
|
||||
.map(entry -> MigrationEntity.fromRedactionLogEntry(entry, fileId, dictionaryService, dossierTemplateId))
|
||||
.toList();
|
||||
|
||||
List<PrecursorEntity> precursorEntities = entitiesToMigrate.stream()
|
||||
|
||||
@ -20,6 +20,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.BaseAnnotation;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntryType;
|
||||
@ -32,6 +33,7 @@ import com.iqser.red.service.redaction.v1.server.model.document.entity.ManualCha
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Image;
|
||||
import com.iqser.red.service.redaction.v1.server.model.document.nodes.ImageType;
|
||||
import com.iqser.red.service.redaction.v1.server.service.DictionaryService;
|
||||
import com.iqser.red.service.redaction.v1.server.service.ManualChangeFactory;
|
||||
import com.iqser.red.service.redaction.v1.server.service.ManualChangesApplicationService;
|
||||
|
||||
@ -50,6 +52,8 @@ public final class MigrationEntity {
|
||||
|
||||
private final PrecursorEntity precursorEntity;
|
||||
private final RedactionLogEntry redactionLogEntry;
|
||||
private final DictionaryService dictionaryService;
|
||||
private final String dossierTemplateId;
|
||||
private IEntity migratedEntity;
|
||||
private String oldId;
|
||||
private String newId;
|
||||
@ -59,10 +63,10 @@ public final class MigrationEntity {
|
||||
List<BaseAnnotation> manualChanges = new LinkedList<>();
|
||||
|
||||
|
||||
public static MigrationEntity fromRedactionLogEntry(RedactionLogEntry redactionLogEntry, boolean hint, String fileId) {
|
||||
public static MigrationEntity fromRedactionLogEntry(RedactionLogEntry redactionLogEntry, String fileId, DictionaryService dictionaryService, String dossierTemplateId) {
|
||||
|
||||
boolean hint = dictionaryService.isHint(redactionLogEntry.getType(), dossierTemplateId);
|
||||
PrecursorEntity precursorEntity = createPrecursorEntity(redactionLogEntry, hint);
|
||||
|
||||
if (precursorEntity.getEntityType().equals(EntityType.HINT) && !redactionLogEntry.isHint() && !redactionLogEntry.isRedacted()) {
|
||||
precursorEntity.ignore(precursorEntity.getRuleIdentifier(), precursorEntity.getReason());
|
||||
} else if (redactionLogEntry.lastChangeIsRemoved()) {
|
||||
@ -77,13 +81,32 @@ public final class MigrationEntity {
|
||||
precursorEntity.skip(precursorEntity.getRuleIdentifier(), precursorEntity.getReason());
|
||||
}
|
||||
|
||||
return MigrationEntity.builder().precursorEntity(precursorEntity).redactionLogEntry(redactionLogEntry).oldId(redactionLogEntry.getId()).fileId(fileId).build();
|
||||
return MigrationEntity.builder()
|
||||
.precursorEntity(precursorEntity)
|
||||
.redactionLogEntry(redactionLogEntry)
|
||||
.oldId(redactionLogEntry.getId())
|
||||
.fileId(fileId)
|
||||
.dictionaryService(dictionaryService)
|
||||
.dossierTemplateId(dossierTemplateId)
|
||||
.build();
|
||||
}
|
||||
|
||||
|
||||
public static MigrationEntity fromRedactionLogImage(RedactionLogEntry redactionLogImage, Image image, String fileId) {
|
||||
public static MigrationEntity fromRedactionLogImage(RedactionLogEntry redactionLogImage,
|
||||
Image image,
|
||||
String fileId,
|
||||
DictionaryService dictionaryService,
|
||||
String dossierTemplateId) {
|
||||
|
||||
return MigrationEntity.builder().redactionLogEntry(redactionLogImage).migratedEntity(image).oldId(redactionLogImage.getId()).newId(image.getId()).fileId(fileId).build();
|
||||
return MigrationEntity.builder()
|
||||
.redactionLogEntry(redactionLogImage)
|
||||
.migratedEntity(image)
|
||||
.oldId(redactionLogImage.getId())
|
||||
.newId(image.getId())
|
||||
.fileId(fileId)
|
||||
.dictionaryService(dictionaryService)
|
||||
.dossierTemplateId(dossierTemplateId)
|
||||
.build();
|
||||
}
|
||||
|
||||
|
||||
@ -160,18 +183,6 @@ public final class MigrationEntity {
|
||||
}
|
||||
|
||||
|
||||
private static EntryType getEntryType(EntityType entityType) {
|
||||
|
||||
return switch (entityType) {
|
||||
case ENTITY -> EntryType.ENTITY;
|
||||
case HINT -> EntryType.HINT;
|
||||
case FALSE_POSITIVE -> EntryType.FALSE_POSITIVE;
|
||||
case RECOMMENDATION -> EntryType.RECOMMENDATION;
|
||||
case FALSE_RECOMMENDATION -> EntryType.FALSE_RECOMMENDATION;
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
public EntityLogEntry toEntityLogEntry(Map<String, String> oldToNewIdMapping) {
|
||||
|
||||
EntityLogEntry entityLogEntry;
|
||||
@ -246,7 +257,7 @@ public final class MigrationEntity {
|
||||
.textBefore(redactionLogEntry.getTextBefore())
|
||||
.imageHasTransparency(image.isTransparent())
|
||||
.state(buildEntryState(image))
|
||||
.entryType(redactionLogEntry.isHint() ? EntryType.IMAGE_HINT : EntryType.IMAGE)
|
||||
.entryType(dictionaryService.isHint(imageType, dossierTemplateId) ? EntryType.IMAGE_HINT : EntryType.IMAGE)
|
||||
.build();
|
||||
|
||||
}
|
||||
@ -357,6 +368,9 @@ public final class MigrationEntity {
|
||||
if (manualChange instanceof ManualResizeRedaction manualResizeRedaction && migratedEntity instanceof TextEntity textEntity) {
|
||||
manualResizeRedaction.setAnnotationId(newId);
|
||||
manualChangesApplicationService.resize(textEntity, manualResizeRedaction);
|
||||
} else if (manualChange instanceof ManualRecategorization manualRecategorization && migratedEntity instanceof Image image) {
|
||||
image.setImageType(ImageType.fromString(manualRecategorization.getType()));
|
||||
migratedEntity.getManualOverwrite().addChange(manualChange);
|
||||
} else {
|
||||
migratedEntity.getManualOverwrite().addChange(manualChange);
|
||||
}
|
||||
|
||||
@ -13,7 +13,6 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;
|
||||
|
||||
import io.micrometer.core.annotation.Timed;
|
||||
import lombok.AccessLevel;
|
||||
@ -39,9 +38,11 @@ public class EntityChangeLogService {
|
||||
boolean hasChanges = false;
|
||||
|
||||
for (EntityLogEntry entityLogEntry : newEntityLogEntries) {
|
||||
|
||||
Optional<EntityLogEntry> optionalPreviousEntity = previousEntityLogEntries.stream()
|
||||
.filter(entry -> entry.getId().equals(entityLogEntry.getId()))
|
||||
.findAny();
|
||||
|
||||
if (optionalPreviousEntity.isEmpty()) {
|
||||
hasChanges = true;
|
||||
entityLogEntry.getChanges().add(new Change(analysisNumber, ChangeType.ADDED, now));
|
||||
@ -49,14 +50,18 @@ public class EntityChangeLogService {
|
||||
}
|
||||
|
||||
EntityLogEntry previousEntity = optionalPreviousEntity.get();
|
||||
|
||||
entityLogEntry.getChanges().addAll(previousEntity.getChanges());
|
||||
|
||||
if (!previousEntity.getState().equals(entityLogEntry.getState())) {
|
||||
hasChanges = true;
|
||||
ChangeType changeType = calculateChangeType(entityLogEntry.getState(), previousEntity.getState());
|
||||
entityLogEntry.getChanges().add(new Change(analysisNumber, changeType, now));
|
||||
}
|
||||
}
|
||||
|
||||
addRemovedEntriesAsRemoved(previousEntityLogEntries, newEntityLogEntries, manualRedactions, analysisNumber, now);
|
||||
|
||||
return hasChanges;
|
||||
}
|
||||
|
||||
@ -70,11 +75,16 @@ public class EntityChangeLogService {
|
||||
Set<String> existingIds = newEntityLogEntries.stream()
|
||||
.map(EntityLogEntry::getId)
|
||||
.collect(Collectors.toSet());
|
||||
|
||||
List<EntityLogEntry> removedEntries = previousEntityLogEntries.stream()
|
||||
.filter(entry -> !existingIds.contains(entry.getId()))
|
||||
.toList();
|
||||
removedEntries.forEach(entry -> entry.getChanges().add(new Change(analysisNumber, ChangeType.REMOVED, now)));
|
||||
removedEntries.forEach(entry -> entry.setState(EntryState.REMOVED));
|
||||
|
||||
removedEntries.stream()
|
||||
.filter(entry -> !entry.getState().equals(EntryState.REMOVED))
|
||||
.peek(entry -> entry.getChanges().add(new Change(analysisNumber, ChangeType.REMOVED, now)))
|
||||
.forEach(entry -> entry.setState(EntryState.REMOVED));
|
||||
|
||||
newEntityLogEntries.addAll(removedEntries);
|
||||
}
|
||||
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user