RED-7384: handle pending dict application in redaction-service instead of persistence

This commit is contained in:
Kilian Schüttler 2024-04-03 16:32:08 +02:00
parent 90ba3239fc
commit df694a90a8
5 changed files with 151 additions and 55 deletions

View File

@ -1,11 +1,15 @@
package com.iqser.red.service.redaction.v1.model;
import java.util.Collections;
import java.util.Set;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
import lombok.NonNull;
@Data
@Builder
@ -13,9 +17,18 @@ import lombok.NoArgsConstructor;
@AllArgsConstructor
public class MigrationRequest {
@NonNull
String dossierTemplateId;
@NonNull
String dossierId;
@NonNull
String fileId;
boolean fileIsApproved;
@NonNull
ManualRedactions manualRedactions;
@NonNull
@Builder.Default
Set<String> entitiesWithComments = Collections.emptySet();
}

View File

@ -57,7 +57,7 @@ public class MigrationMessageReceiver {
if (redactionLog.getAnalysisVersion() == 0) {
redactionLog = legacyVersion0MigrationService.mergeDuplicateAnnotationIds(redactionLog);
} else if (migrationRequest.getManualRedactions() != null) {
} else {
redactionLog = legacyRedactionLogMergeService.addManualAddEntriesAndRemoveSkippedImported(redactionLog,
migrationRequest.getManualRedactions(),
migrationRequest.getDossierTemplateId());
@ -67,7 +67,9 @@ public class MigrationMessageReceiver {
document,
migrationRequest.getDossierTemplateId(),
migrationRequest.getManualRedactions(),
migrationRequest.getFileId());
migrationRequest.getFileId(),
migrationRequest.getEntitiesWithComments(),
migrationRequest.isFileIsApproved());
log.info("Storing migrated entityLog and ids to migrate in DB for file {}", migrationRequest.getFileId());
redactionStorageService.storeObject(migrationRequest.getDossierId(), migrationRequest.getFileId(), FileType.ENTITY_LOG, migratedEntityLog.getEntityLog());

View File

@ -8,6 +8,7 @@ import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.function.Function;
import java.util.stream.Collectors;
import java.util.stream.Stream;
@ -19,7 +20,9 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.migration.MigratedIds;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.BaseAnnotation;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
@ -57,7 +60,13 @@ public class RedactionLogToEntityLogMigrationService {
ManualChangesApplicationService manualChangesApplicationService;
public MigratedEntityLog migrate(RedactionLog redactionLog, Document document, String dossierTemplateId, ManualRedactions manualRedactions, String fileId) {
public MigratedEntityLog migrate(RedactionLog redactionLog,
Document document,
String dossierTemplateId,
ManualRedactions manualRedactions,
String fileId,
Set<String> entitiesWithComments,
boolean fileIsApproved) {
log.info("Migrating entities for file {}", fileId);
List<MigrationEntity> entitiesToMigrate = calculateMigrationEntitiesFromRedactionLog(redactionLog, document, dossierTemplateId, fileId);
@ -66,7 +75,7 @@ public class RedactionLogToEntityLogMigrationService {
.collect(new MigratedIdsCollector());
log.info("applying manual changes to migrated entities for file {}", fileId);
applyManualChanges(entitiesToMigrate, manualRedactions);
applyLocalProcessedManualChanges(entitiesToMigrate, manualRedactions, fileIsApproved);
EntityLog entityLog = new EntityLog();
entityLog.setAnalysisNumber(redactionLog.getAnalysisNumber());
@ -87,7 +96,7 @@ public class RedactionLogToEntityLogMigrationService {
.map(migrationEntity -> migrationEntity.toEntityLogEntry(oldToNewIDMapping))
.toList());
if (getNumberOfApprovedEntries(redactionLog) != entityLog.getEntityLogEntry().size()) {
if (getNumberOfApprovedEntries(redactionLog, document.getNumberOfPages()) != entityLog.getEntityLogEntry().size()) {
String message = String.format("Not all entities have been found during the migration redactionLog has %d entries and new entityLog %d",
redactionLog.getRedactionLogEntry().size(),
entityLog.getEntityLogEntry().size());
@ -95,8 +104,14 @@ public class RedactionLogToEntityLogMigrationService {
throw new AssertionError(message);
}
Set<String> entitiesWithUnprocessedChanges = manualRedactions.buildAll()
.stream()
.filter(manualRedaction -> manualRedaction.getProcessedDate() == null)
.map(BaseAnnotation::getAnnotationId)
.collect(Collectors.toSet());
MigratedIds idsToMigrateInDb = entitiesToMigrate.stream()
.filter(MigrationEntity::hasManualChangesOrComments)
.filter(migrationEntity -> migrationEntity.hasManualChangesOrComments(entitiesWithComments, entitiesWithUnprocessedChanges))
.filter(m -> !m.getOldId().equals(m.getNewId()))
.collect(new MigratedIdsCollector());
@ -111,20 +126,29 @@ public class RedactionLogToEntityLogMigrationService {
}
private void applyManualChanges(List<MigrationEntity> entitiesToMigrate, ManualRedactions manualRedactions) {
private void applyLocalProcessedManualChanges(List<MigrationEntity> entitiesToMigrate, ManualRedactions manualRedactions, boolean fileIsApproved) {
if (manualRedactions == null) {
return;
}
Map<String, List<BaseAnnotation>> manualChangesPerAnnotationId;
Map<String, List<BaseAnnotation>> manualChangesPerAnnotationId = Stream.of(manualRedactions.getIdsToRemove(),
manualRedactions.getEntriesToAdd(),
manualRedactions.getForceRedactions(),
manualRedactions.getResizeRedactions(),
manualRedactions.getLegalBasisChanges(),
manualRedactions.getRecategorizations())
.flatMap(Collection::stream)
.collect(Collectors.groupingBy(BaseAnnotation::getAnnotationId));
if (fileIsApproved) {
manualChangesPerAnnotationId = manualRedactions.buildAll()
.stream()
.filter(manualChange -> (manualChange.getProcessedDate() != null && manualChange.isLocal()) //
// unprocessed dict change of type IdRemoval or ManualResize must be applied for approved documents
|| (manualChange.getProcessedDate() == null && !manualChange.isLocal() //
&& (manualChange instanceof IdRemoval || manualChange instanceof ManualResizeRedaction)))
.map(this::convertPendingDictChangesToLocal)
.collect(Collectors.groupingBy(BaseAnnotation::getAnnotationId));
} else {
manualChangesPerAnnotationId = manualRedactions.buildAll()
.stream()
.filter(manualChange -> manualChange.getProcessedDate() != null)
.filter(BaseAnnotation::isLocal)
.collect(Collectors.groupingBy(BaseAnnotation::getAnnotationId));
}
entitiesToMigrate.forEach(migrationEntity -> migrationEntity.applyManualChanges(manualChangesPerAnnotationId.getOrDefault(migrationEntity.getOldId(),
Collections.emptyList()),
@ -133,15 +157,40 @@ public class RedactionLogToEntityLogMigrationService {
}
private static long getNumberOfApprovedEntries(RedactionLog redactionLog) {
private BaseAnnotation convertPendingDictChangesToLocal(BaseAnnotation baseAnnotation) {
return redactionLog.getRedactionLogEntry().size();
if (baseAnnotation.getProcessedDate() != null) {
return baseAnnotation;
}
if (baseAnnotation.isLocal()) {
return baseAnnotation;
}
if (baseAnnotation instanceof ManualResizeRedaction manualResizeRedaction) {
manualResizeRedaction.setAddToAllDossiers(false);
manualResizeRedaction.setUpdateDictionary(false);
} else if (baseAnnotation instanceof IdRemoval idRemoval) {
idRemoval.setRemoveFromAllDossiers(false);
idRemoval.setRemoveFromDictionary(false);
}
return baseAnnotation;
}
private long getNumberOfApprovedEntries(RedactionLog redactionLog, int numberOfPages) {
return redactionLog.getRedactionLogEntry()
.stream()
.filter(redactionLogEntry -> isOnExistingPage(redactionLogEntry, numberOfPages))
.count();
}
private List<MigrationEntity> calculateMigrationEntitiesFromRedactionLog(RedactionLog redactionLog, Document document, String dossierTemplateId, String fileId) {
List<MigrationEntity> images = getImageBasedMigrationEntities(redactionLog, document, fileId);
List<MigrationEntity> images = getImageBasedMigrationEntities(redactionLog, document, fileId, dossierTemplateId);
List<MigrationEntity> textMigrationEntities = getTextBasedMigrationEntities(redactionLog, document, dossierTemplateId, fileId);
return Stream.of(textMigrationEntities.stream(), images.stream())
.flatMap(Function.identity())
@ -155,7 +204,7 @@ public class RedactionLogToEntityLogMigrationService {
}
private List<MigrationEntity> getImageBasedMigrationEntities(RedactionLog redactionLog, Document document, String fileId) {
private List<MigrationEntity> getImageBasedMigrationEntities(RedactionLog redactionLog, Document document, String fileId, String dossierTemplateId) {
List<Image> images = document.streamAllImages()
.collect(Collectors.toList());
@ -202,7 +251,7 @@ public class RedactionLogToEntityLogMigrationService {
} else {
closestImage.skip(ruleIdentifier, reason);
}
migrationEntities.add(MigrationEntity.fromRedactionLogImage(redactionLogImage, closestImage, fileId));
migrationEntities.add(MigrationEntity.fromRedactionLogImage(redactionLogImage, closestImage, fileId, dictionaryService, dossierTemplateId));
}
return migrationEntities;
}
@ -248,7 +297,8 @@ public class RedactionLogToEntityLogMigrationService {
List<MigrationEntity> entitiesToMigrate = redactionLog.getRedactionLogEntry()
.stream()
.filter(redactionLogEntry -> !redactionLogEntry.isImage())
.map(entry -> MigrationEntity.fromRedactionLogEntry(entry, dictionaryService.isHint(entry.getType(), dossierTemplateId), fileId))
.filter(redactionLogEntry -> isOnExistingPage(redactionLogEntry, document.getNumberOfPages()))
.map(entry -> MigrationEntity.fromRedactionLogEntry(entry, fileId, dictionaryService, dossierTemplateId))
.toList();
List<PrecursorEntity> precursorEntities = entitiesToMigrate.stream()
@ -285,4 +335,20 @@ public class RedactionLogToEntityLogMigrationService {
return entitiesToMigrate;
}
private boolean isOnExistingPage(RedactionLogEntry redactionLogEntry, int numberOfPages) {
var pages = redactionLogEntry.getPositions()
.stream()
.map(Rectangle::getPage)
.collect(Collectors.toSet());
for (int page : pages) {
if (page > numberOfPages) {
return false;
}
}
return true;
}
}

View File

@ -8,6 +8,7 @@ import java.time.OffsetDateTime;
import java.util.Collections;
import java.util.LinkedList;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
@ -19,6 +20,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.BaseAnnotation;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntryType;
@ -30,6 +32,8 @@ import com.iqser.red.service.redaction.v1.server.model.document.entity.IEntity;
import com.iqser.red.service.redaction.v1.server.model.document.entity.ManualChangeOverwrite;
import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Image;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.ImageType;
import com.iqser.red.service.redaction.v1.server.service.DictionaryService;
import com.iqser.red.service.redaction.v1.server.service.ManualChangeFactory;
import com.iqser.red.service.redaction.v1.server.service.ManualChangesApplicationService;
@ -48,6 +52,8 @@ public final class MigrationEntity {
private final PrecursorEntity precursorEntity;
private final RedactionLogEntry redactionLogEntry;
private final DictionaryService dictionaryService;
private final String dossierTemplateId;
private IEntity migratedEntity;
private String oldId;
private String newId;
@ -57,10 +63,10 @@ public final class MigrationEntity {
List<BaseAnnotation> manualChanges = new LinkedList<>();
public static MigrationEntity fromRedactionLogEntry(RedactionLogEntry redactionLogEntry, boolean hint, String fileId) {
public static MigrationEntity fromRedactionLogEntry(RedactionLogEntry redactionLogEntry, String fileId, DictionaryService dictionaryService, String dossierTemplateId) {
boolean hint = dictionaryService.isHint(redactionLogEntry.getType(), dossierTemplateId);
PrecursorEntity precursorEntity = createPrecursorEntity(redactionLogEntry, hint);
if (precursorEntity.getEntityType().equals(EntityType.HINT) && !redactionLogEntry.isHint() && !redactionLogEntry.isRedacted()) {
precursorEntity.ignore(precursorEntity.getRuleIdentifier(), precursorEntity.getReason());
} else if (redactionLogEntry.lastChangeIsRemoved()) {
@ -75,13 +81,32 @@ public final class MigrationEntity {
precursorEntity.skip(precursorEntity.getRuleIdentifier(), precursorEntity.getReason());
}
return MigrationEntity.builder().precursorEntity(precursorEntity).redactionLogEntry(redactionLogEntry).oldId(redactionLogEntry.getId()).fileId(fileId).build();
return MigrationEntity.builder()
.precursorEntity(precursorEntity)
.redactionLogEntry(redactionLogEntry)
.oldId(redactionLogEntry.getId())
.fileId(fileId)
.dictionaryService(dictionaryService)
.dossierTemplateId(dossierTemplateId)
.build();
}
public static MigrationEntity fromRedactionLogImage(RedactionLogEntry redactionLogImage, Image image, String fileId) {
public static MigrationEntity fromRedactionLogImage(RedactionLogEntry redactionLogImage,
Image image,
String fileId,
DictionaryService dictionaryService,
String dossierTemplateId) {
return MigrationEntity.builder().redactionLogEntry(redactionLogImage).migratedEntity(image).oldId(redactionLogImage.getId()).newId(image.getId()).fileId(fileId).build();
return MigrationEntity.builder()
.redactionLogEntry(redactionLogImage)
.migratedEntity(image)
.oldId(redactionLogImage.getId())
.newId(image.getId())
.fileId(fileId)
.dictionaryService(dictionaryService)
.dossierTemplateId(dossierTemplateId)
.build();
}
@ -158,19 +183,6 @@ public final class MigrationEntity {
}
private static EntryType getEntryType(EntityType entityType) {
return switch (entityType) {
case ENTITY -> EntryType.ENTITY;
case HINT -> EntryType.HINT;
case FALSE_POSITIVE -> EntryType.FALSE_POSITIVE;
case RECOMMENDATION -> EntryType.RECOMMENDATION;
case FALSE_RECOMMENDATION -> EntryType.FALSE_RECOMMENDATION;
default -> EntryType.FALSE_POSITIVE;
};
}
public EntityLogEntry toEntityLogEntry(Map<String, String> oldToNewIdMapping) {
EntityLogEntry entityLogEntry;
@ -193,9 +205,6 @@ public final class MigrationEntity {
entityLogEntry.setReference(migrateSetOfIds(redactionLogEntry.getReference(), oldToNewIdMapping));
entityLogEntry.setImportedRedactionIntersections(migrateSetOfIds(redactionLogEntry.getImportedRedactionIntersections(), oldToNewIdMapping));
entityLogEntry.setEngines(MigrationMapper.getMigratedEngines(redactionLogEntry));
if (redactionLogEntry.getLegalBasis() != null) {
entityLogEntry.setLegalBasis(redactionLogEntry.getLegalBasis());
}
if (entityLogEntry.getEntryType().equals(EntryType.HINT) && lastManualChangeIsRemoveLocally(entityLogEntry)) {
entityLogEntry.setState(EntryState.IGNORED);
@ -228,13 +237,15 @@ public final class MigrationEntity {
public EntityLogEntry createEntityLogEntry(Image image) {
String imageType = image.getImageType().equals(ImageType.OTHER) ? "image" : image.getImageType().toString().toLowerCase(Locale.ENGLISH);
List<Position> positions = getPositionsFromOverride(image).orElse(List.of(new Position(image.getPosition(), image.getPage().getNumber())));
return EntityLogEntry.builder()
.id(image.getId())
.value(image.value())
.type(image.type())
.value(image.getValue())
.type(imageType)
.reason(image.buildReasonWithManualChangeDescriptions())
.legalBasis(image.legalBasis())
.legalBasis(image.getManualOverwrite().getLegalBasis()
.orElse(redactionLogEntry.getLegalBasis()))
.matchedRule(image.getMatchedRule().getRuleIdentifier().toString())
.dictionaryEntry(false)
.positions(positions)
@ -246,7 +257,7 @@ public final class MigrationEntity {
.textBefore(redactionLogEntry.getTextBefore())
.imageHasTransparency(image.isTransparent())
.state(buildEntryState(image))
.entryType(redactionLogEntry.isHint() ? EntryType.IMAGE_HINT : EntryType.IMAGE)
.entryType(dictionaryService.isHint(imageType, dossierTemplateId) ? EntryType.IMAGE_HINT : EntryType.IMAGE)
.build();
}
@ -257,7 +268,8 @@ public final class MigrationEntity {
return EntityLogEntry.builder()
.id(precursorEntity.getId())
.reason(precursorEntity.buildReasonWithManualChangeDescriptions())
.legalBasis(precursorEntity.legalBasis())
.legalBasis(precursorEntity.getManualOverwrite().getLegalBasis()
.orElse(redactionLogEntry.getLegalBasis()))
.value(precursorEntity.value())
.type(precursorEntity.type())
.state(buildEntryState(precursorEntity))
@ -291,7 +303,8 @@ public final class MigrationEntity {
.id(entity.getId())
.positions(rectanglesPerLine)
.reason(entity.buildReasonWithManualChangeDescriptions())
.legalBasis(entity.legalBasis())
.legalBasis(entity.getManualOverwrite().getLegalBasis()
.orElse(redactionLogEntry.getLegalBasis()))
.value(entity.getManualOverwrite().getValue()
.orElse(entity.getMatchedRule().isWriteValueWithLineBreaks() ? entity.getValueWithLineBreaks() : entity.getValue()))
.type(entity.type())
@ -334,11 +347,11 @@ public final class MigrationEntity {
}
public boolean hasManualChangesOrComments() {
public boolean hasManualChangesOrComments(Set<String> entitiesWithComments, Set<String> entitiesWithUnprocessedChanges) {
return !(redactionLogEntry.getManualChanges() == null || redactionLogEntry.getManualChanges().isEmpty()) || //
!(redactionLogEntry.getComments() == null || redactionLogEntry.getComments().isEmpty()) //
|| hasManualChanges();
|| hasManualChanges() || entitiesWithComments.contains(oldId) || entitiesWithUnprocessedChanges.contains(oldId);
}
@ -355,6 +368,9 @@ public final class MigrationEntity {
if (manualChange instanceof ManualResizeRedaction manualResizeRedaction && migratedEntity instanceof TextEntity textEntity) {
manualResizeRedaction.setAnnotationId(newId);
manualChangesApplicationService.resize(textEntity, manualResizeRedaction);
} else if (manualChange instanceof ManualRecategorization manualRecategorization && migratedEntity instanceof Image image) {
image.setImageType(ImageType.fromString(manualRecategorization.getType()));
migratedEntity.getManualOverwrite().addChange(manualChange);
} else {
migratedEntity.getManualOverwrite().addChange(manualChange);
}
@ -372,10 +388,7 @@ public final class MigrationEntity {
.findFirst()
.orElse(manualChanges.get(0)).getUser();
OffsetDateTime requestDate = manualChanges.stream()
.filter(mc -> mc instanceof ManualResizeRedaction)
.findFirst()
.orElse(manualChanges.get(0)).getRequestDate();
OffsetDateTime requestDate = manualChanges.get(0).getRequestDate();
return ManualRedactionEntry.builder()
.annotationId(newId)

View File

@ -177,7 +177,9 @@ public class MigrationIntegrationTest extends BuildDocumentIntegrationTest {
document,
TEST_DOSSIER_TEMPLATE_ID,
manualRedactions,
TEST_FILE_ID);
TEST_FILE_ID,
Collections.emptySet(),
false);
redactionStorageService.storeObject(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.ENTITY_LOG, migratedEntityLog.getEntityLog());
assertEquals(mergedRedactionLog.getRedactionLogEntry().size(), migratedEntityLog.getEntityLog().getEntityLogEntry().size());