Compare commits

...
8 Commits
Author SHA1 Message Date
Andrei Isvoran 8187983944 Merge branch 'RED-8776-bp' into 'release/4.244.x'
RED-8776 - Add local redaction when we do a manual change on a non-manual redaction

See merge request redactmanager/redaction-service!358
2024-04-03 14:55:45 +02:00
Andrei Isvoran 99951dd1bb RED-8776 - Add local redaction when we do a manual change on a non-manual redaction 2024-04-03 14:02:25 +03:00
Kilian Schüttler 67184eb861 Merge branch 'RED-7384' into 'release/4.244.x'
RED-7384: also migrate annotation Ids with unprocessed manual changes

See merge request redactmanager/redaction-service!355
2024-04-02 15:01:41 +02:00
Kilian Schuettler a72d96c646 RED-7384: also migrate annotation Ids with unprocessed manual changes 2024-04-02 14:53:24 +02:00
Andrei Isvoran e5641d7097 Merge branch 'RED-8828-fix-npe-bp' into 'release/4.244.x'
RED-8828 - Fix error when resizing dict based redaction

See merge request redactmanager/redaction-service!354
2024-04-02 08:33:37 +02:00
Andrei Isvoran 62c0a7eb09 RED-8828 - Fix error when resizing dict based redaction 2024-03-29 14:32:45 +02:00
Kilian Schüttler 7dbc585274 Merge branch 'RED-7384' into 'release/4.244.x'
RED-7384: migration fixes, ignore non-local manual changes in migration

See merge request redactmanager/redaction-service!344
2024-03-27 18:09:10 +01:00
Kilian Schüttler 981901012c RED-7384: migration fixes, ignore non-local manual changes in migration 2024-03-27 18:09:10 +01:00
14 changed files with 204 additions and 38 deletions
@@ -72,7 +72,7 @@ public class RedactionLogToEntityLogMigrationService {
.collect(new MigratedIdsCollector());
log.info("applying manual changes to migrated entities for file {}", fileId);
applyProcessedManualChanges(entitiesToMigrate, manualRedactions);
applyLocalProcessedManualChanges(entitiesToMigrate, manualRedactions);
EntityLog entityLog = new EntityLog();
entityLog.setAnalysisNumber(redactionLog.getAnalysisNumber());
@@ -101,8 +101,14 @@ public class RedactionLogToEntityLogMigrationService {
throw new AssertionError(message);
}
Set<String> entitiesWithUnprocessedChanges = manualRedactions.buildAll()
.stream()
.filter(manualRedaction -> manualRedaction.getProcessedDate() == null)
.map(BaseAnnotation::getAnnotationId)
.collect(Collectors.toSet());
MigratedIds idsToMigrateInDb = entitiesToMigrate.stream()
.filter(migrationEntity -> migrationEntity.hasManualChangesOrComments(entitiesWithComments))
.filter(migrationEntity -> migrationEntity.hasManualChangesOrComments(entitiesWithComments, entitiesWithUnprocessedChanges))
.filter(m -> !m.getOldId().equals(m.getNewId()))
.collect(new MigratedIdsCollector());
@@ -117,7 +123,7 @@ public class RedactionLogToEntityLogMigrationService {
}
private void applyProcessedManualChanges(List<MigrationEntity> entitiesToMigrate, ManualRedactions manualRedactions) {
private void applyLocalProcessedManualChanges(List<MigrationEntity> entitiesToMigrate, ManualRedactions manualRedactions) {
if (manualRedactions == null) {
return;
@@ -131,6 +137,7 @@ public class RedactionLogToEntityLogMigrationService {
manualRedactions.getRecategorizations())
.flatMap(Collection::stream)
.filter(manualChange -> manualChange.getProcessedDate() != null)
.filter(BaseAnnotation::isLocal)
.collect(Collectors.groupingBy(BaseAnnotation::getAnnotationId));
entitiesToMigrate.forEach(migrationEntity -> migrationEntity.applyManualChanges(manualChangesPerAnnotationId.getOrDefault(migrationEntity.getOldId(),
@@ -151,7 +158,7 @@ public class RedactionLogToEntityLogMigrationService {
private List<MigrationEntity> calculateMigrationEntitiesFromRedactionLog(RedactionLog redactionLog, Document document, String dossierTemplateId, String fileId) {
List<MigrationEntity> images = getImageBasedMigrationEntities(redactionLog, document, fileId);
List<MigrationEntity> images = getImageBasedMigrationEntities(redactionLog, document, fileId, dossierTemplateId);
List<MigrationEntity> textMigrationEntities = getTextBasedMigrationEntities(redactionLog, document, dossierTemplateId, fileId);
return Stream.of(textMigrationEntities.stream(), images.stream())
.flatMap(Function.identity())
@@ -165,7 +172,7 @@ public class RedactionLogToEntityLogMigrationService {
}
private List<MigrationEntity> getImageBasedMigrationEntities(RedactionLog redactionLog, Document document, String fileId) {
private List<MigrationEntity> getImageBasedMigrationEntities(RedactionLog redactionLog, Document document, String fileId, String dossierTemplateId) {
List<Image> images = document.streamAllImages()
.collect(Collectors.toList());
@@ -212,7 +219,7 @@ public class RedactionLogToEntityLogMigrationService {
} else {
closestImage.skip(ruleIdentifier, reason);
}
migrationEntities.add(MigrationEntity.fromRedactionLogImage(redactionLogImage, closestImage, fileId));
migrationEntities.add(MigrationEntity.fromRedactionLogImage(redactionLogImage, closestImage, fileId, dictionaryService, dossierTemplateId));
}
return migrationEntities;
}
@@ -259,7 +266,7 @@ public class RedactionLogToEntityLogMigrationService {
.stream()
.filter(redactionLogEntry -> !redactionLogEntry.isImage())
.filter(redactionLogEntry -> isOnExistingPage(redactionLogEntry, document.getNumberOfPages()))
.map(entry -> MigrationEntity.fromRedactionLogEntry(entry, dictionaryService.isHint(entry.getType(), dossierTemplateId), fileId))
.map(entry -> MigrationEntity.fromRedactionLogEntry(entry, fileId, dictionaryService, dossierTemplateId))
.toList();
List<PrecursorEntity> precursorEntities = entitiesToMigrate.stream()
@@ -20,6 +20,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.BaseAnnotation;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntryType;
@@ -32,6 +33,7 @@ import com.iqser.red.service.redaction.v1.server.model.document.entity.ManualCha
import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Image;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.ImageType;
import com.iqser.red.service.redaction.v1.server.service.DictionaryService;
import com.iqser.red.service.redaction.v1.server.service.ManualChangeFactory;
import com.iqser.red.service.redaction.v1.server.service.ManualChangesApplicationService;
@@ -50,6 +52,8 @@ public final class MigrationEntity {
private final PrecursorEntity precursorEntity;
private final RedactionLogEntry redactionLogEntry;
private final DictionaryService dictionaryService;
private final String dossierTemplateId;
private IEntity migratedEntity;
private String oldId;
private String newId;
@@ -59,10 +63,10 @@ public final class MigrationEntity {
List<BaseAnnotation> manualChanges = new LinkedList<>();
public static MigrationEntity fromRedactionLogEntry(RedactionLogEntry redactionLogEntry, boolean hint, String fileId) {
public static MigrationEntity fromRedactionLogEntry(RedactionLogEntry redactionLogEntry, String fileId, DictionaryService dictionaryService, String dossierTemplateId) {
boolean hint = dictionaryService.isHint(redactionLogEntry.getType(), dossierTemplateId);
PrecursorEntity precursorEntity = createPrecursorEntity(redactionLogEntry, hint);
if (precursorEntity.getEntityType().equals(EntityType.HINT) && !redactionLogEntry.isHint() && !redactionLogEntry.isRedacted()) {
precursorEntity.ignore(precursorEntity.getRuleIdentifier(), precursorEntity.getReason());
} else if (redactionLogEntry.lastChangeIsRemoved()) {
@@ -77,13 +81,32 @@ public final class MigrationEntity {
precursorEntity.skip(precursorEntity.getRuleIdentifier(), precursorEntity.getReason());
}
return MigrationEntity.builder().precursorEntity(precursorEntity).redactionLogEntry(redactionLogEntry).oldId(redactionLogEntry.getId()).fileId(fileId).build();
return MigrationEntity.builder()
.precursorEntity(precursorEntity)
.redactionLogEntry(redactionLogEntry)
.oldId(redactionLogEntry.getId())
.fileId(fileId)
.dictionaryService(dictionaryService)
.dossierTemplateId(dossierTemplateId)
.build();
}
public static MigrationEntity fromRedactionLogImage(RedactionLogEntry redactionLogImage, Image image, String fileId) {
public static MigrationEntity fromRedactionLogImage(RedactionLogEntry redactionLogImage,
Image image,
String fileId,
DictionaryService dictionaryService,
String dossierTemplateId) {
return MigrationEntity.builder().redactionLogEntry(redactionLogImage).migratedEntity(image).oldId(redactionLogImage.getId()).newId(image.getId()).fileId(fileId).build();
return MigrationEntity.builder()
.redactionLogEntry(redactionLogImage)
.migratedEntity(image)
.oldId(redactionLogImage.getId())
.newId(image.getId())
.fileId(fileId)
.dictionaryService(dictionaryService)
.dossierTemplateId(dossierTemplateId)
.build();
}
@@ -160,18 +183,6 @@ public final class MigrationEntity {
}
private static EntryType getEntryType(EntityType entityType) {
return switch (entityType) {
case ENTITY -> EntryType.ENTITY;
case HINT -> EntryType.HINT;
case FALSE_POSITIVE -> EntryType.FALSE_POSITIVE;
case RECOMMENDATION -> EntryType.RECOMMENDATION;
case FALSE_RECOMMENDATION -> EntryType.FALSE_RECOMMENDATION;
};
}
public EntityLogEntry toEntityLogEntry(Map<String, String> oldToNewIdMapping) {
EntityLogEntry entityLogEntry;
@@ -246,7 +257,7 @@ public final class MigrationEntity {
.textBefore(redactionLogEntry.getTextBefore())
.imageHasTransparency(image.isTransparent())
.state(buildEntryState(image))
.entryType(redactionLogEntry.isHint() ? EntryType.IMAGE_HINT : EntryType.IMAGE)
.entryType(dictionaryService.isHint(imageType, dossierTemplateId) ? EntryType.IMAGE_HINT : EntryType.IMAGE)
.build();
}
@@ -336,11 +347,11 @@ public final class MigrationEntity {
}
public boolean hasManualChangesOrComments(Set<String> entitiesWithComments) {
public boolean hasManualChangesOrComments(Set<String> entitiesWithComments, Set<String> entitiesWithUnprocessedChanges) {
return !(redactionLogEntry.getManualChanges() == null || redactionLogEntry.getManualChanges().isEmpty()) || //
!(redactionLogEntry.getComments() == null || redactionLogEntry.getComments().isEmpty()) //
|| hasManualChanges() || entitiesWithComments.contains(oldId);
|| hasManualChanges() || entitiesWithComments.contains(oldId) || entitiesWithUnprocessedChanges.contains(oldId);
}
@@ -357,6 +368,9 @@ public final class MigrationEntity {
if (manualChange instanceof ManualResizeRedaction manualResizeRedaction && migratedEntity instanceof TextEntity textEntity) {
manualResizeRedaction.setAnnotationId(newId);
manualChangesApplicationService.resize(textEntity, manualResizeRedaction);
} else if (manualChange instanceof ManualRecategorization manualRecategorization && migratedEntity instanceof Image image) {
image.setImageType(ImageType.fromString(manualRecategorization.getType()));
migratedEntity.getManualOverwrite().addChange(manualChange);
} else {
migratedEntity.getManualOverwrite().addChange(manualChange);
}
@@ -13,7 +13,6 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;
import io.micrometer.core.annotation.Timed;
import lombok.AccessLevel;
@@ -39,9 +38,11 @@ public class EntityChangeLogService {
boolean hasChanges = false;
for (EntityLogEntry entityLogEntry : newEntityLogEntries) {
Optional<EntityLogEntry> optionalPreviousEntity = previousEntityLogEntries.stream()
.filter(entry -> entry.getId().equals(entityLogEntry.getId()))
.findAny();
if (optionalPreviousEntity.isEmpty()) {
hasChanges = true;
entityLogEntry.getChanges().add(new Change(analysisNumber, ChangeType.ADDED, now));
@@ -49,14 +50,18 @@ public class EntityChangeLogService {
}
EntityLogEntry previousEntity = optionalPreviousEntity.get();
entityLogEntry.getChanges().addAll(previousEntity.getChanges());
if (!previousEntity.getState().equals(entityLogEntry.getState())) {
hasChanges = true;
ChangeType changeType = calculateChangeType(entityLogEntry.getState(), previousEntity.getState());
entityLogEntry.getChanges().add(new Change(analysisNumber, changeType, now));
}
}
addRemovedEntriesAsRemoved(previousEntityLogEntries, newEntityLogEntries, manualRedactions, analysisNumber, now);
return hasChanges;
}
@@ -70,11 +75,16 @@ public class EntityChangeLogService {
Set<String> existingIds = newEntityLogEntries.stream()
.map(EntityLogEntry::getId)
.collect(Collectors.toSet());
List<EntityLogEntry> removedEntries = previousEntityLogEntries.stream()
.filter(entry -> !existingIds.contains(entry.getId()))
.toList();
removedEntries.forEach(entry -> entry.getChanges().add(new Change(analysisNumber, ChangeType.REMOVED, now)));
removedEntries.forEach(entry -> entry.setState(EntryState.REMOVED));
removedEntries.stream()
.filter(entry -> !entry.getState().equals(EntryState.REMOVED))
.peek(entry -> entry.getChanges().add(new Change(analysisNumber, ChangeType.REMOVED, now)))
.forEach(entry -> entry.setState(EntryState.REMOVED));
newEntityLogEntries.addAll(removedEntries);
}
@@ -664,6 +664,8 @@ public class EntityCreationService {
addEntityToGraph(mergedEntity, node);
insertToKieSession(mergedEntity);
entitiesToMerge.stream().filter(e -> !e.equals(mergedEntity)).forEach(node.getEntities()::remove);
return mergedEntity;
}
@@ -747,25 +749,38 @@ public class EntityCreationService {
try {
if (node.getEntities().contains(entity)) {
// If entity already exists and it has a different text range, we add the text range to the list of duplicated text ranges
node.getEntities().stream()//
Optional<TextEntity> optionalTextEntity = node.getEntities()
.stream()//
.filter(e -> e.equals(entity))//
.filter(e -> !e.getTextRange().equals(entity.getTextRange()))//
.findAny()//
.ifPresent(entityToDuplicate -> addDuplicateEntityToGraph(entityToDuplicate, entity.getTextRange(), node));
.findAny();
if (optionalTextEntity.isPresent()) {
addDuplicateEntityToGraph(optionalTextEntity.get(), entity.getTextRange(), node);
} else {
node.getEntities().remove(entity);
addNewEntityToGraph(entity, documentTree);
}
} else {
entity.addIntersectingNode(documentTree.getRoot().getNode());
addEntityToGraph(entity, documentTree);
}
} catch (NoSuchElementException e) {
entity.setDeepestFullyContainingNode(documentTree.getRoot().getNode());
entityEnrichmentService.enrichEntity(entity, entity.getDeepestFullyContainingNode().getTextBlock());
entity.addIntersectingNode(documentTree.getRoot().getNode());
addToPages(entity);
addEntityToNodeEntitySets(entity);
addNewEntityToGraph(entity, documentTree);
}
}
private void addNewEntityToGraph(TextEntity entity, DocumentTree documentTree) {
entity.setDeepestFullyContainingNode(documentTree.getRoot().getNode());
entityEnrichmentService.enrichEntity(entity, entity.getDeepestFullyContainingNode().getTextBlock());
entity.addIntersectingNode(documentTree.getRoot().getNode());
addToPages(entity);
addEntityToNodeEntitySets(entity);
}
private void addDuplicateEntityToGraph(TextEntity entityToDuplicate, TextRange newTextRange, SemanticNode node) {
entityToDuplicate.addTextRange(newTextRange);
@@ -1360,6 +1360,18 @@ rule "X.8.1: Remove Entity when intersected by imported Entity"
end
// Rule unit: X.11
rule "X.11.0: Remove dictionary entity which intersects with a manual entity"
salience 64
when
$manualEntity: TextEntity(engines contains Engine.MANUAL, active())
$dictionaryEntity: TextEntity(intersects($manualEntity), dictionaryEntry, engines not contains Engine.MANUAL)
then
$dictionaryEntity.remove("X.11.0", "remove dictionary entity which intersects with a manual entity");
retract($dictionaryEntity);
end
//------------------------------------ File attributes rules ------------------------------------
// Rule unit: FA.1
@@ -1963,6 +1963,18 @@ rule "X.8.1: Remove Entity when intersected by imported Entity"
end
// Rule unit: X.11
rule "X.11.0: Remove dictionary entity which intersects with a manual entity"
salience 64
when
$manualEntity: TextEntity(engines contains Engine.MANUAL, active())
$dictionaryEntity: TextEntity(intersects($manualEntity), dictionaryEntry, engines not contains Engine.MANUAL)
then
$dictionaryEntity.remove("X.11.0", "remove dictionary entity which intersects with a manual entity");
retract($dictionaryEntity);
end
//------------------------------------ File attributes rules ------------------------------------
// Rule unit: FA.1
@@ -1414,6 +1414,18 @@ rule "X.8.1: Remove Entity when intersected by imported Entity"
end
// Rule unit: X.11
rule "X.11.0: Remove dictionary entity which intersects with a manual entity"
salience 64
when
$manualEntity: TextEntity(engines contains Engine.MANUAL, active())
$dictionaryEntity: TextEntity(intersects($manualEntity), dictionaryEntry, engines not contains Engine.MANUAL)
then
$dictionaryEntity.remove("X.11.0", "remove dictionary entity which intersects with a manual entity");
retract($dictionaryEntity);
end
//------------------------------------ File attributes rules ------------------------------------
// Rule unit: FA.1
@@ -927,6 +927,18 @@ rule "X.8.1: Remove Entity when intersected by imported Entity"
end
// Rule unit: X.11
rule "X.11.0: Remove dictionary entity which intersects with a manual entity"
salience 64
when
$manualEntity: TextEntity(engines contains Engine.MANUAL, active())
$dictionaryEntity: TextEntity(intersects($manualEntity), dictionaryEntry, engines not contains Engine.MANUAL)
then
$dictionaryEntity.remove("X.11.0", "remove dictionary entity which intersects with a manual entity");
retract($dictionaryEntity);
end
//------------------------------------ File attributes rules ------------------------------------
// Rule unit: FA.1
@@ -255,6 +255,18 @@ rule "X.8.1: Remove Entity when intersected by imported Entity"
end
// Rule unit: X.11
rule "X.11.0: Remove dictionary entity which intersects with a manual entity"
salience 64
when
$manualEntity: TextEntity(engines contains Engine.MANUAL, active())
$dictionaryEntity: TextEntity(intersects($manualEntity), dictionaryEntry, engines not contains Engine.MANUAL)
then
$dictionaryEntity.remove("X.11.0", "remove dictionary entity which intersects with a manual entity");
retract($dictionaryEntity);
end
//------------------------------------ Local dictionary search rules ------------------------------------
// Rule unit: LDS.0
@@ -1449,6 +1449,18 @@ rule "X.8.1: Remove Entity when intersected by imported Entity"
end
// Rule unit: X.11
rule "X.11.0: Remove dictionary entity which intersects with a manual entity"
salience 64
when
$manualEntity: TextEntity(engines contains Engine.MANUAL, active())
$dictionaryEntity: TextEntity(intersects($manualEntity), dictionaryEntry, engines not contains Engine.MANUAL)
then
$dictionaryEntity.remove("X.11.0", "remove dictionary entity which intersects with a manual entity");
retract($dictionaryEntity);
end
//------------------------------------ File attributes rules ------------------------------------
// Rule unit: FA.1
@@ -379,6 +379,18 @@ rule "X.8.1: Remove Entity when intersected by imported Entity"
end
// Rule unit: X.11
rule "X.11.0: Remove dictionary entity which intersects with a manual entity"
salience 64
when
$manualEntity: TextEntity(engines contains Engine.MANUAL, active())
$dictionaryEntity: TextEntity(intersects($manualEntity), dictionaryEntry, engines not contains Engine.MANUAL)
then
$dictionaryEntity.remove("X.11.0", "remove dictionary entity which intersects with a manual entity");
retract($dictionaryEntity);
end
//------------------------------------ File attributes rules ------------------------------------
// Rule unit: FA.1
@@ -500,6 +500,18 @@ rule "X.8.1: Remove Entity when intersected by imported Entity"
end
// Rule unit: X.11
rule "X.11.0: Remove dictionary entity which intersects with a manual entity"
salience 64
when
$manualEntity: TextEntity(engines contains Engine.MANUAL, active())
$dictionaryEntity: TextEntity(intersects($manualEntity), dictionaryEntry, engines not contains Engine.MANUAL)
then
$dictionaryEntity.remove("X.11.0", "remove dictionary entity which intersects with a manual entity");
retract($dictionaryEntity);
end
//------------------------------------ File attributes rules ------------------------------------
// Rule unit: FA.1
@@ -1986,6 +1986,18 @@ rule "X.8.1: Remove Entity when intersected by imported Entity"
end
// Rule unit: X.11
rule "X.11.0: Remove dictionary entity which intersects with a manual entity"
salience 64
when
$manualEntity: TextEntity(engines contains Engine.MANUAL, active())
$dictionaryEntity: TextEntity(intersects($manualEntity), dictionaryEntry, engines not contains Engine.MANUAL)
then
$dictionaryEntity.remove("X.11.0", "remove dictionary entity which intersects with a manual entity");
retract($dictionaryEntity);
end
//------------------------------------ File attributes rules ------------------------------------
// Rule unit: FA.1
@@ -1562,6 +1562,18 @@ rule "X.8.1: Remove Entity when intersected by imported Entity"
end
// Rule unit: X.11
rule "X.11.0: Remove dictionary entity which intersects with a manual entity"
salience 64
when
$manualEntity: TextEntity(engines contains Engine.MANUAL, active())
$dictionaryEntity: TextEntity(intersects($manualEntity), dictionaryEntry, engines not contains Engine.MANUAL)
then
$dictionaryEntity.remove("X.11.0", "remove dictionary entity which intersects with a manual entity");
retract($dictionaryEntity);
end
//------------------------------------ File attributes rules ------------------------------------
// Rule unit: FA.1