Marmelator/redaction-service
1
0
Fork 0
Code Issues Pull Requests 14 Actions Packages Projects Releases 622 Wiki Activity

Merge branch 'RED-8372' into 'master'

Browse Source 
RED-8372: Include additional info about redactions in redaction annotations

Closes RED-8372

See merge request redactmanager/redaction-service!263
This commit is contained in:
Dominique Eifländer 2024-02-02 11:11:30 +01:00
commit ce9ed6de9b
35 changed files with 756 additions and 807 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
redaction-service-v1/redaction-service-server-v1/build.gradle.kts
View File

@ -16,7 +16,7 @@ val layoutParserVersion = "0.86.0"
val jacksonVersion = "2.15.2" val jacksonVersion = "2.15.2"
val droolsVersion = "9.44.0.Final" val droolsVersion = "9.44.0.Final"
val pdfBoxVersion = "3.0.0" val pdfBoxVersion = "3.0.0"
val persistenceServiceVersion = "2.324.0" val persistenceServiceVersion = "2.326.0"
val springBootStarterVersion = "3.1.5" val springBootStarterVersion = "3.1.5"
configurations { configurations {

54
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/migration/RedactionLogToEntityLogMigrationService.java
View File

@ -16,13 +16,12 @@ import org.springframework.stereotype.Service;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogLegalBasis; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogLegalBasis;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.migration.MigratedIds; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.migration.MigratedIds;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ManualChange;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ManualRedactionType; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ManualRedactionType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogLegalBasis; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogLegalBasis;
import com.iqser.red.service.redaction.v1.server.model.ManualEntity; import com.iqser.red.service.redaction.v1.server.model.PrecursorEntity;
import com.iqser.red.service.redaction.v1.server.model.MigratedEntityLog; import com.iqser.red.service.redaction.v1.server.model.MigratedEntityLog;
import com.iqser.red.service.redaction.v1.server.model.MigrationEntity; import com.iqser.red.service.redaction.v1.server.model.MigrationEntity;
import com.iqser.red.service.redaction.v1.server.model.RectangleWithPage; import com.iqser.red.service.redaction.v1.server.model.RectangleWithPage;
@ -186,46 +185,47 @@ public class RedactionLogToEntityLogMigrationService {
.filter(redactionLogEntry -> !redactionLogEntry.isImage()) .filter(redactionLogEntry -> !redactionLogEntry.isImage())
.map(entry -> MigrationEntity.fromRedactionLogEntry(entry, dictionaryService.isHint(entry.getType(), dossierTemplateId))) .map(entry -> MigrationEntity.fromRedactionLogEntry(entry, dictionaryService.isHint(entry.getType(), dossierTemplateId)))
.peek(migrationEntity -> { .peek(migrationEntity -> {
if (migrationEntity.getManualEntity().getEntityType().equals(EntityType.HINT) &&// if (migrationEntity.getPrecursorEntity().getEntityType().equals(EntityType.HINT) &&//
!migrationEntity.getRedactionLogEntry().isHint() &&// !migrationEntity.getRedactionLogEntry().isHint() &&//
!migrationEntity.getRedactionLogEntry().isRedacted()) { !migrationEntity.getRedactionLogEntry().isRedacted()) {
migrationEntity.getManualEntity().ignore(migrationEntity.getManualEntity().getRuleIdentifier(), migrationEntity.getRedactionLogEntry().getReason()); migrationEntity.getPrecursorEntity().ignore(migrationEntity.getPrecursorEntity().getRuleIdentifier(), migrationEntity.getRedactionLogEntry().getReason());
} else if (migrationEntity.getRedactionLogEntry().lastChangeIsRemoved()) { } else if (migrationEntity.getRedactionLogEntry().lastChangeIsRemoved()) {
migrationEntity.getManualEntity().remove(migrationEntity.getManualEntity().getRuleIdentifier(), migrationEntity.getRedactionLogEntry().getReason()); migrationEntity.getPrecursorEntity().remove(migrationEntity.getPrecursorEntity().getRuleIdentifier(), migrationEntity.getRedactionLogEntry().getReason());
} else if (lastManualChangeIsRemove(migrationEntity)) { } else if (lastManualChangeIsRemove(migrationEntity)) {
migrationEntity.getManualEntity().ignore(migrationEntity.getManualEntity().getRuleIdentifier(), migrationEntity.getManualEntity().getReason()); migrationEntity.getPrecursorEntity().ignore(migrationEntity.getPrecursorEntity().getRuleIdentifier(), migrationEntity.getPrecursorEntity().getReason());
} else if (migrationEntity.getManualEntity().isApplied() && migrationEntity.getRedactionLogEntry().isRecommendation()) { } else if (migrationEntity.getPrecursorEntity().isApplied() && migrationEntity.getRedactionLogEntry().isRecommendation()) {
migrationEntity.getManualEntity().skip(migrationEntity.getManualEntity().getRuleIdentifier(), migrationEntity.getManualEntity().getReason()); migrationEntity.getPrecursorEntity().skip(migrationEntity.getPrecursorEntity().getRuleIdentifier(), migrationEntity.getPrecursorEntity().getReason());
} else if (migrationEntity.getManualEntity().isApplied()) { } else if (migrationEntity.getPrecursorEntity().isApplied()) {
migrationEntity.getManualEntity() migrationEntity.getPrecursorEntity()
.apply(migrationEntity.getManualEntity().getRuleIdentifier(), .apply(migrationEntity.getPrecursorEntity().getRuleIdentifier(),
migrationEntity.getManualEntity().getReason(), migrationEntity.getPrecursorEntity().getReason(),
migrationEntity.getManualEntity().getLegalBasis()); migrationEntity.getPrecursorEntity().getLegalBasis());
} else { } else {
migrationEntity.getManualEntity().skip(migrationEntity.getManualEntity().getRuleIdentifier(), migrationEntity.getManualEntity().getReason()); migrationEntity.getPrecursorEntity().skip(migrationEntity.getPrecursorEntity().getRuleIdentifier(), migrationEntity.getPrecursorEntity().getReason());
} }
}) })
.toList(); .toList();
Map<String, List<TextEntity>> tempEntitiesByValue = entityFindingUtility.findAllPossibleEntitiesAndGroupByValue(document, Map<String, List<TextEntity>> tempEntitiesByValue = entityFindingUtility.findAllPossibleEntitiesAndGroupByValue(document,
entitiesToMigrate.stream().map(MigrationEntity::getManualEntity).toList()); entitiesToMigrate.stream().map(MigrationEntity::getPrecursorEntity).toList());
for (MigrationEntity migrationEntity : entitiesToMigrate) { for (MigrationEntity migrationEntity : entitiesToMigrate) {
Optional<TextEntity> optionalTextEntity = entityFindingUtility.findClosestEntityAndReturnEmptyIfNotFound(migrationEntity.getManualEntity(), Optional<TextEntity> optionalTextEntity = entityFindingUtility.findClosestEntityAndReturnEmptyIfNotFound(migrationEntity.getPrecursorEntity(),
tempEntitiesByValue, tempEntitiesByValue,
MATCH_THRESHOLD); MATCH_THRESHOLD);
if (optionalTextEntity.isEmpty()) { if (optionalTextEntity.isEmpty()) {
migrationEntity.setMigratedEntity(migrationEntity.getManualEntity()); migrationEntity.setMigratedEntity(migrationEntity.getPrecursorEntity());
migrationEntity.setOldId(migrationEntity.getManualEntity().getId()); migrationEntity.setOldId(migrationEntity.getPrecursorEntity().getId());
migrationEntity.setNewId(migrationEntity.getManualEntity().getId()); migrationEntity.setNewId(migrationEntity.getPrecursorEntity().getId());
continue; continue;
} }
TextEntity entity = createCorrectEntity(migrationEntity.getManualEntity(), document, optionalTextEntity.get().getTextRange()); TextEntity entity = createCorrectEntity(migrationEntity.getPrecursorEntity(), document, optionalTextEntity.get().getTextRange());
migrationEntity.setMigratedEntity(entity); migrationEntity.setMigratedEntity(entity);
migrationEntity.setOldId(migrationEntity.getManualEntity().getId()); migrationEntity.setOldId(migrationEntity.getPrecursorEntity().getId());
migrationEntity.setNewId(entity.getPositionsOnPagePerPage().get(0).getId()); // Can only be on one page, since redactionLogEntries can only be on one page migrationEntity.setNewId(entity.getPositionsOnPagePerPage().get(0).getId()); // Can only be on one page, since redactionLogEntries can only be on one page
} }
tempEntitiesByValue.values().stream().flatMap(Collection::stream).forEach(TextEntity::removeFromGraph); tempEntitiesByValue.values().stream().flatMap(Collection::stream).forEach(TextEntity::removeFromGraph);
@ -248,15 +248,15 @@ public class RedactionLogToEntityLogMigrationService {
} }
private TextEntity createCorrectEntity(ManualEntity manualEntity, SemanticNode node, TextRange closestTextRange) { private TextEntity createCorrectEntity(PrecursorEntity precursorEntity, SemanticNode node, TextRange closestTextRange) {
EntityCreationService entityCreationService = new EntityCreationService(entityEnrichmentService); EntityCreationService entityCreationService = new EntityCreationService(entityEnrichmentService);
TextEntity correctEntity = entityCreationService.forceByTextRange(closestTextRange, manualEntity.getType(), manualEntity.getEntityType(), node); TextEntity correctEntity = entityCreationService.forceByTextRange(closestTextRange, precursorEntity.getType(), precursorEntity.getEntityType(), node);
correctEntity.addMatchedRules(manualEntity.getMatchedRuleList()); correctEntity.addMatchedRules(precursorEntity.getMatchedRuleList());
correctEntity.setDictionaryEntry(manualEntity.isDictionaryEntry()); correctEntity.setDictionaryEntry(precursorEntity.isDictionaryEntry());
correctEntity.setDossierDictionaryEntry(manualEntity.isDossierDictionaryEntry()); correctEntity.setDossierDictionaryEntry(precursorEntity.isDossierDictionaryEntry());
correctEntity.getManualOverwrite().addChanges(manualEntity.getManualOverwrite().getManualChangeLog()); correctEntity.getManualOverwrite().addChanges(precursorEntity.getManualOverwrite().getManualChangeLog());
return correctEntity; return correctEntity;
} }

40
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/MigrationEntity.java
View File

@ -34,7 +34,7 @@ import lombok.RequiredArgsConstructor;
@RequiredArgsConstructor @RequiredArgsConstructor
public final class MigrationEntity { public final class MigrationEntity {
private final ManualEntity manualEntity; private final PrecursorEntity precursorEntity;
private final RedactionLogEntry redactionLogEntry; private final RedactionLogEntry redactionLogEntry;
private IEntity migratedEntity; private IEntity migratedEntity;
private String oldId; private String oldId;
@ -47,12 +47,12 @@ public final class MigrationEntity {
} }
public static ManualEntity createManualEntity(RedactionLogEntry redactionLogEntry, boolean hint) { public static PrecursorEntity createManualEntity(RedactionLogEntry redactionLogEntry, boolean hint) {
String ruleIdentifier = buildRuleIdentifier(redactionLogEntry); String ruleIdentifier = buildRuleIdentifier(redactionLogEntry);
List<RectangleWithPage> rectangleWithPages = redactionLogEntry.getPositions().stream().map(RectangleWithPage::fromRedactionLogRectangle).toList(); List<RectangleWithPage> rectangleWithPages = redactionLogEntry.getPositions().stream().map(RectangleWithPage::fromRedactionLogRectangle).toList();
EntityType entityType = getEntityType(redactionLogEntry, hint); EntityType entityType = getEntityType(redactionLogEntry, hint);
return ManualEntity.builder() return PrecursorEntity.builder()
.id(redactionLogEntry.getId()) .id(redactionLogEntry.getId())
.value(redactionLogEntry.getValue()) .value(redactionLogEntry.getValue())
.entityPosition(rectangleWithPages) .entityPosition(rectangleWithPages)
@ -164,7 +164,7 @@ public final class MigrationEntity {
entityLogEntry = createEntityLogEntry(image); entityLogEntry = createEntityLogEntry(image);
} else if (migratedEntity instanceof TextEntity textEntity) { } else if (migratedEntity instanceof TextEntity textEntity) {
entityLogEntry = createEntityLogEntry(textEntity); entityLogEntry = createEntityLogEntry(textEntity);
} else if (migratedEntity instanceof ManualEntity entity) { } else if (migratedEntity instanceof PrecursorEntity entity) {
entityLogEntry = createEntityLogEntry(entity); entityLogEntry = createEntityLogEntry(entity);
} else { } else {
throw new UnsupportedOperationException("Unknown subclass " + migratedEntity.getClass()); throw new UnsupportedOperationException("Unknown subclass " + migratedEntity.getClass());
@ -261,30 +261,30 @@ public final class MigrationEntity {
} }
public EntityLogEntry createEntityLogEntry(ManualEntity manualEntity) { public EntityLogEntry createEntityLogEntry(PrecursorEntity precursorEntity) {
String type = manualEntity.getManualOverwrite().getType().orElse(manualEntity.getType()); String type = precursorEntity.getManualOverwrite().getType().orElse(precursorEntity.getType());
return EntityLogEntry.builder() return EntityLogEntry.builder()
.id(manualEntity.getId()) .id(precursorEntity.getId())
.reason(manualEntity.buildReasonWithManualChangeDescriptions()) .reason(precursorEntity.buildReasonWithManualChangeDescriptions())
.legalBasis(manualEntity.legalBasis()) .legalBasis(precursorEntity.legalBasis())
.value(manualEntity.value()) .value(precursorEntity.value())
.type(type) .type(type)
.state(buildEntryState(manualEntity)) .state(buildEntryState(precursorEntity))
.entryType(buildEntryType(manualEntity)) .entryType(buildEntryType(precursorEntity))
.section(redactionLogEntry.getSection()) .section(redactionLogEntry.getSection())
.textAfter(redactionLogEntry.getTextAfter()) .textAfter(redactionLogEntry.getTextAfter())
.textBefore(redactionLogEntry.getTextBefore()) .textBefore(redactionLogEntry.getTextBefore())
.containingNodeId(Collections.emptyList()) .containingNodeId(Collections.emptyList())
.closestHeadline("") .closestHeadline("")
.matchedRule(manualEntity.getMatchedRule().getRuleIdentifier().toString()) .matchedRule(precursorEntity.getMatchedRule().getRuleIdentifier().toString())
.dictionaryEntry(manualEntity.isDictionaryEntry()) .dictionaryEntry(precursorEntity.isDictionaryEntry())
.dossierDictionaryEntry(manualEntity.isDossierDictionaryEntry()) .dossierDictionaryEntry(precursorEntity.isDossierDictionaryEntry())
.startOffset(-1) .startOffset(-1)
.endOffset(-1) .endOffset(-1)
.positions(manualEntity.getManualOverwrite() .positions(precursorEntity.getManualOverwrite()
.getPositions() .getPositions()
.orElse(manualEntity.getEntityPosition()) .orElse(precursorEntity.getEntityPosition())
.stream() .stream()
.map(entityPosition -> new Position(entityPosition.rectangle2D(), entityPosition.pageNumber())) .map(entityPosition -> new Position(entityPosition.rectangle2D(), entityPosition.pageNumber()))
.toList()) .toList())
@ -343,11 +343,11 @@ public final class MigrationEntity {
if (entity instanceof TextEntity textEntity) { if (entity instanceof TextEntity textEntity) {
return getEntryType(textEntity.getEntityType()); return getEntryType(textEntity.getEntityType());
} else if (entity instanceof ManualEntity manualEntity) { } else if (entity instanceof PrecursorEntity precursorEntity) {
if (manualEntity.isRectangle()) { if (precursorEntity.isRectangle()) {
return EntryType.AREA; return EntryType.AREA;
} }
return getEntryType(manualEntity.getEntityType()); return getEntryType(precursorEntity.getEntityType());
} else if (entity instanceof Image) { } else if (entity instanceof Image) {
return EntryType.IMAGE; return EntryType.IMAGE;
} }

57
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/ManualEntity.java → redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/PrecursorEntity.java
View File

@ -1,11 +1,17 @@
package com.iqser.red.service.redaction.v1.server.model; package com.iqser.red.service.redaction.v1.server.model;
import static com.iqser.red.service.redaction.v1.server.service.NotFoundImportedEntitiesService.IMPORTED_REDACTION_TYPE;
import java.util.List; import java.util.List;
import java.util.Optional;
import java.util.PriorityQueue; import java.util.PriorityQueue;
import java.util.Set;
import java.util.UUID; import java.util.UUID;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryType; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedaction;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
import com.iqser.red.service.redaction.v1.server.model.document.TextRange; import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
@ -25,7 +31,7 @@ import lombok.experimental.FieldDefaults;
@AllArgsConstructor @AllArgsConstructor
@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE) @FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
@SuppressWarnings("PMD.AvoidFieldNameMatchingMethodName") @SuppressWarnings("PMD.AvoidFieldNameMatchingMethodName")
public class ManualEntity implements IEntity { public class PrecursorEntity implements IEntity {
// The id must be mapped into a TextEntity as is for comments to work correctly // The id must be mapped into a TextEntity as is for comments to work correctly
String id; String id;
@ -37,23 +43,26 @@ public class ManualEntity implements IEntity {
String type; String type;
String section; String section;
EntityType entityType; EntityType entityType;
EntryType entryType;
boolean applied; boolean applied;
boolean isDictionaryEntry; boolean isDictionaryEntry;
boolean isDossierDictionaryEntry; boolean isDossierDictionaryEntry;
boolean rectangle; boolean rectangle;
Set<Engine> engines;
@Builder.Default @Builder.Default
PriorityQueue<MatchedRule> matchedRuleList = new PriorityQueue<>(); PriorityQueue<MatchedRule> matchedRuleList = new PriorityQueue<>();
ManualChangeOverwrite manualOverwrite; ManualChangeOverwrite manualOverwrite;
public static ManualEntity fromManualRedactionEntry(ManualRedactionEntry manualRedactionEntry, boolean hint) { public static PrecursorEntity fromManualRedactionEntry(ManualRedactionEntry manualRedactionEntry, boolean hint) {
List<RectangleWithPage> rectangleWithPages = manualRedactionEntry.getPositions().stream().map(RectangleWithPage::fromAnnotationRectangle).toList(); List<RectangleWithPage> rectangleWithPages = manualRedactionEntry.getPositions().stream().map(RectangleWithPage::fromAnnotationRectangle).toList();
var entityType = hint ? EntityType.HINT : EntityType.ENTITY; var entityType = hint ? EntityType.HINT : EntityType.ENTITY;
var entryType = hint ? EntryType.HINT : EntryType.ENTITY;
ManualChangeOverwrite manualChangeOverwrite = new ManualChangeOverwrite(entityType); ManualChangeOverwrite manualChangeOverwrite = new ManualChangeOverwrite(entityType);
manualChangeOverwrite.addChange(manualRedactionEntry); manualChangeOverwrite.addChange(manualRedactionEntry);
return ManualEntity.builder() return PrecursorEntity.builder()
.id(manualRedactionEntry.getAnnotationId()) .id(manualRedactionEntry.getAnnotationId())
.value(manualRedactionEntry.getValue()) .value(manualRedactionEntry.getValue())
.entityPosition(rectangleWithPages) .entityPosition(rectangleWithPages)
@ -63,20 +72,22 @@ public class ManualEntity implements IEntity {
.type(manualRedactionEntry.getType()) .type(manualRedactionEntry.getType())
.section(manualRedactionEntry.getSection()) .section(manualRedactionEntry.getSection())
.entityType(entityType) .entityType(entityType)
.entryType(entryType)
.applied(true) .applied(true)
.isDictionaryEntry(false) .isDictionaryEntry(false)
.isDossierDictionaryEntry(false) .isDossierDictionaryEntry(false)
.rectangle(manualRedactionEntry.isRectangle()) .rectangle(manualRedactionEntry.isRectangle())
.manualOverwrite(manualChangeOverwrite) .manualOverwrite(manualChangeOverwrite)
.engines(Set.of(Engine.MANUAL))
.build(); .build();
} }
public static ManualEntity fromEntityLogEntry(EntityLogEntry entityLogEntry) { public static PrecursorEntity fromEntityLogEntry(EntityLogEntry entityLogEntry) {
List<RectangleWithPage> rectangleWithPages = entityLogEntry.getPositions().stream().map(RectangleWithPage::fromEntityLogPosition).toList(); List<RectangleWithPage> rectangleWithPages = entityLogEntry.getPositions().stream().map(RectangleWithPage::fromEntityLogPosition).toList();
EntityType entityType = getEntityType(entityLogEntry.getEntryType()); EntityType entityType = getEntityType(entityLogEntry.getEntryType());
return ManualEntity.builder() return PrecursorEntity.builder()
.id(entityLogEntry.getId()) .id(entityLogEntry.getId())
.value(entityLogEntry.getValue()) .value(entityLogEntry.getValue())
.entityPosition(rectangleWithPages) .entityPosition(rectangleWithPages)
@ -86,17 +97,43 @@ public class ManualEntity implements IEntity {
.type(entityLogEntry.getType()) .type(entityLogEntry.getType())
.section(entityLogEntry.getSection()) .section(entityLogEntry.getSection())
.entityType(entityType) .entityType(entityType)
.entryType(entityLogEntry.getEntryType())
.isDictionaryEntry(entityLogEntry.isDictionaryEntry()) .isDictionaryEntry(entityLogEntry.isDictionaryEntry())
.isDossierDictionaryEntry(entityLogEntry.isDossierDictionaryEntry()) .isDossierDictionaryEntry(entityLogEntry.isDossierDictionaryEntry())
.manualOverwrite(new ManualChangeOverwrite(entityType)) .manualOverwrite(new ManualChangeOverwrite(entityType))
.engines(entityLogEntry.getEngines())
.build(); .build();
} }
public static ManualEntity fromManualResizeRedaction(ManualResizeRedaction manualResizeRedaction) { public static PrecursorEntity fromImportedEntry(ImportedRedaction importedRedaction) {
List<RectangleWithPage> rectangleWithPages = importedRedaction.getPositions().stream().map(RectangleWithPage::fromEntityLogPosition).toList();
EntryType entryType = Optional.ofNullable(importedRedaction.getEntryType()).orElse(EntryType.ENTITY);
EntityType entityType = getEntityType(entryType);
String value = Optional.ofNullable(importedRedaction.getValue()).orElse("");
return PrecursorEntity.builder()
.id(importedRedaction.getId())
.value(value)
.entityPosition(rectangleWithPages)
.reason(Optional.ofNullable(importedRedaction.getReason()).orElse(""))
.legalBasis(Optional.ofNullable(importedRedaction.getLegalBasis()).orElse(""))
.type(Optional.ofNullable(importedRedaction.getType()).orElse(IMPORTED_REDACTION_TYPE))
.entityType(entityType)
.entryType(entryType)
.isDictionaryEntry(false)
.isDossierDictionaryEntry(false)
.rectangle(value.isBlank() || entryType.equals(EntryType.IMAGE) || entryType.equals(EntryType.IMAGE_HINT) || entryType.equals(EntryType.AREA))
.manualOverwrite(new ManualChangeOverwrite(entityType))
.engines(Set.of(Engine.IMPORTED))
.build();
}
public static PrecursorEntity fromManualResizeRedaction(ManualResizeRedaction manualResizeRedaction) {
List<RectangleWithPage> rectangleWithPages = manualResizeRedaction.getPositions().stream().map(RectangleWithPage::fromAnnotationRectangle).toList(); List<RectangleWithPage> rectangleWithPages = manualResizeRedaction.getPositions().stream().map(RectangleWithPage::fromAnnotationRectangle).toList();
return ManualEntity.builder() return PrecursorEntity.builder()
.id(UUID.randomUUID().toString()) .id(UUID.randomUUID().toString())
.value(manualResizeRedaction.getValue()) .value(manualResizeRedaction.getValue())
.entityPosition(rectangleWithPages) .entityPosition(rectangleWithPages)
@ -141,10 +178,4 @@ public class ManualEntity implements IEntity {
} }
} }
private EntityType getEntityType(boolean isHint) {
return isHint ? EntityType.HINT : EntityType.ENTITY;
}
} }

3
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/MatchedRule.java
View File

@ -24,7 +24,8 @@ public final class MatchedRule implements Comparable<MatchedRule> {
public static final RuleType FINAL_TYPE = RuleType.fromString("FINAL"); public static final RuleType FINAL_TYPE = RuleType.fromString("FINAL");
public static final RuleType ELIMINATION_RULE_TYPE = RuleType.fromString("X"); public static final RuleType ELIMINATION_RULE_TYPE = RuleType.fromString("X");
private static final List<RuleType> RULE_TYPE_PRIORITIES = List.of(FINAL_TYPE, ELIMINATION_RULE_TYPE); public static final RuleType IMPORTED_TYPE = RuleType.fromString("IMP");
private static final List<RuleType> RULE_TYPE_PRIORITIES = List.of(FINAL_TYPE, ELIMINATION_RULE_TYPE, IMPORTED_TYPE);
RuleIdentifier ruleIdentifier; RuleIdentifier ruleIdentifier;
@Builder.Default @Builder.Default

72
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalyzeService.java
View File

@ -1,12 +1,13 @@
package com.iqser.red.service.redaction.v1.server.service; package com.iqser.red.service.redaction.v1.server.service;
import java.util.Collection;
import java.util.Collections; import java.util.Collections;
import java.util.HashSet; import java.util.HashSet;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Set; import java.util.Set;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import java.util.stream.Stream;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestBody;
@ -18,6 +19,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileTyp
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentLog; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentLog;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogChanges; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogChanges;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedactions;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.legalbasis.LegalBasis; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.legalbasis.LegalBasis;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog;
@ -28,7 +30,7 @@ import com.iqser.red.service.redaction.v1.server.RedactionServiceSettings;
import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient; import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient;
import com.iqser.red.service.redaction.v1.server.client.model.NerEntitiesModel; import com.iqser.red.service.redaction.v1.server.client.model.NerEntitiesModel;
import com.iqser.red.service.redaction.v1.server.model.KieWrapper; import com.iqser.red.service.redaction.v1.server.model.KieWrapper;
import com.iqser.red.service.redaction.v1.server.model.ManualEntity; import com.iqser.red.service.redaction.v1.server.model.PrecursorEntity;
import com.iqser.red.service.redaction.v1.server.model.NerEntities; import com.iqser.red.service.redaction.v1.server.model.NerEntities;
import com.iqser.red.service.redaction.v1.server.model.component.Component; import com.iqser.red.service.redaction.v1.server.model.component.Component;
import com.iqser.red.service.redaction.v1.server.model.dictionary.Dictionary; import com.iqser.red.service.redaction.v1.server.model.dictionary.Dictionary;
@ -37,6 +39,7 @@ import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryVers
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode; import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
import com.iqser.red.service.redaction.v1.server.service.document.DocumentGraphMapper; import com.iqser.red.service.redaction.v1.server.service.document.DocumentGraphMapper;
import com.iqser.red.service.redaction.v1.server.service.document.ImportedRedactionEntryService;
import com.iqser.red.service.redaction.v1.server.service.document.ManualRedactionEntryService; import com.iqser.red.service.redaction.v1.server.service.document.ManualRedactionEntryService;
import com.iqser.red.service.redaction.v1.server.service.document.NerEntitiesAdapter; import com.iqser.red.service.redaction.v1.server.service.document.NerEntitiesAdapter;
import com.iqser.red.service.redaction.v1.server.service.document.SectionFinderService; import com.iqser.red.service.redaction.v1.server.service.document.SectionFinderService;
@ -72,9 +75,10 @@ public class AnalyzeService {
RedactionChangeLogService redactionChangeLogService; RedactionChangeLogService redactionChangeLogService;
LegalBasisClient legalBasisClient; LegalBasisClient legalBasisClient;
RedactionServiceSettings redactionServiceSettings; RedactionServiceSettings redactionServiceSettings;
ImportedRedactionService importedRedactionService; NotFoundImportedEntitiesService notFoundImportedEntitiesService;
SectionFinderService sectionFinderService; SectionFinderService sectionFinderService;
ManualRedactionEntryService manualRedactionEntryService; ManualRedactionEntryService manualRedactionEntryService;
ImportedRedactionEntryService importedRedactionEntryService;
ObservedStorageService observedStorageService; ObservedStorageService observedStorageService;
FunctionTimerValues redactmanagerAnalyzePagewiseValues; FunctionTimerValues redactmanagerAnalyzePagewiseValues;
@ -93,6 +97,9 @@ public class AnalyzeService {
Document document = DocumentGraphMapper.toDocumentGraph(observedStorageService.getDocumentData(analyzeRequest.getDossierId(), analyzeRequest.getFileId())); Document document = DocumentGraphMapper.toDocumentGraph(observedStorageService.getDocumentData(analyzeRequest.getDossierId(), analyzeRequest.getFileId()));
log.info("Loaded Document Graph for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); log.info("Loaded Document Graph for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
ImportedRedactions importedRedactions = redactionStorageService.getImportedRedactions(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
log.info("Loaded Imported Redactions for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
// not yet ready for reanalysis // not yet ready for reanalysis
if (previousEntityLog == null || document == null || document.getNumberOfPages() == 0) { if (previousEntityLog == null || document == null || document.getNumberOfPages() == 0) {
return analyze(analyzeRequest); return analyze(analyzeRequest);
@ -102,13 +109,11 @@ public class AnalyzeService {
new DictionaryVersion(previousEntityLog.getDictionaryVersion(), previousEntityLog.getDossierDictionaryVersion()), new DictionaryVersion(previousEntityLog.getDictionaryVersion(), previousEntityLog.getDossierDictionaryVersion()),
analyzeRequest.getDossierId()); analyzeRequest.getDossierId());
Set<Integer> sectionsToReanalyseIds = getSectionsToReanalyseIds(analyzeRequest, previousEntityLog, document, dictionaryIncrement); Set<Integer> sectionsToReanalyseIds = getSectionsToReanalyseIds(analyzeRequest, previousEntityLog, document, dictionaryIncrement, importedRedactions);
List<SemanticNode> sectionsToReAnalyse = getSectionsToReAnalyse(document, sectionsToReanalyseIds); List<SemanticNode> sectionsToReAnalyse = getSectionsToReAnalyse(document, sectionsToReanalyseIds);
log.info("{} Sections to reanalyze found for file {} in dossier {}", sectionsToReanalyseIds.size(), analyzeRequest.getFileId(), analyzeRequest.getDossierId()); log.info("{} Sections to reanalyze found for file {} in dossier {}", sectionsToReanalyseIds.size(), analyzeRequest.getFileId(), analyzeRequest.getDossierId());
if (sectionsToReAnalyse.isEmpty()) { if (sectionsToReAnalyse.isEmpty()) {
// do this only to update the imported redactions with unprocessed manual changes if there are changes
importedRedactionService.processImportedRedactions(previousEntityLog, analyzeRequest);
EntityLogChanges entityLogChanges = entityLogCreatorService.updateVersionsAndReturnChanges(previousEntityLog, EntityLogChanges entityLogChanges = entityLogCreatorService.updateVersionsAndReturnChanges(previousEntityLog,
dictionaryIncrement.getDictionaryVersion(), dictionaryIncrement.getDictionaryVersion(),
@ -126,15 +131,18 @@ public class AnalyzeService {
true, true,
Collections.emptySet()); Collections.emptySet());
} }
KieWrapper kieWrapperEntityRules = kieContainerCreationService.getLatestKieContainer(analyzeRequest.getDossierTemplateId(), RuleFileType.ENTITY); KieWrapper kieWrapperEntityRules = kieContainerCreationService.getLatestKieContainer(analyzeRequest.getDossierTemplateId(), RuleFileType.ENTITY);
log.info("Updated entity rules to version {} for file {} in dossier {}", kieWrapperEntityRules.rulesVersion(), analyzeRequest.getFileId(), analyzeRequest.getDossierId()); log.info("Updated entity rules to version {} for file {} in dossier {}", kieWrapperEntityRules.rulesVersion(), analyzeRequest.getFileId(), analyzeRequest.getDossierId());
NerEntities nerEntities = getEntityRecognitionEntitiesFilteredBySectionIds(analyzeRequest, document, sectionsToReanalyseIds); NerEntities nerEntities = getEntityRecognitionEntitiesFilteredBySectionIds(analyzeRequest, document, sectionsToReanalyseIds);
log.info("Loaded Ner Entities for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); log.info("Loaded Ner Entities for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
List<ManualEntity> notFoundManualRedactionEntries = manualRedactionEntryService.addManualRedactionEntriesAndReturnNotFoundEntries(analyzeRequest, var notFoundManualRedactionEntries = manualRedactionEntryService.addManualRedactionEntriesAndReturnNotFoundEntries(analyzeRequest,
document, document,
analyzeRequest.getDossierTemplateId()); analyzeRequest.getDossierTemplateId());
var notFoundImportedEntries = importedRedactionEntryService.addImportedEntriesAndReturnNotFoundEntries(analyzeRequest, importedRedactions, document);
var notFoundManualOrImportedEntries = Stream.of(notFoundManualRedactionEntries, notFoundImportedEntries).flatMap(Collection::stream).collect(Collectors.toList());
Dictionary dictionary = dictionaryService.getDeepCopyDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId()); Dictionary dictionary = dictionaryService.getDeepCopyDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId());
log.info("Updated Dictionaries for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); log.info("Updated Dictionaries for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
@ -142,6 +150,7 @@ public class AnalyzeService {
dictionarySearchService.addDictionaryEntities(dictionary, sectionsToReAnalyse); dictionarySearchService.addDictionaryEntities(dictionary, sectionsToReAnalyse);
log.info("Finished Dictionary Search for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); log.info("Finished Dictionary Search for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
// we could add the imported redactions similar to the manual redactions here as well for additional processing
List<FileAttribute> allFileAttributes = entityDroolsExecutionService.executeRules(kieWrapperEntityRules.container(), List<FileAttribute> allFileAttributes = entityDroolsExecutionService.executeRules(kieWrapperEntityRules.container(),
document, document,
sectionsToReAnalyse, sectionsToReAnalyse,
@ -151,14 +160,16 @@ public class AnalyzeService {
nerEntities); nerEntities);
log.info("Finished entity rule execution for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); log.info("Finished entity rule execution for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
RedactionLog redactionLog = updatePreviousRedactionLog(analyzeRequest, document, notFoundManualRedactionEntries, previousRedactionLog, sectionsToReanalyseIds); RedactionLog redactionLog = updatePreviousRedactionLog(analyzeRequest, document, notFoundManualOrImportedEntries, previousRedactionLog, sectionsToReanalyseIds);
EntityLogChanges entityLogChanges = entityLogCreatorService.updatePreviousEntityLog(analyzeRequest, EntityLogChanges entityLogChanges = entityLogCreatorService.updatePreviousEntityLog(analyzeRequest,
document, document,
notFoundManualRedactionEntries, notFoundManualOrImportedEntries,
previousEntityLog, previousEntityLog,
sectionsToReanalyseIds, sectionsToReanalyseIds,
dictionary.getVersion()); dictionary.getVersion());
notFoundImportedEntitiesService.processEntityLog(entityLogChanges.getEntityLog(), analyzeRequest, notFoundImportedEntries);
return finalizeAnalysis(analyzeRequest, return finalizeAnalysis(analyzeRequest,
startTime, startTime,
kieContainerCreationService.getLatestKieContainer(analyzeRequest.getDossierTemplateId(), RuleFileType.COMPONENT), kieContainerCreationService.getLatestKieContainer(analyzeRequest.getDossierTemplateId(), RuleFileType.COMPONENT),
@ -187,6 +198,9 @@ public class AnalyzeService {
Document document = DocumentGraphMapper.toDocumentGraph(observedStorageService.getDocumentData(analyzeRequest.getDossierId(), analyzeRequest.getFileId())); Document document = DocumentGraphMapper.toDocumentGraph(observedStorageService.getDocumentData(analyzeRequest.getDossierId(), analyzeRequest.getFileId()));
log.info("Loaded Document Graph for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); log.info("Loaded Document Graph for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
ImportedRedactions importedRedactions = redactionStorageService.getImportedRedactions(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
log.info("Loaded Imported Redactions for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
NerEntities nerEntities = getEntityRecognitionEntities(analyzeRequest, document); NerEntities nerEntities = getEntityRecognitionEntities(analyzeRequest, document);
log.info("Loaded Ner Entities for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); log.info("Loaded Ner Entities for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
@ -194,13 +208,16 @@ public class AnalyzeService {
Dictionary dictionary = dictionaryService.getDeepCopyDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId()); Dictionary dictionary = dictionaryService.getDeepCopyDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId());
log.info("Updated Dictionaries for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); log.info("Updated Dictionaries for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
List<ManualEntity> notFoundManualRedactionEntries = manualRedactionEntryService.addManualRedactionEntriesAndReturnNotFoundEntries(analyzeRequest, var notFoundManualRedactionEntries = manualRedactionEntryService.addManualRedactionEntriesAndReturnNotFoundEntries(analyzeRequest,
document, document,
analyzeRequest.getDossierTemplateId()); analyzeRequest.getDossierTemplateId());
var notFoundImportedEntries = importedRedactionEntryService.addImportedEntriesAndReturnNotFoundEntries(analyzeRequest, importedRedactions, document);
var notFoundManualOrImportedEntries = Stream.of(notFoundManualRedactionEntries, notFoundImportedEntries).flatMap(Collection::stream).collect(Collectors.toList());
dictionarySearchService.addDictionaryEntities(dictionary, document); dictionarySearchService.addDictionaryEntities(dictionary, document);
log.info("Finished Dictionary Search for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); log.info("Finished Dictionary Search for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
// we could add the imported redactions similar to the manual redactions here as well for additional processing
List<FileAttribute> allFileAttributes = entityDroolsExecutionService.executeRules(kieWrapperEntityRules.container(), List<FileAttribute> allFileAttributes = entityDroolsExecutionService.executeRules(kieWrapperEntityRules.container(),
document, document,
dictionary, dictionary,
@ -209,13 +226,15 @@ public class AnalyzeService {
nerEntities); nerEntities);
log.info("Finished entity rule execution for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); log.info("Finished entity rule execution for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
RedactionLog redactionLog = createRedactionLog(analyzeRequest, document, notFoundManualRedactionEntries, dictionary, kieWrapperEntityRules); RedactionLog redactionLog = createRedactionLog(analyzeRequest, document, notFoundManualOrImportedEntries, dictionary, kieWrapperEntityRules);
EntityLog entityLog = entityLogCreatorService.createInitialEntityLog(analyzeRequest, EntityLog entityLog = entityLogCreatorService.createInitialEntityLog(analyzeRequest,
document, document,
notFoundManualRedactionEntries, notFoundManualOrImportedEntries,
dictionary.getVersion(), dictionary.getVersion(),
kieWrapperEntityRules.rulesVersion()); kieWrapperEntityRules.rulesVersion());
notFoundImportedEntitiesService.processEntityLog(entityLog, analyzeRequest, notFoundImportedEntries);
return finalizeAnalysis(analyzeRequest, return finalizeAnalysis(analyzeRequest,
startTime, startTime,
kieWrapperComponentRules, kieWrapperComponentRules,
@ -232,24 +251,15 @@ public class AnalyzeService {
@Deprecated(forRemoval = true) @Deprecated(forRemoval = true)
private RedactionLog updatePreviousRedactionLog(AnalyzeRequest analyzeRequest, private RedactionLog updatePreviousRedactionLog(AnalyzeRequest analyzeRequest,
Document document, Document document,
List<ManualEntity> notFoundManualRedactionEntries, List<PrecursorEntity> notFoundEntries,
RedactionLog previousRedactionLog, RedactionLog previousRedactionLog,
Set<Integer> sectionsToReanalyseIds) { Set<Integer> sectionsToReanalyseIds) {
List<RedactionLogEntry> newRedactionLogEntries = redactionLogCreatorService.createRedactionLog(document, List<RedactionLogEntry> newRedactionLogEntries = redactionLogCreatorService.createRedactionLog(document, analyzeRequest.getDossierTemplateId(), notFoundEntries);
analyzeRequest.getDossierTemplateId(),
notFoundManualRedactionEntries);
var importedRedactionFilteredEntries = importedRedactionService.processImportedRedactions(analyzeRequest.getDossierTemplateId(),
analyzeRequest.getDossierId(),
analyzeRequest.getFileId(),
newRedactionLogEntries,
false);
previousRedactionLog.getRedactionLogEntry() previousRedactionLog.getRedactionLogEntry()
.removeIf(entry -> sectionsToReanalyseIds.contains(entry.getSectionNumber()) && !entry.getType().equals(ImportedRedactionService.IMPORTED_REDACTION_TYPE)); .removeIf(entry -> sectionsToReanalyseIds.contains(entry.getSectionNumber()) && !entry.getType().equals(NotFoundImportedEntitiesService.IMPORTED_REDACTION_TYPE));
previousRedactionLog.getRedactionLogEntry().addAll(importedRedactionFilteredEntries);
return previousRedactionLog; return previousRedactionLog;
} }
@ -348,11 +358,9 @@ public class AnalyzeService {
} }
private Set<Integer> getSectionsToReanalyseIds(AnalyzeRequest analyzeRequest, EntityLog entityLog, Document document, DictionaryIncrement dictionaryIncrement) { private Set<Integer> getSectionsToReanalyseIds(AnalyzeRequest analyzeRequest, EntityLog entityLog, Document document, DictionaryIncrement dictionaryIncrement, ImportedRedactions importedRedactions) {
return analyzeRequest.getSectionsToReanalyse().isEmpty() // return sectionFinderService.findSectionsToReanalyse(dictionaryIncrement, entityLog, document, analyzeRequest, importedRedactions);
? sectionFinderService.findSectionsToReanalyse(dictionaryIncrement, entityLog, document, analyzeRequest) //
: analyzeRequest.getSectionsToReanalyse();
} }
@ -393,7 +401,7 @@ public class AnalyzeService {
@Deprecated(forRemoval = true) @Deprecated(forRemoval = true)
private RedactionLog createRedactionLog(AnalyzeRequest analyzeRequest, private RedactionLog createRedactionLog(AnalyzeRequest analyzeRequest,
Document document, Document document,
List<ManualEntity> notFoundManualRedactionEntries, List<PrecursorEntity> notFoundManualRedactionEntries,
Dictionary dictionary, Dictionary dictionary,
KieWrapper wrapper) { KieWrapper wrapper) {
@ -411,12 +419,6 @@ public class AnalyzeService {
wrapper.rulesVersion(), wrapper.rulesVersion(),
legalBasisClient.getVersion(analyzeRequest.getDossierTemplateId())); legalBasisClient.getVersion(analyzeRequest.getDossierTemplateId()));
List<RedactionLogEntry> importedRedactionFilteredEntries = importedRedactionService.processImportedRedactions(analyzeRequest.getDossierTemplateId(),
analyzeRequest.getDossierId(),
analyzeRequest.getFileId(),
redactionLog.getRedactionLogEntry(),
true);
redactionLog.setRedactionLogEntry(importedRedactionFilteredEntries);
return redactionLog; return redactionLog;
} }

13
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/DictionarySearchService.java
View File

@ -1,6 +1,7 @@
package com.iqser.red.service.redaction.v1.server.service; package com.iqser.red.service.redaction.v1.server.service;
import java.util.List; import java.util.List;
import java.util.Set;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
@ -29,6 +30,7 @@ public class DictionarySearchService {
@Observed(name = "DictionarySearchService", contextualName = "add-dictionary-entries") @Observed(name = "DictionarySearchService", contextualName = "add-dictionary-entries")
public void addDictionaryEntities(Dictionary dictionary, List<SemanticNode> semanticNodes) { public void addDictionaryEntities(Dictionary dictionary, List<SemanticNode> semanticNodes) {
semanticNodes.forEach(node -> addDictionaryEntities(dictionary, node)); semanticNodes.forEach(node -> addDictionaryEntities(dictionary, node));
} }
@ -52,11 +54,12 @@ public class DictionarySearchService {
EntityCreationService entityCreationService = new EntityCreationService(entityEnrichmentService); EntityCreationService entityCreationService = new EntityCreationService(entityEnrichmentService);
searchImplementation.getBoundaries(node.getTextBlock(), node.getTextRange()) searchImplementation.getBoundaries(node.getTextBlock(), node.getTextRange())
.stream().filter(boundary -> entityCreationService.isValidEntityTextRange(node.getTextBlock(), boundary)) .stream()
.map(bounds -> entityCreationService.forceByTextRange(bounds, type, entityType, node)) .filter(boundary -> entityCreationService.isValidEntityTextRange(node.getTextBlock(), boundary))
.peek(entity -> entity.setDictionaryEntry(true)) .forEach(bounds -> entityCreationService.byTextRangeWithEngine(bounds, type, entityType, node, Set.of(Engine.DICTIONARY)).ifPresent(entity -> {
.peek(entity -> entity.setDossierDictionaryEntry(isDossierDictionaryEntry)) entity.setDictionaryEntry(true);
.forEach(entity -> entity.addEngine(Engine.DICTIONARY)); entity.setDossierDictionaryEntry(isDossierDictionaryEntry);
}));
} }
} }

3
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityChangeLogService.java
View File

@ -60,9 +60,8 @@ public class EntityChangeLogService {
OffsetDateTime now) { OffsetDateTime now) {
Set<String> existingIds = newEntityLogEntries.stream().map(EntityLogEntry::getId).collect(Collectors.toSet()); Set<String> existingIds = newEntityLogEntries.stream().map(EntityLogEntry::getId).collect(Collectors.toSet());
// no need to check imported redactions because they will be added with the merged changes after this step
List<EntityLogEntry> removedEntries = previousEntityLogEntries.stream() List<EntityLogEntry> removedEntries = previousEntityLogEntries.stream()
.filter(entry -> !entry.getType().equals(ImportedRedactionService.IMPORTED_REDACTION_TYPE) && !existingIds.contains(entry.getId())) .filter(entry -> !existingIds.contains(entry.getId()))
.toList(); .toList();
removedEntries.forEach(entry -> entry.getChanges().add(new Change(analysisNumber, ChangeType.REMOVED, now))); removedEntries.forEach(entry -> entry.getChanges().add(new Change(analysisNumber, ChangeType.REMOVED, now)));
removedEntries.forEach(entry -> entry.setState(EntryState.REMOVED)); removedEntries.forEach(entry -> entry.setState(EntryState.REMOVED));

77
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java
View File

@ -21,7 +21,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.legalbasis.LegalBasis; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.legalbasis.LegalBasis;
import com.iqser.red.service.redaction.v1.server.RedactionServiceSettings; import com.iqser.red.service.redaction.v1.server.RedactionServiceSettings;
import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient; import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient;
import com.iqser.red.service.redaction.v1.server.model.ManualEntity; import com.iqser.red.service.redaction.v1.server.model.PrecursorEntity;
import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryVersion; import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryVersion;
import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType; import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
import com.iqser.red.service.redaction.v1.server.model.document.entity.IEntity; import com.iqser.red.service.redaction.v1.server.model.document.entity.IEntity;
@ -45,7 +45,6 @@ public class EntityLogCreatorService {
DictionaryService dictionaryService; DictionaryService dictionaryService;
ManualChangeFactory manualChangeFactory; ManualChangeFactory manualChangeFactory;
ImportedRedactionService importedRedactionService;
RedactionServiceSettings redactionServiceSettings; RedactionServiceSettings redactionServiceSettings;
LegalBasisClient legalBasisClient; LegalBasisClient legalBasisClient;
EntityChangeLogService entityChangeLogService; EntityChangeLogService entityChangeLogService;
@ -60,11 +59,11 @@ public class EntityLogCreatorService {
public EntityLog createInitialEntityLog(AnalyzeRequest analyzeRequest, public EntityLog createInitialEntityLog(AnalyzeRequest analyzeRequest,
Document document, Document document,
List<ManualEntity> notFoundManualEntities, List<PrecursorEntity> notFoundEntities,
DictionaryVersion dictionaryVersion, DictionaryVersion dictionaryVersion,
long rulesVersion) { long rulesVersion) {
List<EntityLogEntry> entityLogEntries = createEntityLogEntries(document, analyzeRequest.getDossierTemplateId(), notFoundManualEntities); List<EntityLogEntry> entityLogEntries = createEntityLogEntries(document, analyzeRequest, notFoundEntities);
List<LegalBasis> legalBasis = legalBasisClient.getLegalBasisMapping(analyzeRequest.getDossierTemplateId()); List<LegalBasis> legalBasis = legalBasisClient.getLegalBasisMapping(analyzeRequest.getDossierTemplateId());
EntityLog entityLog = new EntityLog(redactionServiceSettings.getAnalysisVersion(), EntityLog entityLog = new EntityLog(redactionServiceSettings.getAnalysisVersion(),
@ -77,12 +76,8 @@ public class EntityLogCreatorService {
legalBasisClient.getVersion(analyzeRequest.getDossierTemplateId())); legalBasisClient.getVersion(analyzeRequest.getDossierTemplateId()));
List<EntityLogEntry> previousExistingEntityLogEntries = getPreviousEntityLogEntries(analyzeRequest.getDossierId(), analyzeRequest.getFileId()); List<EntityLogEntry> previousExistingEntityLogEntries = getPreviousEntityLogEntries(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
// compute changes will be done for the new entries without the imported redactions, since previous imported redactions are present in
// previousExistingEntityLogEntries and have to be ignored in this calculation because the changes will be merged in the next step (processImportedRedactions)
entityChangeLogService.computeChanges(previousExistingEntityLogEntries, entityLogEntries, analyzeRequest.getAnalysisNumber());
// check to see if there are imported redactions and only then recreate them entityChangeLogService.computeChanges(previousExistingEntityLogEntries, entityLogEntries, analyzeRequest.getAnalysisNumber());
importedRedactionService.processImportedRedactions(entityLog, analyzeRequest);
return entityLog; return entityLog;
} }
@ -113,34 +108,33 @@ public class EntityLogCreatorService {
public EntityLogChanges updatePreviousEntityLog(AnalyzeRequest analyzeRequest, public EntityLogChanges updatePreviousEntityLog(AnalyzeRequest analyzeRequest,
Document document, Document document,
List<ManualEntity> notFoundManualRedactionEntries, List<PrecursorEntity> notFoundEntries,
EntityLog previousEntityLog, EntityLog previousEntityLog,
Set<Integer> sectionsToReanalyseIds, Set<Integer> sectionsToReanalyseIds,
DictionaryVersion dictionaryVersion) { DictionaryVersion dictionaryVersion) {
List<EntityLogEntry> newEntityLogEntries = createEntityLogEntries(document, analyzeRequest.getDossierTemplateId(), notFoundManualRedactionEntries).stream() List<EntityLogEntry> newEntityLogEntries = createEntityLogEntries(document, analyzeRequest, notFoundEntries).stream()
.filter(entry -> entry.getContainingNodeId().isEmpty() || sectionsToReanalyseIds.contains(entry.getContainingNodeId().get(0))) .filter(entry -> entry.getContainingNodeId().isEmpty() || sectionsToReanalyseIds.contains(entry.getContainingNodeId().get(0)))
.collect(Collectors.toList()); .collect(Collectors.toList());
Set<String> newEntityIds = newEntityLogEntries.stream().map(EntityLogEntry::getId).collect(Collectors.toSet()); Set<String> newEntityIds = newEntityLogEntries.stream().map(EntityLogEntry::getId).collect(Collectors.toSet());
List<EntityLogEntry> previousEntriesFromReAnalyzedSections = previousEntityLog.getEntityLogEntry() List<EntityLogEntry> previousEntriesFromReAnalyzedSections = previousEntityLog.getEntityLogEntry()
.stream() .stream()
.filter(entry -> !entry.getType().equals(ImportedRedactionService.IMPORTED_REDACTION_TYPE) && (newEntityIds.contains(entry.getId()) || entry.getContainingNodeId() .filter(entry -> (newEntityIds.contains(entry.getId()) || entry.getContainingNodeId().isEmpty() || sectionsToReanalyseIds.contains(entry.getContainingNodeId()
.isEmpty() || sectionsToReanalyseIds.contains(entry.getContainingNodeId().get(0)))) .get(0))))
.toList(); .toList();
previousEntityLog.getEntityLogEntry().removeAll(previousEntriesFromReAnalyzedSections); previousEntityLog.getEntityLogEntry().removeAll(previousEntriesFromReAnalyzedSections);
boolean hasChanges = entityChangeLogService.computeChanges(previousEntriesFromReAnalyzedSections, newEntityLogEntries, analyzeRequest.getAnalysisNumber()); boolean hasChanges = entityChangeLogService.computeChanges(previousEntriesFromReAnalyzedSections, newEntityLogEntries, analyzeRequest.getAnalysisNumber());
previousEntityLog.getEntityLogEntry().addAll(newEntityLogEntries); previousEntityLog.getEntityLogEntry().addAll(newEntityLogEntries);
importedRedactionService.processImportedRedactions(previousEntityLog, analyzeRequest);
return updateVersionsAndReturnChanges(previousEntityLog, dictionaryVersion, analyzeRequest, hasChanges); return updateVersionsAndReturnChanges(previousEntityLog, dictionaryVersion, analyzeRequest, hasChanges);
} }
private List<EntityLogEntry> createEntityLogEntries(Document document, String dossierTemplateId, List<ManualEntity> notFoundManualRedactionEntries) { private List<EntityLogEntry> createEntityLogEntries(Document document, AnalyzeRequest analyzeRequest, List<PrecursorEntity> notFoundPrecursorEntries) {
String dossierTemplateId = analyzeRequest.getDossierTemplateId();
List<EntityLogEntry> entries = new ArrayList<>(); List<EntityLogEntry> entries = new ArrayList<>();
document.getEntities() document.getEntities()
.stream() .stream()
@ -149,7 +143,9 @@ public class EntityLogCreatorService {
.filter(entity -> !entity.removed()) .filter(entity -> !entity.removed())
.forEach(entityNode -> entries.addAll(toEntityLogEntries(entityNode))); .forEach(entityNode -> entries.addAll(toEntityLogEntries(entityNode)));
document.streamAllImages().filter(entity -> !entity.removed()).forEach(imageNode -> entries.add(createEntityLogEntry(imageNode, dossierTemplateId))); document.streamAllImages().filter(entity -> !entity.removed()).forEach(imageNode -> entries.add(createEntityLogEntry(imageNode, dossierTemplateId)));
notFoundManualRedactionEntries.stream().filter(entity -> !entity.removed()).forEach(manualEntity -> entries.add(createEntityLogEntry(manualEntity))); notFoundPrecursorEntries.stream()
.filter(entity -> !entity.removed())
.forEach(precursorEntity -> entries.add(createEntityLogEntry(precursorEntity, dossierTemplateId)));
return entries; return entries;
} }
@ -203,37 +199,40 @@ public class EntityLogCreatorService {
} }
private EntityLogEntry createEntityLogEntry(ManualEntity manualEntity) { private EntityLogEntry createEntityLogEntry(PrecursorEntity precursorEntity, String dossierTemplateId) {
String type = manualEntity.getManualOverwrite().getType().orElse(manualEntity.getType()); String type = precursorEntity.getManualOverwrite().getType().orElse(precursorEntity.getType());
boolean isHint = isHint(manualEntity.getEntityType()); boolean isHint = isHint(precursorEntity.getEntityType());
return EntityLogEntry.builder() return EntityLogEntry.builder()
.id(manualEntity.getId()) .id(precursorEntity.getId())
.reason(manualEntity.buildReasonWithManualChangeDescriptions()) .reason(precursorEntity.buildReasonWithManualChangeDescriptions())
.legalBasis(manualEntity.legalBasis()) .legalBasis(precursorEntity.legalBasis())
.value(manualEntity.value()) .value(precursorEntity.value())
.type(type) .type(type)
.state(buildEntryState(manualEntity)) .state(buildEntryState(precursorEntity))
.entryType(buildEntryType(manualEntity)) .entryType(buildEntryType(precursorEntity))
.section(manualEntity.getManualOverwrite().getSection().orElse(manualEntity.getSection())) .section(precursorEntity.getManualOverwrite().getSection().orElse(precursorEntity.getSection()))
.containingNodeId(Collections.emptyList()) .containingNodeId(Collections.emptyList())
.closestHeadline("") .closestHeadline("")
.matchedRule(manualEntity.getMatchedRule().getRuleIdentifier().toString()) .matchedRule(precursorEntity.getMatchedRule().getRuleIdentifier().toString())
.dictionaryEntry(manualEntity.isDictionaryEntry()) .dictionaryEntry(precursorEntity.isDictionaryEntry())
.dossierDictionaryEntry(manualEntity.isDossierDictionaryEntry()) .dossierDictionaryEntry(precursorEntity.isDossierDictionaryEntry())
.textAfter("") .textAfter("")
.textBefore("") .textBefore("")
.startOffset(-1) .startOffset(-1)
.endOffset(-1) .endOffset(-1)
.positions(manualEntity.getManualOverwrite() .positions(precursorEntity.getManualOverwrite()
.getPositions() .getPositions()
.orElse(manualEntity.getEntityPosition()) .orElse(precursorEntity.getEntityPosition())
.stream() .stream()
.map(entityPosition -> new Position(entityPosition.rectangle2D(), entityPosition.pageNumber())) .map(entityPosition -> new Position(entityPosition.rectangle2D(), entityPosition.pageNumber()))
.toList()) .toList())
.engines(Collections.emptySet()) .engines(precursorEntity.getEngines())
//imported is no longer used, frontend should check engines
//(was .imported(precursorEntity.getEngines() != null && precursorEntity.getEngines().contains(Engine.IMPORTED)))
.imported(false)
.reference(Collections.emptySet()) .reference(Collections.emptySet())
.manualChanges(manualChangeFactory.toManualChangeList(manualEntity.getManualOverwrite().getManualChangeLog(), isHint)) .manualChanges(manualChangeFactory.toManualChangeList(precursorEntity.getManualOverwrite().getManualChangeLog(), isHint))
.build(); .build();
} }
@ -259,6 +258,9 @@ public class EntityLogCreatorService {
.endOffset(entity.getTextRange().end()) .endOffset(entity.getTextRange().end())
.dossierDictionaryEntry(entity.isDossierDictionaryEntry()) .dossierDictionaryEntry(entity.isDossierDictionaryEntry())
.engines(entity.getEngines() != null ? entity.getEngines() : Collections.emptySet()) .engines(entity.getEngines() != null ? entity.getEngines() : Collections.emptySet())
//imported is no longer used, frontend should check engines
//(was .imported(entity.getEngines() != null && entity.getEngines().contains(Engine.IMPORTED)))
.imported(false)
.reference(referenceIds) .reference(referenceIds)
.manualChanges(manualChangeFactory.toManualChangeList(entity.getManualOverwrite().getManualChangeLog(), isHint)) .manualChanges(manualChangeFactory.toManualChangeList(entity.getManualOverwrite().getManualChangeLog(), isHint))
.state(buildEntryState(entity)) .state(buildEntryState(entity))
@ -291,11 +293,8 @@ public class EntityLogCreatorService {
if (entity instanceof TextEntity textEntity) { if (entity instanceof TextEntity textEntity) {
return getEntryType(textEntity.getEntityType()); return getEntryType(textEntity.getEntityType());
} else if (entity instanceof ManualEntity manualEntity) { } else if (entity instanceof PrecursorEntity precursorEntity) {
if (manualEntity.isRectangle()) { return precursorEntity.getEntryType();
return EntryType.AREA;
}
return getEntryType(manualEntity.getEntityType());
} }
throw new UnsupportedOperationException(String.format("Entity subclass %s is not implemented!", entity.getClass())); throw new UnsupportedOperationException(String.format("Entity subclass %s is not implemented!", entity.getClass()));
} }

195
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/ImportedRedactionEntriesMergeService.java
View File

@ -1,195 +0,0 @@
package com.iqser.red.service.redaction.v1.server.service;
import java.time.OffsetDateTime;
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import org.springframework.stereotype.Service;
import com.google.common.base.Strings;
import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Change;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ChangeType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualChange;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.BaseAnnotation;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
// This is retrieved from persistence service and updated to the needs for imported redactions
@Service
@Slf4j
@RequiredArgsConstructor
public class ImportedRedactionEntriesMergeService {
public List<EntityLogEntry> mergeManualChangesForImportedRedactions(List<EntityLogEntry> importedRedactionsEntityLogEntries, AnalyzeRequest analyzeRequest) {
log.debug("Merging manual changes for imported redactions with analyze number: {}", analyzeRequest.getAnalysisNumber());
Set<String> importedRedactionsIds = importedRedactionsEntityLogEntries.stream().map(e -> e.getId()).collect(Collectors.toSet());
List<BaseAnnotation> allManualChangesForImportedRedactions = allManualChangesForImportedRedactions(analyzeRequest.getManualRedactions(), importedRedactionsIds);
if (allManualChangesForImportedRedactions.isEmpty()) {
log.debug("no changes for imported redactions");
// no changes for the imported redactions
return importedRedactionsEntityLogEntries;
}
final int analysisNumber = analyzeRequest.getAnalysisNumber();
// Sort manual changes by date, so we process them in order of when they were requested
allManualChangesForImportedRedactions = allManualChangesForImportedRedactions.stream().sorted(Comparator.comparing(BaseAnnotation::getRequestDate)).toList();
allManualChangesForImportedRedactions.forEach(manualChange -> {
// this is ugly and should be replaced with switch pattern matching https://openjdk.org/jeps/406 -> requires Java 17 (preview) or higher
if (manualChange instanceof IdRemoval idRemoval) {
mergeIdsToRemove(idRemoval, importedRedactionsEntityLogEntries, analysisNumber);
} else if (manualChange instanceof ManualResizeRedaction manualResizeRedaction) {
mergeResizeRedactions(manualResizeRedaction, importedRedactionsEntityLogEntries, analysisNumber);
} else if (manualChange instanceof ManualLegalBasisChange manualLegalBasisChange) {
mergeLegalBasisChanges(manualLegalBasisChange, importedRedactionsEntityLogEntries, analysisNumber);
} else if (manualChange instanceof ManualForceRedaction manualForceRedaction) {
mergeForceRedactions(manualForceRedaction, importedRedactionsEntityLogEntries, analysisNumber);
}
});
return importedRedactionsEntityLogEntries;
}
private List<BaseAnnotation> allManualChangesForImportedRedactions(ManualRedactions manualRedactions, Set<String> importedRedactionsIds) {
log.debug("allManualChangesForImportedRedactions {} manualRedactions resize: {} legalChanges {} removals {} force {}", importedRedactionsIds.size(), manualRedactions.getResizeRedactions().size(), manualRedactions.getLegalBasisChanges().size(),
manualRedactions.getIdsToRemove().size(), manualRedactions.getForceRedactions().size());
List<BaseAnnotation> annotations = Stream.of(manualRedactions.getForceRedactions(),
manualRedactions.getResizeRedactions(),
manualRedactions.getIdsToRemove(),
manualRedactions.getLegalBasisChanges()).flatMap(Collection::stream).map(baseAnnotation -> (BaseAnnotation) baseAnnotation)
.filter(baseAnnotation -> importedRedactionsIds.contains(baseAnnotation.getAnnotationId())).toList();
return annotations;
}
public void mergeIdsToRemove(IdRemoval idRemoval, List<EntityLogEntry> entityLogEntries, int analysisNumber) {
var entity = entityLogEntries.stream().filter(entityLogEntry -> entityLogEntry.getId().equals(idRemoval.getAnnotationId())).findAny();
entity.ifPresent(entityLogEntry -> {
entityLogEntry.setState(EntryState.IGNORED);
addChanges(entityLogEntry.getChanges(), ChangeType.REMOVED, analysisNumber, idRemoval.getRequestDate());
entityLogEntry.getManualChanges()
.add(ManualChange.builder()
.manualRedactionType(ManualRedactionType.REMOVE_LOCALLY)
.requestedDate(idRemoval.getRequestDate())
.processedDate(idRemoval.getProcessedDate() == null ? OffsetDateTime.now() : idRemoval.getProcessedDate())
.userId(idRemoval.getUser())
.build());
});
}
public void mergeResizeRedactions(ManualResizeRedaction manualResizeRedaction, List<EntityLogEntry> entityLogEntries, int analysisNumber) {
var entity = entityLogEntries.stream().filter(entityLogEntry -> entityLogEntry.getId().equals(manualResizeRedaction.getAnnotationId())).findAny();
entity.ifPresent(entityLogEntry -> {
entityLogEntry.setTextAfter(manualResizeRedaction.getTextAfter());
entityLogEntry.setTextBefore(manualResizeRedaction.getTextBefore());
entityLogEntry.setPositions(convertPositions(manualResizeRedaction.getPositions()));
addChanges(entityLogEntry.getChanges(), ChangeType.CHANGED, analysisNumber, manualResizeRedaction.getRequestDate());
ManualChange.ManualChangeBuilder manualChange = ManualChange.builder()
.manualRedactionType(ManualRedactionType.RESIZE)
.requestedDate(manualResizeRedaction.getRequestDate())
.processedDate(manualResizeRedaction.getProcessedDate() == null ? OffsetDateTime.now() : manualResizeRedaction.getProcessedDate())
.userId(manualResizeRedaction.getUser());
entityLogEntry.getManualChanges().add(manualChange.build());
});
}
public void mergeLegalBasisChanges(ManualLegalBasisChange manualLegalBasisChange, List<EntityLogEntry> entityLogEntries, int analysisNumber) {
var entity = entityLogEntries.stream().filter(entityLogEntry -> entityLogEntry.getId().equals(manualLegalBasisChange.getAnnotationId())).findAny();
entity.ifPresent(entityLogEntry -> {
entityLogEntry.setLegalBasis(manualLegalBasisChange.getLegalBasis());
entityLogEntry.setSection(manualLegalBasisChange.getSection());
entityLogEntry.setValue(manualLegalBasisChange.getValue());
addChanges(entityLogEntry.getChanges(), ChangeType.CHANGED, analysisNumber, manualLegalBasisChange.getRequestDate());
Map<String, String> propertyChanges = getPropertyChanges(manualLegalBasisChange);
entityLogEntry.getManualChanges().add(ManualChange.builder()
.manualRedactionType(ManualRedactionType.LEGAL_BASIS_CHANGE)
.requestedDate(manualLegalBasisChange.getRequestDate())
.processedDate(manualLegalBasisChange.getProcessedDate() == null ? OffsetDateTime.now() : manualLegalBasisChange.getProcessedDate())
.propertyChanges(propertyChanges)
.userId(manualLegalBasisChange.getUser())
.build());
});
}
private Map<String, String> getPropertyChanges(ManualLegalBasisChange manualLegalBasisChange) {
Map<String, String> propertyChanges = new HashMap<>();
if (!Strings.isNullOrEmpty(manualLegalBasisChange.getLegalBasis())) {
propertyChanges.put("legalBasis", manualLegalBasisChange.getLegalBasis());
}
if (!Strings.isNullOrEmpty(manualLegalBasisChange.getValue())) {
propertyChanges.put("value", manualLegalBasisChange.getValue());
}
if(!Strings.isNullOrEmpty(manualLegalBasisChange.getSection())) {
propertyChanges.put("section", manualLegalBasisChange.getSection());
}
return propertyChanges;
}
public void mergeForceRedactions(ManualForceRedaction forceRedaction, List<EntityLogEntry> entityLogEntries, int analysisNumber) {
var entity = entityLogEntries.stream().filter(entityLogEntry -> entityLogEntry.getId().equals(forceRedaction.getAnnotationId())).findAny();
entity.ifPresent(entityLogEntry -> {
entityLogEntry.setLegalBasis(forceRedaction.getLegalBasis());
entityLogEntry.setState(entityLogEntry.getEntryType().equals(EntryType.HINT) ? EntryState.SKIPPED : EntryState.APPLIED);
addChanges(entityLogEntry.getChanges(), ChangeType.CHANGED, analysisNumber, forceRedaction.getRequestDate());
var forceRedactManualChange = ManualChange.builder()
.manualRedactionType(ManualRedactionType.FORCE_REDACT)
.requestedDate(forceRedaction.getRequestDate())
.processedDate(forceRedaction.getProcessedDate() == null ? OffsetDateTime.now() : forceRedaction.getProcessedDate())
.userId(forceRedaction.getUser());
if (forceRedaction.getLegalBasis() != null && !forceRedaction.getLegalBasis().isEmpty()) {
forceRedactManualChange.propertyChanges(Map.of("legalBasis", forceRedaction.getLegalBasis()));
}
entityLogEntry.getManualChanges().add(forceRedactManualChange.build());
});
}
private void addChanges(List<Change> changes, ChangeType changeType, int analysisNumber, OffsetDateTime offsetDateTime) {
if (!changes.isEmpty()) {
changes.add(Change.builder()
.analysisNumber(analysisNumber)
.dateTime(offsetDateTime)
.type(changeType)
.build());
} else {
changes.add(Change.builder().analysisNumber(analysisNumber).dateTime(OffsetDateTime.now()).type(changeType).build());
}
}
private List<Position> convertPositions(List<Rectangle> rectangles) {
return rectangles.stream().map(rectangle -> new Position(rectangle.getTopLeftX(), rectangle.getTopLeftY(), rectangle.getWidth(), rectangle.getHeight(), rectangle.getPage())).collect(Collectors.toList());
}
}

339
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/ImportedRedactionService.java
View File

@ -1,339 +0,0 @@
package com.iqser.red.service.redaction.v1.server.service;
import java.time.OffsetDateTime;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Change;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ChangeType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedaction;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedactions;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Point;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
import org.springframework.stereotype.Service;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position;
import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
import io.micrometer.core.annotation.Timed;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@Service
@Slf4j
@RequiredArgsConstructor
public class ImportedRedactionService {
public static final String IMPORTED_REDACTION_TYPE = "imported_redaction";
private final DictionaryService dictionaryService;
private final RedactionStorageService redactionStorageService;
private final ImportedRedactionEntriesMergeService importedRedactionEntriesMergeService;
@Deprecated(forRemoval = true)
@Timed("redactmanager_processImportedRedactions")
public List<RedactionLogEntry> processImportedRedactions(String dossierTemplateId,
String dossierId,
String fileId,
List<RedactionLogEntry> redactionLogEntries,
boolean addImportedRedactions) {
var importedRedactions = redactionStorageService.getImportedRedactions(dossierId, fileId);
if (importedRedactions == null) {
return redactionLogEntries;
}
redactionLogEntries.forEach(redactionLogEntry -> addIntersections(redactionLogEntry, importedRedactions));
if (addImportedRedactions) {
return addImportedRedactionsRedactionLogEntries(dossierTemplateId, redactionLogEntries, importedRedactions);
}
return redactionLogEntries;
}
// Always recreate the imported redactions
@Timed("redactmanager_processImportedRedactions")
public void processImportedRedactions(EntityLog entityLog, AnalyzeRequest analyzeRequest) {
// recreate imported redactions with manual changes
ImportedRedactions importedRedactions = redactionStorageService.getImportedRedactions(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
if (importedRedactions == null || importedRedactions.getImportedRedactions() == null || importedRedactions.getImportedRedactions().isEmpty()) {
log.debug("checkForImportedRedactions - no imported redactions");
return;
}
List<EntityLogEntry> initialImportedRedactionsEntityLogEntries = addImportedRedactionsEntityLogEntries(analyzeRequest.getDossierTemplateId(), importedRedactions);
//merge manual changes for imported redactions
var newEntityLogsForImportedEntities = importedRedactionEntriesMergeService.mergeManualChangesForImportedRedactions(initialImportedRedactionsEntityLogEntries, analyzeRequest);
//remove the old imported redactions
List<EntityLogEntry> previousImportedRedactionsEntries = entityLog.getEntityLogEntry()
.stream()
.filter(entry -> entry.getType().equals(ImportedRedactionService.IMPORTED_REDACTION_TYPE))
.toList();
entityLog.getEntityLogEntry().removeAll(previousImportedRedactionsEntries);
// add all changes to entity log
entityLog.getEntityLogEntry().addAll(newEntityLogsForImportedEntities);
// remove all intersections and update with merged imported redactions
calculateIntersections(newEntityLogsForImportedEntities, entityLog, analyzeRequest.getAnalysisNumber());
}
private void calculateIntersections(List<EntityLogEntry> importedRedactionAndMergedEntries, EntityLog entityLog, int analysisNumber) {
if (!importedRedactionAndMergedEntries.isEmpty()) {
// imported redactions present, intersections must be added with merged imported redactions
Map<Integer, List<EntityLogEntry>> importedRedactionsMap = mapImportedRedactionsOnPage(importedRedactionAndMergedEntries);
entityLog.getEntityLogEntry().stream().filter(entry -> !entry.getType().equals(ImportedRedactionService.IMPORTED_REDACTION_TYPE)).forEach(redactionLogEntry -> {
redactionLogEntry.setImportedRedactionIntersections(new HashSet<>());
addIntersections(redactionLogEntry, importedRedactionsMap, analysisNumber);
});
}
}
@Deprecated(forRemoval = true)
private List<RedactionLogEntry> addImportedRedactionsRedactionLogEntries(String dossierTemplateId,
List<RedactionLogEntry> redactionLogEntries,
ImportedRedactions importedRedactions) {
for (var importedRedactionsValues : importedRedactions.getImportedRedactions().values()) {
for (var importedRedaction : importedRedactionsValues) {
RedactionLogEntry redactionLogEntry = RedactionLogEntry.builder()
.id(importedRedaction.getId())
.type(IMPORTED_REDACTION_TYPE)
.imported(true)
.redacted(true)
.positions(importedRedaction.getPositions().stream().map(this::normalize).collect(Collectors.toList()))
.color(getColor(IMPORTED_REDACTION_TYPE, dossierTemplateId))
.build();
redactionLogEntries.add(redactionLogEntry);
}
}
return redactionLogEntries;
}
private List<EntityLogEntry> addImportedRedactionsEntityLogEntries(String dossierTemplateId, ImportedRedactions importedRedactions) {
List<EntityLogEntry> initialImportedRedactionsEntityLogEntries = new ArrayList<>();
var now = OffsetDateTime.now();
for (var importedRedactionsValues : importedRedactions.getImportedRedactions().values()) {
for (var importedRedaction : importedRedactionsValues) {
EntityLogEntry redactionLogEntry = EntityLogEntry.builder()
.id(importedRedaction.getId())
.type(IMPORTED_REDACTION_TYPE)
.entryType(EntryType.ENTITY)
.value(Optional.ofNullable(importedRedaction.getValue()).orElse(""))
.imported(true)
.state(EntryState.APPLIED)
.positions(importedRedaction.getPositions())
.color(getColor(IMPORTED_REDACTION_TYPE, dossierTemplateId))
.containingNodeId(Collections.emptyList())
.build();
redactionLogEntry.getChanges().add(new Change(1, ChangeType.ADDED, now));
initialImportedRedactionsEntityLogEntries.add(redactionLogEntry);
}
}
return initialImportedRedactionsEntityLogEntries;
}
private List<Position> toPositions(List<Rectangle> positions) {
return positions.stream().map(this::toPosition).toList();
}
private Position toPosition(Rectangle rectangle) {
return new Position(new float[]{rectangle.getTopLeft().getX(), rectangle.getTopLeft().getY(), rectangle.getWidth(), rectangle.getHeight()}, rectangle.getPage());
}
@Deprecated(forRemoval = true)
private void addIntersections(RedactionLogEntry redactionLogEntry, ImportedRedactions importedRedactions) {
for (Rectangle rectangle : redactionLogEntry.getPositions()) {
var normalizedRectangle = normalize(rectangle);
if (importedRedactions.getImportedRedactions().containsKey(rectangle.getPage())) {
var importedRedactionsOnPage = importedRedactions.getImportedRedactions().get(rectangle.getPage());
for (ImportedRedaction importedRedaction : importedRedactionsOnPage) {
for (Position importedRedactionPosition : importedRedaction.getPositions()) {
if (rectOverlap(normalizedRectangle, normalize(importedRedactionPosition))) {
if (redactionLogEntry.getImportedRedactionIntersections() == null) {
redactionLogEntry.setImportedRedactionIntersections(new HashSet<>());
}
redactionLogEntry.getImportedRedactionIntersections().add(importedRedaction.getId());
}
}
}
}
}
}
public Map<Integer, List<EntityLogEntry>> mapImportedRedactionsOnPage(List<EntityLogEntry> importedRedactions) {
Map<Integer, List<EntityLogEntry>> importedRedactionsMap = new HashMap<>();
Set<Integer> pageNumbers = importedRedactions.stream().map(EntityLogEntry::getPositions)
.flatMap(Collection::stream)
.map(com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position::getPageNumber)
.collect(Collectors.toSet());
pageNumbers.forEach(pageNumber ->
importedRedactionsMap.put(pageNumber, importedRedactions.stream().filter(i -> pageNumber == i.getPositions().get(0).getPageNumber()).collect(Collectors.toList())));
return importedRedactionsMap;
}
public void addIntersections(EntityLogEntry entityLogEntry, Map<Integer, List<EntityLogEntry>> importedRedactionsMap, int analysisNumber) {
for (Position rectangle : entityLogEntry.getPositions()) {
var normalizedRectangle = normalize(rectangle);
if (importedRedactionsMap.containsKey(rectangle.getPageNumber())) {
var importedRedactionsOnPage = importedRedactionsMap.get(rectangle.getPageNumber());
for (EntityLogEntry importedRedactionLogEntry : importedRedactionsOnPage) {
for (Position importedRedactionPosition : importedRedactionLogEntry.getPositions()) {
if (rectOverlap(normalizedRectangle, normalize(importedRedactionPosition))) {
if (entityLogEntry.getImportedRedactionIntersections() == null) {
entityLogEntry.setImportedRedactionIntersections(new HashSet<>());
}
entityLogEntry.getImportedRedactionIntersections().add(importedRedactionLogEntry.getId());
if(entityLogEntry.getState() != EntryState.REMOVED) {
entityLogEntry.setState(EntryState.REMOVED);
entityLogEntry.getChanges().add(new Change(analysisNumber, ChangeType.REMOVED, OffsetDateTime.now()));
}
}
}
}
}
}
}
private void addIntersections(EntityLogEntry entityLogEntry, ImportedRedactions importedRedactions) {
for (Position rectangle : entityLogEntry.getPositions()) {
var normalizedRectangle = normalize(rectangle);
if (importedRedactions.getImportedRedactions().containsKey(rectangle.getPageNumber())) {
var importedRedactionsOnPage = importedRedactions.getImportedRedactions().get(rectangle.getPageNumber());
for (ImportedRedaction importedRedaction : importedRedactionsOnPage) {
for (Position importedRedactionPosition : importedRedaction.getPositions()) {
if (rectOverlap(normalizedRectangle, normalize(importedRedactionPosition))) {
if (entityLogEntry.getImportedRedactionIntersections() == null) {
entityLogEntry.setImportedRedactionIntersections(new HashSet<>());
}
entityLogEntry.getImportedRedactionIntersections().add(importedRedaction.getId());
}
}
}
}
}
}
boolean valueInRange(float value, float min, float max) {
return round(value) >= round(min) && round(value) <= round(max);
}
boolean rectOverlap(Rectangle a, Rectangle b) {
boolean xOverlap = valueInRange(a.getTopLeft().getX(), b.getTopLeft().getX(), b.getTopLeft().getX() + b.getWidth()) || valueInRange(b.getTopLeft().getX(),
a.getTopLeft().getX(),
a.getTopLeft().getX() + a.getWidth());
boolean yOverlap = valueInRange(a.getTopLeft().getY(), b.getTopLeft().getY(), b.getTopLeft().getY() + b.getHeight()) || valueInRange(b.getTopLeft().getY(),
a.getTopLeft().getY(),
a.getTopLeft().getY() + a.getHeight());
return xOverlap && yOverlap;
}
@Deprecated(forRemoval = true)
private Rectangle normalize(Rectangle rectangle) {
Rectangle r = new Rectangle();
Point p = new Point();
if (rectangle.getWidth() < 0) {
p.setX(rectangle.getTopLeft().getX() + rectangle.getWidth());
r.setWidth(Math.abs(rectangle.getWidth()));
} else {
p.setX(rectangle.getTopLeft().getX());
r.setWidth(rectangle.getWidth());
}
if (rectangle.getHeight() < 0) {
p.setY(rectangle.getTopLeft().getY() + rectangle.getHeight());
r.setHeight(Math.abs(rectangle.getHeight()));
} else {
p.setY(rectangle.getTopLeft().getY());
r.setHeight(rectangle.getHeight());
}
r.setTopLeft(p);
r.setPage(rectangle.getPage());
return r;
}
private Rectangle normalize(Position position) {
Rectangle r = new Rectangle();
Point p = new Point();
if (position.getRectangle()[2] < 0) {
p.setX(position.getRectangle()[0] + position.getRectangle()[2]);
r.setWidth(Math.abs(position.getRectangle()[2]));
} else {
p.setX(position.getRectangle()[0]);
r.setWidth(position.getRectangle()[2]);
}
if (position.getRectangle()[3] < 0) {
p.setY(position.getRectangle()[1] + position.getRectangle()[3]);
r.setHeight(Math.abs(position.getRectangle()[3]));
} else {
p.setY(position.getRectangle()[1]);
r.setHeight(position.getRectangle()[3]);
}
r.setTopLeft(p);
r.setPage(position.getPageNumber());
return r;
}
private float round(float value) {
double d = Math.pow(10, 0);
return (float) (Math.round(value * d) / d);
}
private float[] getColor(String type, String dossierTemplateId) {
return dictionaryService.getColor(type, dossierTemplateId);
}
}

4
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/ManualChangesApplicationService.java
View File

@ -16,7 +16,7 @@ import org.springframework.stereotype.Service;
import com.google.common.collect.Sets; import com.google.common.collect.Sets;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
import com.iqser.red.service.redaction.v1.server.model.ManualEntity; import com.iqser.red.service.redaction.v1.server.model.PrecursorEntity;
import com.iqser.red.service.redaction.v1.server.model.document.entity.IEntity; import com.iqser.red.service.redaction.v1.server.model.document.entity.IEntity;
import com.iqser.red.service.redaction.v1.server.model.document.entity.PositionOnPage; import com.iqser.red.service.redaction.v1.server.model.document.entity.PositionOnPage;
import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity; import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
@ -79,7 +79,7 @@ public class ManualChangesApplicationService {
.collect(Collectors.toList())); .collect(Collectors.toList()));
SemanticNode node = entityToBeResized.getDeepestFullyContainingNode(); SemanticNode node = entityToBeResized.getDeepestFullyContainingNode();
ManualEntity searchEntity = ManualEntity.fromManualResizeRedaction(manualResizeRedaction); PrecursorEntity searchEntity = PrecursorEntity.fromManualResizeRedaction(manualResizeRedaction);
// Loop through nodes starting from the deepest fully containing node all the way to the document node // Loop through nodes starting from the deepest fully containing node all the way to the document node
while (node != null) { while (node != null) {

102
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/NotFoundImportedEntitiesService.java Normal file
View File

@ -0,0 +1,102 @@
package com.iqser.red.service.redaction.v1.server.service;
import java.time.OffsetDateTime;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Change;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ChangeType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position;
import com.iqser.red.service.redaction.v1.server.model.PrecursorEntity;
import com.iqser.red.service.redaction.v1.server.model.RectangleWithPage;
import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
import org.springframework.stereotype.Service;
import io.micrometer.core.annotation.Timed;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@Service
@Slf4j
@RequiredArgsConstructor
public class NotFoundImportedEntitiesService {
public static final String IMPORTED_REDACTION_TYPE = "imported_redaction";
@Timed("redactmanager_processEntityLog")
public void processEntityLog(EntityLog entityLog, AnalyzeRequest analyzeRequest, List<PrecursorEntity> notFoundEntities) {
// recreate imported redactions with manual changes
if (notFoundEntities == null || notFoundEntities.isEmpty()) {
log.debug("checkForImportedRedactions - no not found imported entities");
return;
}
// remove all intersections and update imported entities
calculateIntersectionsForNotFound(notFoundEntities, entityLog, analyzeRequest.getAnalysisNumber());
}
private void calculateIntersectionsForNotFound(List<PrecursorEntity> notFoundEntities, EntityLog entityLog, int analysisNumber) {
if (!notFoundEntities.isEmpty()) {
// imported redactions present, intersections must be added with merged imported redactions
Map<Integer, List<PrecursorEntity>> importedRedactionsMap = mapImportedRedactionsOnPage(notFoundEntities);
entityLog.getEntityLogEntry().stream().filter(entry -> !entry.getEngines().contains(Engine.IMPORTED)).forEach(redactionLogEntry -> {
redactionLogEntry.setImportedRedactionIntersections(new HashSet<>());
addIntersections(redactionLogEntry, importedRedactionsMap, analysisNumber);
});
}
}
public Map<Integer, List<PrecursorEntity>> mapImportedRedactionsOnPage(List<PrecursorEntity> importedEntities) {
Map<Integer, List<PrecursorEntity>> importedRedactionsMap = new HashMap<>();
Set<Integer> pageNumbers = importedEntities.stream()
.map(PrecursorEntity::getEntityPosition)
.flatMap(Collection::stream)
.map(RectangleWithPage::pageNumber)
.collect(Collectors.toSet());
pageNumbers.forEach(pageNumber -> importedRedactionsMap.put(pageNumber,
importedEntities.stream().filter(i -> pageNumber == i.getEntityPosition().get(0).pageNumber()).collect(Collectors.toList())));
return importedRedactionsMap;
}
public void addIntersections(EntityLogEntry entityLogEntry, Map<Integer, List<PrecursorEntity>> importedEntitiesMap, int analysisNumber) {
for (Position rectangle : entityLogEntry.getPositions()) {
if (importedEntitiesMap.containsKey(rectangle.getPageNumber())) {
var importedEntitiesOnPage = importedEntitiesMap.get(rectangle.getPageNumber());
for (PrecursorEntity precursorEntity : importedEntitiesOnPage) {
for (RectangleWithPage rectangleWithPage : precursorEntity.getEntityPosition()) {
if (rectangle.toRectangle2D().intersects(rectangleWithPage.rectangle2D())) {
if (entityLogEntry.getImportedRedactionIntersections() == null) {
entityLogEntry.setImportedRedactionIntersections(new HashSet<>());
}
entityLogEntry.getImportedRedactionIntersections().add(precursorEntity.getId());
if (entityLogEntry.getState() != EntryState.REMOVED) {
entityLogEntry.setState(EntryState.REMOVED);
entityLogEntry.getChanges().add(new Change(analysisNumber, ChangeType.REMOVED, OffsetDateTime.now()));
}
}
}
}
}
}
}
}

42
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RedactionLogCreatorService.java
View File

@ -20,7 +20,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlo
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Point; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Point;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
import com.iqser.red.service.redaction.v1.server.model.ManualEntity; import com.iqser.red.service.redaction.v1.server.model.PrecursorEntity;
import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType; import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
import com.iqser.red.service.redaction.v1.server.model.document.entity.IEntity; import com.iqser.red.service.redaction.v1.server.model.document.entity.IEntity;
import com.iqser.red.service.redaction.v1.server.model.document.entity.ManualChangeOverwrite; import com.iqser.red.service.redaction.v1.server.model.document.entity.ManualChangeOverwrite;
@ -43,7 +43,7 @@ public class RedactionLogCreatorService {
private final ManualChangeFactory manualChangeFactory; private final ManualChangeFactory manualChangeFactory;
public List<RedactionLogEntry> createRedactionLog(Document document, String dossierTemplateId, List<ManualEntity> notFoundManualRedactionEntries) { public List<RedactionLogEntry> createRedactionLog(Document document, String dossierTemplateId, List<PrecursorEntity> notFoundManualRedactionEntries) {
List<RedactionLogEntry> entries = new ArrayList<>(); List<RedactionLogEntry> entries = new ArrayList<>();
Set<String> processIds = new HashSet<>(); Set<String> processIds = new HashSet<>();
@ -219,40 +219,40 @@ public class RedactionLogCreatorService {
} }
public RedactionLogEntry createRedactionLogEntry(ManualEntity manualEntity, String dossierTemplateId) { public RedactionLogEntry createRedactionLogEntry(PrecursorEntity precursorEntity, String dossierTemplateId) {
String type = manualEntity.getManualOverwrite().getType().orElse(manualEntity.getType()); String type = precursorEntity.getManualOverwrite().getType().orElse(precursorEntity.getType());
boolean isHint = isHint(manualEntity.getEntityType()); boolean isHint = isHint(precursorEntity.getEntityType());
return RedactionLogEntry.builder() return RedactionLogEntry.builder()
.id(manualEntity.getId()) .id(precursorEntity.getId())
.color(getColor(type, dossierTemplateId, manualEntity.applied(), isHint)) .color(getColor(type, dossierTemplateId, precursorEntity.applied(), isHint))
.reason(manualEntity.buildReasonWithManualChangeDescriptions()) .reason(precursorEntity.buildReasonWithManualChangeDescriptions())
.legalBasis(manualEntity.legalBasis()) .legalBasis(precursorEntity.legalBasis())
.value(manualEntity.value()) .value(precursorEntity.value())
.type(type) .type(type)
.redacted(manualEntity.applied()) .redacted(precursorEntity.applied())
.isHint(isHint) .isHint(isHint)
.isRecommendation(manualEntity.getEntityType().equals(EntityType.RECOMMENDATION)) .isRecommendation(precursorEntity.getEntityType().equals(EntityType.RECOMMENDATION))
.isFalsePositive(manualEntity.getEntityType().equals(EntityType.FALSE_POSITIVE) || manualEntity.getEntityType().equals(EntityType.FALSE_RECOMMENDATION)) .isFalsePositive(precursorEntity.getEntityType().equals(EntityType.FALSE_POSITIVE) || precursorEntity.getEntityType().equals(EntityType.FALSE_RECOMMENDATION))
.section(manualEntity.getManualOverwrite().getSection().orElse(manualEntity.getSection())) .section(precursorEntity.getManualOverwrite().getSection().orElse(precursorEntity.getSection()))
.sectionNumber(0) .sectionNumber(0)
.matchedRule(manualEntity.getMatchedRule().getRuleIdentifier().toString()) .matchedRule(precursorEntity.getMatchedRule().getRuleIdentifier().toString())
.rectangle(manualEntity.isRectangle()) .rectangle(precursorEntity.isRectangle())
.isDictionaryEntry(manualEntity.isDictionaryEntry()) .isDictionaryEntry(precursorEntity.isDictionaryEntry())
.isDossierDictionaryEntry(manualEntity.isDossierDictionaryEntry()) .isDossierDictionaryEntry(precursorEntity.isDossierDictionaryEntry())
.textAfter("") .textAfter("")
.textBefore("") .textBefore("")
.startOffset(-1) .startOffset(-1)
.endOffset(-1) .endOffset(-1)
.positions(manualEntity.getManualOverwrite() .positions(precursorEntity.getManualOverwrite()
.getPositions() .getPositions()
.orElse(manualEntity.getEntityPosition()) .orElse(precursorEntity.getEntityPosition())
.stream() .stream()
.map(entityPosition -> toRedactionLogRectangle(entityPosition.rectangle2D(), entityPosition.pageNumber())) .map(entityPosition -> toRedactionLogRectangle(entityPosition.rectangle2D(), entityPosition.pageNumber()))
.toList()) .toList())
.engines(Collections.emptySet()) .engines(Collections.emptySet())
.reference(Collections.emptySet()) .reference(Collections.emptySet())
.manualChanges(mapManualChanges(manualEntity.getManualOverwrite(), isHint)) .manualChanges(mapManualChanges(precursorEntity.getManualOverwrite(), isHint))
.build(); .build();
} }

62
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/UnprocessedChangesService.java
View File

@ -21,17 +21,14 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
import com.iqser.red.service.redaction.v1.model.AnalyzeResponse; import com.iqser.red.service.redaction.v1.model.AnalyzeResponse;
import com.iqser.red.service.redaction.v1.model.QueueNames; import com.iqser.red.service.redaction.v1.model.QueueNames;
import com.iqser.red.service.redaction.v1.model.UnprocessedManualEntity; import com.iqser.red.service.redaction.v1.model.UnprocessedManualEntity;
import com.iqser.red.service.redaction.v1.server.model.ManualEntity; import com.iqser.red.service.redaction.v1.server.model.PrecursorEntity;
import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
import com.iqser.red.service.redaction.v1.server.model.document.entity.PositionOnPage;
import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity; import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
import com.iqser.red.service.redaction.v1.server.service.document.DocumentGraphMapper; import com.iqser.red.service.redaction.v1.server.service.document.DocumentGraphMapper;
import com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService; import com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService;
import com.iqser.red.service.redaction.v1.server.service.document.EntityEnrichmentService; import com.iqser.red.service.redaction.v1.server.service.document.EntityEnrichmentService;
import com.iqser.red.service.redaction.v1.server.service.document.EntityFindingUtility; import com.iqser.red.service.redaction.v1.server.service.document.EntityFindingUtility;
import com.iqser.red.service.redaction.v1.server.service.document.ManualEntityCreationService; import com.iqser.red.service.redaction.v1.server.service.document.EntityFromPrecursorCreationService;
import com.iqser.red.service.redaction.v1.server.storage.ObservedStorageService; import com.iqser.red.service.redaction.v1.server.storage.ObservedStorageService;
import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService; import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
@ -55,7 +52,7 @@ public class UnprocessedChangesService {
final EntityFindingUtility entityFindingUtility; final EntityFindingUtility entityFindingUtility;
final RedactionStorageService redactionStorageService; final RedactionStorageService redactionStorageService;
final EntityEnrichmentService entityEnrichmentService; final EntityEnrichmentService entityEnrichmentService;
final ManualEntityCreationService manualEntityCreationService; final EntityFromPrecursorCreationService entityFromPrecursorCreationService;
final DictionaryService dictionaryService; final DictionaryService dictionaryService;
final ManualChangesApplicationService manualChangesApplicationService; final ManualChangesApplicationService manualChangesApplicationService;
@ -84,18 +81,22 @@ public class UnprocessedChangesService {
allAnnotationIds.addAll(resizeIds); allAnnotationIds.addAll(resizeIds);
List<ManualResizeRedaction> manualResizeRedactions = analyzeRequest.getManualRedactions().getResizeRedactions().stream().toList(); List<ManualResizeRedaction> manualResizeRedactions = analyzeRequest.getManualRedactions().getResizeRedactions().stream().toList();
List<ManualEntity> manualEntitiesToBeResized = previousEntityLog.getEntityLogEntry() List<PrecursorEntity> manualEntitiesToBeResized = previousEntityLog.getEntityLogEntry()
.stream().filter(entityLogEntry -> resizeIds.contains(entityLogEntry.getId())).toList() .stream()
.stream().map(ManualEntity::fromEntityLogEntry).toList(); .filter(entityLogEntry -> resizeIds.contains(entityLogEntry.getId()))
.toList()
.stream()
.map(PrecursorEntity::fromEntityLogEntry)
.toList();
if (!manualResizeRedactions.isEmpty()) { if (!manualResizeRedactions.isEmpty()) {
processManualResizeRedactions(document, manualEntitiesToBeResized, unprocessedManualEntities, manualResizeRedactions); processManualResizeRedactions(document, manualEntitiesToBeResized, unprocessedManualEntities, manualResizeRedactions);
} }
List<ManualEntity> notFoundManualEntities = new ArrayList<>(); List<PrecursorEntity> notFoundManualEntities = new ArrayList<>();
List<ManualEntity> manualEntities = manualEntitiesConverter(analyzeRequest.getManualRedactions(), analyzeRequest.getDossierTemplateId()); List<PrecursorEntity> manualEntities = manualEntitiesConverter(analyzeRequest.getManualRedactions(), analyzeRequest.getDossierTemplateId());
if (!manualEntities.isEmpty()) { if (!manualEntities.isEmpty()) {
notFoundManualEntities = manualEntityCreationService.toTextEntity(manualEntities, document); notFoundManualEntities = entityFromPrecursorCreationService.toTextEntity(manualEntities, document);
} }
document.getEntities().forEach(textEntity -> { document.getEntities().forEach(textEntity -> {
@ -127,24 +128,24 @@ public class UnprocessedChangesService {
private void processManualResizeRedactions(Document document, private void processManualResizeRedactions(Document document,
List<ManualEntity> manualEntities, List<PrecursorEntity> manualEntities,
List<UnprocessedManualEntity> unprocessedManualEntities, List<UnprocessedManualEntity> unprocessedManualEntities,
List<ManualResizeRedaction> manualResizeRedactions) { List<ManualResizeRedaction> manualResizeRedactions) {
Map<String, List<TextEntity>> tempEntities = entityFindingUtility.findAllPossibleEntitiesAndGroupByValue(document, manualEntities); Map<String, List<TextEntity>> tempEntities = entityFindingUtility.findAllPossibleEntitiesAndGroupByValue(document, manualEntities);
for (ManualEntity manualEntity : manualEntities) { for (PrecursorEntity precursorEntity : manualEntities) {
Optional<TextEntity> optionalTextEntity = entityFindingUtility.findClosestEntityAndReturnEmptyIfNotFound(manualEntity, tempEntities, THRESHOLD); Optional<TextEntity> optionalTextEntity = entityFindingUtility.findClosestEntityAndReturnEmptyIfNotFound(precursorEntity, tempEntities, THRESHOLD);
if (optionalTextEntity.isEmpty()) { if (optionalTextEntity.isEmpty()) {
unprocessedManualEntities.add(builDefaultUnprocessedManualEntity(manualEntity)); unprocessedManualEntities.add(builDefaultUnprocessedManualEntity(precursorEntity));
continue; continue;
} }
TextEntity correctEntity = createCorrectEntity(manualEntity, optionalTextEntity.get()); TextEntity correctEntity = createCorrectEntity(precursorEntity, optionalTextEntity.get());
Optional<ManualResizeRedaction> optionalManualResizeRedaction = manualResizeRedactions.stream() Optional<ManualResizeRedaction> optionalManualResizeRedaction = manualResizeRedactions.stream()
.filter(manualResizeRedaction -> manualResizeRedaction.getAnnotationId().equals(manualEntity.getId())) .filter(manualResizeRedaction -> manualResizeRedaction.getAnnotationId().equals(precursorEntity.getId()))
.findFirst(); .findFirst();
if (optionalManualResizeRedaction.isPresent()) { if (optionalManualResizeRedaction.isPresent()) {
ManualResizeRedaction manualResizeRedaction = optionalManualResizeRedaction.get(); ManualResizeRedaction manualResizeRedaction = optionalManualResizeRedaction.get();
@ -156,7 +157,6 @@ public class UnprocessedChangesService {
correctEntity.removeFromGraph(); correctEntity.removeFromGraph();
} }
} }
} }
// remove all temp entities from the graph // remove all temp entities from the graph
@ -164,9 +164,9 @@ public class UnprocessedChangesService {
} }
private TextEntity createCorrectEntity(ManualEntity manualEntity, TextEntity closestEntity) { private TextEntity createCorrectEntity(PrecursorEntity precursorEntity, TextEntity closestEntity) {
TextEntity correctEntity = TextEntity.initialEntityNode(closestEntity.getTextRange(), manualEntity.type(), manualEntity.getEntityType(), manualEntity.getId()); TextEntity correctEntity = TextEntity.initialEntityNode(closestEntity.getTextRange(), precursorEntity.type(), precursorEntity.getEntityType(), precursorEntity.getId());
correctEntity.setDeepestFullyContainingNode(closestEntity.getDeepestFullyContainingNode()); correctEntity.setDeepestFullyContainingNode(closestEntity.getDeepestFullyContainingNode());
correctEntity.setIntersectingNodes(new ArrayList<>(closestEntity.getIntersectingNodes())); correctEntity.setIntersectingNodes(new ArrayList<>(closestEntity.getIntersectingNodes()));
@ -180,25 +180,25 @@ public class UnprocessedChangesService {
correctEntity.getIntersectingNodes().forEach(n -> n.getEntities().add(correctEntity)); correctEntity.getIntersectingNodes().forEach(n -> n.getEntities().add(correctEntity));
correctEntity.getPages().forEach(page -> page.getEntities().add(correctEntity)); correctEntity.getPages().forEach(page -> page.getEntities().add(correctEntity));
correctEntity.addMatchedRules(manualEntity.getMatchedRuleList()); correctEntity.addMatchedRules(precursorEntity.getMatchedRuleList());
correctEntity.setDictionaryEntry(manualEntity.isDictionaryEntry()); correctEntity.setDictionaryEntry(precursorEntity.isDictionaryEntry());
correctEntity.setDossierDictionaryEntry(manualEntity.isDossierDictionaryEntry()); correctEntity.setDossierDictionaryEntry(precursorEntity.isDossierDictionaryEntry());
correctEntity.getManualOverwrite().addChanges(manualEntity.getManualOverwrite().getManualChangeLog()); correctEntity.getManualOverwrite().addChanges(precursorEntity.getManualOverwrite().getManualChangeLog());
return correctEntity; return correctEntity;
} }
private UnprocessedManualEntity builDefaultUnprocessedManualEntity(ManualEntity manualEntity) { private UnprocessedManualEntity builDefaultUnprocessedManualEntity(PrecursorEntity precursorEntity) {
return UnprocessedManualEntity.builder() return UnprocessedManualEntity.builder()
.annotationId(manualEntity.getId()) .annotationId(precursorEntity.getId())
.textAfter("") .textAfter("")
.textBefore("") .textBefore("")
.section("") .section("")
.positions(manualEntity.getManualOverwrite() .positions(precursorEntity.getManualOverwrite()
.getPositions() .getPositions()
.orElse(manualEntity.getEntityPosition()) .orElse(precursorEntity.getEntityPosition())
.stream() .stream()
.map(entityPosition -> new Position(entityPosition.rectangle2D(), entityPosition.pageNumber())) .map(entityPosition -> new Position(entityPosition.rectangle2D(), entityPosition.pageNumber()))
.toList()) .toList())
@ -206,12 +206,12 @@ public class UnprocessedChangesService {
} }
private List<ManualEntity> manualEntitiesConverter(ManualRedactions manualRedactions, String dossierTemplateId) { private List<PrecursorEntity> manualEntitiesConverter(ManualRedactions manualRedactions, String dossierTemplateId) {
return manualRedactions.getEntriesToAdd() return manualRedactions.getEntriesToAdd()
.stream() .stream()
.filter(manualRedactionEntry -> manualRedactionEntry.getPositions() != null && !manualRedactionEntry.getPositions().isEmpty()) .filter(manualRedactionEntry -> manualRedactionEntry.getPositions() != null && !manualRedactionEntry.getPositions().isEmpty())
.map(manualRedactionEntry -> ManualEntity.fromManualRedactionEntry(manualRedactionEntry, .map(manualRedactionEntry -> PrecursorEntity.fromManualRedactionEntry(manualRedactionEntry,
dictionaryService.isHint(manualRedactionEntry.getType(), dossierTemplateId))) dictionaryService.isHint(manualRedactionEntry.getType(), dossierTemplateId)))
.toList(); .toList();
} }

26
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityFindingUtility.java
View File

@ -19,7 +19,7 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import com.iqser.red.service.redaction.v1.server.model.ClosestEntity; import com.iqser.red.service.redaction.v1.server.model.ClosestEntity;
import com.iqser.red.service.redaction.v1.server.model.ManualEntity; import com.iqser.red.service.redaction.v1.server.model.PrecursorEntity;
import com.iqser.red.service.redaction.v1.server.model.RectangleWithPage; import com.iqser.red.service.redaction.v1.server.model.RectangleWithPage;
import com.iqser.red.service.redaction.v1.server.model.dictionary.SearchImplementation; import com.iqser.red.service.redaction.v1.server.model.dictionary.SearchImplementation;
import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType; import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
@ -47,35 +47,35 @@ public class EntityFindingUtility {
} }
public Optional<TextEntity> findClosestEntityAndReturnEmptyIfNotFound(ManualEntity manualEntity, Map<String, List<TextEntity>> entitiesWithSameValue, double matchThreshold) { public Optional<TextEntity> findClosestEntityAndReturnEmptyIfNotFound(PrecursorEntity precursorEntity, Map<String, List<TextEntity>> entitiesWithSameValue, double matchThreshold) {
if (manualEntity.getValue() == null) { if (precursorEntity.getValue() == null) {
return Optional.empty(); return Optional.empty();
} }
List<TextEntity> possibleEntities = entitiesWithSameValue.get(manualEntity.getValue().toLowerCase(Locale.ENGLISH)); List<TextEntity> possibleEntities = entitiesWithSameValue.get(precursorEntity.getValue().toLowerCase(Locale.ENGLISH));
if (entityIdentifierValueNotFound(possibleEntities)) { if (entityIdentifierValueNotFound(possibleEntities)) {
log.warn("Entity could not be created with manualEntity: {}, due to the value {} not being found anywhere.", manualEntity, manualEntity.getValue()); log.warn("Entity could not be created with precursorEntity: {}, due to the value {} not being found anywhere.", precursorEntity, precursorEntity.getValue());
return Optional.empty(); return Optional.empty();
} }
Optional<ClosestEntity> optionalClosestEntity = possibleEntities.stream() Optional<ClosestEntity> optionalClosestEntity = possibleEntities.stream()
.filter(entity -> pagesMatch(entity, manualEntity.getEntityPosition())) .filter(entity -> pagesMatch(entity, precursorEntity.getEntityPosition()))
.map(entity -> ClosestEntity.builder().distance(calculateMinDistance(manualEntity.getEntityPosition(), entity)).textEntity(entity).build()) .map(entity -> ClosestEntity.builder().distance(calculateMinDistance(precursorEntity.getEntityPosition(), entity)).textEntity(entity).build())
.min(Comparator.comparingDouble(ClosestEntity::getDistance)); .min(Comparator.comparingDouble(ClosestEntity::getDistance));
if (optionalClosestEntity.isEmpty()) { if (optionalClosestEntity.isEmpty()) {
log.warn("No Entity with value {} found on page {}", manualEntity.getValue(), manualEntity.getEntityPosition()); log.warn("No Entity with value {} found on page {}", precursorEntity.getValue(), precursorEntity.getEntityPosition());
return Optional.empty(); return Optional.empty();
} }
ClosestEntity closestEntity = optionalClosestEntity.get(); ClosestEntity closestEntity = optionalClosestEntity.get();
if (closestEntity.getDistance() > matchThreshold) { if (closestEntity.getDistance() > matchThreshold) {
log.warn("For entity {} on page {} with positions {} distance to closest found entity is {} and therefore higher than the threshold of {}", log.warn("For entity {} on page {} with positions {} distance to closest found entity is {} and therefore higher than the threshold of {}",
manualEntity.getValue(), precursorEntity.getValue(),
manualEntity.getEntityPosition().get(0).pageNumber(), precursorEntity.getEntityPosition().get(0).pageNumber(),
manualEntity.getEntityPosition().stream().map(RectangleWithPage::rectangle2D).toList(), precursorEntity.getEntityPosition().stream().map(RectangleWithPage::rectangle2D).toList(),
closestEntity.getDistance(), closestEntity.getDistance(),
matchThreshold); matchThreshold);
return Optional.empty(); return Optional.empty();
@ -150,10 +150,10 @@ public class EntityFindingUtility {
} }
public Map<String, List<TextEntity>> findAllPossibleEntitiesAndGroupByValue(SemanticNode node, List<ManualEntity> manualEntities) { public Map<String, List<TextEntity>> findAllPossibleEntitiesAndGroupByValue(SemanticNode node, List<PrecursorEntity> manualEntities) {
Set<Integer> pageNumbers = manualEntities.stream().flatMap(entry -> entry.getEntityPosition().stream().map(RectangleWithPage::pageNumber)).collect(Collectors.toSet()); Set<Integer> pageNumbers = manualEntities.stream().flatMap(entry -> entry.getEntityPosition().stream().map(RectangleWithPage::pageNumber)).collect(Collectors.toSet());
Set<String> entryValues = manualEntities.stream().map(ManualEntity::getValue).filter(Objects::nonNull).map(String::toLowerCase).collect(Collectors.toSet()); Set<String> entryValues = manualEntities.stream().map(PrecursorEntity::getValue).filter(Objects::nonNull).map(String::toLowerCase).collect(Collectors.toSet());
if (!pageNumbers.stream().allMatch(node::onPage)) { if (!pageNumbers.stream().allMatch(node::onPage)) {
throw new IllegalArgumentException(format("SemanticNode \"%s\" does not contain these pages %s, it has pages: %s", throw new IllegalArgumentException(format("SemanticNode \"%s\" does not contain these pages %s, it has pages: %s",

62
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/ManualEntityCreationService.java → redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityFromPrecursorCreationService.java
View File

@ -3,22 +3,21 @@ package com.iqser.red.service.redaction.v1.server.service.document;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Collection; import java.util.Collection;
import java.util.HashSet; import java.util.HashSet;
import java.util.LinkedList;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Optional; import java.util.Optional;
import java.util.Set; import java.util.Set;
import java.util.stream.Collectors;
import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedactions;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.BaseAnnotation; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.BaseAnnotation;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;
import com.iqser.red.service.redaction.v1.server.model.ManualEntity; import com.iqser.red.service.redaction.v1.server.model.PrecursorEntity;
import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType; import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
import com.iqser.red.service.redaction.v1.server.model.document.entity.PositionOnPage;
import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity; import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode; import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
import com.iqser.red.service.redaction.v1.server.service.DictionaryService; import com.iqser.red.service.redaction.v1.server.service.DictionaryService;
@ -30,7 +29,7 @@ import lombok.extern.slf4j.Slf4j;
@Slf4j @Slf4j
@Service @Service
@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE) @FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
public class ManualEntityCreationService { public class EntityFromPrecursorCreationService {
static double MATCH_THRESHOLD = 10; // Is compared to the average sum of distances in pdf coordinates for each corner of the bounding box of the entities static double MATCH_THRESHOLD = 10; // Is compared to the average sum of distances in pdf coordinates for each corner of the bounding box of the entities
EntityFindingUtility entityFindingUtility; EntityFindingUtility entityFindingUtility;
@ -39,7 +38,7 @@ public class ManualEntityCreationService {
@Autowired @Autowired
public ManualEntityCreationService(EntityEnrichmentService entityEnrichmentService, DictionaryService dictionaryService, EntityFindingUtility entityFindingUtility) { public EntityFromPrecursorCreationService(EntityEnrichmentService entityEnrichmentService, DictionaryService dictionaryService, EntityFindingUtility entityFindingUtility) {
this.entityFindingUtility = entityFindingUtility; this.entityFindingUtility = entityFindingUtility;
entityCreationService = new EntityCreationService(entityEnrichmentService); entityCreationService = new EntityCreationService(entityEnrichmentService);
@ -47,10 +46,10 @@ public class ManualEntityCreationService {
} }
public List<ManualEntity> createRedactionEntitiesIfFoundAndReturnNotFoundEntries(ManualRedactions manualRedactions, SemanticNode node, String dossierTemplateId) { public List<PrecursorEntity> createEntitiesIfFoundAndReturnNotFoundEntries(ManualRedactions manualRedactions, SemanticNode node, String dossierTemplateId) {
Set<IdRemoval> idRemovals = manualRedactions.getIdsToRemove(); Set<IdRemoval> idRemovals = manualRedactions.getIdsToRemove();
List<ManualEntity> manualEntities = manualRedactions.getEntriesToAdd() List<PrecursorEntity> manualEntities = manualRedactions.getEntriesToAdd()
.stream() .stream()
.filter(manualRedactionEntry -> !(idRemovals.stream() .filter(manualRedactionEntry -> !(idRemovals.stream()
.map(BaseAnnotation::getAnnotationId) .map(BaseAnnotation::getAnnotationId)
@ -62,7 +61,7 @@ public class ManualEntityCreationService {
.get() .get()
.getRequestDate()))) .getRequestDate())))
.filter(manualRedactionEntry -> !(manualRedactionEntry.isAddToDictionary() || manualRedactionEntry.isAddToDossierDictionary())) .filter(manualRedactionEntry -> !(manualRedactionEntry.isAddToDictionary() || manualRedactionEntry.isAddToDossierDictionary()))
.map(manualRedactionEntry -> ManualEntity.fromManualRedactionEntry(manualRedactionEntry, .map(manualRedactionEntry -> PrecursorEntity.fromManualRedactionEntry(manualRedactionEntry,
dictionaryService.isHint(manualRedactionEntry.getType(), dossierTemplateId))) dictionaryService.isHint(manualRedactionEntry.getType(), dossierTemplateId)))
.peek(manualEntity -> { .peek(manualEntity -> {
if (manualEntity.getEntityType().equals(EntityType.HINT)) { if (manualEntity.getEntityType().equals(EntityType.HINT)) {
@ -77,34 +76,46 @@ public class ManualEntityCreationService {
} }
public List<ManualEntity> toTextEntity(List<ManualEntity> manualEntities, SemanticNode node) { public List<PrecursorEntity> createEntitiesIfFoundAndReturnNotFoundEntries(ImportedRedactions importedRedactions, SemanticNode node) {
Map<String, List<TextEntity>> tempEntitiesByValue = entityFindingUtility.findAllPossibleEntitiesAndGroupByValue(node, manualEntities); List<PrecursorEntity> importedEntities = importedRedactions.getImportedRedactions()
.stream()
.map(PrecursorEntity::fromImportedEntry)
.peek(importedEntity -> importedEntity.apply("IMP.0.0", importedEntity.getReason(), importedEntity.getLegalBasis()))
.toList();
//we could add image handling here as well and create not only text entities
return toTextEntity(importedEntities, node);
}
List<ManualEntity> notFoundManualEntities = new LinkedList<>();
for (ManualEntity manualEntity : manualEntities) { public List<PrecursorEntity> toTextEntity(List<PrecursorEntity> precursorEntities, SemanticNode node) {
Optional<TextEntity> optionalClosestEntity = entityFindingUtility.findClosestEntityAndReturnEmptyIfNotFound(manualEntity, tempEntitiesByValue, MATCH_THRESHOLD);
var notFoundEntities = precursorEntities.stream().filter(PrecursorEntity::isRectangle).collect(Collectors.toList());
var findableEntities = precursorEntities.stream().filter(precursorEntity -> !precursorEntity.isRectangle()).toList();
Map<String, List<TextEntity>> tempEntitiesByValue = entityFindingUtility.findAllPossibleEntitiesAndGroupByValue(node, findableEntities);
for (PrecursorEntity precursorEntity : findableEntities) {
Optional<TextEntity> optionalClosestEntity = entityFindingUtility.findClosestEntityAndReturnEmptyIfNotFound(precursorEntity, tempEntitiesByValue, MATCH_THRESHOLD);
if (optionalClosestEntity.isEmpty()) { if (optionalClosestEntity.isEmpty()) {
notFoundManualEntities.add(manualEntity); notFoundEntities.add(precursorEntity);
continue; continue;
} }
createCorrectEntity(manualEntity, optionalClosestEntity.get()); createCorrectEntity(precursorEntity, optionalClosestEntity.get());
} }
tempEntitiesByValue.values().stream().flatMap(Collection::stream).forEach(TextEntity::removeFromGraph); tempEntitiesByValue.values().stream().flatMap(Collection::stream).forEach(TextEntity::removeFromGraph);
return notFoundManualEntities; return notFoundEntities;
} }
/** /**
* Deletes the temp Entity and creates a RedactionEntity with correct values, based on the given parameters. * Deletes the temp Entity and creates a RedactionEntity with correct values, based on the given parameters.
* *
* @param manualEntity The entity identifier for the RedactionEntity. * @param precursorEntity The entity identifier for the RedactionEntity.
* @param closestEntity The closest Boundary to the RedactionEntity. * @param closestEntity The closest Boundary to the RedactionEntity.
*/ */
private void createCorrectEntity(ManualEntity manualEntity, TextEntity closestEntity) { private void createCorrectEntity(PrecursorEntity precursorEntity, TextEntity closestEntity) {
TextEntity correctEntity = TextEntity.initialEntityNode(closestEntity.getTextRange(), manualEntity.type(), manualEntity.getEntityType(), manualEntity.getId());
TextEntity correctEntity = TextEntity.initialEntityNode(closestEntity.getTextRange(), precursorEntity.type(), precursorEntity.getEntityType(), precursorEntity.getId());
correctEntity.setDeepestFullyContainingNode(closestEntity.getDeepestFullyContainingNode()); correctEntity.setDeepestFullyContainingNode(closestEntity.getDeepestFullyContainingNode());
correctEntity.setIntersectingNodes(new ArrayList<>(closestEntity.getIntersectingNodes())); correctEntity.setIntersectingNodes(new ArrayList<>(closestEntity.getIntersectingNodes()));
correctEntity.setDuplicateTextRanges(new ArrayList<>(closestEntity.getDuplicateTextRanges())); correctEntity.setDuplicateTextRanges(new ArrayList<>(closestEntity.getDuplicateTextRanges()));
@ -117,10 +128,11 @@ public class ManualEntityCreationService {
correctEntity.getIntersectingNodes().forEach(n -> n.getEntities().add(correctEntity)); correctEntity.getIntersectingNodes().forEach(n -> n.getEntities().add(correctEntity));
correctEntity.getPages().forEach(page -> page.getEntities().add(correctEntity)); correctEntity.getPages().forEach(page -> page.getEntities().add(correctEntity));
correctEntity.addMatchedRules(manualEntity.getMatchedRuleList()); correctEntity.addMatchedRules(precursorEntity.getMatchedRuleList());
correctEntity.setDictionaryEntry(manualEntity.isDictionaryEntry()); correctEntity.setDictionaryEntry(precursorEntity.isDictionaryEntry());
correctEntity.setDossierDictionaryEntry(manualEntity.isDossierDictionaryEntry()); correctEntity.setDossierDictionaryEntry(precursorEntity.isDossierDictionaryEntry());
correctEntity.getManualOverwrite().addChanges(manualEntity.getManualOverwrite().getManualChangeLog()); correctEntity.getManualOverwrite().addChanges(precursorEntity.getManualOverwrite().getManualChangeLog());
correctEntity.addEngines(precursorEntity.getEngines());
} }
} }

60
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/ImportedRedactionEntryService.java Normal file
View File

@ -0,0 +1,60 @@
package com.iqser.red.service.redaction.v1.server.service.document;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.stream.Stream;
import org.springframework.stereotype.Service;
import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedactions;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.BaseAnnotation;
import com.iqser.red.service.redaction.v1.server.model.PrecursorEntity;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
import io.micrometer.observation.annotation.Observed;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@Service
@RequiredArgsConstructor
public class ImportedRedactionEntryService {
private final EntityFromPrecursorCreationService entityFromPrecursorCreationService;
@Observed(name = "ImportedRedactionEntryService", contextualName = "add-imported-redaction-entries")
public List<PrecursorEntity> addImportedEntriesAndReturnNotFoundEntries(AnalyzeRequest analyzeRequest, ImportedRedactions importedRedactions, Document document) {
List<PrecursorEntity> notFoundImportedEntries = Collections.emptyList();
if (!importedRedactions.getImportedRedactions().isEmpty()) {
notFoundImportedEntries = entityFromPrecursorCreationService.createEntitiesIfFoundAndReturnNotFoundEntries(importedRedactions, document);
log.info("Added Imported entries for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
}
if (notFoundImportedEntries.isEmpty()) {
return Collections.emptyList();
}
List<BaseAnnotation> manualChanges = allManualChangesExceptAdd(analyzeRequest.getManualRedactions());
for (PrecursorEntity notFoundManualRedactionEntry : notFoundImportedEntries) {
manualChanges.stream()
.filter(change -> change.getAnnotationId().equals(notFoundManualRedactionEntry.getId()))
.forEach(change -> notFoundManualRedactionEntry.getManualOverwrite().addChange(change));
}
return notFoundImportedEntries;
}
private List<BaseAnnotation> allManualChangesExceptAdd(ManualRedactions manualRedactions) {
return Stream.of(manualRedactions.getForceRedactions(),
manualRedactions.getResizeRedactions(),
manualRedactions.getRecategorizations(),
manualRedactions.getIdsToRemove(),
manualRedactions.getLegalBasisChanges()).flatMap(Collection::stream).map(baseAnnotation -> (BaseAnnotation) baseAnnotation).toList();
}
}

13
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/ManualRedactionEntryService.java
View File

@ -10,8 +10,8 @@ import org.springframework.stereotype.Service;
import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest; import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.BaseAnnotation; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.BaseAnnotation;
import com.iqser.red.service.redaction.v1.server.model.ManualEntity;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
import com.iqser.red.service.redaction.v1.server.model.PrecursorEntity;
import io.micrometer.observation.annotation.Observed; import io.micrometer.observation.annotation.Observed;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
@ -22,23 +22,24 @@ import lombok.extern.slf4j.Slf4j;
@RequiredArgsConstructor @RequiredArgsConstructor
public class ManualRedactionEntryService { public class ManualRedactionEntryService {
private final ManualEntityCreationService manualEntityCreationService; private final EntityFromPrecursorCreationService entityFromPrecursorCreationService;
@Observed(name = "ManualRedactionEntryService", contextualName = "add-manual-redaction-entries") @Observed(name = "ManualRedactionEntryService", contextualName = "add-manual-redaction-entries")
public List<ManualEntity> addManualRedactionEntriesAndReturnNotFoundEntries(AnalyzeRequest analyzeRequest, Document document, String dossierTemplateId) { public List<PrecursorEntity> addManualRedactionEntriesAndReturnNotFoundEntries(AnalyzeRequest analyzeRequest, Document document, String dossierTemplateId) {
List<ManualEntity> notFoundManualRedactionEntries = Collections.emptyList(); List<PrecursorEntity> notFoundManualRedactionEntries = Collections.emptyList();
if (analyzeRequest.getManualRedactions() != null) { if (analyzeRequest.getManualRedactions() != null) {
notFoundManualRedactionEntries = manualEntityCreationService.createRedactionEntitiesIfFoundAndReturnNotFoundEntries(analyzeRequest.getManualRedactions(), notFoundManualRedactionEntries = entityFromPrecursorCreationService.createEntitiesIfFoundAndReturnNotFoundEntries(analyzeRequest.getManualRedactions(),
document, dossierTemplateId); document, dossierTemplateId);
log.info("Added Manual redaction entries for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); log.info("Added Manual redaction entries for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
} }
if (notFoundManualRedactionEntries.isEmpty()) { if (notFoundManualRedactionEntries.isEmpty()) {
return Collections.emptyList(); return Collections.emptyList();
} }
List<BaseAnnotation> manualChanges = allManualChangesExceptAdd(analyzeRequest.getManualRedactions()); List<BaseAnnotation> manualChanges = allManualChangesExceptAdd(analyzeRequest.getManualRedactions());
for (ManualEntity notFoundManualRedactionEntry : notFoundManualRedactionEntries) { for (PrecursorEntity notFoundManualRedactionEntry : notFoundManualRedactionEntries) {
manualChanges.stream() manualChanges.stream()
.filter(change -> change.getAnnotationId().equals(notFoundManualRedactionEntry.getId())) .filter(change -> change.getAnnotationId().equals(notFoundManualRedactionEntry.getId()))
.forEach(change -> notFoundManualRedactionEntry.getManualOverwrite().addChange(change)); .forEach(change -> notFoundManualRedactionEntry.getManualOverwrite().addChange(change));

26
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/SectionFinderService.java
View File

@ -13,6 +13,9 @@ import org.springframework.stereotype.Service;
import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest; import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedaction;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedactions;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;
@ -40,7 +43,11 @@ import lombok.extern.slf4j.Slf4j;
public class SectionFinderService { public class SectionFinderService {
@Timed("redactmanager_findSectionsToReanalyse") @Timed("redactmanager_findSectionsToReanalyse")
public Set<Integer> findSectionsToReanalyse(DictionaryIncrement dictionaryIncrement, EntityLog entityLog, Document document, AnalyzeRequest analyzeRequest) { public Set<Integer> findSectionsToReanalyse(DictionaryIncrement dictionaryIncrement,
EntityLog entityLog,
Document document,
AnalyzeRequest analyzeRequest,
ImportedRedactions importedRedactions) {
long start = System.currentTimeMillis(); long start = System.currentTimeMillis();
Set<String> relevantManuallyModifiedAnnotationIds = getRelevantManuallyModifiedAnnotationIds(analyzeRequest.getManualRedactions()); Set<String> relevantManuallyModifiedAnnotationIds = getRelevantManuallyModifiedAnnotationIds(analyzeRequest.getManualRedactions());
@ -64,6 +71,7 @@ public class SectionFinderService {
}); });
Set<Integer> relevantPagesForReanalysis = getRelevantPageNumbersForAddRedactions(analyzeRequest); Set<Integer> relevantPagesForReanalysis = getRelevantPageNumbersForAddRedactions(analyzeRequest);
relevantPagesForReanalysis.addAll(getRelevantPageNumbersForImportedRedactions(importedRedactions, relevantManuallyModifiedAnnotationIds));
if (!relevantPagesForReanalysis.isEmpty()) { if (!relevantPagesForReanalysis.isEmpty()) {
sectionsToReanalyse.addAll(getSectionNumbersOnPages(document, relevantPagesForReanalysis)); sectionsToReanalyse.addAll(getSectionNumbersOnPages(document, relevantPagesForReanalysis));
} }
@ -102,6 +110,22 @@ public class SectionFinderService {
} }
private static Set<Integer> getRelevantPageNumbersForImportedRedactions(ImportedRedactions importedRedactions, Set<String> idsWithUnprocessedManualChanges) {
if (importedRedactions == null || importedRedactions.getImportedRedactions() == null) {
return Collections.emptySet();
}
return importedRedactions.getImportedRedactions()
.stream()
.filter(importedRedaction -> idsWithUnprocessedManualChanges.contains(importedRedaction.getId()))
.map(ImportedRedaction::getPositions)
.flatMap(Collection::stream)
.map(Position::getPageNumber)
.collect(Collectors.toSet());
}
private static Set<String> getRelevantManuallyModifiedAnnotationIds(ManualRedactions manualRedactions) { private static Set<String> getRelevantManuallyModifiedAnnotationIds(ManualRedactions manualRedactions) {
if (manualRedactions == null) { if (manualRedactions == null) {

20
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/RedactionStorageService.java
View File

@ -3,6 +3,7 @@ package com.iqser.red.service.redaction.v1.server.storage;
import java.io.File; import java.io.File;
import java.io.FileInputStream; import java.io.FileInputStream;
import java.io.InputStream; import java.io.InputStream;
import java.util.List;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import org.springframework.cache.annotation.Cacheable; import org.springframework.cache.annotation.Cacheable;
@ -11,6 +12,7 @@ import org.springframework.stereotype.Service;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentLog; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentLog;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedactions; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedactions;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedactionsPerPage;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog;
import com.iqser.red.service.redaction.v1.server.client.model.NerEntitiesModel; import com.iqser.red.service.redaction.v1.server.client.model.NerEntitiesModel;
@ -76,10 +78,26 @@ public class RedactionStorageService {
@Timed("redactmanager_getImportedRedactions") @Timed("redactmanager_getImportedRedactions")
public ImportedRedactions getImportedRedactions(String dossierId, String fileId) { public ImportedRedactions getImportedRedactions(String dossierId, String fileId) {
try {
ImportedRedactionsPerPage importedRedactionsPerPage = storageService.readJSONObject(TenantContext.getTenantId(),
StorageIdUtils.getStorageId(dossierId, fileId, FileType.IMPORTED_REDACTIONS),
ImportedRedactionsPerPage.class);
return new ImportedRedactions(importedRedactionsPerPage.getImportedRedactions().values().stream().flatMap(List::stream).collect(Collectors.toList()));
} catch (StorageObjectDoesNotExist e) {
log.debug("Imported redactions not available.");
return new ImportedRedactions();
}
}
@Timed("redactmanager_getImportedRedactions")
public ImportedRedactionsPerPage getImportedRedactionsPerPage(String dossierId, String fileId) {
try { try {
return storageService.readJSONObject(TenantContext.getTenantId(), return storageService.readJSONObject(TenantContext.getTenantId(),
StorageIdUtils.getStorageId(dossierId, fileId, FileType.IMPORTED_REDACTIONS), StorageIdUtils.getStorageId(dossierId, fileId, FileType.IMPORTED_REDACTIONS),
ImportedRedactions.class); ImportedRedactionsPerPage.class);
} catch (StorageObjectDoesNotExist e) { } catch (StorageObjectDoesNotExist e) {
log.debug("Imported redactions not available."); log.debug("Imported redactions not available.");
return null; return null;

6
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/entity/TextEntityTest.java
View File

@ -5,7 +5,7 @@ import static org.junit.jupiter.api.Assertions.assertThrows;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
import com.iqser.red.service.redaction.v1.server.model.ManualEntity; import com.iqser.red.service.redaction.v1.server.model.PrecursorEntity;
import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType; import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
public class TextEntityTest { public class TextEntityTest {
@ -13,7 +13,7 @@ public class TextEntityTest {
@Test @Test
public void testMatchedRule() { public void testMatchedRule() {
ManualEntity entity = ManualEntity.builder() PrecursorEntity entity = PrecursorEntity.builder()
.type("PII") .type("PII")
.entityType(EntityType.ENTITY) .entityType(EntityType.ENTITY)
.build(); .build();
@ -31,7 +31,7 @@ public class TextEntityTest {
@Test @Test
public void testMatchedRuleWithNonsense() { public void testMatchedRuleWithNonsense() {
ManualEntity entity = ManualEntity.builder() PrecursorEntity entity = PrecursorEntity.builder()
.type("PII") .type("PII")
.entityType(EntityType.ENTITY) .entityType(EntityType.ENTITY)
.build(); .build();

14
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualEntityTest.java → redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/PrecursorEntityTest.java
View File

@ -25,7 +25,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Point; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Point;
import com.iqser.red.service.redaction.v1.server.document.graph.BuildDocumentIntegrationTest; import com.iqser.red.service.redaction.v1.server.document.graph.BuildDocumentIntegrationTest;
import com.iqser.red.service.redaction.v1.server.model.ManualEntity; import com.iqser.red.service.redaction.v1.server.model.PrecursorEntity;
import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryVersion; import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryVersion;
import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType; import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
import com.iqser.red.service.redaction.v1.server.model.document.entity.IEntity; import com.iqser.red.service.redaction.v1.server.model.document.entity.IEntity;
@ -35,17 +35,17 @@ import com.iqser.red.service.redaction.v1.server.service.DictionaryService;
import com.iqser.red.service.redaction.v1.server.service.EntityLogCreatorService; import com.iqser.red.service.redaction.v1.server.service.EntityLogCreatorService;
import com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService; import com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService;
import com.iqser.red.service.redaction.v1.server.service.document.EntityEnrichmentService; import com.iqser.red.service.redaction.v1.server.service.document.EntityEnrichmentService;
import com.iqser.red.service.redaction.v1.server.service.document.ManualEntityCreationService; import com.iqser.red.service.redaction.v1.server.service.document.EntityFromPrecursorCreationService;
import lombok.SneakyThrows; import lombok.SneakyThrows;
public class ManualEntityTest extends BuildDocumentIntegrationTest { public class PrecursorEntityTest extends BuildDocumentIntegrationTest {
@Autowired @Autowired
private EntityEnrichmentService entityEnrichmentService; private EntityEnrichmentService entityEnrichmentService;
@Autowired @Autowired
private ManualEntityCreationService manualEntityCreationService; private EntityFromPrecursorCreationService entityFromPrecursorCreationService;
@Autowired @Autowired
private EntityLogCreatorService entityLogCreatorService; private EntityLogCreatorService entityLogCreatorService;
@ -122,7 +122,7 @@ public class ManualEntityTest extends BuildDocumentIntegrationTest {
assertTrue(document.getEntities().isEmpty()); assertTrue(document.getEntities().isEmpty());
List<ManualEntity> notFoundManualEntities = manualEntityCreationService.createRedactionEntitiesIfFoundAndReturnNotFoundEntries(ManualRedactions.builder().entriesToAdd(Set.of(manualRedactionEntry)).build(), List<PrecursorEntity> notFoundManualEntities = entityFromPrecursorCreationService.createEntitiesIfFoundAndReturnNotFoundEntries(ManualRedactions.builder().entriesToAdd(Set.of(manualRedactionEntry)).build(),
document, document,
TEST_DOSSIER_TEMPLATE_ID); TEST_DOSSIER_TEMPLATE_ID);
assertEquals(1, notFoundManualEntities.size()); assertEquals(1, notFoundManualEntities.size());
@ -162,7 +162,7 @@ public class ManualEntityTest extends BuildDocumentIntegrationTest {
.reason("reason") .reason("reason")
.legalBasis("n-a") .legalBasis("n-a")
.section(tempEntity.getDeepestFullyContainingNode().toString()) .section(tempEntity.getDeepestFullyContainingNode().toString())
.rectangle(true) .rectangle(false)
.positions(positions) .positions(positions)
.requestDate(OffsetDateTime.now()) .requestDate(OffsetDateTime.now())
.textAfter("") .textAfter("")
@ -172,7 +172,7 @@ public class ManualEntityTest extends BuildDocumentIntegrationTest {
tempEntity.removeFromGraph(); tempEntity.removeFromGraph();
assertTrue(document.getEntities().isEmpty()); assertTrue(document.getEntities().isEmpty());
List<ManualEntity> notFoundManualEntities = manualEntityCreationService.createRedactionEntitiesIfFoundAndReturnNotFoundEntries(ManualRedactions.builder().entriesToAdd(Set.of(manualRedactionEntry)).build(), List<PrecursorEntity> notFoundManualEntities = entityFromPrecursorCreationService.createEntitiesIfFoundAndReturnNotFoundEntries(ManualRedactions.builder().entriesToAdd(Set.of(manualRedactionEntry)).build(),
document, document,
TEST_DOSSIER_TEMPLATE_ID); TEST_DOSSIER_TEMPLATE_ID);
assertTrue(notFoundManualEntities.isEmpty()); assertTrue(notFoundManualEntities.isEmpty());

23
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/acceptance_rules.drl
View File

@ -990,6 +990,29 @@ rule "X.6.1: remove Entity of higher rank, when intersected by entity of type EN
end end
// Rule unit: X.8
rule "X.8.0: Remove Entity when textRange is equal to imported Entity"
salience 257
when
$entity: TextEntity($type: type, engines contains Engine.IMPORTED, active())
$other: TextEntity(getTextRange().equals($entity.getTextRange()), this != $entity, engines not contains Engine.IMPORTED)
then
$other.remove("X.8.0", "remove Entity when equal to imported Entity");
$entity.addEngines($other.getEngines());
retract($other);
end
rule "X.8.1: Remove Entity when intersected by imported Entity"
salience 256
when
$entity: TextEntity($type: type, engines contains Engine.IMPORTED, active())
$other: TextEntity(intersects($entity), this != $entity, engines not contains Engine.IMPORTED)
then
$other.remove("X.8.1", "remove Entity when intersected by imported Entity");
retract($other);
end
//------------------------------------ File attributes rules ------------------------------------ //------------------------------------ File attributes rules ------------------------------------
// Rule unit: FA.1 // Rule unit: FA.1

23
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/all_redact_manager_rules.drl
View File

@ -1560,6 +1560,29 @@ rule "X.6.1: remove Entity of higher rank, when intersected by entity of type EN
end end
// Rule unit: X.8
rule "X.8.0: Remove Entity when textRange is equal to imported Entity"
salience 257
when
$entity: TextEntity($type: type, engines contains Engine.IMPORTED, active())
$other: TextEntity(getTextRange().equals($entity.getTextRange()), this != $entity, engines not contains Engine.IMPORTED)
then
$other.remove("X.8.0", "remove Entity when equal to imported Entity");
$entity.addEngines($other.getEngines());
retract($other);
end
rule "X.8.1: Remove Entity when intersected by imported Entity"
salience 256
when
$entity: TextEntity($type: type, engines contains Engine.IMPORTED, active())
$other: TextEntity(intersects($entity), this != $entity, engines not contains Engine.IMPORTED)
then
$other.remove("X.8.1", "remove Entity when intersected by imported Entity");
retract($other);
end
//------------------------------------ File attributes rules ------------------------------------ //------------------------------------ File attributes rules ------------------------------------
// Rule unit: FA.1 // Rule unit: FA.1

23
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora.drl
View File

@ -1391,6 +1391,29 @@ rule "X.7.0: Remove all images"
end end
// Rule unit: X.8
rule "X.8.0: Remove Entity when textRange is equal to imported Entity"
salience 257
when
$entity: TextEntity($type: type, engines contains Engine.IMPORTED, active())
$other: TextEntity(getTextRange().equals($entity.getTextRange()), this != $entity, engines not contains Engine.IMPORTED)
then
$other.remove("X.8.0", "remove Entity when equal to imported Entity");
$entity.addEngines($other.getEngines());
retract($other);
end
rule "X.8.1: Remove Entity when intersected by imported Entity"
salience 256
when
$entity: TextEntity($type: type, engines contains Engine.IMPORTED, active())
$other: TextEntity(intersects($entity), this != $entity, engines not contains Engine.IMPORTED)
then
$other.remove("X.8.1", "remove Entity when intersected by imported Entity");
retract($other);
end
//------------------------------------ File attributes rules ------------------------------------ //------------------------------------ File attributes rules ------------------------------------
// Rule unit: FA.1 // Rule unit: FA.1

23
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/manual_redaction_rules.drl
View File

@ -232,6 +232,29 @@ rule "X.5.1: Remove Entity of type RECOMMENDATION when contained by RECOMMENDATI
end end
// Rule unit: X.8
rule "X.8.0: Remove Entity when textRange is equal to imported Entity"
salience 257
when
$entity: TextEntity($type: type, engines contains Engine.IMPORTED, active())
$other: TextEntity(getTextRange().equals($entity.getTextRange()), this != $entity, engines not contains Engine.IMPORTED)
then
$other.remove("X.8.0", "remove Entity when equal to imported Entity");
$entity.addEngines($other.getEngines());
retract($other);
end
rule "X.8.1: Remove Entity when intersected by imported Entity"
salience 256
when
$entity: TextEntity($type: type, engines contains Engine.IMPORTED, active())
$other: TextEntity(intersects($entity), this != $entity, engines not contains Engine.IMPORTED)
then
$other.remove("X.8.1", "remove Entity when intersected by imported Entity");
retract($other);
end
//------------------------------------ Local dictionary search rules ------------------------------------ //------------------------------------ Local dictionary search rules ------------------------------------
// Rule unit: LDS.0 // Rule unit: LDS.0

23
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl
View File

@ -1150,6 +1150,29 @@ rule "X.6.1: remove Entity of higher rank, when intersected by entity of type EN
end end
// Rule unit: X.8
rule "X.8.0: Remove Entity when textRange is equal to imported Entity"
salience 257
when
$entity: TextEntity($type: type, engines contains Engine.IMPORTED, active())
$other: TextEntity(getTextRange().equals($entity.getTextRange()), this != $entity, engines not contains Engine.IMPORTED)
then
$other.remove("X.8.0", "remove Entity when equal to imported Entity");
$entity.addEngines($other.getEngines());
retract($other);
end
rule "X.8.1: Remove Entity when intersected by imported Entity"
salience 256
when
$entity: TextEntity($type: type, engines contains Engine.IMPORTED, active())
$other: TextEntity(intersects($entity), this != $entity, engines not contains Engine.IMPORTED)
then
$other.remove("X.8.1", "remove Entity when intersected by imported Entity");
retract($other);
end
//------------------------------------ File attributes rules ------------------------------------ //------------------------------------ File attributes rules ------------------------------------
// Rule unit: FA.1 // Rule unit: FA.1

23
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_v2.drl
View File

@ -356,6 +356,29 @@ rule "X.6.1: remove Entity of higher rank, when intersected by entity of type EN
end end
// Rule unit: X.8
rule "X.8.0: Remove Entity when textRange is equal to imported Entity"
salience 257
when
$entity: TextEntity($type: type, engines contains Engine.IMPORTED, active())
$other: TextEntity(getTextRange().equals($entity.getTextRange()), this != $entity, engines not contains Engine.IMPORTED)
then
$other.remove("X.8.0", "remove Entity when equal to imported Entity");
$entity.addEngines($other.getEngines());
retract($other);
end
rule "X.8.1: Remove Entity when intersected by imported Entity"
salience 256
when
$entity: TextEntity($type: type, engines contains Engine.IMPORTED, active())
$other: TextEntity(intersects($entity), this != $entity, engines not contains Engine.IMPORTED)
then
$other.remove("X.8.1", "remove Entity when intersected by imported Entity");
retract($other);
end
//------------------------------------ File attributes rules ------------------------------------ //------------------------------------ File attributes rules ------------------------------------
// Rule unit: FA.1 // Rule unit: FA.1

23
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/table_demo.drl
View File

@ -477,6 +477,29 @@ rule "X.7.0: Remove all images"
end end
// Rule unit: X.8
rule "X.8.0: Remove Entity when textRange is equal to imported Entity"
salience 257
when
$entity: TextEntity($type: type, engines contains Engine.IMPORTED, active())
$other: TextEntity(getTextRange().equals($entity.getTextRange()), this != $entity, engines not contains Engine.IMPORTED)
then
$other.remove("X.8.0", "remove Entity when equal to imported Entity");
$entity.addEngines($other.getEngines());
retract($other);
end
rule "X.8.1: Remove Entity when intersected by imported Entity"
salience 256
when
$entity: TextEntity($type: type, engines contains Engine.IMPORTED, active())
$other: TextEntity(intersects($entity), this != $entity, engines not contains Engine.IMPORTED)
then
$other.remove("X.8.1", "remove Entity when intersected by imported Entity");
retract($other);
end
//------------------------------------ File attributes rules ------------------------------------ //------------------------------------ File attributes rules ------------------------------------
// Rule unit: FA.1 // Rule unit: FA.1

23
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/test_rules.drl
View File

@ -393,6 +393,29 @@ rule "X.7.0: Remove all images"
end end
// Rule unit: X.8
rule "X.8.0: Remove Entity when textRange is equal to imported Entity"
salience 257
when
$entity: TextEntity($type: type, engines contains Engine.IMPORTED, active())
$other: TextEntity(getTextRange().equals($entity.getTextRange()), this != $entity, engines not contains Engine.IMPORTED)
then
$other.remove("X.8.0", "remove Entity when equal to imported Entity");
$entity.addEngines($other.getEngines());
retract($other);
end
rule "X.8.1: Remove Entity when intersected by imported Entity"
salience 256
when
$entity: TextEntity($type: type, engines contains Engine.IMPORTED, active())
$other: TextEntity(intersects($entity), this != $entity, engines not contains Engine.IMPORTED)
then
$other.remove("X.8.1", "remove Entity when intersected by imported Entity");
retract($other);
end
//------------------------------------ File attributes rules ------------------------------------ //------------------------------------ File attributes rules ------------------------------------
// Rule unit: FA.1 // Rule unit: FA.1

23
redaction-service-v1/rules-management/src/main/resources/all_redact_manager_rules.drl
View File

@ -1584,6 +1584,29 @@ rule "X.6.1: remove Entity of higher rank, when intersected by entity of type EN
end end
// Rule unit: X.8
rule "X.8.0: Remove Entity when textRange is equal to imported Entity"
salience 257
when
$entity: TextEntity($type: type, engines contains Engine.IMPORTED, active())
$other: TextEntity(getTextRange().equals($entity.getTextRange()), this != $entity, engines not contains Engine.IMPORTED)
then
$other.remove("X.8.0", "remove Entity when equal to imported Entity");
$entity.addEngines($other.getEngines());
retract($other);
end
rule "X.8.1: Remove Entity when intersected by imported Entity"
salience 256
when
$entity: TextEntity($type: type, engines contains Engine.IMPORTED, active())
$other: TextEntity(intersects($entity), this != $entity, engines not contains Engine.IMPORTED)
then
$other.remove("X.8.1", "remove Entity when intersected by imported Entity");
retract($other);
end
//------------------------------------ File attributes rules ------------------------------------ //------------------------------------ File attributes rules ------------------------------------
// Rule unit: FA.1 // Rule unit: FA.1

22
redaction-service-v1/rules-management/src/main/resources/all_rules_documine.drl
View File

@ -1539,6 +1539,28 @@ rule "X.7.0: Remove all images"
end end
// Rule unit: X.8
rule "X.8.0: Remove Entity when textRange is equal to imported Entity"
salience 257
when
$entity: TextEntity($type: type, engines contains Engine.IMPORTED, active())
$other: TextEntity(getTextRange().equals($entity.getTextRange()), this != $entity, engines not contains Engine.IMPORTED)
then
$other.remove("X.8.0", "remove Entity when equal to imported Entity");
$entity.addEngines($other.getEngines());
retract($other);
end
rule "X.8.1: Remove Entity when intersected by imported Entity"
salience 256
when
$entity: TextEntity($type: type, engines contains Engine.IMPORTED, active())
$other: TextEntity(intersects($entity), this != $entity, engines not contains Engine.IMPORTED)
then
$other.remove("X.8.1", "remove Entity when intersected by imported Entity");
retract($other);
end
//------------------------------------ File attributes rules ------------------------------------ //------------------------------------ File attributes rules ------------------------------------
// Rule unit: FA.1 // Rule unit: FA.1

1
redaction-service-v1/rules-management/src/main/resources/default_rule_identifiers.txt
View File

@ -7,5 +7,6 @@ X.4.0
X.5.0 X.5.0
X.5.1 X.5.1
X.6.* X.6.*
X.8.*
FA.*.* FA.*.*
LDS.*.* LDS.*.*

1
redaction-service-v1/rules-management/src/main/resources/default_rule_identifiers_dm.txt
View File

@ -6,5 +6,6 @@ X.4.0
X.5.0 X.5.0
X.5.1 X.5.1
X.7.0 X.7.0
X.8.*
FA.*.* FA.*.*
LDS.*.* LDS.*.*