diff --git a/redaction-service-v1/redaction-service-server-v1/build.gradle.kts b/redaction-service-v1/redaction-service-server-v1/build.gradle.kts index d9c0485d..1ef837b8 100644 --- a/redaction-service-v1/redaction-service-server-v1/build.gradle.kts +++ b/redaction-service-v1/redaction-service-server-v1/build.gradle.kts @@ -16,7 +16,7 @@ val layoutParserVersion = "0.86.0" val jacksonVersion = "2.15.2" val droolsVersion = "9.44.0.Final" val pdfBoxVersion = "3.0.0" -val persistenceServiceVersion = "2.324.0" +val persistenceServiceVersion = "2.326.0" val springBootStarterVersion = "3.1.5" configurations { diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/migration/RedactionLogToEntityLogMigrationService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/migration/RedactionLogToEntityLogMigrationService.java index 25219f29..e1635cfe 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/migration/RedactionLogToEntityLogMigrationService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/migration/RedactionLogToEntityLogMigrationService.java @@ -16,13 +16,12 @@ import org.springframework.stereotype.Service; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogLegalBasis; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.migration.MigratedIds; -import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ManualChange; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ManualRedactionType; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogLegalBasis; -import com.iqser.red.service.redaction.v1.server.model.ManualEntity; +import com.iqser.red.service.redaction.v1.server.model.PrecursorEntity; import com.iqser.red.service.redaction.v1.server.model.MigratedEntityLog; import com.iqser.red.service.redaction.v1.server.model.MigrationEntity; import com.iqser.red.service.redaction.v1.server.model.RectangleWithPage; @@ -186,46 +185,47 @@ public class RedactionLogToEntityLogMigrationService { .filter(redactionLogEntry -> !redactionLogEntry.isImage()) .map(entry -> MigrationEntity.fromRedactionLogEntry(entry, dictionaryService.isHint(entry.getType(), dossierTemplateId))) .peek(migrationEntity -> { - if (migrationEntity.getManualEntity().getEntityType().equals(EntityType.HINT) &&// + if (migrationEntity.getPrecursorEntity().getEntityType().equals(EntityType.HINT) &&// !migrationEntity.getRedactionLogEntry().isHint() &&// !migrationEntity.getRedactionLogEntry().isRedacted()) { - migrationEntity.getManualEntity().ignore(migrationEntity.getManualEntity().getRuleIdentifier(), migrationEntity.getRedactionLogEntry().getReason()); + migrationEntity.getPrecursorEntity().ignore(migrationEntity.getPrecursorEntity().getRuleIdentifier(), migrationEntity.getRedactionLogEntry().getReason()); } else if (migrationEntity.getRedactionLogEntry().lastChangeIsRemoved()) { - migrationEntity.getManualEntity().remove(migrationEntity.getManualEntity().getRuleIdentifier(), migrationEntity.getRedactionLogEntry().getReason()); + migrationEntity.getPrecursorEntity().remove(migrationEntity.getPrecursorEntity().getRuleIdentifier(), migrationEntity.getRedactionLogEntry().getReason()); } else if (lastManualChangeIsRemove(migrationEntity)) { - migrationEntity.getManualEntity().ignore(migrationEntity.getManualEntity().getRuleIdentifier(), migrationEntity.getManualEntity().getReason()); - } else if (migrationEntity.getManualEntity().isApplied() && migrationEntity.getRedactionLogEntry().isRecommendation()) { - migrationEntity.getManualEntity().skip(migrationEntity.getManualEntity().getRuleIdentifier(), migrationEntity.getManualEntity().getReason()); - } else if (migrationEntity.getManualEntity().isApplied()) { - migrationEntity.getManualEntity() - .apply(migrationEntity.getManualEntity().getRuleIdentifier(), - migrationEntity.getManualEntity().getReason(), - migrationEntity.getManualEntity().getLegalBasis()); + migrationEntity.getPrecursorEntity().ignore(migrationEntity.getPrecursorEntity().getRuleIdentifier(), migrationEntity.getPrecursorEntity().getReason()); + } else if (migrationEntity.getPrecursorEntity().isApplied() && migrationEntity.getRedactionLogEntry().isRecommendation()) { + migrationEntity.getPrecursorEntity().skip(migrationEntity.getPrecursorEntity().getRuleIdentifier(), migrationEntity.getPrecursorEntity().getReason()); + } else if (migrationEntity.getPrecursorEntity().isApplied()) { + migrationEntity.getPrecursorEntity() + .apply(migrationEntity.getPrecursorEntity().getRuleIdentifier(), + migrationEntity.getPrecursorEntity().getReason(), + migrationEntity.getPrecursorEntity().getLegalBasis()); } else { - migrationEntity.getManualEntity().skip(migrationEntity.getManualEntity().getRuleIdentifier(), migrationEntity.getManualEntity().getReason()); + migrationEntity.getPrecursorEntity().skip(migrationEntity.getPrecursorEntity().getRuleIdentifier(), migrationEntity.getPrecursorEntity().getReason()); } }) .toList(); Map> tempEntitiesByValue = entityFindingUtility.findAllPossibleEntitiesAndGroupByValue(document, - entitiesToMigrate.stream().map(MigrationEntity::getManualEntity).toList()); + entitiesToMigrate.stream().map(MigrationEntity::getPrecursorEntity).toList()); for (MigrationEntity migrationEntity : entitiesToMigrate) { - Optional optionalTextEntity = entityFindingUtility.findClosestEntityAndReturnEmptyIfNotFound(migrationEntity.getManualEntity(), + Optional optionalTextEntity = entityFindingUtility.findClosestEntityAndReturnEmptyIfNotFound(migrationEntity.getPrecursorEntity(), tempEntitiesByValue, MATCH_THRESHOLD); if (optionalTextEntity.isEmpty()) { - migrationEntity.setMigratedEntity(migrationEntity.getManualEntity()); - migrationEntity.setOldId(migrationEntity.getManualEntity().getId()); - migrationEntity.setNewId(migrationEntity.getManualEntity().getId()); + migrationEntity.setMigratedEntity(migrationEntity.getPrecursorEntity()); + migrationEntity.setOldId(migrationEntity.getPrecursorEntity().getId()); + migrationEntity.setNewId(migrationEntity.getPrecursorEntity().getId()); continue; } - TextEntity entity = createCorrectEntity(migrationEntity.getManualEntity(), document, optionalTextEntity.get().getTextRange()); + TextEntity entity = createCorrectEntity(migrationEntity.getPrecursorEntity(), document, optionalTextEntity.get().getTextRange()); migrationEntity.setMigratedEntity(entity); - migrationEntity.setOldId(migrationEntity.getManualEntity().getId()); + migrationEntity.setOldId(migrationEntity.getPrecursorEntity().getId()); migrationEntity.setNewId(entity.getPositionsOnPagePerPage().get(0).getId()); // Can only be on one page, since redactionLogEntries can only be on one page + } tempEntitiesByValue.values().stream().flatMap(Collection::stream).forEach(TextEntity::removeFromGraph); @@ -248,15 +248,15 @@ public class RedactionLogToEntityLogMigrationService { } - private TextEntity createCorrectEntity(ManualEntity manualEntity, SemanticNode node, TextRange closestTextRange) { + private TextEntity createCorrectEntity(PrecursorEntity precursorEntity, SemanticNode node, TextRange closestTextRange) { EntityCreationService entityCreationService = new EntityCreationService(entityEnrichmentService); - TextEntity correctEntity = entityCreationService.forceByTextRange(closestTextRange, manualEntity.getType(), manualEntity.getEntityType(), node); + TextEntity correctEntity = entityCreationService.forceByTextRange(closestTextRange, precursorEntity.getType(), precursorEntity.getEntityType(), node); - correctEntity.addMatchedRules(manualEntity.getMatchedRuleList()); - correctEntity.setDictionaryEntry(manualEntity.isDictionaryEntry()); - correctEntity.setDossierDictionaryEntry(manualEntity.isDossierDictionaryEntry()); - correctEntity.getManualOverwrite().addChanges(manualEntity.getManualOverwrite().getManualChangeLog()); + correctEntity.addMatchedRules(precursorEntity.getMatchedRuleList()); + correctEntity.setDictionaryEntry(precursorEntity.isDictionaryEntry()); + correctEntity.setDossierDictionaryEntry(precursorEntity.isDossierDictionaryEntry()); + correctEntity.getManualOverwrite().addChanges(precursorEntity.getManualOverwrite().getManualChangeLog()); return correctEntity; } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/MigrationEntity.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/MigrationEntity.java index dd208f56..4abe6f03 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/MigrationEntity.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/MigrationEntity.java @@ -34,7 +34,7 @@ import lombok.RequiredArgsConstructor; @RequiredArgsConstructor public final class MigrationEntity { - private final ManualEntity manualEntity; + private final PrecursorEntity precursorEntity; private final RedactionLogEntry redactionLogEntry; private IEntity migratedEntity; private String oldId; @@ -47,12 +47,12 @@ public final class MigrationEntity { } - public static ManualEntity createManualEntity(RedactionLogEntry redactionLogEntry, boolean hint) { + public static PrecursorEntity createManualEntity(RedactionLogEntry redactionLogEntry, boolean hint) { String ruleIdentifier = buildRuleIdentifier(redactionLogEntry); List rectangleWithPages = redactionLogEntry.getPositions().stream().map(RectangleWithPage::fromRedactionLogRectangle).toList(); EntityType entityType = getEntityType(redactionLogEntry, hint); - return ManualEntity.builder() + return PrecursorEntity.builder() .id(redactionLogEntry.getId()) .value(redactionLogEntry.getValue()) .entityPosition(rectangleWithPages) @@ -164,7 +164,7 @@ public final class MigrationEntity { entityLogEntry = createEntityLogEntry(image); } else if (migratedEntity instanceof TextEntity textEntity) { entityLogEntry = createEntityLogEntry(textEntity); - } else if (migratedEntity instanceof ManualEntity entity) { + } else if (migratedEntity instanceof PrecursorEntity entity) { entityLogEntry = createEntityLogEntry(entity); } else { throw new UnsupportedOperationException("Unknown subclass " + migratedEntity.getClass()); @@ -261,30 +261,30 @@ public final class MigrationEntity { } - public EntityLogEntry createEntityLogEntry(ManualEntity manualEntity) { + public EntityLogEntry createEntityLogEntry(PrecursorEntity precursorEntity) { - String type = manualEntity.getManualOverwrite().getType().orElse(manualEntity.getType()); + String type = precursorEntity.getManualOverwrite().getType().orElse(precursorEntity.getType()); return EntityLogEntry.builder() - .id(manualEntity.getId()) - .reason(manualEntity.buildReasonWithManualChangeDescriptions()) - .legalBasis(manualEntity.legalBasis()) - .value(manualEntity.value()) + .id(precursorEntity.getId()) + .reason(precursorEntity.buildReasonWithManualChangeDescriptions()) + .legalBasis(precursorEntity.legalBasis()) + .value(precursorEntity.value()) .type(type) - .state(buildEntryState(manualEntity)) - .entryType(buildEntryType(manualEntity)) + .state(buildEntryState(precursorEntity)) + .entryType(buildEntryType(precursorEntity)) .section(redactionLogEntry.getSection()) .textAfter(redactionLogEntry.getTextAfter()) .textBefore(redactionLogEntry.getTextBefore()) .containingNodeId(Collections.emptyList()) .closestHeadline("") - .matchedRule(manualEntity.getMatchedRule().getRuleIdentifier().toString()) - .dictionaryEntry(manualEntity.isDictionaryEntry()) - .dossierDictionaryEntry(manualEntity.isDossierDictionaryEntry()) + .matchedRule(precursorEntity.getMatchedRule().getRuleIdentifier().toString()) + .dictionaryEntry(precursorEntity.isDictionaryEntry()) + .dossierDictionaryEntry(precursorEntity.isDossierDictionaryEntry()) .startOffset(-1) .endOffset(-1) - .positions(manualEntity.getManualOverwrite() + .positions(precursorEntity.getManualOverwrite() .getPositions() - .orElse(manualEntity.getEntityPosition()) + .orElse(precursorEntity.getEntityPosition()) .stream() .map(entityPosition -> new Position(entityPosition.rectangle2D(), entityPosition.pageNumber())) .toList()) @@ -343,11 +343,11 @@ public final class MigrationEntity { if (entity instanceof TextEntity textEntity) { return getEntryType(textEntity.getEntityType()); - } else if (entity instanceof ManualEntity manualEntity) { - if (manualEntity.isRectangle()) { + } else if (entity instanceof PrecursorEntity precursorEntity) { + if (precursorEntity.isRectangle()) { return EntryType.AREA; } - return getEntryType(manualEntity.getEntityType()); + return getEntryType(precursorEntity.getEntityType()); } else if (entity instanceof Image) { return EntryType.IMAGE; } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/ManualEntity.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/PrecursorEntity.java similarity index 66% rename from redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/ManualEntity.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/PrecursorEntity.java index d33f588e..beb8de15 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/ManualEntity.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/PrecursorEntity.java @@ -1,11 +1,17 @@ package com.iqser.red.service.redaction.v1.server.model; +import static com.iqser.red.service.redaction.v1.server.service.NotFoundImportedEntitiesService.IMPORTED_REDACTION_TYPE; + import java.util.List; +import java.util.Optional; import java.util.PriorityQueue; +import java.util.Set; import java.util.UUID; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryType; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedaction; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction; import com.iqser.red.service.redaction.v1.server.model.document.TextRange; @@ -25,7 +31,7 @@ import lombok.experimental.FieldDefaults; @AllArgsConstructor @FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE) @SuppressWarnings("PMD.AvoidFieldNameMatchingMethodName") -public class ManualEntity implements IEntity { +public class PrecursorEntity implements IEntity { // The id must be mapped into a TextEntity as is for comments to work correctly String id; @@ -37,23 +43,26 @@ public class ManualEntity implements IEntity { String type; String section; EntityType entityType; + EntryType entryType; boolean applied; boolean isDictionaryEntry; boolean isDossierDictionaryEntry; boolean rectangle; + Set engines; @Builder.Default PriorityQueue matchedRuleList = new PriorityQueue<>(); ManualChangeOverwrite manualOverwrite; - public static ManualEntity fromManualRedactionEntry(ManualRedactionEntry manualRedactionEntry, boolean hint) { + public static PrecursorEntity fromManualRedactionEntry(ManualRedactionEntry manualRedactionEntry, boolean hint) { List rectangleWithPages = manualRedactionEntry.getPositions().stream().map(RectangleWithPage::fromAnnotationRectangle).toList(); var entityType = hint ? EntityType.HINT : EntityType.ENTITY; + var entryType = hint ? EntryType.HINT : EntryType.ENTITY; ManualChangeOverwrite manualChangeOverwrite = new ManualChangeOverwrite(entityType); manualChangeOverwrite.addChange(manualRedactionEntry); - return ManualEntity.builder() + return PrecursorEntity.builder() .id(manualRedactionEntry.getAnnotationId()) .value(manualRedactionEntry.getValue()) .entityPosition(rectangleWithPages) @@ -63,20 +72,22 @@ public class ManualEntity implements IEntity { .type(manualRedactionEntry.getType()) .section(manualRedactionEntry.getSection()) .entityType(entityType) + .entryType(entryType) .applied(true) .isDictionaryEntry(false) .isDossierDictionaryEntry(false) .rectangle(manualRedactionEntry.isRectangle()) .manualOverwrite(manualChangeOverwrite) + .engines(Set.of(Engine.MANUAL)) .build(); } - public static ManualEntity fromEntityLogEntry(EntityLogEntry entityLogEntry) { + public static PrecursorEntity fromEntityLogEntry(EntityLogEntry entityLogEntry) { List rectangleWithPages = entityLogEntry.getPositions().stream().map(RectangleWithPage::fromEntityLogPosition).toList(); EntityType entityType = getEntityType(entityLogEntry.getEntryType()); - return ManualEntity.builder() + return PrecursorEntity.builder() .id(entityLogEntry.getId()) .value(entityLogEntry.getValue()) .entityPosition(rectangleWithPages) @@ -86,17 +97,43 @@ public class ManualEntity implements IEntity { .type(entityLogEntry.getType()) .section(entityLogEntry.getSection()) .entityType(entityType) + .entryType(entityLogEntry.getEntryType()) .isDictionaryEntry(entityLogEntry.isDictionaryEntry()) .isDossierDictionaryEntry(entityLogEntry.isDossierDictionaryEntry()) .manualOverwrite(new ManualChangeOverwrite(entityType)) + .engines(entityLogEntry.getEngines()) .build(); } - public static ManualEntity fromManualResizeRedaction(ManualResizeRedaction manualResizeRedaction) { + public static PrecursorEntity fromImportedEntry(ImportedRedaction importedRedaction) { + + List rectangleWithPages = importedRedaction.getPositions().stream().map(RectangleWithPage::fromEntityLogPosition).toList(); + EntryType entryType = Optional.ofNullable(importedRedaction.getEntryType()).orElse(EntryType.ENTITY); + EntityType entityType = getEntityType(entryType); + String value = Optional.ofNullable(importedRedaction.getValue()).orElse(""); + return PrecursorEntity.builder() + .id(importedRedaction.getId()) + .value(value) + .entityPosition(rectangleWithPages) + .reason(Optional.ofNullable(importedRedaction.getReason()).orElse("")) + .legalBasis(Optional.ofNullable(importedRedaction.getLegalBasis()).orElse("")) + .type(Optional.ofNullable(importedRedaction.getType()).orElse(IMPORTED_REDACTION_TYPE)) + .entityType(entityType) + .entryType(entryType) + .isDictionaryEntry(false) + .isDossierDictionaryEntry(false) + .rectangle(value.isBlank() || entryType.equals(EntryType.IMAGE) || entryType.equals(EntryType.IMAGE_HINT) || entryType.equals(EntryType.AREA)) + .manualOverwrite(new ManualChangeOverwrite(entityType)) + .engines(Set.of(Engine.IMPORTED)) + .build(); + } + + + public static PrecursorEntity fromManualResizeRedaction(ManualResizeRedaction manualResizeRedaction) { List rectangleWithPages = manualResizeRedaction.getPositions().stream().map(RectangleWithPage::fromAnnotationRectangle).toList(); - return ManualEntity.builder() + return PrecursorEntity.builder() .id(UUID.randomUUID().toString()) .value(manualResizeRedaction.getValue()) .entityPosition(rectangleWithPages) @@ -141,10 +178,4 @@ public class ManualEntity implements IEntity { } } - - private EntityType getEntityType(boolean isHint) { - - return isHint ? EntityType.HINT : EntityType.ENTITY; - } - } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/MatchedRule.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/MatchedRule.java index 7a04467e..e45d74f4 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/MatchedRule.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/MatchedRule.java @@ -24,7 +24,8 @@ public final class MatchedRule implements Comparable { public static final RuleType FINAL_TYPE = RuleType.fromString("FINAL"); public static final RuleType ELIMINATION_RULE_TYPE = RuleType.fromString("X"); - private static final List RULE_TYPE_PRIORITIES = List.of(FINAL_TYPE, ELIMINATION_RULE_TYPE); + public static final RuleType IMPORTED_TYPE = RuleType.fromString("IMP"); + private static final List RULE_TYPE_PRIORITIES = List.of(FINAL_TYPE, ELIMINATION_RULE_TYPE, IMPORTED_TYPE); RuleIdentifier ruleIdentifier; @Builder.Default diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalyzeService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalyzeService.java index fbda3175..85872703 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalyzeService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalyzeService.java @@ -1,12 +1,13 @@ package com.iqser.red.service.redaction.v1.server.service; +import java.util.Collection; import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; import java.util.stream.Collectors; - +import java.util.stream.Stream; import org.springframework.stereotype.Service; import org.springframework.web.bind.annotation.RequestBody; @@ -18,6 +19,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileTyp import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentLog; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogChanges; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedactions; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.legalbasis.LegalBasis; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog; @@ -28,7 +30,7 @@ import com.iqser.red.service.redaction.v1.server.RedactionServiceSettings; import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient; import com.iqser.red.service.redaction.v1.server.client.model.NerEntitiesModel; import com.iqser.red.service.redaction.v1.server.model.KieWrapper; -import com.iqser.red.service.redaction.v1.server.model.ManualEntity; +import com.iqser.red.service.redaction.v1.server.model.PrecursorEntity; import com.iqser.red.service.redaction.v1.server.model.NerEntities; import com.iqser.red.service.redaction.v1.server.model.component.Component; import com.iqser.red.service.redaction.v1.server.model.dictionary.Dictionary; @@ -37,6 +39,7 @@ import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryVers import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document; import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode; import com.iqser.red.service.redaction.v1.server.service.document.DocumentGraphMapper; +import com.iqser.red.service.redaction.v1.server.service.document.ImportedRedactionEntryService; import com.iqser.red.service.redaction.v1.server.service.document.ManualRedactionEntryService; import com.iqser.red.service.redaction.v1.server.service.document.NerEntitiesAdapter; import com.iqser.red.service.redaction.v1.server.service.document.SectionFinderService; @@ -72,9 +75,10 @@ public class AnalyzeService { RedactionChangeLogService redactionChangeLogService; LegalBasisClient legalBasisClient; RedactionServiceSettings redactionServiceSettings; - ImportedRedactionService importedRedactionService; + NotFoundImportedEntitiesService notFoundImportedEntitiesService; SectionFinderService sectionFinderService; ManualRedactionEntryService manualRedactionEntryService; + ImportedRedactionEntryService importedRedactionEntryService; ObservedStorageService observedStorageService; FunctionTimerValues redactmanagerAnalyzePagewiseValues; @@ -93,6 +97,9 @@ public class AnalyzeService { Document document = DocumentGraphMapper.toDocumentGraph(observedStorageService.getDocumentData(analyzeRequest.getDossierId(), analyzeRequest.getFileId())); log.info("Loaded Document Graph for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); + ImportedRedactions importedRedactions = redactionStorageService.getImportedRedactions(analyzeRequest.getDossierId(), analyzeRequest.getFileId()); + log.info("Loaded Imported Redactions for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); + // not yet ready for reanalysis if (previousEntityLog == null || document == null || document.getNumberOfPages() == 0) { return analyze(analyzeRequest); @@ -102,13 +109,11 @@ public class AnalyzeService { new DictionaryVersion(previousEntityLog.getDictionaryVersion(), previousEntityLog.getDossierDictionaryVersion()), analyzeRequest.getDossierId()); - Set sectionsToReanalyseIds = getSectionsToReanalyseIds(analyzeRequest, previousEntityLog, document, dictionaryIncrement); + Set sectionsToReanalyseIds = getSectionsToReanalyseIds(analyzeRequest, previousEntityLog, document, dictionaryIncrement, importedRedactions); List sectionsToReAnalyse = getSectionsToReAnalyse(document, sectionsToReanalyseIds); log.info("{} Sections to reanalyze found for file {} in dossier {}", sectionsToReanalyseIds.size(), analyzeRequest.getFileId(), analyzeRequest.getDossierId()); if (sectionsToReAnalyse.isEmpty()) { - // do this only to update the imported redactions with unprocessed manual changes if there are changes - importedRedactionService.processImportedRedactions(previousEntityLog, analyzeRequest); EntityLogChanges entityLogChanges = entityLogCreatorService.updateVersionsAndReturnChanges(previousEntityLog, dictionaryIncrement.getDictionaryVersion(), @@ -126,15 +131,18 @@ public class AnalyzeService { true, Collections.emptySet()); } + KieWrapper kieWrapperEntityRules = kieContainerCreationService.getLatestKieContainer(analyzeRequest.getDossierTemplateId(), RuleFileType.ENTITY); log.info("Updated entity rules to version {} for file {} in dossier {}", kieWrapperEntityRules.rulesVersion(), analyzeRequest.getFileId(), analyzeRequest.getDossierId()); NerEntities nerEntities = getEntityRecognitionEntitiesFilteredBySectionIds(analyzeRequest, document, sectionsToReanalyseIds); log.info("Loaded Ner Entities for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); - List notFoundManualRedactionEntries = manualRedactionEntryService.addManualRedactionEntriesAndReturnNotFoundEntries(analyzeRequest, + var notFoundManualRedactionEntries = manualRedactionEntryService.addManualRedactionEntriesAndReturnNotFoundEntries(analyzeRequest, document, analyzeRequest.getDossierTemplateId()); + var notFoundImportedEntries = importedRedactionEntryService.addImportedEntriesAndReturnNotFoundEntries(analyzeRequest, importedRedactions, document); + var notFoundManualOrImportedEntries = Stream.of(notFoundManualRedactionEntries, notFoundImportedEntries).flatMap(Collection::stream).collect(Collectors.toList()); Dictionary dictionary = dictionaryService.getDeepCopyDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId()); log.info("Updated Dictionaries for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); @@ -142,6 +150,7 @@ public class AnalyzeService { dictionarySearchService.addDictionaryEntities(dictionary, sectionsToReAnalyse); log.info("Finished Dictionary Search for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); + // we could add the imported redactions similar to the manual redactions here as well for additional processing List allFileAttributes = entityDroolsExecutionService.executeRules(kieWrapperEntityRules.container(), document, sectionsToReAnalyse, @@ -151,14 +160,16 @@ public class AnalyzeService { nerEntities); log.info("Finished entity rule execution for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); - RedactionLog redactionLog = updatePreviousRedactionLog(analyzeRequest, document, notFoundManualRedactionEntries, previousRedactionLog, sectionsToReanalyseIds); + RedactionLog redactionLog = updatePreviousRedactionLog(analyzeRequest, document, notFoundManualOrImportedEntries, previousRedactionLog, sectionsToReanalyseIds); EntityLogChanges entityLogChanges = entityLogCreatorService.updatePreviousEntityLog(analyzeRequest, document, - notFoundManualRedactionEntries, + notFoundManualOrImportedEntries, previousEntityLog, sectionsToReanalyseIds, dictionary.getVersion()); + notFoundImportedEntitiesService.processEntityLog(entityLogChanges.getEntityLog(), analyzeRequest, notFoundImportedEntries); + return finalizeAnalysis(analyzeRequest, startTime, kieContainerCreationService.getLatestKieContainer(analyzeRequest.getDossierTemplateId(), RuleFileType.COMPONENT), @@ -187,6 +198,9 @@ public class AnalyzeService { Document document = DocumentGraphMapper.toDocumentGraph(observedStorageService.getDocumentData(analyzeRequest.getDossierId(), analyzeRequest.getFileId())); log.info("Loaded Document Graph for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); + ImportedRedactions importedRedactions = redactionStorageService.getImportedRedactions(analyzeRequest.getDossierId(), analyzeRequest.getFileId()); + log.info("Loaded Imported Redactions for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); + NerEntities nerEntities = getEntityRecognitionEntities(analyzeRequest, document); log.info("Loaded Ner Entities for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); @@ -194,13 +208,16 @@ public class AnalyzeService { Dictionary dictionary = dictionaryService.getDeepCopyDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId()); log.info("Updated Dictionaries for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); - List notFoundManualRedactionEntries = manualRedactionEntryService.addManualRedactionEntriesAndReturnNotFoundEntries(analyzeRequest, + var notFoundManualRedactionEntries = manualRedactionEntryService.addManualRedactionEntriesAndReturnNotFoundEntries(analyzeRequest, document, analyzeRequest.getDossierTemplateId()); + var notFoundImportedEntries = importedRedactionEntryService.addImportedEntriesAndReturnNotFoundEntries(analyzeRequest, importedRedactions, document); + var notFoundManualOrImportedEntries = Stream.of(notFoundManualRedactionEntries, notFoundImportedEntries).flatMap(Collection::stream).collect(Collectors.toList()); dictionarySearchService.addDictionaryEntities(dictionary, document); log.info("Finished Dictionary Search for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); + // we could add the imported redactions similar to the manual redactions here as well for additional processing List allFileAttributes = entityDroolsExecutionService.executeRules(kieWrapperEntityRules.container(), document, dictionary, @@ -209,13 +226,15 @@ public class AnalyzeService { nerEntities); log.info("Finished entity rule execution for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); - RedactionLog redactionLog = createRedactionLog(analyzeRequest, document, notFoundManualRedactionEntries, dictionary, kieWrapperEntityRules); + RedactionLog redactionLog = createRedactionLog(analyzeRequest, document, notFoundManualOrImportedEntries, dictionary, kieWrapperEntityRules); EntityLog entityLog = entityLogCreatorService.createInitialEntityLog(analyzeRequest, document, - notFoundManualRedactionEntries, + notFoundManualOrImportedEntries, dictionary.getVersion(), kieWrapperEntityRules.rulesVersion()); + notFoundImportedEntitiesService.processEntityLog(entityLog, analyzeRequest, notFoundImportedEntries); + return finalizeAnalysis(analyzeRequest, startTime, kieWrapperComponentRules, @@ -232,24 +251,15 @@ public class AnalyzeService { @Deprecated(forRemoval = true) private RedactionLog updatePreviousRedactionLog(AnalyzeRequest analyzeRequest, Document document, - List notFoundManualRedactionEntries, + List notFoundEntries, RedactionLog previousRedactionLog, Set sectionsToReanalyseIds) { - List newRedactionLogEntries = redactionLogCreatorService.createRedactionLog(document, - analyzeRequest.getDossierTemplateId(), - notFoundManualRedactionEntries); - - var importedRedactionFilteredEntries = importedRedactionService.processImportedRedactions(analyzeRequest.getDossierTemplateId(), - analyzeRequest.getDossierId(), - analyzeRequest.getFileId(), - newRedactionLogEntries, - false); + List newRedactionLogEntries = redactionLogCreatorService.createRedactionLog(document, analyzeRequest.getDossierTemplateId(), notFoundEntries); previousRedactionLog.getRedactionLogEntry() - .removeIf(entry -> sectionsToReanalyseIds.contains(entry.getSectionNumber()) && !entry.getType().equals(ImportedRedactionService.IMPORTED_REDACTION_TYPE)); + .removeIf(entry -> sectionsToReanalyseIds.contains(entry.getSectionNumber()) && !entry.getType().equals(NotFoundImportedEntitiesService.IMPORTED_REDACTION_TYPE)); - previousRedactionLog.getRedactionLogEntry().addAll(importedRedactionFilteredEntries); return previousRedactionLog; } @@ -348,11 +358,9 @@ public class AnalyzeService { } - private Set getSectionsToReanalyseIds(AnalyzeRequest analyzeRequest, EntityLog entityLog, Document document, DictionaryIncrement dictionaryIncrement) { + private Set getSectionsToReanalyseIds(AnalyzeRequest analyzeRequest, EntityLog entityLog, Document document, DictionaryIncrement dictionaryIncrement, ImportedRedactions importedRedactions) { - return analyzeRequest.getSectionsToReanalyse().isEmpty() // - ? sectionFinderService.findSectionsToReanalyse(dictionaryIncrement, entityLog, document, analyzeRequest) // - : analyzeRequest.getSectionsToReanalyse(); + return sectionFinderService.findSectionsToReanalyse(dictionaryIncrement, entityLog, document, analyzeRequest, importedRedactions); } @@ -393,7 +401,7 @@ public class AnalyzeService { @Deprecated(forRemoval = true) private RedactionLog createRedactionLog(AnalyzeRequest analyzeRequest, Document document, - List notFoundManualRedactionEntries, + List notFoundManualRedactionEntries, Dictionary dictionary, KieWrapper wrapper) { @@ -411,12 +419,6 @@ public class AnalyzeService { wrapper.rulesVersion(), legalBasisClient.getVersion(analyzeRequest.getDossierTemplateId())); - List importedRedactionFilteredEntries = importedRedactionService.processImportedRedactions(analyzeRequest.getDossierTemplateId(), - analyzeRequest.getDossierId(), - analyzeRequest.getFileId(), - redactionLog.getRedactionLogEntry(), - true); - redactionLog.setRedactionLogEntry(importedRedactionFilteredEntries); return redactionLog; } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/DictionarySearchService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/DictionarySearchService.java index 48a2184b..7766d5f4 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/DictionarySearchService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/DictionarySearchService.java @@ -1,6 +1,7 @@ package com.iqser.red.service.redaction.v1.server.service; import java.util.List; +import java.util.Set; import org.springframework.stereotype.Service; @@ -28,7 +29,8 @@ public class DictionarySearchService { @Observed(name = "DictionarySearchService", contextualName = "add-dictionary-entries") - public void addDictionaryEntities(Dictionary dictionary, List semanticNodes){ + public void addDictionaryEntities(Dictionary dictionary, List semanticNodes) { + semanticNodes.forEach(node -> addDictionaryEntities(dictionary, node)); } @@ -52,11 +54,12 @@ public class DictionarySearchService { EntityCreationService entityCreationService = new EntityCreationService(entityEnrichmentService); searchImplementation.getBoundaries(node.getTextBlock(), node.getTextRange()) - .stream().filter(boundary -> entityCreationService.isValidEntityTextRange(node.getTextBlock(), boundary)) - .map(bounds -> entityCreationService.forceByTextRange(bounds, type, entityType, node)) - .peek(entity -> entity.setDictionaryEntry(true)) - .peek(entity -> entity.setDossierDictionaryEntry(isDossierDictionaryEntry)) - .forEach(entity -> entity.addEngine(Engine.DICTIONARY)); + .stream() + .filter(boundary -> entityCreationService.isValidEntityTextRange(node.getTextBlock(), boundary)) + .forEach(bounds -> entityCreationService.byTextRangeWithEngine(bounds, type, entityType, node, Set.of(Engine.DICTIONARY)).ifPresent(entity -> { + entity.setDictionaryEntry(true); + entity.setDossierDictionaryEntry(isDossierDictionaryEntry); + })); } } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityChangeLogService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityChangeLogService.java index cb967c15..ef023fcb 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityChangeLogService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityChangeLogService.java @@ -60,9 +60,8 @@ public class EntityChangeLogService { OffsetDateTime now) { Set existingIds = newEntityLogEntries.stream().map(EntityLogEntry::getId).collect(Collectors.toSet()); - // no need to check imported redactions because they will be added with the merged changes after this step List removedEntries = previousEntityLogEntries.stream() - .filter(entry -> !entry.getType().equals(ImportedRedactionService.IMPORTED_REDACTION_TYPE) && !existingIds.contains(entry.getId())) + .filter(entry -> !existingIds.contains(entry.getId())) .toList(); removedEntries.forEach(entry -> entry.getChanges().add(new Change(analysisNumber, ChangeType.REMOVED, now))); removedEntries.forEach(entry -> entry.setState(EntryState.REMOVED)); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java index 5cf13382..6da9525a 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java @@ -21,7 +21,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.legalbasis.LegalBasis; import com.iqser.red.service.redaction.v1.server.RedactionServiceSettings; import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient; -import com.iqser.red.service.redaction.v1.server.model.ManualEntity; +import com.iqser.red.service.redaction.v1.server.model.PrecursorEntity; import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryVersion; import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType; import com.iqser.red.service.redaction.v1.server.model.document.entity.IEntity; @@ -45,7 +45,6 @@ public class EntityLogCreatorService { DictionaryService dictionaryService; ManualChangeFactory manualChangeFactory; - ImportedRedactionService importedRedactionService; RedactionServiceSettings redactionServiceSettings; LegalBasisClient legalBasisClient; EntityChangeLogService entityChangeLogService; @@ -60,11 +59,11 @@ public class EntityLogCreatorService { public EntityLog createInitialEntityLog(AnalyzeRequest analyzeRequest, Document document, - List notFoundManualEntities, + List notFoundEntities, DictionaryVersion dictionaryVersion, long rulesVersion) { - List entityLogEntries = createEntityLogEntries(document, analyzeRequest.getDossierTemplateId(), notFoundManualEntities); + List entityLogEntries = createEntityLogEntries(document, analyzeRequest, notFoundEntities); List legalBasis = legalBasisClient.getLegalBasisMapping(analyzeRequest.getDossierTemplateId()); EntityLog entityLog = new EntityLog(redactionServiceSettings.getAnalysisVersion(), @@ -77,12 +76,8 @@ public class EntityLogCreatorService { legalBasisClient.getVersion(analyzeRequest.getDossierTemplateId())); List previousExistingEntityLogEntries = getPreviousEntityLogEntries(analyzeRequest.getDossierId(), analyzeRequest.getFileId()); - // compute changes will be done for the new entries without the imported redactions, since previous imported redactions are present in - // previousExistingEntityLogEntries and have to be ignored in this calculation because the changes will be merged in the next step (processImportedRedactions) - entityChangeLogService.computeChanges(previousExistingEntityLogEntries, entityLogEntries, analyzeRequest.getAnalysisNumber()); - // check to see if there are imported redactions and only then recreate them - importedRedactionService.processImportedRedactions(entityLog, analyzeRequest); + entityChangeLogService.computeChanges(previousExistingEntityLogEntries, entityLogEntries, analyzeRequest.getAnalysisNumber()); return entityLog; } @@ -113,34 +108,33 @@ public class EntityLogCreatorService { public EntityLogChanges updatePreviousEntityLog(AnalyzeRequest analyzeRequest, Document document, - List notFoundManualRedactionEntries, + List notFoundEntries, EntityLog previousEntityLog, Set sectionsToReanalyseIds, DictionaryVersion dictionaryVersion) { - List newEntityLogEntries = createEntityLogEntries(document, analyzeRequest.getDossierTemplateId(), notFoundManualRedactionEntries).stream() + List newEntityLogEntries = createEntityLogEntries(document, analyzeRequest, notFoundEntries).stream() .filter(entry -> entry.getContainingNodeId().isEmpty() || sectionsToReanalyseIds.contains(entry.getContainingNodeId().get(0))) .collect(Collectors.toList()); Set newEntityIds = newEntityLogEntries.stream().map(EntityLogEntry::getId).collect(Collectors.toSet()); List previousEntriesFromReAnalyzedSections = previousEntityLog.getEntityLogEntry() .stream() - .filter(entry -> !entry.getType().equals(ImportedRedactionService.IMPORTED_REDACTION_TYPE) && (newEntityIds.contains(entry.getId()) || entry.getContainingNodeId() - .isEmpty() || sectionsToReanalyseIds.contains(entry.getContainingNodeId().get(0)))) + .filter(entry -> (newEntityIds.contains(entry.getId()) || entry.getContainingNodeId().isEmpty() || sectionsToReanalyseIds.contains(entry.getContainingNodeId() + .get(0)))) .toList(); previousEntityLog.getEntityLogEntry().removeAll(previousEntriesFromReAnalyzedSections); boolean hasChanges = entityChangeLogService.computeChanges(previousEntriesFromReAnalyzedSections, newEntityLogEntries, analyzeRequest.getAnalysisNumber()); previousEntityLog.getEntityLogEntry().addAll(newEntityLogEntries); - importedRedactionService.processImportedRedactions(previousEntityLog, analyzeRequest); - return updateVersionsAndReturnChanges(previousEntityLog, dictionaryVersion, analyzeRequest, hasChanges); } - private List createEntityLogEntries(Document document, String dossierTemplateId, List notFoundManualRedactionEntries) { + private List createEntityLogEntries(Document document, AnalyzeRequest analyzeRequest, List notFoundPrecursorEntries) { + String dossierTemplateId = analyzeRequest.getDossierTemplateId(); List entries = new ArrayList<>(); document.getEntities() .stream() @@ -149,7 +143,9 @@ public class EntityLogCreatorService { .filter(entity -> !entity.removed()) .forEach(entityNode -> entries.addAll(toEntityLogEntries(entityNode))); document.streamAllImages().filter(entity -> !entity.removed()).forEach(imageNode -> entries.add(createEntityLogEntry(imageNode, dossierTemplateId))); - notFoundManualRedactionEntries.stream().filter(entity -> !entity.removed()).forEach(manualEntity -> entries.add(createEntityLogEntry(manualEntity))); + notFoundPrecursorEntries.stream() + .filter(entity -> !entity.removed()) + .forEach(precursorEntity -> entries.add(createEntityLogEntry(precursorEntity, dossierTemplateId))); return entries; } @@ -203,37 +199,40 @@ public class EntityLogCreatorService { } - private EntityLogEntry createEntityLogEntry(ManualEntity manualEntity) { + private EntityLogEntry createEntityLogEntry(PrecursorEntity precursorEntity, String dossierTemplateId) { - String type = manualEntity.getManualOverwrite().getType().orElse(manualEntity.getType()); - boolean isHint = isHint(manualEntity.getEntityType()); + String type = precursorEntity.getManualOverwrite().getType().orElse(precursorEntity.getType()); + boolean isHint = isHint(precursorEntity.getEntityType()); return EntityLogEntry.builder() - .id(manualEntity.getId()) - .reason(manualEntity.buildReasonWithManualChangeDescriptions()) - .legalBasis(manualEntity.legalBasis()) - .value(manualEntity.value()) + .id(precursorEntity.getId()) + .reason(precursorEntity.buildReasonWithManualChangeDescriptions()) + .legalBasis(precursorEntity.legalBasis()) + .value(precursorEntity.value()) .type(type) - .state(buildEntryState(manualEntity)) - .entryType(buildEntryType(manualEntity)) - .section(manualEntity.getManualOverwrite().getSection().orElse(manualEntity.getSection())) + .state(buildEntryState(precursorEntity)) + .entryType(buildEntryType(precursorEntity)) + .section(precursorEntity.getManualOverwrite().getSection().orElse(precursorEntity.getSection())) .containingNodeId(Collections.emptyList()) .closestHeadline("") - .matchedRule(manualEntity.getMatchedRule().getRuleIdentifier().toString()) - .dictionaryEntry(manualEntity.isDictionaryEntry()) - .dossierDictionaryEntry(manualEntity.isDossierDictionaryEntry()) + .matchedRule(precursorEntity.getMatchedRule().getRuleIdentifier().toString()) + .dictionaryEntry(precursorEntity.isDictionaryEntry()) + .dossierDictionaryEntry(precursorEntity.isDossierDictionaryEntry()) .textAfter("") .textBefore("") .startOffset(-1) .endOffset(-1) - .positions(manualEntity.getManualOverwrite() + .positions(precursorEntity.getManualOverwrite() .getPositions() - .orElse(manualEntity.getEntityPosition()) + .orElse(precursorEntity.getEntityPosition()) .stream() .map(entityPosition -> new Position(entityPosition.rectangle2D(), entityPosition.pageNumber())) .toList()) - .engines(Collections.emptySet()) + .engines(precursorEntity.getEngines()) + //imported is no longer used, frontend should check engines + //(was .imported(precursorEntity.getEngines() != null && precursorEntity.getEngines().contains(Engine.IMPORTED))) + .imported(false) .reference(Collections.emptySet()) - .manualChanges(manualChangeFactory.toManualChangeList(manualEntity.getManualOverwrite().getManualChangeLog(), isHint)) + .manualChanges(manualChangeFactory.toManualChangeList(precursorEntity.getManualOverwrite().getManualChangeLog(), isHint)) .build(); } @@ -259,6 +258,9 @@ public class EntityLogCreatorService { .endOffset(entity.getTextRange().end()) .dossierDictionaryEntry(entity.isDossierDictionaryEntry()) .engines(entity.getEngines() != null ? entity.getEngines() : Collections.emptySet()) + //imported is no longer used, frontend should check engines + //(was .imported(entity.getEngines() != null && entity.getEngines().contains(Engine.IMPORTED))) + .imported(false) .reference(referenceIds) .manualChanges(manualChangeFactory.toManualChangeList(entity.getManualOverwrite().getManualChangeLog(), isHint)) .state(buildEntryState(entity)) @@ -291,11 +293,8 @@ public class EntityLogCreatorService { if (entity instanceof TextEntity textEntity) { return getEntryType(textEntity.getEntityType()); - } else if (entity instanceof ManualEntity manualEntity) { - if (manualEntity.isRectangle()) { - return EntryType.AREA; - } - return getEntryType(manualEntity.getEntityType()); + } else if (entity instanceof PrecursorEntity precursorEntity) { + return precursorEntity.getEntryType(); } throw new UnsupportedOperationException(String.format("Entity subclass %s is not implemented!", entity.getClass())); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/ImportedRedactionEntriesMergeService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/ImportedRedactionEntriesMergeService.java deleted file mode 100644 index c2c71747..00000000 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/ImportedRedactionEntriesMergeService.java +++ /dev/null @@ -1,195 +0,0 @@ -package com.iqser.red.service.redaction.v1.server.service; - -import java.time.OffsetDateTime; -import java.util.Collection; -import java.util.Collections; -import java.util.Comparator; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -import org.springframework.stereotype.Service; - -import com.google.common.base.Strings; -import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest; -import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Change; -import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ChangeType; -import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry; -import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState; -import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryType; -import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualChange; -import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType; -import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position; -import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions; -import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle; -import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.BaseAnnotation; -import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval; -import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction; -import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange; -import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction; - -import lombok.RequiredArgsConstructor; -import lombok.extern.slf4j.Slf4j; - -// This is retrieved from persistence service and updated to the needs for imported redactions -@Service -@Slf4j -@RequiredArgsConstructor -public class ImportedRedactionEntriesMergeService { - - public List mergeManualChangesForImportedRedactions(List importedRedactionsEntityLogEntries, AnalyzeRequest analyzeRequest) { - - log.debug("Merging manual changes for imported redactions with analyze number: {}", analyzeRequest.getAnalysisNumber()); - Set importedRedactionsIds = importedRedactionsEntityLogEntries.stream().map(e -> e.getId()).collect(Collectors.toSet()); - List allManualChangesForImportedRedactions = allManualChangesForImportedRedactions(analyzeRequest.getManualRedactions(), importedRedactionsIds); - - if (allManualChangesForImportedRedactions.isEmpty()) { - log.debug("no changes for imported redactions"); - // no changes for the imported redactions - return importedRedactionsEntityLogEntries; - } - - final int analysisNumber = analyzeRequest.getAnalysisNumber(); - - // Sort manual changes by date, so we process them in order of when they were requested - allManualChangesForImportedRedactions = allManualChangesForImportedRedactions.stream().sorted(Comparator.comparing(BaseAnnotation::getRequestDate)).toList(); - allManualChangesForImportedRedactions.forEach(manualChange -> { - // this is ugly and should be replaced with switch pattern matching https://openjdk.org/jeps/406 -> requires Java 17 (preview) or higher - if (manualChange instanceof IdRemoval idRemoval) { - mergeIdsToRemove(idRemoval, importedRedactionsEntityLogEntries, analysisNumber); - } else if (manualChange instanceof ManualResizeRedaction manualResizeRedaction) { - mergeResizeRedactions(manualResizeRedaction, importedRedactionsEntityLogEntries, analysisNumber); - } else if (manualChange instanceof ManualLegalBasisChange manualLegalBasisChange) { - mergeLegalBasisChanges(manualLegalBasisChange, importedRedactionsEntityLogEntries, analysisNumber); - } else if (manualChange instanceof ManualForceRedaction manualForceRedaction) { - mergeForceRedactions(manualForceRedaction, importedRedactionsEntityLogEntries, analysisNumber); - } - }); - - return importedRedactionsEntityLogEntries; - } - - private List allManualChangesForImportedRedactions(ManualRedactions manualRedactions, Set importedRedactionsIds) { - - log.debug("allManualChangesForImportedRedactions {} manualRedactions resize: {} legalChanges {} removals {} force {}", importedRedactionsIds.size(), manualRedactions.getResizeRedactions().size(), manualRedactions.getLegalBasisChanges().size(), - manualRedactions.getIdsToRemove().size(), manualRedactions.getForceRedactions().size()); - - List annotations = Stream.of(manualRedactions.getForceRedactions(), - manualRedactions.getResizeRedactions(), - manualRedactions.getIdsToRemove(), - manualRedactions.getLegalBasisChanges()).flatMap(Collection::stream).map(baseAnnotation -> (BaseAnnotation) baseAnnotation) - .filter(baseAnnotation -> importedRedactionsIds.contains(baseAnnotation.getAnnotationId())).toList(); - return annotations; - } - - public void mergeIdsToRemove(IdRemoval idRemoval, List entityLogEntries, int analysisNumber) { - - var entity = entityLogEntries.stream().filter(entityLogEntry -> entityLogEntry.getId().equals(idRemoval.getAnnotationId())).findAny(); - entity.ifPresent(entityLogEntry -> { - entityLogEntry.setState(EntryState.IGNORED); - addChanges(entityLogEntry.getChanges(), ChangeType.REMOVED, analysisNumber, idRemoval.getRequestDate()); - entityLogEntry.getManualChanges() - .add(ManualChange.builder() - .manualRedactionType(ManualRedactionType.REMOVE_LOCALLY) - .requestedDate(idRemoval.getRequestDate()) - .processedDate(idRemoval.getProcessedDate() == null ? OffsetDateTime.now() : idRemoval.getProcessedDate()) - .userId(idRemoval.getUser()) - .build()); - }); - } - - public void mergeResizeRedactions(ManualResizeRedaction manualResizeRedaction, List entityLogEntries, int analysisNumber) { - - var entity = entityLogEntries.stream().filter(entityLogEntry -> entityLogEntry.getId().equals(manualResizeRedaction.getAnnotationId())).findAny(); - entity.ifPresent(entityLogEntry -> { - entityLogEntry.setTextAfter(manualResizeRedaction.getTextAfter()); - entityLogEntry.setTextBefore(manualResizeRedaction.getTextBefore()); - entityLogEntry.setPositions(convertPositions(manualResizeRedaction.getPositions())); - addChanges(entityLogEntry.getChanges(), ChangeType.CHANGED, analysisNumber, manualResizeRedaction.getRequestDate()); - ManualChange.ManualChangeBuilder manualChange = ManualChange.builder() - .manualRedactionType(ManualRedactionType.RESIZE) - .requestedDate(manualResizeRedaction.getRequestDate()) - .processedDate(manualResizeRedaction.getProcessedDate() == null ? OffsetDateTime.now() : manualResizeRedaction.getProcessedDate()) - .userId(manualResizeRedaction.getUser()); - entityLogEntry.getManualChanges().add(manualChange.build()); - }); - } - - public void mergeLegalBasisChanges(ManualLegalBasisChange manualLegalBasisChange, List entityLogEntries, int analysisNumber) { - - var entity = entityLogEntries.stream().filter(entityLogEntry -> entityLogEntry.getId().equals(manualLegalBasisChange.getAnnotationId())).findAny(); - entity.ifPresent(entityLogEntry -> { - entityLogEntry.setLegalBasis(manualLegalBasisChange.getLegalBasis()); - entityLogEntry.setSection(manualLegalBasisChange.getSection()); - entityLogEntry.setValue(manualLegalBasisChange.getValue()); - addChanges(entityLogEntry.getChanges(), ChangeType.CHANGED, analysisNumber, manualLegalBasisChange.getRequestDate()); - Map propertyChanges = getPropertyChanges(manualLegalBasisChange); - entityLogEntry.getManualChanges().add(ManualChange.builder() - .manualRedactionType(ManualRedactionType.LEGAL_BASIS_CHANGE) - .requestedDate(manualLegalBasisChange.getRequestDate()) - .processedDate(manualLegalBasisChange.getProcessedDate() == null ? OffsetDateTime.now() : manualLegalBasisChange.getProcessedDate()) - .propertyChanges(propertyChanges) - .userId(manualLegalBasisChange.getUser()) - .build()); - }); - - } - - - private Map getPropertyChanges(ManualLegalBasisChange manualLegalBasisChange) { - - Map propertyChanges = new HashMap<>(); - if (!Strings.isNullOrEmpty(manualLegalBasisChange.getLegalBasis())) { - propertyChanges.put("legalBasis", manualLegalBasisChange.getLegalBasis()); - } - if (!Strings.isNullOrEmpty(manualLegalBasisChange.getValue())) { - propertyChanges.put("value", manualLegalBasisChange.getValue()); - } - if(!Strings.isNullOrEmpty(manualLegalBasisChange.getSection())) { - propertyChanges.put("section", manualLegalBasisChange.getSection()); - } - return propertyChanges; - } - - public void mergeForceRedactions(ManualForceRedaction forceRedaction, List entityLogEntries, int analysisNumber) { - - var entity = entityLogEntries.stream().filter(entityLogEntry -> entityLogEntry.getId().equals(forceRedaction.getAnnotationId())).findAny(); - entity.ifPresent(entityLogEntry -> { - entityLogEntry.setLegalBasis(forceRedaction.getLegalBasis()); - entityLogEntry.setState(entityLogEntry.getEntryType().equals(EntryType.HINT) ? EntryState.SKIPPED : EntryState.APPLIED); - addChanges(entityLogEntry.getChanges(), ChangeType.CHANGED, analysisNumber, forceRedaction.getRequestDate()); - var forceRedactManualChange = ManualChange.builder() - .manualRedactionType(ManualRedactionType.FORCE_REDACT) - .requestedDate(forceRedaction.getRequestDate()) - .processedDate(forceRedaction.getProcessedDate() == null ? OffsetDateTime.now() : forceRedaction.getProcessedDate()) - .userId(forceRedaction.getUser()); - if (forceRedaction.getLegalBasis() != null && !forceRedaction.getLegalBasis().isEmpty()) { - forceRedactManualChange.propertyChanges(Map.of("legalBasis", forceRedaction.getLegalBasis())); - } - entityLogEntry.getManualChanges().add(forceRedactManualChange.build()); - }); - } - - private void addChanges(List changes, ChangeType changeType, int analysisNumber, OffsetDateTime offsetDateTime) { - - if (!changes.isEmpty()) { - changes.add(Change.builder() - .analysisNumber(analysisNumber) - .dateTime(offsetDateTime) - .type(changeType) - .build()); - } else { - changes.add(Change.builder().analysisNumber(analysisNumber).dateTime(OffsetDateTime.now()).type(changeType).build()); - } - } - - private List convertPositions(List rectangles) { - - return rectangles.stream().map(rectangle -> new Position(rectangle.getTopLeftX(), rectangle.getTopLeftY(), rectangle.getWidth(), rectangle.getHeight(), rectangle.getPage())).collect(Collectors.toList()); - } - -} - diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/ImportedRedactionService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/ImportedRedactionService.java index f2793281..e69de29b 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/ImportedRedactionService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/ImportedRedactionService.java @@ -1,339 +0,0 @@ -package com.iqser.red.service.redaction.v1.server.service; - -import java.time.OffsetDateTime; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.Set; -import java.util.stream.Collectors; - -import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest; -import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Change; -import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ChangeType; -import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog; -import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedaction; -import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedactions; -import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Point; -import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle; -import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry; -import org.springframework.stereotype.Service; - -import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry; -import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState; -import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryType; -import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position; - - -import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService; - -import io.micrometer.core.annotation.Timed; -import lombok.RequiredArgsConstructor; -import lombok.extern.slf4j.Slf4j; - -@Service -@Slf4j -@RequiredArgsConstructor -public class ImportedRedactionService { - - public static final String IMPORTED_REDACTION_TYPE = "imported_redaction"; - - private final DictionaryService dictionaryService; - private final RedactionStorageService redactionStorageService; - private final ImportedRedactionEntriesMergeService importedRedactionEntriesMergeService; - - - @Deprecated(forRemoval = true) - @Timed("redactmanager_processImportedRedactions") - public List processImportedRedactions(String dossierTemplateId, - String dossierId, - String fileId, - List redactionLogEntries, - boolean addImportedRedactions) { - - var importedRedactions = redactionStorageService.getImportedRedactions(dossierId, fileId); - if (importedRedactions == null) { - return redactionLogEntries; - } - - redactionLogEntries.forEach(redactionLogEntry -> addIntersections(redactionLogEntry, importedRedactions)); - - if (addImportedRedactions) { - return addImportedRedactionsRedactionLogEntries(dossierTemplateId, redactionLogEntries, importedRedactions); - } - - return redactionLogEntries; - } - - // Always recreate the imported redactions - @Timed("redactmanager_processImportedRedactions") - public void processImportedRedactions(EntityLog entityLog, AnalyzeRequest analyzeRequest) { - // recreate imported redactions with manual changes - ImportedRedactions importedRedactions = redactionStorageService.getImportedRedactions(analyzeRequest.getDossierId(), analyzeRequest.getFileId()); - if (importedRedactions == null || importedRedactions.getImportedRedactions() == null || importedRedactions.getImportedRedactions().isEmpty()) { - log.debug("checkForImportedRedactions - no imported redactions"); - return; - } - - List initialImportedRedactionsEntityLogEntries = addImportedRedactionsEntityLogEntries(analyzeRequest.getDossierTemplateId(), importedRedactions); - //merge manual changes for imported redactions - var newEntityLogsForImportedEntities = importedRedactionEntriesMergeService.mergeManualChangesForImportedRedactions(initialImportedRedactionsEntityLogEntries, analyzeRequest); - - //remove the old imported redactions - List previousImportedRedactionsEntries = entityLog.getEntityLogEntry() - .stream() - .filter(entry -> entry.getType().equals(ImportedRedactionService.IMPORTED_REDACTION_TYPE)) - .toList(); - entityLog.getEntityLogEntry().removeAll(previousImportedRedactionsEntries); - - // add all changes to entity log - entityLog.getEntityLogEntry().addAll(newEntityLogsForImportedEntities); - // remove all intersections and update with merged imported redactions - calculateIntersections(newEntityLogsForImportedEntities, entityLog, analyzeRequest.getAnalysisNumber()); - } - - private void calculateIntersections(List importedRedactionAndMergedEntries, EntityLog entityLog, int analysisNumber) { - if (!importedRedactionAndMergedEntries.isEmpty()) { - // imported redactions present, intersections must be added with merged imported redactions - Map> importedRedactionsMap = mapImportedRedactionsOnPage(importedRedactionAndMergedEntries); - entityLog.getEntityLogEntry().stream().filter(entry -> !entry.getType().equals(ImportedRedactionService.IMPORTED_REDACTION_TYPE)).forEach(redactionLogEntry -> { - redactionLogEntry.setImportedRedactionIntersections(new HashSet<>()); - addIntersections(redactionLogEntry, importedRedactionsMap, analysisNumber); - }); - } - } - - - @Deprecated(forRemoval = true) - private List addImportedRedactionsRedactionLogEntries(String dossierTemplateId, - List redactionLogEntries, - ImportedRedactions importedRedactions) { - - for (var importedRedactionsValues : importedRedactions.getImportedRedactions().values()) { - for (var importedRedaction : importedRedactionsValues) { - RedactionLogEntry redactionLogEntry = RedactionLogEntry.builder() - .id(importedRedaction.getId()) - .type(IMPORTED_REDACTION_TYPE) - .imported(true) - .redacted(true) - .positions(importedRedaction.getPositions().stream().map(this::normalize).collect(Collectors.toList())) - .color(getColor(IMPORTED_REDACTION_TYPE, dossierTemplateId)) - .build(); - - redactionLogEntries.add(redactionLogEntry); - } - } - - return redactionLogEntries; - } - - - private List addImportedRedactionsEntityLogEntries(String dossierTemplateId, ImportedRedactions importedRedactions) { - - List initialImportedRedactionsEntityLogEntries = new ArrayList<>(); - var now = OffsetDateTime.now(); - for (var importedRedactionsValues : importedRedactions.getImportedRedactions().values()) { - for (var importedRedaction : importedRedactionsValues) { - EntityLogEntry redactionLogEntry = EntityLogEntry.builder() - .id(importedRedaction.getId()) - .type(IMPORTED_REDACTION_TYPE) - .entryType(EntryType.ENTITY) - .value(Optional.ofNullable(importedRedaction.getValue()).orElse("")) - .imported(true) - .state(EntryState.APPLIED) - .positions(importedRedaction.getPositions()) - .color(getColor(IMPORTED_REDACTION_TYPE, dossierTemplateId)) - .containingNodeId(Collections.emptyList()) - .build(); - redactionLogEntry.getChanges().add(new Change(1, ChangeType.ADDED, now)); - initialImportedRedactionsEntityLogEntries.add(redactionLogEntry); - } - } - - return initialImportedRedactionsEntityLogEntries; - } - - - private List toPositions(List positions) { - - return positions.stream().map(this::toPosition).toList(); - } - - - private Position toPosition(Rectangle rectangle) { - - return new Position(new float[]{rectangle.getTopLeft().getX(), rectangle.getTopLeft().getY(), rectangle.getWidth(), rectangle.getHeight()}, rectangle.getPage()); - } - - - @Deprecated(forRemoval = true) - private void addIntersections(RedactionLogEntry redactionLogEntry, ImportedRedactions importedRedactions) { - - for (Rectangle rectangle : redactionLogEntry.getPositions()) { - var normalizedRectangle = normalize(rectangle); - if (importedRedactions.getImportedRedactions().containsKey(rectangle.getPage())) { - var importedRedactionsOnPage = importedRedactions.getImportedRedactions().get(rectangle.getPage()); - for (ImportedRedaction importedRedaction : importedRedactionsOnPage) { - for (Position importedRedactionPosition : importedRedaction.getPositions()) { - if (rectOverlap(normalizedRectangle, normalize(importedRedactionPosition))) { - if (redactionLogEntry.getImportedRedactionIntersections() == null) { - redactionLogEntry.setImportedRedactionIntersections(new HashSet<>()); - } - redactionLogEntry.getImportedRedactionIntersections().add(importedRedaction.getId()); - } - } - } - } - } - } - - public Map> mapImportedRedactionsOnPage(List importedRedactions) { - - Map> importedRedactionsMap = new HashMap<>(); - Set pageNumbers = importedRedactions.stream().map(EntityLogEntry::getPositions) - .flatMap(Collection::stream) - .map(com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position::getPageNumber) - .collect(Collectors.toSet()); - pageNumbers.forEach(pageNumber -> - importedRedactionsMap.put(pageNumber, importedRedactions.stream().filter(i -> pageNumber == i.getPositions().get(0).getPageNumber()).collect(Collectors.toList()))); - return importedRedactionsMap; - } - - public void addIntersections(EntityLogEntry entityLogEntry, Map> importedRedactionsMap, int analysisNumber) { - - for (Position rectangle : entityLogEntry.getPositions()) { - var normalizedRectangle = normalize(rectangle); - if (importedRedactionsMap.containsKey(rectangle.getPageNumber())) { - var importedRedactionsOnPage = importedRedactionsMap.get(rectangle.getPageNumber()); - for (EntityLogEntry importedRedactionLogEntry : importedRedactionsOnPage) { - for (Position importedRedactionPosition : importedRedactionLogEntry.getPositions()) { - if (rectOverlap(normalizedRectangle, normalize(importedRedactionPosition))) { - if (entityLogEntry.getImportedRedactionIntersections() == null) { - entityLogEntry.setImportedRedactionIntersections(new HashSet<>()); - } - entityLogEntry.getImportedRedactionIntersections().add(importedRedactionLogEntry.getId()); - - if(entityLogEntry.getState() != EntryState.REMOVED) { - entityLogEntry.setState(EntryState.REMOVED); - entityLogEntry.getChanges().add(new Change(analysisNumber, ChangeType.REMOVED, OffsetDateTime.now())); - } - } - } - } - } - } - } - - private void addIntersections(EntityLogEntry entityLogEntry, ImportedRedactions importedRedactions) { - - for (Position rectangle : entityLogEntry.getPositions()) { - var normalizedRectangle = normalize(rectangle); - if (importedRedactions.getImportedRedactions().containsKey(rectangle.getPageNumber())) { - var importedRedactionsOnPage = importedRedactions.getImportedRedactions().get(rectangle.getPageNumber()); - for (ImportedRedaction importedRedaction : importedRedactionsOnPage) { - for (Position importedRedactionPosition : importedRedaction.getPositions()) { - if (rectOverlap(normalizedRectangle, normalize(importedRedactionPosition))) { - if (entityLogEntry.getImportedRedactionIntersections() == null) { - entityLogEntry.setImportedRedactionIntersections(new HashSet<>()); - } - entityLogEntry.getImportedRedactionIntersections().add(importedRedaction.getId()); - } - } - } - } - } - } - - - boolean valueInRange(float value, float min, float max) { - - return round(value) >= round(min) && round(value) <= round(max); - } - - - boolean rectOverlap(Rectangle a, Rectangle b) { - - boolean xOverlap = valueInRange(a.getTopLeft().getX(), b.getTopLeft().getX(), b.getTopLeft().getX() + b.getWidth()) || valueInRange(b.getTopLeft().getX(), - a.getTopLeft().getX(), - a.getTopLeft().getX() + a.getWidth()); - - boolean yOverlap = valueInRange(a.getTopLeft().getY(), b.getTopLeft().getY(), b.getTopLeft().getY() + b.getHeight()) || valueInRange(b.getTopLeft().getY(), - a.getTopLeft().getY(), - a.getTopLeft().getY() + a.getHeight()); - - return xOverlap && yOverlap; - } - - - @Deprecated(forRemoval = true) - private Rectangle normalize(Rectangle rectangle) { - - Rectangle r = new Rectangle(); - Point p = new Point(); - - if (rectangle.getWidth() < 0) { - p.setX(rectangle.getTopLeft().getX() + rectangle.getWidth()); - r.setWidth(Math.abs(rectangle.getWidth())); - } else { - p.setX(rectangle.getTopLeft().getX()); - r.setWidth(rectangle.getWidth()); - } - - if (rectangle.getHeight() < 0) { - p.setY(rectangle.getTopLeft().getY() + rectangle.getHeight()); - r.setHeight(Math.abs(rectangle.getHeight())); - } else { - p.setY(rectangle.getTopLeft().getY()); - r.setHeight(rectangle.getHeight()); - } - - r.setTopLeft(p); - r.setPage(rectangle.getPage()); - return r; - } - - - private Rectangle normalize(Position position) { - - Rectangle r = new Rectangle(); - Point p = new Point(); - - if (position.getRectangle()[2] < 0) { - p.setX(position.getRectangle()[0] + position.getRectangle()[2]); - r.setWidth(Math.abs(position.getRectangle()[2])); - } else { - p.setX(position.getRectangle()[0]); - r.setWidth(position.getRectangle()[2]); - } - - if (position.getRectangle()[3] < 0) { - p.setY(position.getRectangle()[1] + position.getRectangle()[3]); - r.setHeight(Math.abs(position.getRectangle()[3])); - } else { - p.setY(position.getRectangle()[1]); - r.setHeight(position.getRectangle()[3]); - } - - r.setTopLeft(p); - r.setPage(position.getPageNumber()); - return r; - } - - - private float round(float value) { - - double d = Math.pow(10, 0); - return (float) (Math.round(value * d) / d); - } - - - private float[] getColor(String type, String dossierTemplateId) { - - return dictionaryService.getColor(type, dossierTemplateId); - } - -} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/ManualChangesApplicationService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/ManualChangesApplicationService.java index c0f9c402..aed3320f 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/ManualChangesApplicationService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/ManualChangesApplicationService.java @@ -16,7 +16,7 @@ import org.springframework.stereotype.Service; import com.google.common.collect.Sets; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction; -import com.iqser.red.service.redaction.v1.server.model.ManualEntity; +import com.iqser.red.service.redaction.v1.server.model.PrecursorEntity; import com.iqser.red.service.redaction.v1.server.model.document.entity.IEntity; import com.iqser.red.service.redaction.v1.server.model.document.entity.PositionOnPage; import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity; @@ -79,7 +79,7 @@ public class ManualChangesApplicationService { .collect(Collectors.toList())); SemanticNode node = entityToBeResized.getDeepestFullyContainingNode(); - ManualEntity searchEntity = ManualEntity.fromManualResizeRedaction(manualResizeRedaction); + PrecursorEntity searchEntity = PrecursorEntity.fromManualResizeRedaction(manualResizeRedaction); // Loop through nodes starting from the deepest fully containing node all the way to the document node while (node != null) { diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/NotFoundImportedEntitiesService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/NotFoundImportedEntitiesService.java new file mode 100644 index 00000000..159d7c6b --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/NotFoundImportedEntitiesService.java @@ -0,0 +1,102 @@ +package com.iqser.red.service.redaction.v1.server.service; + +import java.time.OffsetDateTime; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Change; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ChangeType; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position; +import com.iqser.red.service.redaction.v1.server.model.PrecursorEntity; +import com.iqser.red.service.redaction.v1.server.model.RectangleWithPage; +import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService; + +import org.springframework.stereotype.Service; + +import io.micrometer.core.annotation.Timed; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +@Service +@Slf4j +@RequiredArgsConstructor +public class NotFoundImportedEntitiesService { + + public static final String IMPORTED_REDACTION_TYPE = "imported_redaction"; + + + @Timed("redactmanager_processEntityLog") + public void processEntityLog(EntityLog entityLog, AnalyzeRequest analyzeRequest, List notFoundEntities) { + // recreate imported redactions with manual changes + if (notFoundEntities == null || notFoundEntities.isEmpty()) { + log.debug("checkForImportedRedactions - no not found imported entities"); + return; + } + + // remove all intersections and update imported entities + calculateIntersectionsForNotFound(notFoundEntities, entityLog, analyzeRequest.getAnalysisNumber()); + } + + + private void calculateIntersectionsForNotFound(List notFoundEntities, EntityLog entityLog, int analysisNumber) { + + if (!notFoundEntities.isEmpty()) { + // imported redactions present, intersections must be added with merged imported redactions + Map> importedRedactionsMap = mapImportedRedactionsOnPage(notFoundEntities); + entityLog.getEntityLogEntry().stream().filter(entry -> !entry.getEngines().contains(Engine.IMPORTED)).forEach(redactionLogEntry -> { + redactionLogEntry.setImportedRedactionIntersections(new HashSet<>()); + addIntersections(redactionLogEntry, importedRedactionsMap, analysisNumber); + }); + } + } + + + public Map> mapImportedRedactionsOnPage(List importedEntities) { + + Map> importedRedactionsMap = new HashMap<>(); + Set pageNumbers = importedEntities.stream() + .map(PrecursorEntity::getEntityPosition) + .flatMap(Collection::stream) + .map(RectangleWithPage::pageNumber) + .collect(Collectors.toSet()); + pageNumbers.forEach(pageNumber -> importedRedactionsMap.put(pageNumber, + importedEntities.stream().filter(i -> pageNumber == i.getEntityPosition().get(0).pageNumber()).collect(Collectors.toList()))); + return importedRedactionsMap; + } + + + public void addIntersections(EntityLogEntry entityLogEntry, Map> importedEntitiesMap, int analysisNumber) { + + for (Position rectangle : entityLogEntry.getPositions()) { + if (importedEntitiesMap.containsKey(rectangle.getPageNumber())) { + var importedEntitiesOnPage = importedEntitiesMap.get(rectangle.getPageNumber()); + for (PrecursorEntity precursorEntity : importedEntitiesOnPage) { + for (RectangleWithPage rectangleWithPage : precursorEntity.getEntityPosition()) { + if (rectangle.toRectangle2D().intersects(rectangleWithPage.rectangle2D())) { + if (entityLogEntry.getImportedRedactionIntersections() == null) { + entityLogEntry.setImportedRedactionIntersections(new HashSet<>()); + } + entityLogEntry.getImportedRedactionIntersections().add(precursorEntity.getId()); + + if (entityLogEntry.getState() != EntryState.REMOVED) { + entityLogEntry.setState(EntryState.REMOVED); + entityLogEntry.getChanges().add(new Change(analysisNumber, ChangeType.REMOVED, OffsetDateTime.now())); + } + } + } + } + } + } + } + +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RedactionLogCreatorService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RedactionLogCreatorService.java index 596167ea..2b3dab77 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RedactionLogCreatorService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RedactionLogCreatorService.java @@ -20,7 +20,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlo import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Point; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry; -import com.iqser.red.service.redaction.v1.server.model.ManualEntity; +import com.iqser.red.service.redaction.v1.server.model.PrecursorEntity; import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType; import com.iqser.red.service.redaction.v1.server.model.document.entity.IEntity; import com.iqser.red.service.redaction.v1.server.model.document.entity.ManualChangeOverwrite; @@ -43,7 +43,7 @@ public class RedactionLogCreatorService { private final ManualChangeFactory manualChangeFactory; - public List createRedactionLog(Document document, String dossierTemplateId, List notFoundManualRedactionEntries) { + public List createRedactionLog(Document document, String dossierTemplateId, List notFoundManualRedactionEntries) { List entries = new ArrayList<>(); Set processIds = new HashSet<>(); @@ -219,40 +219,40 @@ public class RedactionLogCreatorService { } - public RedactionLogEntry createRedactionLogEntry(ManualEntity manualEntity, String dossierTemplateId) { + public RedactionLogEntry createRedactionLogEntry(PrecursorEntity precursorEntity, String dossierTemplateId) { - String type = manualEntity.getManualOverwrite().getType().orElse(manualEntity.getType()); - boolean isHint = isHint(manualEntity.getEntityType()); + String type = precursorEntity.getManualOverwrite().getType().orElse(precursorEntity.getType()); + boolean isHint = isHint(precursorEntity.getEntityType()); return RedactionLogEntry.builder() - .id(manualEntity.getId()) - .color(getColor(type, dossierTemplateId, manualEntity.applied(), isHint)) - .reason(manualEntity.buildReasonWithManualChangeDescriptions()) - .legalBasis(manualEntity.legalBasis()) - .value(manualEntity.value()) + .id(precursorEntity.getId()) + .color(getColor(type, dossierTemplateId, precursorEntity.applied(), isHint)) + .reason(precursorEntity.buildReasonWithManualChangeDescriptions()) + .legalBasis(precursorEntity.legalBasis()) + .value(precursorEntity.value()) .type(type) - .redacted(manualEntity.applied()) + .redacted(precursorEntity.applied()) .isHint(isHint) - .isRecommendation(manualEntity.getEntityType().equals(EntityType.RECOMMENDATION)) - .isFalsePositive(manualEntity.getEntityType().equals(EntityType.FALSE_POSITIVE) || manualEntity.getEntityType().equals(EntityType.FALSE_RECOMMENDATION)) - .section(manualEntity.getManualOverwrite().getSection().orElse(manualEntity.getSection())) + .isRecommendation(precursorEntity.getEntityType().equals(EntityType.RECOMMENDATION)) + .isFalsePositive(precursorEntity.getEntityType().equals(EntityType.FALSE_POSITIVE) || precursorEntity.getEntityType().equals(EntityType.FALSE_RECOMMENDATION)) + .section(precursorEntity.getManualOverwrite().getSection().orElse(precursorEntity.getSection())) .sectionNumber(0) - .matchedRule(manualEntity.getMatchedRule().getRuleIdentifier().toString()) - .rectangle(manualEntity.isRectangle()) - .isDictionaryEntry(manualEntity.isDictionaryEntry()) - .isDossierDictionaryEntry(manualEntity.isDossierDictionaryEntry()) + .matchedRule(precursorEntity.getMatchedRule().getRuleIdentifier().toString()) + .rectangle(precursorEntity.isRectangle()) + .isDictionaryEntry(precursorEntity.isDictionaryEntry()) + .isDossierDictionaryEntry(precursorEntity.isDossierDictionaryEntry()) .textAfter("") .textBefore("") .startOffset(-1) .endOffset(-1) - .positions(manualEntity.getManualOverwrite() + .positions(precursorEntity.getManualOverwrite() .getPositions() - .orElse(manualEntity.getEntityPosition()) + .orElse(precursorEntity.getEntityPosition()) .stream() .map(entityPosition -> toRedactionLogRectangle(entityPosition.rectangle2D(), entityPosition.pageNumber())) .toList()) .engines(Collections.emptySet()) .reference(Collections.emptySet()) - .manualChanges(mapManualChanges(manualEntity.getManualOverwrite(), isHint)) + .manualChanges(mapManualChanges(precursorEntity.getManualOverwrite(), isHint)) .build(); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/UnprocessedChangesService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/UnprocessedChangesService.java index dc1c49aa..b7dfd989 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/UnprocessedChangesService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/UnprocessedChangesService.java @@ -21,17 +21,14 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations import com.iqser.red.service.redaction.v1.model.AnalyzeResponse; import com.iqser.red.service.redaction.v1.model.QueueNames; import com.iqser.red.service.redaction.v1.model.UnprocessedManualEntity; -import com.iqser.red.service.redaction.v1.server.model.ManualEntity; -import com.iqser.red.service.redaction.v1.server.model.document.TextRange; -import com.iqser.red.service.redaction.v1.server.model.document.entity.PositionOnPage; +import com.iqser.red.service.redaction.v1.server.model.PrecursorEntity; import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document; -import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode; import com.iqser.red.service.redaction.v1.server.service.document.DocumentGraphMapper; import com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService; import com.iqser.red.service.redaction.v1.server.service.document.EntityEnrichmentService; import com.iqser.red.service.redaction.v1.server.service.document.EntityFindingUtility; -import com.iqser.red.service.redaction.v1.server.service.document.ManualEntityCreationService; +import com.iqser.red.service.redaction.v1.server.service.document.EntityFromPrecursorCreationService; import com.iqser.red.service.redaction.v1.server.storage.ObservedStorageService; import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService; @@ -55,7 +52,7 @@ public class UnprocessedChangesService { final EntityFindingUtility entityFindingUtility; final RedactionStorageService redactionStorageService; final EntityEnrichmentService entityEnrichmentService; - final ManualEntityCreationService manualEntityCreationService; + final EntityFromPrecursorCreationService entityFromPrecursorCreationService; final DictionaryService dictionaryService; final ManualChangesApplicationService manualChangesApplicationService; @@ -84,18 +81,22 @@ public class UnprocessedChangesService { allAnnotationIds.addAll(resizeIds); List manualResizeRedactions = analyzeRequest.getManualRedactions().getResizeRedactions().stream().toList(); - List manualEntitiesToBeResized = previousEntityLog.getEntityLogEntry() - .stream().filter(entityLogEntry -> resizeIds.contains(entityLogEntry.getId())).toList() - .stream().map(ManualEntity::fromEntityLogEntry).toList(); + List manualEntitiesToBeResized = previousEntityLog.getEntityLogEntry() + .stream() + .filter(entityLogEntry -> resizeIds.contains(entityLogEntry.getId())) + .toList() + .stream() + .map(PrecursorEntity::fromEntityLogEntry) + .toList(); if (!manualResizeRedactions.isEmpty()) { processManualResizeRedactions(document, manualEntitiesToBeResized, unprocessedManualEntities, manualResizeRedactions); } - List notFoundManualEntities = new ArrayList<>(); - List manualEntities = manualEntitiesConverter(analyzeRequest.getManualRedactions(), analyzeRequest.getDossierTemplateId()); + List notFoundManualEntities = new ArrayList<>(); + List manualEntities = manualEntitiesConverter(analyzeRequest.getManualRedactions(), analyzeRequest.getDossierTemplateId()); if (!manualEntities.isEmpty()) { - notFoundManualEntities = manualEntityCreationService.toTextEntity(manualEntities, document); + notFoundManualEntities = entityFromPrecursorCreationService.toTextEntity(manualEntities, document); } document.getEntities().forEach(textEntity -> { @@ -127,24 +128,24 @@ public class UnprocessedChangesService { private void processManualResizeRedactions(Document document, - List manualEntities, + List manualEntities, List unprocessedManualEntities, List manualResizeRedactions) { Map> tempEntities = entityFindingUtility.findAllPossibleEntitiesAndGroupByValue(document, manualEntities); - for (ManualEntity manualEntity : manualEntities) { + for (PrecursorEntity precursorEntity : manualEntities) { - Optional optionalTextEntity = entityFindingUtility.findClosestEntityAndReturnEmptyIfNotFound(manualEntity, tempEntities, THRESHOLD); + Optional optionalTextEntity = entityFindingUtility.findClosestEntityAndReturnEmptyIfNotFound(precursorEntity, tempEntities, THRESHOLD); if (optionalTextEntity.isEmpty()) { - unprocessedManualEntities.add(builDefaultUnprocessedManualEntity(manualEntity)); + unprocessedManualEntities.add(builDefaultUnprocessedManualEntity(precursorEntity)); continue; } - TextEntity correctEntity = createCorrectEntity(manualEntity, optionalTextEntity.get()); + TextEntity correctEntity = createCorrectEntity(precursorEntity, optionalTextEntity.get()); Optional optionalManualResizeRedaction = manualResizeRedactions.stream() - .filter(manualResizeRedaction -> manualResizeRedaction.getAnnotationId().equals(manualEntity.getId())) + .filter(manualResizeRedaction -> manualResizeRedaction.getAnnotationId().equals(precursorEntity.getId())) .findFirst(); if (optionalManualResizeRedaction.isPresent()) { ManualResizeRedaction manualResizeRedaction = optionalManualResizeRedaction.get(); @@ -156,7 +157,6 @@ public class UnprocessedChangesService { correctEntity.removeFromGraph(); } } - } // remove all temp entities from the graph @@ -164,9 +164,9 @@ public class UnprocessedChangesService { } - private TextEntity createCorrectEntity(ManualEntity manualEntity, TextEntity closestEntity) { + private TextEntity createCorrectEntity(PrecursorEntity precursorEntity, TextEntity closestEntity) { - TextEntity correctEntity = TextEntity.initialEntityNode(closestEntity.getTextRange(), manualEntity.type(), manualEntity.getEntityType(), manualEntity.getId()); + TextEntity correctEntity = TextEntity.initialEntityNode(closestEntity.getTextRange(), precursorEntity.type(), precursorEntity.getEntityType(), precursorEntity.getId()); correctEntity.setDeepestFullyContainingNode(closestEntity.getDeepestFullyContainingNode()); correctEntity.setIntersectingNodes(new ArrayList<>(closestEntity.getIntersectingNodes())); @@ -180,40 +180,40 @@ public class UnprocessedChangesService { correctEntity.getIntersectingNodes().forEach(n -> n.getEntities().add(correctEntity)); correctEntity.getPages().forEach(page -> page.getEntities().add(correctEntity)); - correctEntity.addMatchedRules(manualEntity.getMatchedRuleList()); - correctEntity.setDictionaryEntry(manualEntity.isDictionaryEntry()); - correctEntity.setDossierDictionaryEntry(manualEntity.isDossierDictionaryEntry()); - correctEntity.getManualOverwrite().addChanges(manualEntity.getManualOverwrite().getManualChangeLog()); + correctEntity.addMatchedRules(precursorEntity.getMatchedRuleList()); + correctEntity.setDictionaryEntry(precursorEntity.isDictionaryEntry()); + correctEntity.setDossierDictionaryEntry(precursorEntity.isDossierDictionaryEntry()); + correctEntity.getManualOverwrite().addChanges(precursorEntity.getManualOverwrite().getManualChangeLog()); - return correctEntity; - } + return correctEntity; +} - private UnprocessedManualEntity builDefaultUnprocessedManualEntity(ManualEntity manualEntity) { +private UnprocessedManualEntity builDefaultUnprocessedManualEntity(PrecursorEntity precursorEntity) { - return UnprocessedManualEntity.builder() - .annotationId(manualEntity.getId()) - .textAfter("") - .textBefore("") - .section("") - .positions(manualEntity.getManualOverwrite() - .getPositions() - .orElse(manualEntity.getEntityPosition()) - .stream() - .map(entityPosition -> new Position(entityPosition.rectangle2D(), entityPosition.pageNumber())) - .toList()) - .build(); - } + return UnprocessedManualEntity.builder() + .annotationId(precursorEntity.getId()) + .textAfter("") + .textBefore("") + .section("") + .positions(precursorEntity.getManualOverwrite() + .getPositions() + .orElse(precursorEntity.getEntityPosition()) + .stream() + .map(entityPosition -> new Position(entityPosition.rectangle2D(), entityPosition.pageNumber())) + .toList()) + .build(); +} - private List manualEntitiesConverter(ManualRedactions manualRedactions, String dossierTemplateId) { +private List manualEntitiesConverter(ManualRedactions manualRedactions, String dossierTemplateId) { - return manualRedactions.getEntriesToAdd() - .stream() - .filter(manualRedactionEntry -> manualRedactionEntry.getPositions() != null && !manualRedactionEntry.getPositions().isEmpty()) - .map(manualRedactionEntry -> ManualEntity.fromManualRedactionEntry(manualRedactionEntry, - dictionaryService.isHint(manualRedactionEntry.getType(), dossierTemplateId))) - .toList(); - } + return manualRedactions.getEntriesToAdd() + .stream() + .filter(manualRedactionEntry -> manualRedactionEntry.getPositions() != null && !manualRedactionEntry.getPositions().isEmpty()) + .map(manualRedactionEntry -> PrecursorEntity.fromManualRedactionEntry(manualRedactionEntry, + dictionaryService.isHint(manualRedactionEntry.getType(), dossierTemplateId))) + .toList(); +} } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityFindingUtility.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityFindingUtility.java index 61f94301..76b9cfb2 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityFindingUtility.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityFindingUtility.java @@ -19,7 +19,7 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import com.iqser.red.service.redaction.v1.server.model.ClosestEntity; -import com.iqser.red.service.redaction.v1.server.model.ManualEntity; +import com.iqser.red.service.redaction.v1.server.model.PrecursorEntity; import com.iqser.red.service.redaction.v1.server.model.RectangleWithPage; import com.iqser.red.service.redaction.v1.server.model.dictionary.SearchImplementation; import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType; @@ -47,35 +47,35 @@ public class EntityFindingUtility { } - public Optional findClosestEntityAndReturnEmptyIfNotFound(ManualEntity manualEntity, Map> entitiesWithSameValue, double matchThreshold) { + public Optional findClosestEntityAndReturnEmptyIfNotFound(PrecursorEntity precursorEntity, Map> entitiesWithSameValue, double matchThreshold) { - if (manualEntity.getValue() == null) { + if (precursorEntity.getValue() == null) { return Optional.empty(); } - List possibleEntities = entitiesWithSameValue.get(manualEntity.getValue().toLowerCase(Locale.ENGLISH)); + List possibleEntities = entitiesWithSameValue.get(precursorEntity.getValue().toLowerCase(Locale.ENGLISH)); if (entityIdentifierValueNotFound(possibleEntities)) { - log.warn("Entity could not be created with manualEntity: {}, due to the value {} not being found anywhere.", manualEntity, manualEntity.getValue()); + log.warn("Entity could not be created with precursorEntity: {}, due to the value {} not being found anywhere.", precursorEntity, precursorEntity.getValue()); return Optional.empty(); } Optional optionalClosestEntity = possibleEntities.stream() - .filter(entity -> pagesMatch(entity, manualEntity.getEntityPosition())) - .map(entity -> ClosestEntity.builder().distance(calculateMinDistance(manualEntity.getEntityPosition(), entity)).textEntity(entity).build()) + .filter(entity -> pagesMatch(entity, precursorEntity.getEntityPosition())) + .map(entity -> ClosestEntity.builder().distance(calculateMinDistance(precursorEntity.getEntityPosition(), entity)).textEntity(entity).build()) .min(Comparator.comparingDouble(ClosestEntity::getDistance)); if (optionalClosestEntity.isEmpty()) { - log.warn("No Entity with value {} found on page {}", manualEntity.getValue(), manualEntity.getEntityPosition()); + log.warn("No Entity with value {} found on page {}", precursorEntity.getValue(), precursorEntity.getEntityPosition()); return Optional.empty(); } ClosestEntity closestEntity = optionalClosestEntity.get(); if (closestEntity.getDistance() > matchThreshold) { log.warn("For entity {} on page {} with positions {} distance to closest found entity is {} and therefore higher than the threshold of {}", - manualEntity.getValue(), - manualEntity.getEntityPosition().get(0).pageNumber(), - manualEntity.getEntityPosition().stream().map(RectangleWithPage::rectangle2D).toList(), + precursorEntity.getValue(), + precursorEntity.getEntityPosition().get(0).pageNumber(), + precursorEntity.getEntityPosition().stream().map(RectangleWithPage::rectangle2D).toList(), closestEntity.getDistance(), matchThreshold); return Optional.empty(); @@ -150,10 +150,10 @@ public class EntityFindingUtility { } - public Map> findAllPossibleEntitiesAndGroupByValue(SemanticNode node, List manualEntities) { + public Map> findAllPossibleEntitiesAndGroupByValue(SemanticNode node, List manualEntities) { Set pageNumbers = manualEntities.stream().flatMap(entry -> entry.getEntityPosition().stream().map(RectangleWithPage::pageNumber)).collect(Collectors.toSet()); - Set entryValues = manualEntities.stream().map(ManualEntity::getValue).filter(Objects::nonNull).map(String::toLowerCase).collect(Collectors.toSet()); + Set entryValues = manualEntities.stream().map(PrecursorEntity::getValue).filter(Objects::nonNull).map(String::toLowerCase).collect(Collectors.toSet()); if (!pageNumbers.stream().allMatch(node::onPage)) { throw new IllegalArgumentException(format("SemanticNode \"%s\" does not contain these pages %s, it has pages: %s", diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/ManualEntityCreationService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityFromPrecursorCreationService.java similarity index 59% rename from redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/ManualEntityCreationService.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityFromPrecursorCreationService.java index 6b0379f1..faddf9bd 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/ManualEntityCreationService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityFromPrecursorCreationService.java @@ -3,22 +3,21 @@ package com.iqser.red.service.redaction.v1.server.service.document; import java.util.ArrayList; import java.util.Collection; import java.util.HashSet; -import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; +import java.util.stream.Collectors; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedactions; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.BaseAnnotation; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval; -import com.iqser.red.service.redaction.v1.server.model.ManualEntity; -import com.iqser.red.service.redaction.v1.server.model.document.TextRange; +import com.iqser.red.service.redaction.v1.server.model.PrecursorEntity; import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType; -import com.iqser.red.service.redaction.v1.server.model.document.entity.PositionOnPage; import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity; import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode; import com.iqser.red.service.redaction.v1.server.service.DictionaryService; @@ -30,7 +29,7 @@ import lombok.extern.slf4j.Slf4j; @Slf4j @Service @FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE) -public class ManualEntityCreationService { +public class EntityFromPrecursorCreationService { static double MATCH_THRESHOLD = 10; // Is compared to the average sum of distances in pdf coordinates for each corner of the bounding box of the entities EntityFindingUtility entityFindingUtility; @@ -39,7 +38,7 @@ public class ManualEntityCreationService { @Autowired - public ManualEntityCreationService(EntityEnrichmentService entityEnrichmentService, DictionaryService dictionaryService, EntityFindingUtility entityFindingUtility) { + public EntityFromPrecursorCreationService(EntityEnrichmentService entityEnrichmentService, DictionaryService dictionaryService, EntityFindingUtility entityFindingUtility) { this.entityFindingUtility = entityFindingUtility; entityCreationService = new EntityCreationService(entityEnrichmentService); @@ -47,10 +46,10 @@ public class ManualEntityCreationService { } - public List createRedactionEntitiesIfFoundAndReturnNotFoundEntries(ManualRedactions manualRedactions, SemanticNode node, String dossierTemplateId) { + public List createEntitiesIfFoundAndReturnNotFoundEntries(ManualRedactions manualRedactions, SemanticNode node, String dossierTemplateId) { Set idRemovals = manualRedactions.getIdsToRemove(); - List manualEntities = manualRedactions.getEntriesToAdd() + List manualEntities = manualRedactions.getEntriesToAdd() .stream() .filter(manualRedactionEntry -> !(idRemovals.stream() .map(BaseAnnotation::getAnnotationId) @@ -62,7 +61,7 @@ public class ManualEntityCreationService { .get() .getRequestDate()))) .filter(manualRedactionEntry -> !(manualRedactionEntry.isAddToDictionary() || manualRedactionEntry.isAddToDossierDictionary())) - .map(manualRedactionEntry -> ManualEntity.fromManualRedactionEntry(manualRedactionEntry, + .map(manualRedactionEntry -> PrecursorEntity.fromManualRedactionEntry(manualRedactionEntry, dictionaryService.isHint(manualRedactionEntry.getType(), dossierTemplateId))) .peek(manualEntity -> { if (manualEntity.getEntityType().equals(EntityType.HINT)) { @@ -77,34 +76,46 @@ public class ManualEntityCreationService { } - public List toTextEntity(List manualEntities, SemanticNode node) { + public List createEntitiesIfFoundAndReturnNotFoundEntries(ImportedRedactions importedRedactions, SemanticNode node) { - Map> tempEntitiesByValue = entityFindingUtility.findAllPossibleEntitiesAndGroupByValue(node, manualEntities); + List importedEntities = importedRedactions.getImportedRedactions() + .stream() + .map(PrecursorEntity::fromImportedEntry) + .peek(importedEntity -> importedEntity.apply("IMP.0.0", importedEntity.getReason(), importedEntity.getLegalBasis())) + .toList(); + //we could add image handling here as well and create not only text entities + return toTextEntity(importedEntities, node); + } - List notFoundManualEntities = new LinkedList<>(); - for (ManualEntity manualEntity : manualEntities) { - Optional optionalClosestEntity = entityFindingUtility.findClosestEntityAndReturnEmptyIfNotFound(manualEntity, tempEntitiesByValue, MATCH_THRESHOLD); + + public List toTextEntity(List precursorEntities, SemanticNode node) { + + var notFoundEntities = precursorEntities.stream().filter(PrecursorEntity::isRectangle).collect(Collectors.toList()); + var findableEntities = precursorEntities.stream().filter(precursorEntity -> !precursorEntity.isRectangle()).toList(); + Map> tempEntitiesByValue = entityFindingUtility.findAllPossibleEntitiesAndGroupByValue(node, findableEntities); + + for (PrecursorEntity precursorEntity : findableEntities) { + Optional optionalClosestEntity = entityFindingUtility.findClosestEntityAndReturnEmptyIfNotFound(precursorEntity, tempEntitiesByValue, MATCH_THRESHOLD); if (optionalClosestEntity.isEmpty()) { - notFoundManualEntities.add(manualEntity); + notFoundEntities.add(precursorEntity); continue; } - createCorrectEntity(manualEntity, optionalClosestEntity.get()); + createCorrectEntity(precursorEntity, optionalClosestEntity.get()); } tempEntitiesByValue.values().stream().flatMap(Collection::stream).forEach(TextEntity::removeFromGraph); - return notFoundManualEntities; + return notFoundEntities; } /** * Deletes the temp Entity and creates a RedactionEntity with correct values, based on the given parameters. * - * @param manualEntity The entity identifier for the RedactionEntity. - * @param closestEntity The closest Boundary to the RedactionEntity. + * @param precursorEntity The entity identifier for the RedactionEntity. + * @param closestEntity The closest Boundary to the RedactionEntity. */ - private void createCorrectEntity(ManualEntity manualEntity, TextEntity closestEntity) { - - TextEntity correctEntity = TextEntity.initialEntityNode(closestEntity.getTextRange(), manualEntity.type(), manualEntity.getEntityType(), manualEntity.getId()); + private void createCorrectEntity(PrecursorEntity precursorEntity, TextEntity closestEntity) { + TextEntity correctEntity = TextEntity.initialEntityNode(closestEntity.getTextRange(), precursorEntity.type(), precursorEntity.getEntityType(), precursorEntity.getId()); correctEntity.setDeepestFullyContainingNode(closestEntity.getDeepestFullyContainingNode()); correctEntity.setIntersectingNodes(new ArrayList<>(closestEntity.getIntersectingNodes())); correctEntity.setDuplicateTextRanges(new ArrayList<>(closestEntity.getDuplicateTextRanges())); @@ -117,10 +128,11 @@ public class ManualEntityCreationService { correctEntity.getIntersectingNodes().forEach(n -> n.getEntities().add(correctEntity)); correctEntity.getPages().forEach(page -> page.getEntities().add(correctEntity)); - correctEntity.addMatchedRules(manualEntity.getMatchedRuleList()); - correctEntity.setDictionaryEntry(manualEntity.isDictionaryEntry()); - correctEntity.setDossierDictionaryEntry(manualEntity.isDossierDictionaryEntry()); - correctEntity.getManualOverwrite().addChanges(manualEntity.getManualOverwrite().getManualChangeLog()); + correctEntity.addMatchedRules(precursorEntity.getMatchedRuleList()); + correctEntity.setDictionaryEntry(precursorEntity.isDictionaryEntry()); + correctEntity.setDossierDictionaryEntry(precursorEntity.isDossierDictionaryEntry()); + correctEntity.getManualOverwrite().addChanges(precursorEntity.getManualOverwrite().getManualChangeLog()); + correctEntity.addEngines(precursorEntity.getEngines()); } } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/ImportedRedactionEntryService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/ImportedRedactionEntryService.java new file mode 100644 index 00000000..8c5015d1 --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/ImportedRedactionEntryService.java @@ -0,0 +1,60 @@ +package com.iqser.red.service.redaction.v1.server.service.document; + +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.stream.Stream; + +import org.springframework.stereotype.Service; + +import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedactions; +import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions; +import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.BaseAnnotation; +import com.iqser.red.service.redaction.v1.server.model.PrecursorEntity; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document; + +import io.micrometer.observation.annotation.Observed; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +@Service +@RequiredArgsConstructor +public class ImportedRedactionEntryService { + + private final EntityFromPrecursorCreationService entityFromPrecursorCreationService; + + + @Observed(name = "ImportedRedactionEntryService", contextualName = "add-imported-redaction-entries") + public List addImportedEntriesAndReturnNotFoundEntries(AnalyzeRequest analyzeRequest, ImportedRedactions importedRedactions, Document document) { + + List notFoundImportedEntries = Collections.emptyList(); + if (!importedRedactions.getImportedRedactions().isEmpty()) { + notFoundImportedEntries = entityFromPrecursorCreationService.createEntitiesIfFoundAndReturnNotFoundEntries(importedRedactions, document); + log.info("Added Imported entries for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); + } + + if (notFoundImportedEntries.isEmpty()) { + return Collections.emptyList(); + } + List manualChanges = allManualChangesExceptAdd(analyzeRequest.getManualRedactions()); + for (PrecursorEntity notFoundManualRedactionEntry : notFoundImportedEntries) { + manualChanges.stream() + .filter(change -> change.getAnnotationId().equals(notFoundManualRedactionEntry.getId())) + .forEach(change -> notFoundManualRedactionEntry.getManualOverwrite().addChange(change)); + } + + return notFoundImportedEntries; + } + + + private List allManualChangesExceptAdd(ManualRedactions manualRedactions) { + + return Stream.of(manualRedactions.getForceRedactions(), + manualRedactions.getResizeRedactions(), + manualRedactions.getRecategorizations(), + manualRedactions.getIdsToRemove(), + manualRedactions.getLegalBasisChanges()).flatMap(Collection::stream).map(baseAnnotation -> (BaseAnnotation) baseAnnotation).toList(); + } +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/ManualRedactionEntryService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/ManualRedactionEntryService.java index efdc7d24..d9f82c86 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/ManualRedactionEntryService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/ManualRedactionEntryService.java @@ -10,8 +10,8 @@ import org.springframework.stereotype.Service; import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.BaseAnnotation; -import com.iqser.red.service.redaction.v1.server.model.ManualEntity; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document; +import com.iqser.red.service.redaction.v1.server.model.PrecursorEntity; import io.micrometer.observation.annotation.Observed; import lombok.RequiredArgsConstructor; @@ -22,23 +22,24 @@ import lombok.extern.slf4j.Slf4j; @RequiredArgsConstructor public class ManualRedactionEntryService { - private final ManualEntityCreationService manualEntityCreationService; + private final EntityFromPrecursorCreationService entityFromPrecursorCreationService; @Observed(name = "ManualRedactionEntryService", contextualName = "add-manual-redaction-entries") - public List addManualRedactionEntriesAndReturnNotFoundEntries(AnalyzeRequest analyzeRequest, Document document, String dossierTemplateId) { + public List addManualRedactionEntriesAndReturnNotFoundEntries(AnalyzeRequest analyzeRequest, Document document, String dossierTemplateId) { - List notFoundManualRedactionEntries = Collections.emptyList(); + List notFoundManualRedactionEntries = Collections.emptyList(); if (analyzeRequest.getManualRedactions() != null) { - notFoundManualRedactionEntries = manualEntityCreationService.createRedactionEntitiesIfFoundAndReturnNotFoundEntries(analyzeRequest.getManualRedactions(), + notFoundManualRedactionEntries = entityFromPrecursorCreationService.createEntitiesIfFoundAndReturnNotFoundEntries(analyzeRequest.getManualRedactions(), document, dossierTemplateId); log.info("Added Manual redaction entries for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); } + if (notFoundManualRedactionEntries.isEmpty()) { return Collections.emptyList(); } List manualChanges = allManualChangesExceptAdd(analyzeRequest.getManualRedactions()); - for (ManualEntity notFoundManualRedactionEntry : notFoundManualRedactionEntries) { + for (PrecursorEntity notFoundManualRedactionEntry : notFoundManualRedactionEntries) { manualChanges.stream() .filter(change -> change.getAnnotationId().equals(notFoundManualRedactionEntry.getId())) .forEach(change -> notFoundManualRedactionEntry.getManualOverwrite().addChange(change)); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/SectionFinderService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/SectionFinderService.java index 18fd4fd3..f8f4e47c 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/SectionFinderService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/SectionFinderService.java @@ -13,6 +13,9 @@ import org.springframework.stereotype.Service; import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedaction; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedactions; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval; @@ -40,7 +43,11 @@ import lombok.extern.slf4j.Slf4j; public class SectionFinderService { @Timed("redactmanager_findSectionsToReanalyse") - public Set findSectionsToReanalyse(DictionaryIncrement dictionaryIncrement, EntityLog entityLog, Document document, AnalyzeRequest analyzeRequest) { + public Set findSectionsToReanalyse(DictionaryIncrement dictionaryIncrement, + EntityLog entityLog, + Document document, + AnalyzeRequest analyzeRequest, + ImportedRedactions importedRedactions) { long start = System.currentTimeMillis(); Set relevantManuallyModifiedAnnotationIds = getRelevantManuallyModifiedAnnotationIds(analyzeRequest.getManualRedactions()); @@ -64,6 +71,7 @@ public class SectionFinderService { }); Set relevantPagesForReanalysis = getRelevantPageNumbersForAddRedactions(analyzeRequest); + relevantPagesForReanalysis.addAll(getRelevantPageNumbersForImportedRedactions(importedRedactions, relevantManuallyModifiedAnnotationIds)); if (!relevantPagesForReanalysis.isEmpty()) { sectionsToReanalyse.addAll(getSectionNumbersOnPages(document, relevantPagesForReanalysis)); } @@ -102,6 +110,22 @@ public class SectionFinderService { } + private static Set getRelevantPageNumbersForImportedRedactions(ImportedRedactions importedRedactions, Set idsWithUnprocessedManualChanges) { + + if (importedRedactions == null || importedRedactions.getImportedRedactions() == null) { + return Collections.emptySet(); + } + + return importedRedactions.getImportedRedactions() + .stream() + .filter(importedRedaction -> idsWithUnprocessedManualChanges.contains(importedRedaction.getId())) + .map(ImportedRedaction::getPositions) + .flatMap(Collection::stream) + .map(Position::getPageNumber) + .collect(Collectors.toSet()); + } + + private static Set getRelevantManuallyModifiedAnnotationIds(ManualRedactions manualRedactions) { if (manualRedactions == null) { diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/RedactionStorageService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/RedactionStorageService.java index d5dc1923..0aaeaac9 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/RedactionStorageService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/RedactionStorageService.java @@ -3,6 +3,7 @@ package com.iqser.red.service.redaction.v1.server.storage; import java.io.File; import java.io.FileInputStream; import java.io.InputStream; +import java.util.List; import java.util.stream.Collectors; import org.springframework.cache.annotation.Cacheable; @@ -11,6 +12,7 @@ import org.springframework.stereotype.Service; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentLog; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedactions; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedactionsPerPage; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog; import com.iqser.red.service.redaction.v1.server.client.model.NerEntitiesModel; @@ -76,10 +78,26 @@ public class RedactionStorageService { @Timed("redactmanager_getImportedRedactions") public ImportedRedactions getImportedRedactions(String dossierId, String fileId) { + try { + ImportedRedactionsPerPage importedRedactionsPerPage = storageService.readJSONObject(TenantContext.getTenantId(), + StorageIdUtils.getStorageId(dossierId, fileId, FileType.IMPORTED_REDACTIONS), + ImportedRedactionsPerPage.class); + return new ImportedRedactions(importedRedactionsPerPage.getImportedRedactions().values().stream().flatMap(List::stream).collect(Collectors.toList())); + } catch (StorageObjectDoesNotExist e) { + log.debug("Imported redactions not available."); + return new ImportedRedactions(); + } + } + + + + @Timed("redactmanager_getImportedRedactions") + public ImportedRedactionsPerPage getImportedRedactionsPerPage(String dossierId, String fileId) { + try { return storageService.readJSONObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.IMPORTED_REDACTIONS), - ImportedRedactions.class); + ImportedRedactionsPerPage.class); } catch (StorageObjectDoesNotExist e) { log.debug("Imported redactions not available."); return null; diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/entity/TextEntityTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/entity/TextEntityTest.java index 95174da1..b6f4b2d9 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/entity/TextEntityTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/entity/TextEntityTest.java @@ -5,7 +5,7 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import org.junit.jupiter.api.Test; -import com.iqser.red.service.redaction.v1.server.model.ManualEntity; +import com.iqser.red.service.redaction.v1.server.model.PrecursorEntity; import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType; public class TextEntityTest { @@ -13,7 +13,7 @@ public class TextEntityTest { @Test public void testMatchedRule() { - ManualEntity entity = ManualEntity.builder() + PrecursorEntity entity = PrecursorEntity.builder() .type("PII") .entityType(EntityType.ENTITY) .build(); @@ -31,7 +31,7 @@ public class TextEntityTest { @Test public void testMatchedRuleWithNonsense() { - ManualEntity entity = ManualEntity.builder() + PrecursorEntity entity = PrecursorEntity.builder() .type("PII") .entityType(EntityType.ENTITY) .build(); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualEntityTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/PrecursorEntityTest.java similarity index 91% rename from redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualEntityTest.java rename to redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/PrecursorEntityTest.java index 19cf1ad8..b10c8a6a 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualEntityTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/PrecursorEntityTest.java @@ -25,7 +25,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Point; import com.iqser.red.service.redaction.v1.server.document.graph.BuildDocumentIntegrationTest; -import com.iqser.red.service.redaction.v1.server.model.ManualEntity; +import com.iqser.red.service.redaction.v1.server.model.PrecursorEntity; import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryVersion; import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType; import com.iqser.red.service.redaction.v1.server.model.document.entity.IEntity; @@ -35,17 +35,17 @@ import com.iqser.red.service.redaction.v1.server.service.DictionaryService; import com.iqser.red.service.redaction.v1.server.service.EntityLogCreatorService; import com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService; import com.iqser.red.service.redaction.v1.server.service.document.EntityEnrichmentService; -import com.iqser.red.service.redaction.v1.server.service.document.ManualEntityCreationService; +import com.iqser.red.service.redaction.v1.server.service.document.EntityFromPrecursorCreationService; import lombok.SneakyThrows; -public class ManualEntityTest extends BuildDocumentIntegrationTest { +public class PrecursorEntityTest extends BuildDocumentIntegrationTest { @Autowired private EntityEnrichmentService entityEnrichmentService; @Autowired - private ManualEntityCreationService manualEntityCreationService; + private EntityFromPrecursorCreationService entityFromPrecursorCreationService; @Autowired private EntityLogCreatorService entityLogCreatorService; @@ -122,7 +122,7 @@ public class ManualEntityTest extends BuildDocumentIntegrationTest { assertTrue(document.getEntities().isEmpty()); - List notFoundManualEntities = manualEntityCreationService.createRedactionEntitiesIfFoundAndReturnNotFoundEntries(ManualRedactions.builder().entriesToAdd(Set.of(manualRedactionEntry)).build(), + List notFoundManualEntities = entityFromPrecursorCreationService.createEntitiesIfFoundAndReturnNotFoundEntries(ManualRedactions.builder().entriesToAdd(Set.of(manualRedactionEntry)).build(), document, TEST_DOSSIER_TEMPLATE_ID); assertEquals(1, notFoundManualEntities.size()); @@ -162,7 +162,7 @@ public class ManualEntityTest extends BuildDocumentIntegrationTest { .reason("reason") .legalBasis("n-a") .section(tempEntity.getDeepestFullyContainingNode().toString()) - .rectangle(true) + .rectangle(false) .positions(positions) .requestDate(OffsetDateTime.now()) .textAfter("") @@ -172,7 +172,7 @@ public class ManualEntityTest extends BuildDocumentIntegrationTest { tempEntity.removeFromGraph(); assertTrue(document.getEntities().isEmpty()); - List notFoundManualEntities = manualEntityCreationService.createRedactionEntitiesIfFoundAndReturnNotFoundEntries(ManualRedactions.builder().entriesToAdd(Set.of(manualRedactionEntry)).build(), + List notFoundManualEntities = entityFromPrecursorCreationService.createEntitiesIfFoundAndReturnNotFoundEntries(ManualRedactions.builder().entriesToAdd(Set.of(manualRedactionEntry)).build(), document, TEST_DOSSIER_TEMPLATE_ID); assertTrue(notFoundManualEntities.isEmpty()); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/acceptance_rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/acceptance_rules.drl index 7c19f205..257c0441 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/acceptance_rules.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/acceptance_rules.drl @@ -990,6 +990,29 @@ rule "X.6.1: remove Entity of higher rank, when intersected by entity of type EN end +// Rule unit: X.8 +rule "X.8.0: Remove Entity when textRange is equal to imported Entity" + salience 257 + when + $entity: TextEntity($type: type, engines contains Engine.IMPORTED, active()) + $other: TextEntity(getTextRange().equals($entity.getTextRange()), this != $entity, engines not contains Engine.IMPORTED) + then + $other.remove("X.8.0", "remove Entity when equal to imported Entity"); + $entity.addEngines($other.getEngines()); + retract($other); + end + +rule "X.8.1: Remove Entity when intersected by imported Entity" + salience 256 + when + $entity: TextEntity($type: type, engines contains Engine.IMPORTED, active()) + $other: TextEntity(intersects($entity), this != $entity, engines not contains Engine.IMPORTED) + then + $other.remove("X.8.1", "remove Entity when intersected by imported Entity"); + retract($other); + end + + //------------------------------------ File attributes rules ------------------------------------ // Rule unit: FA.1 diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/all_redact_manager_rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/all_redact_manager_rules.drl index 156f3f09..a3205ab4 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/all_redact_manager_rules.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/all_redact_manager_rules.drl @@ -1560,6 +1560,29 @@ rule "X.6.1: remove Entity of higher rank, when intersected by entity of type EN end +// Rule unit: X.8 +rule "X.8.0: Remove Entity when textRange is equal to imported Entity" + salience 257 + when + $entity: TextEntity($type: type, engines contains Engine.IMPORTED, active()) + $other: TextEntity(getTextRange().equals($entity.getTextRange()), this != $entity, engines not contains Engine.IMPORTED) + then + $other.remove("X.8.0", "remove Entity when equal to imported Entity"); + $entity.addEngines($other.getEngines()); + retract($other); + end + +rule "X.8.1: Remove Entity when intersected by imported Entity" + salience 256 + when + $entity: TextEntity($type: type, engines contains Engine.IMPORTED, active()) + $other: TextEntity(intersects($entity), this != $entity, engines not contains Engine.IMPORTED) + then + $other.remove("X.8.1", "remove Entity when intersected by imported Entity"); + retract($other); + end + + //------------------------------------ File attributes rules ------------------------------------ // Rule unit: FA.1 diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora.drl index 9cf983bb..9d6e9c9b 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora.drl @@ -1391,6 +1391,29 @@ rule "X.7.0: Remove all images" end +// Rule unit: X.8 +rule "X.8.0: Remove Entity when textRange is equal to imported Entity" + salience 257 + when + $entity: TextEntity($type: type, engines contains Engine.IMPORTED, active()) + $other: TextEntity(getTextRange().equals($entity.getTextRange()), this != $entity, engines not contains Engine.IMPORTED) + then + $other.remove("X.8.0", "remove Entity when equal to imported Entity"); + $entity.addEngines($other.getEngines()); + retract($other); + end + +rule "X.8.1: Remove Entity when intersected by imported Entity" + salience 256 + when + $entity: TextEntity($type: type, engines contains Engine.IMPORTED, active()) + $other: TextEntity(intersects($entity), this != $entity, engines not contains Engine.IMPORTED) + then + $other.remove("X.8.1", "remove Entity when intersected by imported Entity"); + retract($other); + end + + //------------------------------------ File attributes rules ------------------------------------ // Rule unit: FA.1 diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/manual_redaction_rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/manual_redaction_rules.drl index 50272f51..f3af91ad 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/manual_redaction_rules.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/manual_redaction_rules.drl @@ -232,6 +232,29 @@ rule "X.5.1: Remove Entity of type RECOMMENDATION when contained by RECOMMENDATI end +// Rule unit: X.8 +rule "X.8.0: Remove Entity when textRange is equal to imported Entity" + salience 257 + when + $entity: TextEntity($type: type, engines contains Engine.IMPORTED, active()) + $other: TextEntity(getTextRange().equals($entity.getTextRange()), this != $entity, engines not contains Engine.IMPORTED) + then + $other.remove("X.8.0", "remove Entity when equal to imported Entity"); + $entity.addEngines($other.getEngines()); + retract($other); + end + +rule "X.8.1: Remove Entity when intersected by imported Entity" + salience 256 + when + $entity: TextEntity($type: type, engines contains Engine.IMPORTED, active()) + $other: TextEntity(intersects($entity), this != $entity, engines not contains Engine.IMPORTED) + then + $other.remove("X.8.1", "remove Entity when intersected by imported Entity"); + retract($other); + end + + //------------------------------------ Local dictionary search rules ------------------------------------ // Rule unit: LDS.0 diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl index 3e5a6bb5..13655872 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl @@ -1150,6 +1150,29 @@ rule "X.6.1: remove Entity of higher rank, when intersected by entity of type EN end +// Rule unit: X.8 +rule "X.8.0: Remove Entity when textRange is equal to imported Entity" + salience 257 + when + $entity: TextEntity($type: type, engines contains Engine.IMPORTED, active()) + $other: TextEntity(getTextRange().equals($entity.getTextRange()), this != $entity, engines not contains Engine.IMPORTED) + then + $other.remove("X.8.0", "remove Entity when equal to imported Entity"); + $entity.addEngines($other.getEngines()); + retract($other); + end + +rule "X.8.1: Remove Entity when intersected by imported Entity" + salience 256 + when + $entity: TextEntity($type: type, engines contains Engine.IMPORTED, active()) + $other: TextEntity(intersects($entity), this != $entity, engines not contains Engine.IMPORTED) + then + $other.remove("X.8.1", "remove Entity when intersected by imported Entity"); + retract($other); + end + + //------------------------------------ File attributes rules ------------------------------------ // Rule unit: FA.1 diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_v2.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_v2.drl index e7e940d5..65f097b8 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_v2.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_v2.drl @@ -356,6 +356,29 @@ rule "X.6.1: remove Entity of higher rank, when intersected by entity of type EN end +// Rule unit: X.8 +rule "X.8.0: Remove Entity when textRange is equal to imported Entity" + salience 257 + when + $entity: TextEntity($type: type, engines contains Engine.IMPORTED, active()) + $other: TextEntity(getTextRange().equals($entity.getTextRange()), this != $entity, engines not contains Engine.IMPORTED) + then + $other.remove("X.8.0", "remove Entity when equal to imported Entity"); + $entity.addEngines($other.getEngines()); + retract($other); + end + +rule "X.8.1: Remove Entity when intersected by imported Entity" + salience 256 + when + $entity: TextEntity($type: type, engines contains Engine.IMPORTED, active()) + $other: TextEntity(intersects($entity), this != $entity, engines not contains Engine.IMPORTED) + then + $other.remove("X.8.1", "remove Entity when intersected by imported Entity"); + retract($other); + end + + //------------------------------------ File attributes rules ------------------------------------ // Rule unit: FA.1 diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/table_demo.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/table_demo.drl index 73c345d6..e5f2ef32 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/table_demo.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/table_demo.drl @@ -477,6 +477,29 @@ rule "X.7.0: Remove all images" end +// Rule unit: X.8 +rule "X.8.0: Remove Entity when textRange is equal to imported Entity" + salience 257 + when + $entity: TextEntity($type: type, engines contains Engine.IMPORTED, active()) + $other: TextEntity(getTextRange().equals($entity.getTextRange()), this != $entity, engines not contains Engine.IMPORTED) + then + $other.remove("X.8.0", "remove Entity when equal to imported Entity"); + $entity.addEngines($other.getEngines()); + retract($other); + end + +rule "X.8.1: Remove Entity when intersected by imported Entity" + salience 256 + when + $entity: TextEntity($type: type, engines contains Engine.IMPORTED, active()) + $other: TextEntity(intersects($entity), this != $entity, engines not contains Engine.IMPORTED) + then + $other.remove("X.8.1", "remove Entity when intersected by imported Entity"); + retract($other); + end + + //------------------------------------ File attributes rules ------------------------------------ // Rule unit: FA.1 diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/test_rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/test_rules.drl index ab04310c..3e9b25f5 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/test_rules.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/test_rules.drl @@ -393,6 +393,29 @@ rule "X.7.0: Remove all images" end +// Rule unit: X.8 +rule "X.8.0: Remove Entity when textRange is equal to imported Entity" + salience 257 + when + $entity: TextEntity($type: type, engines contains Engine.IMPORTED, active()) + $other: TextEntity(getTextRange().equals($entity.getTextRange()), this != $entity, engines not contains Engine.IMPORTED) + then + $other.remove("X.8.0", "remove Entity when equal to imported Entity"); + $entity.addEngines($other.getEngines()); + retract($other); + end + +rule "X.8.1: Remove Entity when intersected by imported Entity" + salience 256 + when + $entity: TextEntity($type: type, engines contains Engine.IMPORTED, active()) + $other: TextEntity(intersects($entity), this != $entity, engines not contains Engine.IMPORTED) + then + $other.remove("X.8.1", "remove Entity when intersected by imported Entity"); + retract($other); + end + + //------------------------------------ File attributes rules ------------------------------------ // Rule unit: FA.1 diff --git a/redaction-service-v1/rules-management/src/main/resources/all_redact_manager_rules.drl b/redaction-service-v1/rules-management/src/main/resources/all_redact_manager_rules.drl index 7214d931..e931b9e0 100644 --- a/redaction-service-v1/rules-management/src/main/resources/all_redact_manager_rules.drl +++ b/redaction-service-v1/rules-management/src/main/resources/all_redact_manager_rules.drl @@ -1584,6 +1584,29 @@ rule "X.6.1: remove Entity of higher rank, when intersected by entity of type EN end +// Rule unit: X.8 +rule "X.8.0: Remove Entity when textRange is equal to imported Entity" + salience 257 + when + $entity: TextEntity($type: type, engines contains Engine.IMPORTED, active()) + $other: TextEntity(getTextRange().equals($entity.getTextRange()), this != $entity, engines not contains Engine.IMPORTED) + then + $other.remove("X.8.0", "remove Entity when equal to imported Entity"); + $entity.addEngines($other.getEngines()); + retract($other); + end + +rule "X.8.1: Remove Entity when intersected by imported Entity" + salience 256 + when + $entity: TextEntity($type: type, engines contains Engine.IMPORTED, active()) + $other: TextEntity(intersects($entity), this != $entity, engines not contains Engine.IMPORTED) + then + $other.remove("X.8.1", "remove Entity when intersected by imported Entity"); + retract($other); + end + + //------------------------------------ File attributes rules ------------------------------------ // Rule unit: FA.1 diff --git a/redaction-service-v1/rules-management/src/main/resources/all_rules_documine.drl b/redaction-service-v1/rules-management/src/main/resources/all_rules_documine.drl index a8ac8685..ffe81c37 100644 --- a/redaction-service-v1/rules-management/src/main/resources/all_rules_documine.drl +++ b/redaction-service-v1/rules-management/src/main/resources/all_rules_documine.drl @@ -1539,6 +1539,28 @@ rule "X.7.0: Remove all images" end +// Rule unit: X.8 +rule "X.8.0: Remove Entity when textRange is equal to imported Entity" + salience 257 + when + $entity: TextEntity($type: type, engines contains Engine.IMPORTED, active()) + $other: TextEntity(getTextRange().equals($entity.getTextRange()), this != $entity, engines not contains Engine.IMPORTED) + then + $other.remove("X.8.0", "remove Entity when equal to imported Entity"); + $entity.addEngines($other.getEngines()); + retract($other); + end + +rule "X.8.1: Remove Entity when intersected by imported Entity" + salience 256 + when + $entity: TextEntity($type: type, engines contains Engine.IMPORTED, active()) + $other: TextEntity(intersects($entity), this != $entity, engines not contains Engine.IMPORTED) + then + $other.remove("X.8.1", "remove Entity when intersected by imported Entity"); + retract($other); + end + //------------------------------------ File attributes rules ------------------------------------ // Rule unit: FA.1 diff --git a/redaction-service-v1/rules-management/src/main/resources/default_rule_identifiers.txt b/redaction-service-v1/rules-management/src/main/resources/default_rule_identifiers.txt index f31074ae..fcc625f2 100644 --- a/redaction-service-v1/rules-management/src/main/resources/default_rule_identifiers.txt +++ b/redaction-service-v1/rules-management/src/main/resources/default_rule_identifiers.txt @@ -7,5 +7,6 @@ X.4.0 X.5.0 X.5.1 X.6.* +X.8.* FA.*.* LDS.*.* \ No newline at end of file diff --git a/redaction-service-v1/rules-management/src/main/resources/default_rule_identifiers_dm.txt b/redaction-service-v1/rules-management/src/main/resources/default_rule_identifiers_dm.txt index f021f041..2cce8ab4 100644 --- a/redaction-service-v1/rules-management/src/main/resources/default_rule_identifiers_dm.txt +++ b/redaction-service-v1/rules-management/src/main/resources/default_rule_identifiers_dm.txt @@ -6,5 +6,6 @@ X.4.0 X.5.0 X.5.1 X.7.0 +X.8.* FA.*.* LDS.*.* \ No newline at end of file