From 1a6e678b0040c16f673d3ff37001579e6c3625e6 Mon Sep 17 00:00:00 2001 From: Kilian Schuettler Date: Wed, 3 May 2023 11:25:15 +0200 Subject: [PATCH] RED-6009 - Document Tree Structure * remove Add ManualRedactions handling --- .../redaction/adapter/NerEntitiesAdapter.java | 4 +- .../service/analyze/AnalyzeService.java | 1 - .../service/analyze/SectionFinder.java | 8 +-- .../EntityRedactionService.java | 55 ------------------- 4 files changed, 3 insertions(+), 65 deletions(-) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/adapter/NerEntitiesAdapter.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/adapter/NerEntitiesAdapter.java index 2db2c741..c8fd1225 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/adapter/NerEntitiesAdapter.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/adapter/NerEntitiesAdapter.java @@ -47,7 +47,7 @@ public class NerEntitiesAdapter { protected Stream getValidatedEntities(NerEntities nerEntities, DocumentGraph documentGraph) { return addOffsetsAndFlatten(getStringStartOffsetsForMainSections(documentGraph), nerEntities) // - .filter(nerEntity -> nerEntityOffsetMatchesDocumentGraph(nerEntity, documentGraph.buildTextBlock())); + .filter(nerEntity -> nerEntityOffsetMatchesDocumentGraphOffsets(nerEntity, documentGraph.buildTextBlock())); } @@ -116,7 +116,7 @@ public class NerEntitiesAdapter { } - private boolean nerEntityOffsetMatchesDocumentGraph(EntityRecognitionEntity nerEntity, TextBlock textBlock) { + private boolean nerEntityOffsetMatchesDocumentGraphOffsets(EntityRecognitionEntity nerEntity, TextBlock textBlock) { return nerEntity.getValue().contentEquals(textBlock.subSequence(nerEntity.getStartOffset(), nerEntity.getEndOffset())); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/analyze/AnalyzeService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/analyze/AnalyzeService.java index 393d1160..3e7b771c 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/analyze/AnalyzeService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/analyze/AnalyzeService.java @@ -164,7 +164,6 @@ public class AnalyzeService { Dictionary dictionary = dictionaryService.getDeepCopyDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId()); sectionsToReAnalyse.forEach(node -> entityRedactionService.addDictionaryEntities(dictionary, node)); - entityRedactionService.applyManualRedactionEntries(analyzeRequest.getManualRedactions(), documentGraph); Set addedFileAttributes = entityRedactionService.addRuleEntities(dictionary, documentGraph, sectionsToReAnalyse, kieContainer, analyzeRequest, nerEntities); List newRedactionLogEntries = redactionLogCreatorService.createRedactionLog(documentGraph, analyzeRequest.getDossierTemplateId()); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/analyze/SectionFinder.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/analyze/SectionFinder.java index 82bbd9cd..ae6d897b 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/analyze/SectionFinder.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/analyze/SectionFinder.java @@ -3,7 +3,6 @@ package com.iqser.red.service.redaction.v1.server.redaction.service.analyze; import java.util.HashMap; import java.util.HashSet; import java.util.Map; -import java.util.Objects; import java.util.Set; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -16,7 +15,6 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualImageRecategorization; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange; -import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog; @@ -82,11 +80,7 @@ class SectionFinder { Stream.concat(manualRedactions.getLegalBasisChanges().stream().map(ManualLegalBasisChange::getAnnotationId), Stream.concat(manualRedactions.getImageRecategorization().stream().map(ManualImageRecategorization::getAnnotationId), Stream.concat(manualRedactions.getIdsToRemove().stream().map(IdRemoval::getAnnotationId), - Stream.concat(manualRedactions.getEntriesToAdd() - .stream() - .map(ManualRedactionEntry::getSourceId) - .filter(sourceId -> !Objects.isNull(sourceId)), - manualRedactions.getForceRedactions().stream().map(ManualForceRedaction::getAnnotationId)))))).collect(Collectors.toSet()); + manualRedactions.getForceRedactions().stream().map(ManualForceRedaction::getAnnotationId))))).collect(Collectors.toSet()); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/entityredaction/EntityRedactionService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/entityredaction/EntityRedactionService.java index 68567d93..e2c4eff5 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/entityredaction/EntityRedactionService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/entityredaction/EntityRedactionService.java @@ -1,8 +1,6 @@ package com.iqser.red.service.redaction.v1.server.redaction.service.entityredaction; -import java.util.Collection; import java.util.List; -import java.util.Objects; import java.util.Set; import java.util.stream.Collectors; @@ -11,18 +9,13 @@ import org.springframework.stereotype.Service; import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest; import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute; -import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions; -import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle; -import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine; import com.iqser.red.service.redaction.v1.server.client.model.EntityRecognitionEntity; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionEntity; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.DocumentGraph; -import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.PageNode; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.SemanticNode; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.TextBlock; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.services.EntityCreationService; -import com.iqser.red.service.redaction.v1.server.layoutparsing.document.services.ManualRedactionApplicationService; import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType; import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.Dictionary; import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.SearchImplementation; @@ -41,7 +34,6 @@ public class EntityRedactionService { DroolsExecutionService droolsExecutionService; EntityCreationService entityCreationService; - ManualRedactionApplicationService manualRedactionApplicationService; public Set addRuleEntities(Dictionary dictionary, @@ -85,53 +77,6 @@ public class EntityRedactionService { } - public void applyManualRedactionEntries(ManualRedactions manualRedactions, DocumentGraph documentGraph) { - - if (manualRedactions == null) { - return; - } - manualRedactions.getEntriesToAdd() - .stream() - .filter(manualRedactionEntry -> Objects.nonNull(manualRedactionEntry.getSourceId())) - .forEach(manualRedactionEntry -> applyManualRedactionEntry(manualRedactionEntry, documentGraph)); - - } - - - private void applyManualRedactionEntry(ManualRedactionEntry manualRedactionEntry, DocumentGraph documentGraph) { - - EntityType entityType = parseEntityType(manualRedactionEntry.getReason()); - documentGraph.getPages() - .stream() - .filter(page -> isInManualRedactionEntry(manualRedactionEntry, page)) - .map(PageNode::getEntities) - .flatMap(Collection::stream) - .filter(entity -> entity.matchesAnnotationId(manualRedactionEntry.getSourceId())) - .filter(entity -> entity.getValue().equals(manualRedactionEntry.getValue())) - .map(RedactionEntity::getBoundary) - .map(bound -> entityCreationService.byBoundary(bound, manualRedactionEntry.getType(), entityType, documentGraph)) - .peek(entity -> entity.setLegalBasis(manualRedactionEntry.getLegalBasis())) - .forEach(entity -> entity.setRedactionReason(manualRedactionEntry.getReason())); - } - - - private EntityType parseEntityType(String reason) { - - return switch (reason.toLowerCase()) { - case "false positive" -> EntityType.FALSE_POSITIVE; - case "false recommendation" -> EntityType.FALSE_RECOMMENDATION; - case "recommendation" -> EntityType.RECOMMENDATION; - default -> EntityType.ENTITY; - }; - } - - - private static boolean isInManualRedactionEntry(ManualRedactionEntry manualRedactionEntry, PageNode page) { - - return manualRedactionEntry.getPositions().stream().map(Rectangle::getPage).collect(Collectors.toSet()).contains(page.getNumber()); - } - - public void addDictionaryEntities(Dictionary dictionary, SemanticNode node) { for (var model : dictionary.getDictionaryModels()) {