From 826fce455f36e5a9e5eaaad1b4f0407e7598a2d0 Mon Sep 17 00:00:00 2001 From: Kilian Schuettler Date: Wed, 3 May 2023 16:01:46 +0200 Subject: [PATCH] RED-6009 - Document Tree Structure * moved NER Entity combination to drools --- ...NerEntities.java => NerEntitiesModel.java} | 3 +- .../services/EntityCreationService.java | 7 + .../server/redaction/adapter/NerEntities.java | 44 ++++++ .../redaction/adapter/NerEntitiesAdapter.java | 138 +++++++++++------- .../service/DroolsExecutionService.java | 11 +- .../service/analyze/AnalyzeService.java | 17 ++- .../service/entityredaction/EntityFinder.java | 12 +- .../EntityRedactionService.java | 6 +- .../storage/RedactionStorageService.java | 6 +- ...cumentGraphPerformanceIntegrationTest.java | 3 +- .../adapter/NerEntitiesAdapterTest.java | 31 ++-- .../src/test/resources/drools/rules.drl | 67 +++++---- .../src/test/resources/drools/rules_v2.drl | 29 ++-- 13 files changed, 245 insertions(+), 129 deletions(-) rename redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/client/model/{NerEntities.java => NerEntitiesModel.java} (91%) create mode 100644 redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/adapter/NerEntities.java diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/client/model/NerEntities.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/client/model/NerEntitiesModel.java similarity index 91% rename from redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/client/model/NerEntities.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/client/model/NerEntitiesModel.java index 81bd7faf..8b7b9018 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/client/model/NerEntities.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/client/model/NerEntitiesModel.java @@ -4,7 +4,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; - import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; @@ -12,7 +11,7 @@ import lombok.NoArgsConstructor; @Data @NoArgsConstructor @AllArgsConstructor -public class NerEntities { +public class NerEntitiesModel { private Map> data = new HashMap<>(); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/services/EntityCreationService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/services/EntityCreationService.java index ba910ff7..3d63807c 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/services/EntityCreationService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/services/EntityCreationService.java @@ -27,6 +27,7 @@ import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.no import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.TableCellNode; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.TextBlock; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RedactionSearchUtils; +import com.iqser.red.service.redaction.v1.server.redaction.adapter.NerEntities; import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType; import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.SearchImplementation; import com.iqser.red.service.redaction.v1.server.redaction.utils.IdBuilder; @@ -177,6 +178,12 @@ public class EntityCreationService { } + public RedactionEntity byNerEntity(NerEntities.NerEntity nerEntity, EntityType entityType, SemanticNode semanticNode) { + + return byBoundary(nerEntity.boundary(), nerEntity.type(), entityType, semanticNode); + } + + public RedactionEntity byTableCellAsHighlight(TableCellNode tableCellNode, String type, EntityType entityType) { RedactionEntity highlightEntity = RedactionEntity.initialEntityNode(new Boundary(tableCellNode.getBoundary().start(), tableCellNode.getBoundary().start()), diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/adapter/NerEntities.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/adapter/NerEntities.java new file mode 100644 index 00000000..95e5f5fa --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/adapter/NerEntities.java @@ -0,0 +1,44 @@ +package com.iqser.red.service.redaction.v1.server.redaction.adapter; + +import java.util.LinkedList; +import java.util.List; +import java.util.stream.Stream; + +import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.Boundary; + +import lombok.AccessLevel; +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.experimental.FieldDefaults; + +@Getter +@AllArgsConstructor +@FieldDefaults(level = AccessLevel.PRIVATE, makeFinal = true) +public class NerEntities { + + List nerEntityList; + + + public NerEntities() { + + nerEntityList = new LinkedList<>(); + } + + + public boolean hasEntitiesOfType(String type) { + + return nerEntityList.stream().anyMatch(nerEntity -> nerEntity.type.equals(type)); + } + + + public Stream streamEntitiesOfType(String type) { + + return nerEntityList.stream().filter(nerEntity -> nerEntity.type().equals(type)); + } + + + public record NerEntity(String value, Boundary boundary, String type) { + + } + +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/adapter/NerEntitiesAdapter.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/adapter/NerEntitiesAdapter.java index c8fd1225..18b30f4d 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/adapter/NerEntitiesAdapter.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/adapter/NerEntitiesAdapter.java @@ -10,12 +10,11 @@ import java.util.stream.Stream; import org.springframework.stereotype.Service; import com.iqser.red.service.redaction.v1.server.client.model.EntityRecognitionEntity; -import com.iqser.red.service.redaction.v1.server.client.model.NerEntities; +import com.iqser.red.service.redaction.v1.server.client.model.NerEntitiesModel; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.Boundary; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.DocumentGraph; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.SectionNode; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.TextBlock; -import com.iqser.red.service.redaction.v1.server.redaction.utils.SeparatorUtils; import lombok.AccessLevel; import lombok.experimental.FieldDefaults; @@ -25,94 +24,129 @@ import lombok.experimental.FieldDefaults; public class NerEntitiesAdapter { // imported values from SpaCy model, for explanations look here: https://wiki.knecon.com/display/RES/Address+Recognition - static Set CBI_ADDRESS_PARTS = Set.of("ORG", "STREET", "POSTAL", "COUNTRY", "CARDINAL", "CITY", "STATE"); - static Set CBI_ADDRESS_ESSENTIAL_PARTS = Set.of("ORG", "STREET", "CITY"); + static Set CBI_ADDRESS_TYPES_TO_COMBINE = Set.of("ORG", "STREET", "POSTAL", "COUNTRY", "CARDINAL", "CITY", "STATE"); + static Set CBI_ADDRESS_ESSENTIAL_TYPES = Set.of("ORG", "STREET", "CITY"); static int MAX_DISTANCE_BETWEEN_PARTS = 20; - static int MIN_PART_MATCHES = 3; + static int MIN_PARTS_TO_COMBINE = 3; static boolean ALLOW_DUPLICATES; - public List validateAndCombine(NerEntities nerEntities, DocumentGraph documentGraph) { + /** + * Adds the appropriate Offsets to the entities from the NER Service and validates, that the values match. + * + * @param nerEntitiesModel the Entities just as the NER Service returns them + * @param documentGraph the document structure, from which the NER Service found the entities + * @return a stream of validated entities + */ + public List getValidatedEntities(NerEntitiesModel nerEntitiesModel, DocumentGraph documentGraph) { - List entityRecognitionEntities = getValidatedEntities(nerEntities, documentGraph).toList(); - - List cbiAuthors = entityRecognitionEntities.stream().filter(entity -> entity.getType().equals("CBI_author")).toList(); - List cbiAddress = combineToCBIAddressEntities(entityRecognitionEntities, documentGraph.buildTextBlock()); - - return Stream.concat(cbiAuthors.stream(), cbiAddress.stream()).toList(); + return addOffsetsAndFlatten(getStringStartOffsetsForMainSections(documentGraph), nerEntitiesModel) // + .filter(nerEntityModel -> nerEntityOffsetMatchesDocumentGraphOffsets(nerEntityModel, documentGraph.buildTextBlock())) + .map(nerEntityModel -> new NerEntities.NerEntity(nerEntityModel.getValue(), + new Boundary(nerEntityModel.getStartOffset(), nerEntityModel.getEndOffset()), + nerEntityModel.getType())) + .toList(); } - protected Stream getValidatedEntities(NerEntities nerEntities, DocumentGraph documentGraph) { + /** + * This function combines all EntityRecognitionEntities to a list of Boundaries. + * Each part must be at most maxDistanceBetweenParts apart from one another. + * The first part must be an EntityRecognitionEntity of an essential type. + * The resulting list must consist of at least minPartsToCombine parts. + * + * @param nerEntities already validated entities from the NER Service + * @param essentialTypes the combined entities must contain at least one of these types + * @param typesToCombine all types which should be used to combine, must contain all essentialTypes + * @param maxDistanceBetweenParts all parts used to combine should be at most this value apart in string offset coordinates + * @param minPartsToCombine minimum number of parts to combine + * @param allowDuplicates allow combining multiple parts of same type + * @return A Stream of the combined boundaries + */ + public Stream combineNerEntities(NerEntities nerEntities, + Set essentialTypes, + Set typesToCombine, + int maxDistanceBetweenParts, + int minPartsToCombine, + boolean allowDuplicates) { - return addOffsetsAndFlatten(getStringStartOffsetsForMainSections(documentGraph), nerEntities) // - .filter(nerEntity -> nerEntityOffsetMatchesDocumentGraphOffsets(nerEntity, documentGraph.buildTextBlock())); - } - - - private List combineToCBIAddressEntities(List entityRecognitionEntities, TextBlock textBlock) { - - List sortedEntities = entityRecognitionEntities.stream() - .filter(entity -> CBI_ADDRESS_PARTS.contains(entity.getType())) - .sorted(Comparator.comparingInt(EntityRecognitionEntity::getStartOffset)) + List sortedEntities = nerEntities.getNerEntityList() + .stream() + .filter(entity -> typesToCombine.contains(entity.type())) + .sorted(Comparator.comparingInt(entity -> entity.boundary().start())) .toList(); if (sortedEntities.isEmpty()) { - return sortedEntities; + return Stream.empty(); } - List> entityClusters = new LinkedList<>(); + List> entityClusters = new LinkedList<>(); - List startEntitiesOfEssentialType = sortedEntities.stream().filter(e -> CBI_ADDRESS_ESSENTIAL_PARTS.contains(e.getType())).toList(); - for (EntityRecognitionEntity startEntity : startEntitiesOfEssentialType) { - List currentCluster = new LinkedList<>(); + List startEntitiesOfEssentialType = sortedEntities.stream().filter(e -> essentialTypes.contains(e.type())).toList(); + for (NerEntities.NerEntity startEntity : startEntitiesOfEssentialType) { + List currentCluster = new LinkedList<>(); entityClusters.add(currentCluster); - int lastEndOffset = startEntity.getStartOffset(); + int lastEndOffset = startEntity.boundary().end(); - for (EntityRecognitionEntity entity : sortedEntities) { - if (entity.getStartOffset() < lastEndOffset) { + for (NerEntities.NerEntity entity : sortedEntities) { + if (entity.boundary().start() < lastEndOffset) { continue; } - if (disctanceIsLargerThanMaxDistance(lastEndOffset, entity) || isDuplicate(currentCluster, entity)) { + if (distanceIsLargerThanMaxDistance(lastEndOffset, entity, maxDistanceBetweenParts) || isDuplicate(currentCluster, entity, allowDuplicates)) { currentCluster = new LinkedList<>(); entityClusters.add(currentCluster); currentCluster.add(entity); - lastEndOffset = entity.getEndOffset(); + lastEndOffset = entity.boundary().end(); } else { currentCluster.add(entity); - lastEndOffset = entity.getEndOffset(); + lastEndOffset = entity.boundary().end(); } } } - return entityClusters.stream() - .filter(cluster -> cluster.size() >= MIN_PART_MATCHES) - .map(this::toContainingBoundary) - .distinct() - .filter(boundary -> SeparatorUtils.boundaryIsSurroundedBySeparators(textBlock, boundary)) - .map(boundary -> new EntityRecognitionEntity(textBlock.subSequence(boundary).toString(), boundary.start(), boundary.end(), "CBI_address")) - .toList(); + return entityClusters.stream().filter(cluster -> cluster.size() >= minPartsToCombine).map(NerEntitiesAdapter::toContainingBoundary).distinct(); } - private static boolean isDuplicate(List currentCluster, EntityRecognitionEntity entity) { + /** + * Calls combine NerEntities with the following settings. + *

+ * essentialTypes Set.of("ORG", "STREET", "CITY") + * typesToCombine Set.of("ORG", "STREET", "POSTAL", "COUNTRY", "CARDINAL", "CITY", "STATE") + * resultType "CBI_address" + * maxDistanceBetweenParts 20 + * minPartsToCombine 3 + * allowDuplicates false + * + * @return A Stream of the combined entities of type "CBI_address" + */ + public Stream combineNerEntitiesToCbiAddressDefaults(NerEntities entityRecognitionEntities) { - return ALLOW_DUPLICATES || currentCluster.stream().anyMatch(e -> e.getType().equals(entity.getType())); + return combineNerEntities(entityRecognitionEntities, + CBI_ADDRESS_ESSENTIAL_TYPES, + CBI_ADDRESS_TYPES_TO_COMBINE, + MAX_DISTANCE_BETWEEN_PARTS, + MIN_PARTS_TO_COMBINE, + ALLOW_DUPLICATES); } - private static boolean disctanceIsLargerThanMaxDistance(int lastEndOffset, EntityRecognitionEntity entity) { + private static boolean isDuplicate(List currentCluster, NerEntities.NerEntity entity, boolean allowDuplicates) { - return (entity.getStartOffset() - lastEndOffset) > MAX_DISTANCE_BETWEEN_PARTS; + return allowDuplicates || currentCluster.stream().anyMatch(e -> e.type().equals(entity.type())); } - public Boundary toContainingBoundary(Collection entityRecognitionEntities) { + private static boolean distanceIsLargerThanMaxDistance(int lastEndOffset, NerEntities.NerEntity entity, int maxDistance) { - int minStart = entityRecognitionEntities.stream().mapToInt(EntityRecognitionEntity::getStartOffset).min().orElseThrow(IllegalArgumentException::new); - int maxEnd = entityRecognitionEntities.stream().mapToInt(EntityRecognitionEntity::getEndOffset).max().orElseThrow(IllegalArgumentException::new); - return new Boundary(minStart, maxEnd - 1); + return (entity.boundary().start() - lastEndOffset) > maxDistance; + } + + + private static Boundary toContainingBoundary(List nerEntities) { + + return Boundary.merge(nerEntities.stream().map(NerEntities.NerEntity::boundary).toList()); } @@ -122,14 +156,14 @@ public class NerEntitiesAdapter { } - private static Stream addOffsetsAndFlatten(List stringOffsetsForMainSections, NerEntities nerEntities) { + private static Stream addOffsetsAndFlatten(List stringOffsetsForMainSections, NerEntitiesModel nerEntitiesModel) { - nerEntities.getData().forEach((key, value) -> value.forEach(entityRecognitionEntity -> { + nerEntitiesModel.getData().forEach((key, value) -> value.forEach(entityRecognitionEntity -> { int newStartOffset = entityRecognitionEntity.getStartOffset() + stringOffsetsForMainSections.get(key); entityRecognitionEntity.setStartOffset(newStartOffset); entityRecognitionEntity.setEndOffset(newStartOffset + entityRecognitionEntity.getValue().length()); })); - return nerEntities.getData().values().stream().flatMap(Collection::stream); + return nerEntitiesModel.getData().values().stream().flatMap(Collection::stream); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DroolsExecutionService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DroolsExecutionService.java index 0df03a95..e0a8cf58 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DroolsExecutionService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DroolsExecutionService.java @@ -22,12 +22,13 @@ import org.springframework.stereotype.Service; import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions; import com.iqser.red.service.redaction.v1.server.client.RulesClient; -import com.iqser.red.service.redaction.v1.server.client.model.EntityRecognitionEntity; import com.iqser.red.service.redaction.v1.server.exception.RulesValidationException; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.DocumentGraph; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.SemanticNode; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.services.EntityCreationService; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.services.ManualRedactionApplicationService; +import com.iqser.red.service.redaction.v1.server.redaction.adapter.NerEntities; +import com.iqser.red.service.redaction.v1.server.redaction.adapter.NerEntitiesAdapter; import com.iqser.red.service.redaction.v1.server.redaction.model.Entity; import com.iqser.red.service.redaction.v1.server.redaction.model.Section; import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.Dictionary; @@ -49,6 +50,7 @@ public class DroolsExecutionService { Map rulesVersionPerDossierTemplateId = new HashMap<>(); EntityCreationService entityCreationService; ManualRedactionApplicationService manualRedactionApplicationService; + NerEntitiesAdapter nerEntitiesAdapter; public KieContainer getKieContainer(String dossierTemplateId) { @@ -68,7 +70,7 @@ public class DroolsExecutionService { Dictionary dictionary, List fileAttributes, ManualRedactions manualRedactions, - List nerEntities) { + NerEntities nerEntities) { return executeRules(kieContainer, document, document.streamChildren().toList(), dictionary, fileAttributes, manualRedactions, nerEntities); } @@ -81,13 +83,14 @@ public class DroolsExecutionService { Dictionary dictionary, List fileAttributes, ManualRedactions manualRedactions, - List nerEntities) { + NerEntities nerEntities) { KieSession kieSession = kieContainer.newKieSession(); kieSession.setGlobal("document", document); kieSession.setGlobal("entityCreationService", entityCreationService); kieSession.setGlobal("manualRedactionApplicationService", manualRedactionApplicationService); kieSession.setGlobal("dictionary", dictionary); + kieSession.setGlobal("nerEntitiesAdapter", nerEntitiesAdapter); document.getEntities().forEach(kieSession::insert); sectionsToAnalyze.forEach(kieSession::insert); @@ -103,7 +106,7 @@ public class DroolsExecutionService { manualRedactions.getEntriesToAdd().forEach(kieSession::insert); } - nerEntities.forEach(kieSession::insert); + kieSession.insert(nerEntities); kieSession.getAgenda().getAgendaGroup("LOCAL_DICTIONARY_ADDS").setFocus(); kieSession.fireAllRules(); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/analyze/AnalyzeService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/analyze/AnalyzeService.java index f23e82e2..1684392d 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/analyze/AnalyzeService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/analyze/AnalyzeService.java @@ -24,7 +24,6 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlo import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogLegalBasis; import com.iqser.red.service.redaction.v1.model.StructureAnalyzeRequest; import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient; -import com.iqser.red.service.redaction.v1.server.client.model.EntityRecognitionEntity; import com.iqser.red.service.redaction.v1.server.exception.RedactionException; import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.adapter.ImageServiceResponseAdapter; import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.Document; @@ -39,6 +38,7 @@ import com.iqser.red.service.redaction.v1.server.layoutparsing.document.factory. import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.DocumentGraph; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.SectionNode; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.SemanticNode; +import com.iqser.red.service.redaction.v1.server.redaction.adapter.NerEntities; import com.iqser.red.service.redaction.v1.server.redaction.adapter.NerEntitiesAdapter; import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.Dictionary; import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.DictionaryIncrement; @@ -159,8 +159,8 @@ public class AnalyzeService { return finalizeAnalysis(analyzeRequest, startTime, redactionLog, documentGraph.getNumberOfPages(), dictionaryIncrement.getDictionaryVersion(), true, new HashSet<>()); } - List nerEntities = getEntityRecognitionEntities(analyzeRequest, documentGraph); - log.info("Found {} Entity Recognition Service Entities", nerEntities.size()); + NerEntities nerEntities = getEntityRecognitionEntities(analyzeRequest, documentGraph); + log.info("Found {} Entity Recognition Service Entities", nerEntities.getNerEntityList().size()); KieContainer kieContainer = droolsExecutionService.updateRules(analyzeRequest.getDossierTemplateId()); log.info("Rules updated"); Dictionary dictionary = dictionaryService.getDeepCopyDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId()); @@ -200,13 +200,14 @@ public class AnalyzeService { } - private List getEntityRecognitionEntities(AnalyzeRequest analyzeRequest, DocumentGraph documentGraph) { + private NerEntities getEntityRecognitionEntities(AnalyzeRequest analyzeRequest, DocumentGraph documentGraph) { - List nerEntities; + NerEntities nerEntities; if (redactionServiceSettings.isNerServiceEnabled()) { - nerEntities = nerEntitiesAdapter.validateAndCombine(redactionStorageService.getNerEntities(analyzeRequest.getDossierId(), analyzeRequest.getFileId()), documentGraph); + nerEntities = new NerEntities(nerEntitiesAdapter.getValidatedEntities(redactionStorageService.getNerEntities(analyzeRequest.getDossierId(), analyzeRequest.getFileId()), + documentGraph)); } else { - nerEntities = Collections.emptyList(); + nerEntities = new NerEntities(Collections.emptyList()); } return nerEntities; } @@ -218,7 +219,7 @@ public class AnalyzeService { long startTime = System.currentTimeMillis(); DocumentGraph documentGraph = DocumentGraphMapper.toDocumentGraph(redactionStorageService.getDocumentData(analyzeRequest.getDossierId(), analyzeRequest.getFileId())); - List nerEntities = getEntityRecognitionEntities(analyzeRequest, documentGraph); + NerEntities nerEntities = getEntityRecognitionEntities(analyzeRequest, documentGraph); dictionaryService.updateDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId()); KieContainer kieContainer = droolsExecutionService.updateRules(analyzeRequest.getDossierTemplateId()); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/entityredaction/EntityFinder.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/entityredaction/EntityFinder.java index 182bbf31..c318ec31 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/entityredaction/EntityFinder.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/entityredaction/EntityFinder.java @@ -10,7 +10,7 @@ import org.springframework.stereotype.Component; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine; -import com.iqser.red.service.redaction.v1.server.client.model.NerEntities; +import com.iqser.red.service.redaction.v1.server.client.model.NerEntitiesModel; import com.iqser.red.service.redaction.v1.server.redaction.model.Entities; import com.iqser.red.service.redaction.v1.server.redaction.model.Entity; import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType; @@ -42,7 +42,7 @@ class EntityFinder { int sectionNumber, Dictionary dictionary, boolean local, - NerEntities nerEntities, + NerEntitiesModel nerEntitiesModel, List cellStarts, ManualRedactions manualRedactions) { @@ -73,7 +73,7 @@ class EntityFinder { Set nerFound = new HashSet<>(); if (!local) { - nerFound.addAll(getNerValues(sectionNumber, nerEntities, cellStarts, headline)); + nerFound.addAll(getNerValues(sectionNumber, nerEntitiesModel, cellStarts, headline)); } var cleared = EntitySearchUtils.clearAndFindPositions(found, searchableText, dictionary, manualRedactions); @@ -81,12 +81,12 @@ class EntityFinder { } - private Set getNerValues(int sectionNumber, NerEntities nerEntities, List cellStarts, String headline) { + private Set getNerValues(int sectionNumber, NerEntitiesModel nerEntitiesModel, List cellStarts, String headline) { Set entities = new HashSet<>(); - if (redactionServiceSettings.isNerServiceEnabled() && nerEntities.getData().containsKey(sectionNumber)) { - nerEntities.getData().get(sectionNumber).forEach(res -> { + if (redactionServiceSettings.isNerServiceEnabled() && nerEntitiesModel.getData().containsKey(sectionNumber)) { + nerEntitiesModel.getData().get(sectionNumber).forEach(res -> { if (cellStarts == null || cellStarts.isEmpty()) { entities.add(new Entity(res.getValue(), res.getType(), diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/entityredaction/EntityRedactionService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/entityredaction/EntityRedactionService.java index e2c4eff5..d2f75d45 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/entityredaction/EntityRedactionService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/entityredaction/EntityRedactionService.java @@ -10,12 +10,12 @@ import org.springframework.stereotype.Service; import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest; import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine; -import com.iqser.red.service.redaction.v1.server.client.model.EntityRecognitionEntity; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionEntity; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.DocumentGraph; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.SemanticNode; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.TextBlock; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.services.EntityCreationService; +import com.iqser.red.service.redaction.v1.server.redaction.adapter.NerEntities; import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType; import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.Dictionary; import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.SearchImplementation; @@ -40,7 +40,7 @@ public class EntityRedactionService { DocumentGraph documentGraph, KieContainer kieContainer, AnalyzeRequest analyzeRequest, - List nerEntities) { + NerEntities nerEntities) { log.debug("Starting Drools Execution"); long droolsStart = System.currentTimeMillis(); @@ -60,7 +60,7 @@ public class EntityRedactionService { List sectionsToReanalyze, KieContainer kieContainer, AnalyzeRequest analyzeRequest, - List nerEntities) { + NerEntities nerEntities) { log.debug("Starting Drools execution"); long ruleExecutionStart = System.currentTimeMillis(); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/RedactionStorageService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/RedactionStorageService.java index 4d835fe9..80c447bb 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/RedactionStorageService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/RedactionStorageService.java @@ -8,7 +8,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemp import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.imported.ImportedRedactions; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.SectionGrid; -import com.iqser.red.service.redaction.v1.server.client.model.NerEntities; +import com.iqser.red.service.redaction.v1.server.client.model.NerEntitiesModel; import com.iqser.red.service.redaction.v1.server.exception.NotFoundException; import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.Text; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.data.DocumentData; @@ -104,10 +104,10 @@ public class RedactionStorageService { @Timed("redactmanager_getNerEntities") - public NerEntities getNerEntities(String dossierId, String fileId) { + public NerEntitiesModel getNerEntities(String dossierId, String fileId) { try { - return storageService.readJSONObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.NER_ENTITIES), NerEntities.class); + return storageService.readJSONObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.NER_ENTITIES), NerEntitiesModel.class); } catch (StorageObjectDoesNotExist e) { throw new NotFoundException("NER Entities are not available."); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DocumentGraphPerformanceIntegrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DocumentGraphPerformanceIntegrationTest.java index f5707f2f..04f0ebf6 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DocumentGraphPerformanceIntegrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DocumentGraphPerformanceIntegrationTest.java @@ -41,6 +41,7 @@ import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.no import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.TextBlock; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.services.EntityCreationService; import com.iqser.red.service.redaction.v1.server.multitenancy.TenantContext; +import com.iqser.red.service.redaction.v1.server.redaction.adapter.NerEntities; import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType; import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.Dictionary; import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.DictionaryModel; @@ -153,7 +154,7 @@ public class DocumentGraphPerformanceIntegrationTest extends BuildDocumentGraphI dictionary, Collections.emptyList(), new ManualRedactions(), - Collections.emptyList()); + new NerEntities()); System.out.printf("Firing rules took %d ms\n", System.currentTimeMillis() - droolsStart); System.out.printf("Total time %d ms\n", System.currentTimeMillis() - dictionarySearchStart); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/adapter/NerEntitiesAdapterTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/adapter/NerEntitiesAdapterTest.java index 9640c37f..2023bed2 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/adapter/NerEntitiesAdapterTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/adapter/NerEntitiesAdapterTest.java @@ -17,10 +17,8 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.core.io.ClassPathResource; import com.iqser.red.commons.jackson.ObjectMapperFactory; -import com.iqser.red.service.redaction.v1.server.client.model.EntityRecognitionEntity; -import com.iqser.red.service.redaction.v1.server.client.model.NerEntities; +import com.iqser.red.service.redaction.v1.server.client.model.NerEntitiesModel; import com.iqser.red.service.redaction.v1.server.document.graph.BuildDocumentGraphIntegrationTest; -import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.Boundary; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionEntity; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionPosition; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.DocumentGraph; @@ -39,10 +37,10 @@ class NerEntitiesAdapterTest extends BuildDocumentGraphIntegrationTest { @SneakyThrows - private static NerEntities parseNerEntities(String filePath) { + private static NerEntitiesModel parseNerEntities(String filePath) { ClassPathResource resource = new ClassPathResource(filePath); - return ObjectMapperFactory.create().readValue(resource.getInputStream(), NerEntities.class); + return ObjectMapperFactory.create().readValue(resource.getInputStream(), NerEntitiesModel.class); } @@ -53,16 +51,17 @@ class NerEntitiesAdapterTest extends BuildDocumentGraphIntegrationTest { String filePath = "files/new/crafted document.pdf"; String nerEntitiesFilePath = "ner_entities/crafted document.NER_ENTITIES.json"; DocumentGraph documentGraph = buildGraphNoImages(filePath); - List entityRecognitionEntities = nerEntitiesAdapter.validateAndCombine(parseNerEntities(nerEntitiesFilePath), documentGraph); + List entityRecognitionEntities = validateAndCombine(parseNerEntities(nerEntitiesFilePath), documentGraph); assertFalse(entityRecognitionEntities.isEmpty()); - assertTrue(entityRecognitionEntities.stream().allMatch(entity -> entity.getStartOffset() < entity.getEndOffset())); + assertTrue(entityRecognitionEntities.stream().allMatch(entity -> entity.boundary().start() < entity.boundary().end())); ClassPathResource resource = new ClassPathResource(filePath); PDDocument pdDocument = PDDocument.load(resource.getInputStream()); - Stream unchangedAddressParts = nerEntitiesAdapter.getValidatedEntities(parseNerEntities(nerEntitiesFilePath), documentGraph) - .filter(e -> !e.getType().equals("CBI_author")); + Stream unchangedAddressParts = nerEntitiesAdapter.getValidatedEntities(parseNerEntities(nerEntitiesFilePath), documentGraph) + .stream() + .filter(e -> !e.type().equals("CBI_author")); List redactionEntities = Stream.concat(entityRecognitionEntities.stream(), unchangedAddressParts) - .map(e -> entityCreationService.byBoundary(new Boundary(e.getStartOffset(), e.getEndOffset()), e.getType(), EntityType.ENTITY, documentGraph)) + .map(e -> entityCreationService.byBoundary(e.boundary(), e.type(), EntityType.ENTITY, documentGraph)) .toList(); redactionEntities.stream() .collect(Collectors.groupingBy(e -> e.getPages().stream().findFirst().get().getNumber())) @@ -79,6 +78,18 @@ class NerEntitiesAdapterTest extends BuildDocumentGraphIntegrationTest { } + private List validateAndCombine(NerEntitiesModel nerEntitiesModel, DocumentGraph documentGraph) { + + NerEntities nerEntities = new NerEntities(nerEntitiesAdapter.getValidatedEntities(nerEntitiesModel, documentGraph)); + + List cbiAuthors = nerEntities.streamEntitiesOfType("CBI_author").toList(); + Stream cbiAddress = nerEntitiesAdapter.combineNerEntitiesToCbiAddressDefaults(nerEntities) + .map(boundary -> new NerEntities.NerEntity(documentGraph.buildTextBlock().subSequence(boundary).toString(), boundary, "CBI_address")); + + return Stream.concat(cbiAuthors.stream(), cbiAddress).toList(); + } + + private List getPositionsFromEntities(Stream entities) { return entities.map(RedactionEntity::getRedactionPositionsPerPage) diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl index aa27dea2..0d4c2e9b 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl @@ -10,15 +10,15 @@ import com.iqser.red.service.redaction.v1.server.redaction.utils.Liszt; import java.util.LinkedList; import java.util.HashSet; -import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.* -import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.* -import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.* -import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.* +import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.*; +import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.*; +import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.*; +import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.*; import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType; import com.iqser.red.service.redaction.v1.server.redaction.model.ImageType; import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute; import java.util.Set -import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine +import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.services.EntityCreationService; import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.Dictionary; import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.DictionaryModel; @@ -28,20 +28,21 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualImageRecategorization; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.services.ManualRedactionApplicationService; -import com.iqser.red.service.redaction.v1.server.client.model.EntityRecognitionEntity -import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionEntity -import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.Boundary -import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionEntity -import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionEntity -import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.Boundary -import java.util.stream.Collectors -import java.util.Collection -import java.util.stream.Stream +import com.iqser.red.service.redaction.v1.server.client.model.EntityRecognitionEntity; +import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.Boundary; +import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionEntity; +import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.Boundary; +import com.iqser.red.service.redaction.v1.server.redaction.adapter.NerEntitiesAdapter; +import com.iqser.red.service.redaction.v1.server.redaction.adapter.NerEntities; +import java.util.stream.Collectors; +import java.util.Collection; +import java.util.stream.Stream; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RedactionSearchUtils; global DocumentGraph document global EntityCreationService entityCreationService global ManualRedactionApplicationService manualRedactionApplicationService +global NerEntitiesAdapter nerEntitiesAdapter global Dictionary dictionary // --------------------------------------- queries ------------------------------------------------------------------- @@ -50,18 +51,6 @@ query "getFileAttributes" $fileAttribute: FileAttribute() end -// --------------------------------------- NER Entities rules ------------------------------------------------------------------- - -rule "add NER Entities of type CBI_author or CBI_address" - salience 999 - when - $nerEntity: EntityRecognitionEntity($type: type, (type == "CBI_author" || type == "CBI_address")) - then - RedactionEntity redactionEntity = entityCreationService.byBoundary(new Boundary($nerEntity.getStartOffset(), $nerEntity.getEndOffset()), $type, EntityType.RECOMMENDATION, document); - redactionEntity.addEngine(Engine.NER); - insert(redactionEntity); - end - // --------------------------------------- CBI rules ------------------------------------------------------------------- rule "0: Expand CBI_author entities with firstname initials" @@ -526,6 +515,31 @@ rule "102: Guidelines FileAttributes" .forEach(fileAttribute -> insert(fileAttribute)); end +// --------------------------------------- NER Entities rules ------------------------------------------------------------------- + +rule "add NER Entities of type CBI_author" + salience 999 + when + nerEntities: NerEntities(hasEntitiesOfType("CBI_author")) + then + nerEntities.streamEntitiesOfType("CBI_author") + .map(nerEntity -> entityCreationService.byNerEntity(nerEntity, EntityType.RECOMMENDATION, document)) + .forEach(entity -> insert(entity)); + end + +rule "combine and add NER Entities as CBI_address" + salience 999 + when + nerEntities: NerEntities(hasEntitiesOfType("ORG") || hasEntitiesOfType("STREET") || hasEntitiesOfType("CITY")) + then + nerEntitiesAdapter.combineNerEntitiesToCbiAddressDefaults(nerEntities) + .map(boundary -> entityCreationService.byBoundary(boundary, "CBI_address", EntityType.RECOMMENDATION, document)) + .forEach(entity -> { + entity.addEngine(Engine.NER); + insert(entity); + }); + end + // --------------------------------------- manual redaction rules ------------------------------------------------------------------- rule "Apply manual resize redaction" @@ -632,6 +646,7 @@ rule "remove Entity of type RECOMMENDATION when contained by ENTITY" $entity: RedactionEntity($type: type, entityType == EntityType.ENTITY) $recommendation: RedactionEntity(containedBy($entity), type == $type, entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger) then + $entity.addEngines($recommendation.getEngines()); $recommendation.removeFromGraph(); retract($recommendation); end diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_v2.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_v2.drl index 893c2d1f..a6c6d03f 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_v2.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_v2.drl @@ -10,15 +10,15 @@ import com.iqser.red.service.redaction.v1.server.redaction.utils.Liszt; import java.util.LinkedList; import java.util.HashSet; -import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.* -import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.* -import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.* -import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.* +import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.*; +import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.*; +import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.*; +import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.*; import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType; import com.iqser.red.service.redaction.v1.server.redaction.model.ImageType; import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute; import java.util.Set -import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine +import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.services.EntityCreationService; import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.Dictionary; import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.DictionaryModel; @@ -28,20 +28,21 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualImageRecategorization; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.services.ManualRedactionApplicationService; -import com.iqser.red.service.redaction.v1.server.client.model.EntityRecognitionEntity -import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionEntity -import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.Boundary -import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionEntity -import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionEntity -import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.Boundary -import java.util.stream.Collectors -import java.util.Collection -import java.util.stream.Stream +import com.iqser.red.service.redaction.v1.server.client.model.EntityRecognitionEntity; +import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.Boundary; +import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionEntity; +import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.Boundary; +import com.iqser.red.service.redaction.v1.server.redaction.adapter.NerEntitiesAdapter; +import com.iqser.red.service.redaction.v1.server.redaction.adapter.NerEntities; +import java.util.stream.Collectors; +import java.util.Collection; +import java.util.stream.Stream; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RedactionSearchUtils; global DocumentGraph document global EntityCreationService entityCreationService global ManualRedactionApplicationService manualRedactionApplicationService +global NerEntitiesAdapter nerEntitiesAdapter global Dictionary dictionary // --------------------------------------- queries -------------------------------------------------------------------