From 9c48d36b05b7e338b44f6bd6779d231709b111a0 Mon Sep 17 00:00:00 2001 From: Kilian Schuettler Date: Thu, 20 Apr 2023 18:03:20 +0200 Subject: [PATCH] RED-6369: Rules Refactor * minor renaming * added NerEntityAdapter to include the Ner Entities * moved combine logic outside of rules --- .../graph/factory/DocumentGraphFactory.java | 2 +- .../document/graph/nodes/ImageNode.java | 2 +- .../document/graph/nodes/SemanticNode.java | 16 ++ .../document/graph/textblock/TextBlock.java | 9 + .../document/mapper/PropertiesMapper.java | 4 +- .../services/EntityCreationService.java | 26 +- .../redaction/adapter/NerEntitiesAdapter.java | 146 ++++++++++++ .../service/DroolsExecutionService.java | 56 ++--- .../service/RedactionLogCreatorService.java | 223 ++---------------- .../service/SectionGridCreatorService.java | 23 +- .../service/analyze/AnalyzeService.java | 55 ++--- .../EntityRedactionService.java | 12 +- .../server/DocumentGraphIntegrationTest.java | 2 +- .../src/test/resources/drools/rules.drl | 19 +- 14 files changed, 293 insertions(+), 302 deletions(-) create mode 100644 redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/adapter/NerEntitiesAdapter.java diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/factory/DocumentGraphFactory.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/factory/DocumentGraphFactory.java index 5cbc27bb..a3aff77c 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/factory/DocumentGraphFactory.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/factory/DocumentGraphFactory.java @@ -262,7 +262,7 @@ public class DocumentGraphFactory { .id(IdBuilder.buildId(Set.of(page), List.of(position))) .imageType(image.getImageType()) .position(position) - .transparency(image.isHasTransparency()) + .transparent(image.isHasTransparency()) .page(page) .tableOfContents(context.tableOfContents()) .build(); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/ImageNode.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/ImageNode.java index 4fcaa5e3..4eac106b 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/ImageNode.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/ImageNode.java @@ -33,7 +33,7 @@ public class ImageNode implements SemanticNode { String id; ImageType imageType; - boolean transparency; + boolean transparent; Rectangle2D position; boolean redaction; diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/SemanticNode.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/SemanticNode.java index ba011ff3..3b4f716c 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/SemanticNode.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/SemanticNode.java @@ -1,5 +1,7 @@ package com.iqser.red.service.redaction.v1.server.document.graph.nodes; +import static java.lang.String.format; + import java.awt.geom.Rectangle2D; import java.util.HashMap; import java.util.List; @@ -47,6 +49,20 @@ public interface SemanticNode { } + /** + * Each AtomicTextBlock is assigned a page, so to get the pages for this boundary, it collects the PageNodes from each AtomicTextBlock belonging to this node's ClassificationTextBlock. + * + * @return Set of PageNodes this node appears on. + */ + default Set getPages(Boundary boundary) { + + if (!getBoundary().contains(boundary)) { + throw new IllegalArgumentException(format("%s which was used to query foir pages is not contained in the %s of this node!", getBoundary(), boundary)); + } + return buildTextBlock().getPages(boundary); + } + + /** * @return the TableOfContents of the ClassificationDocument this node belongs to */ diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/textblock/TextBlock.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/textblock/TextBlock.java index bd4d2f91..3aeb526b 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/textblock/TextBlock.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/textblock/TextBlock.java @@ -57,6 +57,15 @@ public interface TextBlock extends CharSequence { } + default Set getPages(Boundary boundary) { + + return getAtomicTextBlocks().stream() + .filter(atomicTextBlock -> atomicTextBlock.getBoundary().intersects(boundary)) + .map(AtomicTextBlock::getPage) + .collect(Collectors.toUnmodifiableSet()); + } + + default int indexOf(String searchTerm, int startOffset) { int start = getSearchText().indexOf(searchTerm, startOffset - getBoundary().start()); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/mapper/PropertiesMapper.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/mapper/PropertiesMapper.java index 0582c581..23066c85 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/mapper/PropertiesMapper.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/mapper/PropertiesMapper.java @@ -17,7 +17,7 @@ public class PropertiesMapper { Map properties = new HashMap<>(); properties.put("imageType", image.getImageType().toString()); - properties.put("transparency", String.valueOf(image.isTransparency())); + properties.put("transparency", String.valueOf(image.isTransparent())); properties.put("position", RectangleTransformations.toString(image.getPosition())); properties.put("id", image.getId()); return properties; @@ -53,7 +53,7 @@ public class PropertiesMapper { public static void parseImageProperties(Map properties, ImageNode.ImageNodeBuilder builder) { builder.imageType(parseImageType(properties.get("imageType"))); - builder.transparency(Boolean.parseBoolean(properties.get("transparency"))); + builder.transparent(Boolean.parseBoolean(properties.get("transparency"))); builder.position(parseRectangle2D(properties.get("position"))); builder.id(properties.get("id")); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/EntityCreationService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/EntityCreationService.java index 9ebaedf3..051cbd46 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/EntityCreationService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/EntityCreationService.java @@ -8,7 +8,6 @@ import java.util.Collections; import java.util.Comparator; import java.util.LinkedList; import java.util.List; -import java.util.NoSuchElementException; import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; @@ -148,35 +147,26 @@ public class EntityCreationService { public void addEntityToGraph(EntityNode entity, TableOfContents tableOfContents) { - try { - SemanticNode containingNode = tableOfContents.streamChildrenNodes(Collections.emptyList()) - .filter(node -> node.buildTextBlock().containsBoundary(entity.getBoundary())) - .findFirst() - .orElseThrow(() -> new NoSuchElementException("No containing Node found!")); + tableOfContents.getRoot().getNode().addThisToEntityIfIntersects(entity); - containingNode.addThisToEntityIfIntersects(entity); + TextBlock textBlock = entity.getDeepestFullyContainingNode().buildTextBlock(); + entityEnrichmentService.enrichEntity(entity, textBlock); - TextBlock textBlock = entity.getDeepestFullyContainingNode().buildTextBlock(); - entityEnrichmentService.enrichEntity(entity, textBlock); + addToPages(entity); + addEntityToNodeEntitySets(entity); - addToPages(entity); - addToNodeEntitySets(entity); - - } catch (NoSuchElementException e) { - entity.removeFromGraph(); - } } - private void addToPages(EntityNode entity) { + private static void addToPages(EntityNode entity) { - Set pages = entity.getDeepestFullyContainingNode().getPages(); + Set pages = entity.getDeepestFullyContainingNode().getPages(entity.getBoundary()); entity.getPages().addAll(pages); pages.forEach(page -> page.getEntities().add(entity)); } - private void addToNodeEntitySets(EntityNode entity) { + private static void addEntityToNodeEntitySets(EntityNode entity) { entity.getIntersectingNodes().forEach(node -> node.getEntities().add(entity)); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/adapter/NerEntitiesAdapter.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/adapter/NerEntitiesAdapter.java new file mode 100644 index 00000000..8b90b079 --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/adapter/NerEntitiesAdapter.java @@ -0,0 +1,146 @@ +package com.iqser.red.service.redaction.v1.server.redaction.adapter; + +import java.util.Collection; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import org.springframework.stereotype.Service; + +import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest; +import com.iqser.red.service.redaction.v1.server.client.model.EntityRecognitionEntity; +import com.iqser.red.service.redaction.v1.server.client.model.NerEntities; +import com.iqser.red.service.redaction.v1.server.document.graph.Boundary; +import com.iqser.red.service.redaction.v1.server.document.graph.DocumentGraph; +import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SectionNode; +import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock; +import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings; +import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService; + +import lombok.AccessLevel; +import lombok.RequiredArgsConstructor; +import lombok.experimental.FieldDefaults; + +@Service +@RequiredArgsConstructor +@FieldDefaults(level = AccessLevel.PRIVATE, makeFinal = true) +public class NerEntitiesAdapter { + + RedactionStorageService redactionStorageService; + RedactionServiceSettings redactionServiceSettings; + + // imported values from SpaCy model, for explanations look here: https://wiki.knecon.com/display/RES/Address+Recognition + private static final Set CBI_ADDRESS_PARTS = Set.of("ORG", "STREET", "POSTAL", "COUNTRY", "CARDINAL", "CITY", "STATE"); + private static final Set CBI_ADDRESS_ESSENTIAL_PARTS = Set.of("ORG", "STREET", "CITY"); + private static final int MAX_DISTANCE_BETWEEN_PARTS = 20; + private static final int MIN_PART_MATCHES = 3; + private static final boolean ALLOW_DUPLICATES = false; + + + public List getNerEntities(AnalyzeRequest analyzeRequest, DocumentGraph documentGraph) { + + NerEntities nerEntities; + if (redactionServiceSettings.isNerServiceEnabled()) { + nerEntities = redactionStorageService.getNerEntities(analyzeRequest.getDossierId(), analyzeRequest.getFileId()); + } else { + nerEntities = new NerEntities(); + } + + addSectionStartOffsetToEachNerEntity(getStringStartOffsetsForMainSections(documentGraph), nerEntities); + + List entityRecognitionEntities = validateForCorrectOffsets(documentGraph, nerEntities); + + List cbiAuthors = entityRecognitionEntities.stream().filter(entity -> entity.getType().equals("CBI_author")).toList(); + List cbiAddress = combineToCBIAddressEntities(entityRecognitionEntities, documentGraph.buildTextBlock()); + + return Stream.concat(cbiAuthors.stream(), cbiAddress.stream()).toList(); + } + + + private List combineToCBIAddressEntities(List entityRecognitionEntities, TextBlock textBlock) { + + Set entitiesOfTypeCBIAddressParts = entityRecognitionEntities.stream() + .filter(entity -> CBI_ADDRESS_PARTS.contains(entity.getType())) + .collect(Collectors.toSet()); + + return entitiesOfTypeCBIAddressParts.stream() + .filter(entity -> CBI_ADDRESS_ESSENTIAL_PARTS.contains(entity.getType())) + .map(entity -> findFollowingEntitiesOfTypeCBIAddressPart(entity, entitiesOfTypeCBIAddressParts)) + .filter(entities -> entities.size() < MIN_PART_MATCHES) + .map(this::toContainingBoundary) + .distinct() + .map(boundary -> new EntityRecognitionEntity(textBlock.subSequence(boundary).toString(), boundary.start(), boundary.end(), "CBI_address")) + .toList(); + } + + + private List validateForCorrectOffsets(DocumentGraph documentGraph, NerEntities nerEntities) { + + return nerEntities.getData().values().stream().flatMap(Collection::stream).filter(nerEntity -> nerEntityOffsetMatches(nerEntity, documentGraph.buildTextBlock())).toList(); + } + + + public Boundary toContainingBoundary(Collection entityRecognitionEntities) { + + int minStart = entityRecognitionEntities.stream().mapToInt(EntityRecognitionEntity::getStartOffset).min().orElseThrow(IllegalArgumentException::new); + int maxEnd = entityRecognitionEntities.stream().mapToInt(EntityRecognitionEntity::getStartOffset).max().orElseThrow(IllegalArgumentException::new); + return new Boundary(minStart, maxEnd); + } + + + public Set findFollowingEntitiesOfTypeCBIAddressPart(EntityRecognitionEntity entityOfCBIAddressEssentialType, + Set entitiesOfTypeCBIAddressPart) { + + Set concurrentParts = new HashSet<>(); + concurrentParts.add(entityOfCBIAddressEssentialType); + + int currentNumberOfConcurrentParts; + do { + currentNumberOfConcurrentParts = concurrentParts.size(); + concurrentParts.addAll(entitiesOfTypeCBIAddressPart.stream() + .filter(entity -> entity.getStartOffset() >= toContainingBoundary(concurrentParts).end()) + .filter(entity -> offsetBetweenPartsIsLowerThanMax(toContainingBoundary(concurrentParts), entity)) + .filter(entity -> ALLOW_DUPLICATES || entityTypeInAlreadyFound(concurrentParts, entity.getType())) + .collect(Collectors.toSet())); + } while (currentNumberOfConcurrentParts != concurrentParts.size()); + + return concurrentParts; + } + + + private static boolean entityTypeInAlreadyFound(Set alreadyFound, String type) { + + return !alreadyFound.stream().map(EntityRecognitionEntity::getType).collect(Collectors.toSet()).contains(type); + } + + + private static boolean offsetBetweenPartsIsLowerThanMax(Boundary currentBoundary, EntityRecognitionEntity entity) { + + return Math.abs(entity.getStartOffset() - currentBoundary.end()) < MAX_DISTANCE_BETWEEN_PARTS; + } + + + private boolean nerEntityOffsetMatches(EntityRecognitionEntity nerEntity, TextBlock textBlock) { + + return nerEntity.getValue().contentEquals(textBlock.subSequence(nerEntity.getStartOffset(), nerEntity.getEndOffset())); + } + + + private static void addSectionStartOffsetToEachNerEntity(List stringOffsetsForMainSections, NerEntities nerEntities) { + + nerEntities.getData().forEach((key, value) -> value.forEach(entityRecognitionEntity -> { + int newStartOffset = entityRecognitionEntity.getStartOffset() + stringOffsetsForMainSections.get(key); + entityRecognitionEntity.setStartOffset(newStartOffset); + entityRecognitionEntity.setEndOffset(newStartOffset + entityRecognitionEntity.getValue().length()); + })); + } + + + private static List getStringStartOffsetsForMainSections(DocumentGraph documentGraph) { + + return documentGraph.getMainSections().stream().map(SectionNode::buildTextBlock).map(TextBlock::getBoundary).map(Boundary::start).toList(); + } + +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DroolsExecutionService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DroolsExecutionService.java index 6ed1213d..1bc14b37 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DroolsExecutionService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DroolsExecutionService.java @@ -22,6 +22,7 @@ import org.springframework.stereotype.Service; import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions; import com.iqser.red.service.redaction.v1.server.client.RulesClient; +import com.iqser.red.service.redaction.v1.server.client.model.EntityRecognitionEntity; import com.iqser.red.service.redaction.v1.server.document.graph.DocumentGraph; import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SectionNode; import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SemanticNode; @@ -32,19 +33,19 @@ import com.iqser.red.service.redaction.v1.server.redaction.model.Section; import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.Dictionary; import io.micrometer.core.annotation.Timed; +import lombok.AccessLevel; import lombok.RequiredArgsConstructor; +import lombok.experimental.FieldDefaults; @Service @RequiredArgsConstructor +@FieldDefaults(level = AccessLevel.PRIVATE, makeFinal = true) public class DroolsExecutionService { - private final RulesClient rulesClient; - - private final Map kieContainers = new HashMap<>(); - - private final Map rulesVersionPerDossierTemplateId = new HashMap<>(); - - private final EntityCreationService entityCreationService; + RulesClient rulesClient; + Map kieContainers = new HashMap<>(); + Map rulesVersionPerDossierTemplateId = new HashMap<>(); + EntityCreationService entityCreationService; public KieContainer getKieContainer(String dossierTemplateId) { @@ -63,35 +64,21 @@ public class DroolsExecutionService { DocumentGraph document, Dictionary dictionary, List fileAttributes, - ManualRedactions resizeRedactions) { + ManualRedactions manualRedactions, + List nerEntities) { - KieSession kieSession = kieContainer.newKieSession(); - kieSession.setGlobal("document", document); - kieSession.setGlobal("entityCreationService", entityCreationService); - kieSession.setGlobal("dictionary", dictionary); - - document.getEntities().forEach(kieSession::insert); - document.getTableOfContents().streamAllEntriesInOrder().forEach(entry -> kieSession.insert(entry.getNode())); - document.getPages().forEach(kieSession::insert); - fileAttributes.forEach(kieSession::insert); - if (resizeRedactions != null) { - resizeRedactions.getResizeRedactions().forEach(kieSession::insert); - } - - kieSession.getAgenda().getAgendaGroup("LOCAL_DICTIONARY_ADDS").setFocus(); - kieSession.fireAllRules(); - - return getFileAttributes(kieSession); + return executeRules(kieContainer, document, document.getMainSections(), dictionary, fileAttributes, manualRedactions, nerEntities); } @Timed("redactmanager_executeRules") public List executeRules(KieContainer kieContainer, DocumentGraph document, - List sectionsToReanalyze, + List sectionsToAnalyze, Dictionary dictionary, List fileAttributes, - ManualRedactions resizeRedactions) { + ManualRedactions manualRedactions, + List nerEntities) { KieSession kieSession = kieContainer.newKieSession(); kieSession.setGlobal("document", document); @@ -99,14 +86,21 @@ public class DroolsExecutionService { kieSession.setGlobal("dictionary", dictionary); document.getEntities().forEach(kieSession::insert); - sectionsToReanalyze.forEach(kieSession::insert); - sectionsToReanalyze.stream().flatMap(SemanticNode::streamAllSubNodes).forEach(kieSession::insert); + sectionsToAnalyze.forEach(kieSession::insert); + sectionsToAnalyze.stream().flatMap(SemanticNode::streamAllSubNodes).forEach(kieSession::insert); document.getPages().forEach(kieSession::insert); fileAttributes.forEach(kieSession::insert); - if (resizeRedactions != null) { - resizeRedactions.getResizeRedactions().forEach(kieSession::insert); + + if (manualRedactions != null) { + manualRedactions.getResizeRedactions().forEach(kieSession::insert); + manualRedactions.getForceRedactions().forEach(kieSession::insert); + manualRedactions.getIdsToRemove().forEach(kieSession::insert); + manualRedactions.getImageRecategorization().forEach(kieSession::insert); + manualRedactions.getEntriesToAdd().forEach(kieSession::insert); } + nerEntities.forEach(kieSession::insert); + kieSession.getAgenda().getAgendaGroup("LOCAL_DICTIONARY_ADDS").setFocus(); kieSession.fireAllRules(); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java index b24a70c2..178c0107 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java @@ -5,26 +5,19 @@ import static com.iqser.red.service.redaction.v1.server.document.graph.factory.R import java.util.ArrayList; import java.util.HashSet; import java.util.List; -import java.util.Map; import java.util.Set; -import org.apache.commons.collections4.CollectionUtils; import org.springframework.stereotype.Service; -import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Point; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry; +import com.iqser.red.service.redaction.v1.server.document.graph.DocumentGraph; import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityNode; import com.iqser.red.service.redaction.v1.server.document.graph.entity.RedactionPosition; -import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence; -import com.iqser.red.service.redaction.v1.server.redaction.model.Entity; -import com.iqser.red.service.redaction.v1.server.redaction.model.EntityPositionSequence; +import com.iqser.red.service.redaction.v1.server.document.graph.factory.RectangleTransformations; +import com.iqser.red.service.redaction.v1.server.document.graph.nodes.ImageNode; import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType; -import com.iqser.red.service.redaction.v1.server.redaction.model.Image; -import com.iqser.red.service.redaction.v1.server.redaction.model.PageEntities; -import com.iqser.red.service.redaction.v1.server.redaction.utils.IdBuilder; -import io.micrometer.core.annotation.Timed; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; @@ -36,12 +29,15 @@ public class RedactionLogCreatorService { private final DictionaryService dictionaryService; - public List createRedactionLog(Set entityNodes, String dossierTemplateId) { + public List createRedactionLog(DocumentGraph documentGraph, String dossierTemplateId) { List entries = new ArrayList<>(); Set processedIds = new HashSet<>(); - entityNodes.forEach(entityNode -> entries.addAll(toRedactionLogEntries(entityNode, processedIds, dossierTemplateId))); - + documentGraph.getEntities().forEach(entityNode -> entries.addAll(toRedactionLogEntries(entityNode, processedIds, dossierTemplateId))); + documentGraph.streamAllSubNodes() + .filter(node -> node instanceof ImageNode) + .map(node -> (ImageNode) node) + .forEach(imageNode -> entries.add(createRedactionLogEntry(imageNode, dossierTemplateId))); return entries; } @@ -106,193 +102,26 @@ public class RedactionLogCreatorService { } - @Timed("redactmanager_createRedactionLog") - public List createRedactionLog(PageEntities pageEntities, int numberOfPages, String dossierTemplateId) { - - List entries = new ArrayList<>(); - - for (int page = 1; page <= numberOfPages; page++) { - if (pageEntities.getEntitiesPerPage().get(page) != null) { - entries.addAll(addEntries(pageEntities.getEntitiesPerPage(), page, dossierTemplateId)); - } - - if (pageEntities.getImagesPerPage().get(page) != null) { - entries.addAll(addImageEntries(pageEntities.getImagesPerPage(), page, dossierTemplateId)); - } - } - - return entries; - } - - - public List addImageEntries(Map> images, int pageNumber, String dossierTemplateId) { - - List redactionLogEntities = new ArrayList<>(); - - for (Image image : images.get(pageNumber)) { - - String id = IdBuilder.buildId(image.getPosition(), pageNumber); - - RedactionLogEntry redactionLogEntry = RedactionLogEntry.builder() - .id(id) - .color(getColor(image.getType(), dossierTemplateId, image.isRedaction())) - .isImage(true) - .type(image.getType()) - .redacted(image.isRedaction()) - .reason(image.getRedactionReason()) - .legalBasis(image.getLegalBasis()) - .matchedRule(image.getMatchedRule()) - .isHint(dictionaryService.isHint(image.getType(), dossierTemplateId)) - .isDictionaryEntry(false) - .isRecommendation(false) - .positions(List.of(new Rectangle(new Point((float) image.getPosition().getX(), (float) image.getPosition().getY()), - (float) image.getPosition().getWidth(), - (float) image.getPosition().getHeight(), - pageNumber))) - .sectionNumber(image.getSectionNumber()) - .section(image.getSection()) - .imageHasTransparency(image.isHasTransparency()) - .build(); - - redactionLogEntities.add(redactionLogEntry); - } - - return redactionLogEntities; - } - - - public List addEntries(Map> entities, int page, String dossierTemplateId) { - - List redactionLogEntities = new ArrayList<>(); - - // Duplicates can exist due table extraction columns over multiple rows. - Set processedIds = new HashSet<>(); - - entityLoop: - for (Entity entity : entities.get(page)) { - - for (EntityPositionSequence entityPositionSequence : entity.getPositionSequences()) { - - RedactionLogEntry redactionLogEntry = createRedactionLogEntry(entity, dossierTemplateId); - if (processedIds.contains(entityPositionSequence.getId())) { - - // TODO refactor this outer loop jump as soon as we have the time. - continue entityLoop; - } else { - processedIds.add(entityPositionSequence.getId()); - } - - redactionLogEntry.setId(entityPositionSequence.getId()); - - if (CollectionUtils.isNotEmpty(entityPositionSequence.getSequences())) { - List rectanglesPerLine = getRectanglesPerLine(entityPositionSequence.getSequences()); - - redactionLogEntry.getPositions().addAll(rectanglesPerLine); - - } - - // FIXME ids should never be null. Figure out why this happens. - if (redactionLogEntry.getId() != null) { - redactionLogEntities.add(redactionLogEntry); - } - } - } - - return redactionLogEntities; - } - - - private List getRectanglesPerLine(List textPositionSequences) { - - List rectangles = new ArrayList<>(); - - if (textPositionSequences.size() == 1) { - rectangles.add(textPositionSequences.get(0).getRectangle()); - return rectangles; - } - - TextPositionSequence combinedSequence = new TextPositionSequence(); - for (int i = 0; i < textPositionSequences.size(); i++) { - - if (combinedSequence.getTextPositions().isEmpty()) { - combinedSequence = textPositionSequences.get(i); - continue; - } - - float lastSeqX = textPositionSequences.get(i - 1).getTextPositions().get(textPositionSequences.get(i - 1).getTextPositions().size() - 1).getXDirAdj(); - float lastSeqY = textPositionSequences.get(i - 1).getTextPositions().get(textPositionSequences.get(i - 1).getTextPositions().size() - 1).getYDirAdj(); - float lastSeqWidth = textPositionSequences.get(i - 1).getTextPositions().get(textPositionSequences.get(i - 1).getTextPositions().size() - 1).getWidthDirAdj(); - float lastSeqHeight = textPositionSequences.get(i - 1).getTextPositions().get(textPositionSequences.get(i - 1).getTextPositions().size() - 1).getHeightDir(); - - float currentSeqX = textPositionSequences.get(i).getTextPositions().get(0).getXDirAdj(); - float currentSeqY = textPositionSequences.get(i).getTextPositions().get(0).getYDirAdj(); - float currentSeqHeight = textPositionSequences.get(i).getTextPositions().get(0).getHeightDir(); - - if (isCharInSameLine(lastSeqY, currentSeqY, lastSeqHeight, currentSeqHeight) && isCharClose(lastSeqX, currentSeqX, lastSeqWidth)) { - combinedSequence.getTextPositions().addAll(textPositionSequences.get(i).getTextPositions()); - } else { - rectangles.add(combinedSequence.getRectangle()); - combinedSequence = textPositionSequences.get(i); - } - - } - - if (!combinedSequence.getTextPositions().isEmpty()) { - rectangles.add(combinedSequence.getRectangle()); - } - - return rectangles; - } - - - private boolean isCharClose(float x, float xDirAdj, float width) { - - float max = x + (5 * width); - return xDirAdj < max; - } - - - private boolean isCharInSameLine(float y, float yCompare, float height, float heightCompare) { - - float offsetHeight = height / 2; - float minHeight = height - offsetHeight; - float maxHeight = height + offsetHeight; - - float offsetY = height / 10; - float minY = y - offsetY; - float maxY = y + offsetY; - - return yCompare > minY && yCompare < maxY && heightCompare > minHeight && heightCompare < maxHeight; - } - - - private RedactionLogEntry createRedactionLogEntry(Entity entity, String dossierTemplateId) { - - Set referenceIds = new HashSet<>(); - entity.getReferences().forEach(ref -> ref.getPositionSequences().forEach(pos -> referenceIds.add(pos.getId()))); + public RedactionLogEntry createRedactionLogEntry(ImageNode image, String dossierTemplateId) { return RedactionLogEntry.builder() - .color(getColor(entity.getType(), dossierTemplateId, entity.isRedaction())) - .reason(entity.getRedactionReason()) - .legalBasis(entity.getLegalBasis()) - .value(entity.getWord()) - .type(entity.getType()) - .redacted(entity.isRedaction()) - .isHint(isHint(entity.getType(), dossierTemplateId)) - .isRecommendation(entity.getEntityType().equals(EntityType.RECOMMENDATION)) - .isFalsePositive(entity.getEntityType().equals(EntityType.FALSE_POSITIVE) || entity.getEntityType().equals(EntityType.FALSE_RECOMMENDATION)) - .section(entity.getHeadline()) - .sectionNumber(entity.getSectionNumber()) - .matchedRule(entity.getMatchedRule()) - .isDictionaryEntry(entity.isDictionaryEntry()) - .textAfter(entity.getTextAfter()) - .textBefore(entity.getTextBefore()) - .startOffset(entity.getStart()) - .endOffset(entity.getEnd()) - .isDossierDictionaryEntry(entity.isDossierDictionaryEntry()) - .engines(entity.getEngines()) - .reference(referenceIds) + .id(image.getId()) + .color(getColor(image.getImageType().toString(), dossierTemplateId, image.isRedaction())) + .isImage(true) + .type(image.getImageType().toString()) + .redacted(image.isRedaction()) + .reason(image.getRedactionReason()) + .legalBasis(image.getLegalBasis()) + .matchedRule(image.getMatchedRule()) + .isHint(dictionaryService.isHint(image.getImageType().toString(), dossierTemplateId)) + .isDictionaryEntry(false) + .isRecommendation(false) + .positions(List.of(RectangleTransformations.toRectangle(image.getPosition(), image.getPage().getNumber()))) + .sectionNumber(image.getTocId().get(0)) + .section(image.getParent().buildTextBlock().getSearchText()) + .imageHasTransparency(image.isTransparent()) .build(); + } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/SectionGridCreatorService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/SectionGridCreatorService.java index b88f5770..da7d3b14 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/SectionGridCreatorService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/SectionGridCreatorService.java @@ -1,10 +1,19 @@ package com.iqser.red.service.redaction.v1.server.redaction.service; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +import org.springframework.stereotype.Service; + import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Point; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.CellRectangle; +import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.SectionArea; +import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.SectionGrid; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.SectionRectangle; import com.iqser.red.service.redaction.v1.server.classification.model.Document; import com.iqser.red.service.redaction.v1.server.classification.model.Section; +import com.iqser.red.service.redaction.v1.server.classification.model.SectionText; import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock; import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer; import com.iqser.red.service.redaction.v1.server.tableextraction.model.Cell; @@ -12,20 +21,22 @@ import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table; import lombok.RequiredArgsConstructor; -import org.springframework.stereotype.Service; - -import java.util.ArrayList; -import java.util.List; - @Service @RequiredArgsConstructor public class SectionGridCreatorService { - public void createSectionGrid(Document classifiedDoc, int numberOfPages) { + public void createSectionGrid(Document classifiedDoc, int numberOfPages, List sectionTexts) { for (int page = 1; page <= numberOfPages; page++) { addSectionGrid(classifiedDoc, page); } + // enhance section grid with headline data + sectionTexts.forEach(sectionText -> classifiedDoc.getSectionGrid() + .getSections() + .add(new SectionGrid.SectionGridSection(sectionText.getSectionNumber(), + sectionText.getHeadline(), + sectionText.getSectionAreas().stream().map(SectionArea::getPage).collect(Collectors.toSet()), + sectionText.getSectionAreas()))); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/analyze/AnalyzeService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/analyze/AnalyzeService.java index 6cfbc963..4b19b956 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/analyze/AnalyzeService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/analyze/AnalyzeService.java @@ -2,11 +2,9 @@ package com.iqser.red.service.redaction.v1.server.redaction.service.analyze; import static com.iqser.red.service.redaction.v1.server.redaction.service.ImportedRedactionService.IMPORTED_REDACTION_TYPE; -import java.util.ArrayList; import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.Objects; import java.util.Set; import java.util.stream.Collectors; @@ -23,8 +21,6 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemp import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogLegalBasis; -import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.SectionArea; -import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.SectionGrid; import com.iqser.red.service.redaction.v1.model.StructureAnalyzeRequest; import com.iqser.red.service.redaction.v1.server.classification.model.Document; import com.iqser.red.service.redaction.v1.server.classification.model.SectionText; @@ -32,13 +28,14 @@ import com.iqser.red.service.redaction.v1.server.classification.model.Simplified import com.iqser.red.service.redaction.v1.server.classification.model.SimplifiedText; import com.iqser.red.service.redaction.v1.server.classification.model.Text; import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient; -import com.iqser.red.service.redaction.v1.server.client.model.NerEntities; +import com.iqser.red.service.redaction.v1.server.client.model.EntityRecognitionEntity; import com.iqser.red.service.redaction.v1.server.document.graph.DocumentGraph; import com.iqser.red.service.redaction.v1.server.document.graph.factory.DocumentGraphFactory; import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SectionNode; import com.iqser.red.service.redaction.v1.server.document.mapper.DocumentDataMapper; import com.iqser.red.service.redaction.v1.server.document.mapper.DocumentGraphMapper; import com.iqser.red.service.redaction.v1.server.exception.RedactionException; +import com.iqser.red.service.redaction.v1.server.redaction.adapter.NerEntitiesAdapter; import com.iqser.red.service.redaction.v1.server.redaction.model.PdfImage; import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.Dictionary; import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.DictionaryIncrement; @@ -72,6 +69,7 @@ public class AnalyzeService { private static final String REDACTMANAGER_ANALYZE_PAGEWISE_METRIC_NAME = "redactmanager_analyze.pagewise"; DictionaryService dictionaryService; + NerEntitiesAdapter nerEntitiesAdapter; DroolsExecutionService droolsExecutionService; EntityRedactionService entityRedactionService; RedactionLogCreatorService redactionLogCreatorService; @@ -118,21 +116,13 @@ public class AnalyzeService { List sectionTexts = sectionTextBuilderService.buildSectionText(classifiedDoc); - sectionGridCreatorService.createSectionGrid(classifiedDoc, pageCount); + sectionGridCreatorService.createSectionGrid(classifiedDoc, pageCount, sectionTexts); Text text = new Text(pageCount, sectionTexts); - // enhance section grid with headline data - sectionTexts.forEach(sectionText -> classifiedDoc.getSectionGrid() - .getSections() - .add(new SectionGrid.SectionGridSection(sectionText.getSectionNumber(), - sectionText.getHeadline(), - sectionText.getSectionAreas().stream().map(SectionArea::getPage).collect(Collectors.toSet()), - sectionText.getSectionAreas()))); - log.info("Store document graph, text, simplified text, and section grid for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.TEXT, DocumentDataMapper.toDocumentData(documentGraph)); - redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.SIMPLIFIED_TEXT, convert(text)); + redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.SIMPLIFIED_TEXT, toSimplifiedText(documentGraph)); redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.SECTION_GRID, classifiedDoc.getSectionGrid()); return AnalyzeResult.builder() @@ -179,9 +169,7 @@ public class AnalyzeService { Dictionary dictionary = dictionaryService.getDeepCopyDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId()); - NerEntities nerEntities = redactionServiceSettings.isNerServiceEnabled() // - ? redactionStorageService.getNerEntities(analyzeRequest.getDossierId(), analyzeRequest.getFileId()) // - : new NerEntities(); + List nerEntities = nerEntitiesAdapter.getNerEntities(analyzeRequest, documentGraph); KieContainer kieContainer = droolsExecutionService.updateRules(analyzeRequest.getDossierTemplateId()); @@ -192,7 +180,7 @@ public class AnalyzeService { analyzeRequest, nerEntities); - List newRedactionLogEntries = redactionLogCreatorService.createRedactionLog(documentGraph.getEntities(), analyzeRequest.getDossierTemplateId()); + List newRedactionLogEntries = redactionLogCreatorService.createRedactionLog(documentGraph, analyzeRequest.getDossierTemplateId()); var importedRedactionFilteredEntries = importedRedactionService.processImportedRedactions(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId(), @@ -213,12 +201,7 @@ public class AnalyzeService { long startTime = System.currentTimeMillis(); DocumentGraph documentGraph = DocumentGraphMapper.toDocumentGraph(redactionStorageService.getDocumentData(analyzeRequest.getDossierId(), analyzeRequest.getFileId())); - NerEntities nerEntities; - if (redactionServiceSettings.isNerServiceEnabled()) { - nerEntities = redactionStorageService.getNerEntities(analyzeRequest.getDossierId(), analyzeRequest.getFileId()); - } else { - nerEntities = new NerEntities(); - } + List nerEntities = nerEntitiesAdapter.getNerEntities(analyzeRequest, documentGraph); dictionaryService.updateDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId()); KieContainer kieContainer = droolsExecutionService.updateRules(analyzeRequest.getDossierTemplateId()); long rulesVersion = droolsExecutionService.getRulesVersion(analyzeRequest.getDossierTemplateId()); @@ -231,13 +214,13 @@ public class AnalyzeService { analyzeRequest, nerEntities); - List redactionLogEntries = redactionLogCreatorService.createRedactionLog(documentGraph.getEntities(), analyzeRequest.getDossierTemplateId()); + List redactionLogEntries = redactionLogCreatorService.createRedactionLog(documentGraph, analyzeRequest.getDossierTemplateId()); List legalBasis = legalBasisClient.getLegalBasisMapping(analyzeRequest.getDossierTemplateId()); RedactionLog redactionLog = new RedactionLog(redactionServiceSettings.getAnalysisVersion(), analyzeRequest.getAnalysisNumber(), redactionLogEntries, - convert(legalBasis), + toSimplifiedSectionText(legalBasis), dictionary.getVersion().getDossierTemplateVersion(), dictionary.getVersion().getDossierVersion(), rulesVersion, @@ -296,7 +279,7 @@ public class AnalyzeService { } - public List convert(List legalBasis) { + public List toSimplifiedSectionText(List legalBasis) { return legalBasis.stream().map(l -> new RedactionLogLegalBasis(l.getName(), l.getDescription(), l.getReason())).collect(Collectors.toList()); } @@ -314,16 +297,16 @@ public class AnalyzeService { } - private SimplifiedText convert(Text text) { + private SimplifiedText toSimplifiedText(DocumentGraph documentGraph) { - List sectionTexts = new ArrayList<>(); - int numberOfPages = 0; + List simplifiedSectionTexts = documentGraph.getMainSections().stream().map(this::toSimplifiedSectionText).toList(); + return SimplifiedText.builder().numberOfPages(documentGraph.getNumberOfPages()).sectionTexts(simplifiedSectionTexts).build(); + } - if (text != null) { - numberOfPages = text.getNumberOfPages(); - text.getSectionTexts().stream().filter(Objects::nonNull).forEach(st -> sectionTexts.add(new SimplifiedSectionText(st.getSectionNumber(), st.getText()))); - } - return SimplifiedText.builder().numberOfPages(numberOfPages).sectionTexts(sectionTexts).build(); + + private SimplifiedSectionText toSimplifiedSectionText(SectionNode sectionNode) { + + return SimplifiedSectionText.builder().sectionNumber(sectionNode.getTocId().get(0)).text(sectionNode.buildTextBlock().getSearchText()).build(); } } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/entityredaction/EntityRedactionService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/entityredaction/EntityRedactionService.java index 5264b652..eb35ab07 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/entityredaction/EntityRedactionService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/entityredaction/EntityRedactionService.java @@ -9,7 +9,7 @@ import org.springframework.stereotype.Service; import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest; import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute; -import com.iqser.red.service.redaction.v1.server.client.model.NerEntities; +import com.iqser.red.service.redaction.v1.server.client.model.EntityRecognitionEntity; import com.iqser.red.service.redaction.v1.server.document.graph.DocumentGraph; import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SectionNode; import com.iqser.red.service.redaction.v1.server.document.services.EntityCreationService; @@ -37,13 +37,14 @@ public class EntityRedactionService { DocumentGraph documentGraph, KieContainer kieContainer, AnalyzeRequest analyzeRequest, - NerEntities nerEntities) { + List nerEntities) { List allFileAttributes = droolsExecutionService.executeRules(kieContainer, documentGraph, dictionary, analyzeRequest.getFileAttributes(), - analyzeRequest.getManualRedactions()); + analyzeRequest.getManualRedactions(), + nerEntities); return allFileAttributes.stream().filter(fileAttribute -> !analyzeRequest.getFileAttributes().contains(fileAttribute)).collect(Collectors.toUnmodifiableSet()); } @@ -53,14 +54,15 @@ public class EntityRedactionService { List sectionsToReanalyze, KieContainer kieContainer, AnalyzeRequest analyzeRequest, - NerEntities nerEntities) { + List nerEntities) { List allFileAttributes = droolsExecutionService.executeRules(kieContainer, documentGraph, sectionsToReanalyze, dictionary, analyzeRequest.getFileAttributes(), - analyzeRequest.getManualRedactions()); + analyzeRequest.getManualRedactions(), + nerEntities); return allFileAttributes.stream().filter(fileAttribute -> !analyzeRequest.getFileAttributes().contains(fileAttribute)).collect(Collectors.toUnmodifiableSet()); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DocumentGraphIntegrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DocumentGraphIntegrationTest.java index bbd4fcdb..a6e3bb4e 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DocumentGraphIntegrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DocumentGraphIntegrationTest.java @@ -108,7 +108,7 @@ public class DocumentGraphIntegrationTest extends AbstractTestWithDictionaries { kieSession.fireAllRules(); System.out.printf("Firing rules took %d ms\n", System.currentTimeMillis() - dictionaryAddsStart); System.out.printf("Total time %d ms\n", System.currentTimeMillis() - dictionarySearchStart); - List redactionLogEntries = redactionLogCreatorService.createRedactionLog(document.getEntities(), TEST_DOSSIER_TEMPLATE_ID); + List redactionLogEntries = redactionLogCreatorService.createRedactionLog(document, TEST_DOSSIER_TEMPLATE_ID); drawAllEntities(filename, fileResource, document); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl index eec10869..5d7f83e4 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl @@ -27,6 +27,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualImageRecategorization; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus; import com.iqser.red.service.redaction.v1.server.document.services.ManualRedactionApplicationService; +import com.iqser.red.service.redaction.v1.server.client.model.EntityRecognitionEntity; global DocumentGraph document global EntityCreationService entityCreationService @@ -162,12 +163,22 @@ rule "run local dictionary search" entityNodes.forEach(entityNode -> insert(entityNode)); end +// --------------------------------------- NER Entities rules ------------------------------------------------------------------- + +rule "add NER Entities of type CBI_author or CBI_address" + when + $nerEntity: EntityRecognitionEntity($type: type, (type == "CBI_author" || type == "CBI_address")) + then + EntityNode entityNode = entityCreationService.byBoundary(new Boundary($nerEntity.getStartOffset(), $nerEntity.getEndOffset()), $type, EntityType.RECOMMENDATION, document); + insert(entityNode); + end + // --------------------------------------- CBI rules ------------------------------------------------------------------- rule "1: Redact CBI Authors (Non vertebrate study)" no-loop true when - not FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "yes") + not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") $entity: EntityNode(type == "CBI_author", entityType == EntityType.ENTITY) then $entity.setRedaction(true); @@ -178,7 +189,7 @@ rule "1: Redact CBI Authors (Non vertebrate study)" rule "2: Redact CBI Authors (Vertebrate study)" no-loop true when - FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "yes") + FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") $entity: EntityNode(type == "CBI_author", entityType == EntityType.ENTITY) then $entity.setRedaction(true); @@ -189,7 +200,7 @@ rule "2: Redact CBI Authors (Vertebrate study)" rule "3: Don't redact CBI Address (Non vertebrate study)" no-loop true when - not FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "yes") + not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") $entity: EntityNode(type == "CBI_address", entityType == EntityType.ENTITY) then setFields($entity, 3, "Address found", "Article 39(e)(2) of Regulation (EC) No 178/2002", null); @@ -199,7 +210,7 @@ rule "3: Don't redact CBI Address (Non vertebrate study)" rule "4: Redact CBI Address (Vertebrate study)" no-loop true when - FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "yes") + FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") $entity: EntityNode(type == "CBI_address", entityType == EntityType.ENTITY) then $entity.setRedaction(true);