From 59827eab81dd8744cea9130a1f3f12a863299f2a Mon Sep 17 00:00:00 2001 From: Kilian Schuettler Date: Wed, 5 Apr 2023 15:54:20 +0200 Subject: [PATCH] RED-6369: Rules Refactor * refactored DocumentData to 4 different jsons to make reading only text possible * wip integration of new rules into workflow --- .../data/AtomicPositionBlockData.java | 19 + .../document/data/AtomicTextBlockData.java | 3 +- .../v1/server/document/data/DocumentData.java | 7 +- .../v1/server/document/data/PageData.java | 1 + .../document/data/TableOfContentsData.java | 8 +- .../document/graph/TableOfContents.java | 14 +- .../document/graph/entity/EntityNode.java | 54 +- .../graph/factory/DocumentGraphFactory.java | 13 +- .../SearchTextWithTextPositionFactory.java | 3 - .../document/graph/nodes/FooterNode.java | 2 +- .../document/graph/nodes/HeaderNode.java | 2 +- .../document/graph/nodes/HeadlineNode.java | 2 +- .../graph/{entity => nodes}/ImageNode.java | 6 +- .../server/document/graph/nodes/NodeType.java | 56 +- .../server/document/graph/nodes/PageNode.java | 1 - .../document/graph/nodes/ParagraphNode.java | 2 +- .../document/graph/nodes/SectionNode.java | 2 +- .../document/graph/nodes/SemanticNode.java | 15 +- .../document/graph/nodes/TableCellNode.java | 2 +- .../document/graph/nodes/TableNode.java | 2 +- .../graph/textblock/AtomicTextBlock.java | 2 +- .../textblock/ConcatenatedTextBlock.java | 10 +- .../document/graph/textblock/TextBlock.java | 17 +- .../document/mapper/DocumentDataMapper.java | 33 +- .../document/mapper/DocumentGraphMapper.java | 50 +- .../document/mapper/PropertiesMapper.java | 2 +- .../services/CharSequenceSearchUtils.java | 4 +- .../services/EntityCreationService.java | 32 +- ...vice.java => EntityEnrichmentService.java} | 34 +- .../document/services/EntityFieldSetter.java | 32 - .../v1/server/redaction/model/Dictionary.java | 2 +- .../service/DroolsExecutionService.java | 59 +- .../service/RedactionLogCreatorService.java | 8 +- .../service/analyze/AnalyzeService.java | 69 +- .../service/analyze/SectionFinder.java | 17 +- .../EntityRedactionService.java | 313 +-------- .../redaction/utils/SeparatorUtils.java | 31 +- .../storage/RedactionStorageService.java | 17 +- .../server/DocumentGraphIntegrationTest.java | 10 +- .../graph/DocumentGraphMappingTest.java | 19 +- .../test/resources/drools/entity_rules.drl | 119 +++- .../resources/drools/merge_entity_rules.drl | 5 +- .../src/test/resources/drools/rules.drl | 601 +++++++++-------- .../src/test/resources/drools/rules2.drl | 1 - .../src/test/resources/drools/testRules.drl | 619 ++++++++---------- .../src/test/resources/drools/test_rules.drl | 35 - 46 files changed, 1135 insertions(+), 1220 deletions(-) create mode 100644 redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/data/AtomicPositionBlockData.java rename redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/{entity => nodes}/ImageNode.java (92%) rename redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/{EntityTextEnrichmentService.java => EntityEnrichmentService.java} (69%) delete mode 100644 redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/EntityFieldSetter.java delete mode 100644 redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules2.drl delete mode 100644 redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/test_rules.drl diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/data/AtomicPositionBlockData.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/data/AtomicPositionBlockData.java new file mode 100644 index 00000000..5b50f66b --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/data/AtomicPositionBlockData.java @@ -0,0 +1,19 @@ +package com.iqser.red.service.redaction.v1.server.document.data; + +import lombok.AccessLevel; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.experimental.FieldDefaults; + +@Data +@Builder +@AllArgsConstructor +@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE) +public class AtomicPositionBlockData { + + Long id; + int[] stringIdxToPositionIdx; + float[][] positions; + +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/data/AtomicTextBlockData.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/data/AtomicTextBlockData.java index e4bb25ec..7cbf45d6 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/data/AtomicTextBlockData.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/data/AtomicTextBlockData.java @@ -15,10 +15,9 @@ public class AtomicTextBlockData { Long id; Long page; String searchText; + int numberOnPage; int start; int end; int[] lineBreaks; - int[] stringIdxToPositionIdx; - float[][] positions; } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/data/DocumentData.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/data/DocumentData.java index 9287d604..3660f450 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/data/DocumentData.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/data/DocumentData.java @@ -1,7 +1,5 @@ package com.iqser.red.service.redaction.v1.server.document.data; -import java.util.List; - import lombok.AccessLevel; import lombok.AllArgsConstructor; import lombok.Builder; @@ -14,8 +12,9 @@ import lombok.experimental.FieldDefaults; @FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE) public class DocumentData { - List pages; - List atomicTextBlocks; + PageData[] pages; + AtomicTextBlockData[] atomicTextBlocks; + AtomicPositionBlockData[] atomicPositionBlocks; TableOfContentsData tableOfContents; } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/data/PageData.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/data/PageData.java index 04a1434f..661c625a 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/data/PageData.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/data/PageData.java @@ -15,5 +15,6 @@ public class PageData { int number; int height; int width; + int rotation; } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/data/TableOfContentsData.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/data/TableOfContentsData.java index f842b073..e725c2ef 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/data/TableOfContentsData.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/data/TableOfContentsData.java @@ -15,12 +15,14 @@ import lombok.AccessLevel; import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; +import lombok.NoArgsConstructor; import lombok.experimental.FieldDefaults; @Data @Builder @AllArgsConstructor -@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE) +@NoArgsConstructor +@FieldDefaults(level = AccessLevel.PRIVATE) public class TableOfContentsData { List entries; @@ -64,7 +66,7 @@ public class TableOfContentsData { @Builder - public record EntryData(int[] tocId, List subEntries, NodeType type, Long[] atomicTextBlocks, Long[] pages, int numberOnPage, Map properties) { + public record EntryData(NodeType type, int[] tocId, Long[] atomicBlocks, Long[] pages, Map properties, List subEntries) { @Override public String toString() { @@ -80,7 +82,7 @@ public class TableOfContentsData { sb.append(type); sb.append(" atbs = "); - sb.append(atomicTextBlocks.length); + sb.append(atomicBlocks.length); return sb.toString(); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/TableOfContents.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/TableOfContents.java index 2bbce104..74068f66 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/TableOfContents.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/TableOfContents.java @@ -3,8 +3,6 @@ package com.iqser.red.service.redaction.v1.server.document.graph; import static java.lang.String.format; import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.Arrays; import java.util.Collections; import java.util.LinkedList; import java.util.List; @@ -153,17 +151,7 @@ public class TableOfContents { @Override public String toString() { - return tocId + ": " + type + ".: " + buildSummary(node().buildTextBlock().getSearchText()); - } - - - private static String buildSummary(CharSequence text) { - - String[] words = text.toString().split(" "); - int bound = Math.min(words.length, 4); - List list = new ArrayList<>(Arrays.asList(words).subList(0, bound)); - - return String.join(" ", list); + return node().toString(); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/entity/EntityNode.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/entity/EntityNode.java index 58608056..5195c658 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/entity/EntityNode.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/entity/EntityNode.java @@ -25,28 +25,6 @@ import lombok.experimental.FieldDefaults; @FieldDefaults(level = AccessLevel.PRIVATE) public class EntityNode { - public static EntityNode initialEntityNode(Boundary boundary, String type, EntityType entityType) { - - return EntityNode.builder() - .type(type) - .entityType(entityType) - .boundary(boundary) - .redaction(false) - .removed(false) - .ignored(false) - .resized(false) - .skipRemoveEntitiesContainedInLarger(false) - .dictionaryEntry(false) - .dossierDictionaryEntry(false) - .engines(new HashSet<>()) - .references(new HashSet<>()) - .matchedRule(-1) - .redactionReason("") - .legalBasis("") - .build(); - } - - // initial values final Boundary boundary; final String type; @@ -72,12 +50,34 @@ public class EntityNode { CharSequence textAfter; @Builder.Default Set pages = new HashSet<>(); - List entityPositions; + List entityPositionsPerPage; @Builder.Default List intersectingNodes = new LinkedList<>(); SemanticNode deepestFullyContainingNode; + public static EntityNode initialEntityNode(Boundary boundary, String type, EntityType entityType) { + + return EntityNode.builder() + .type(type) + .entityType(entityType) + .boundary(boundary) + .redaction(false) + .removed(false) + .ignored(false) + .resized(false) + .skipRemoveEntitiesContainedInLarger(false) + .dictionaryEntry(false) + .dossierDictionaryEntry(false) + .engines(new HashSet<>()) + .references(new HashSet<>()) + .matchedRule(-1) + .redactionReason("") + .legalBasis("") + .build(); + } + + public void addIntersectingNode(SemanticNode containingNode) { intersectingNodes.add(containingNode); @@ -93,13 +93,13 @@ public class EntityNode { } - public List getEntityPositions() { + public List getEntityPositionsPerPage() { - if (entityPositions == null || entityPositions.isEmpty()) { - entityPositions = deepestFullyContainingNode.buildTextBlock().getEntityPositions(boundary); + if (entityPositionsPerPage == null || entityPositionsPerPage.isEmpty()) { + entityPositionsPerPage = deepestFullyContainingNode.buildTextBlock().getEntityPositionsPerPage(boundary); } - return entityPositions; + return entityPositionsPerPage; } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/factory/DocumentGraphFactory.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/factory/DocumentGraphFactory.java index d1de728c..2e72ff68 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/factory/DocumentGraphFactory.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/factory/DocumentGraphFactory.java @@ -28,10 +28,10 @@ import com.iqser.red.service.redaction.v1.server.classification.model.Page; import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock; import com.iqser.red.service.redaction.v1.server.document.graph.DocumentGraph; import com.iqser.red.service.redaction.v1.server.document.graph.TableOfContents; -import com.iqser.red.service.redaction.v1.server.document.graph.entity.ImageNode; import com.iqser.red.service.redaction.v1.server.document.graph.nodes.FooterNode; import com.iqser.red.service.redaction.v1.server.document.graph.nodes.HeaderNode; import com.iqser.red.service.redaction.v1.server.document.graph.nodes.HeadlineNode; +import com.iqser.red.service.redaction.v1.server.document.graph.nodes.ImageNode; import com.iqser.red.service.redaction.v1.server.document.graph.nodes.NodeType; import com.iqser.red.service.redaction.v1.server.document.graph.nodes.PageNode; import com.iqser.red.service.redaction.v1.server.document.graph.nodes.ParagraphNode; @@ -40,7 +40,6 @@ import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SemanticNo import com.iqser.red.service.redaction.v1.server.document.graph.nodes.TableCellNode; import com.iqser.red.service.redaction.v1.server.document.graph.nodes.TableNode; import com.iqser.red.service.redaction.v1.server.document.graph.textblock.AtomicTextBlock; -import com.iqser.red.service.redaction.v1.server.document.services.ImageSortService; import com.iqser.red.service.redaction.v1.server.exception.NotFoundException; import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence; import com.iqser.red.service.redaction.v1.server.redaction.model.PdfImage; @@ -48,20 +47,14 @@ import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractT import com.iqser.red.service.redaction.v1.server.tableextraction.model.Cell; import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table; -import lombok.RequiredArgsConstructor; - @Service -@RequiredArgsConstructor public class DocumentGraphFactory { public static final double TABLE_CELL_MERGE_CONTENTS_SIZE_THRESHOLD = 0.05; - private final ImageSortService imageSortService; - public DocumentGraph buildDocumentGraph(Document document) { - ImageSortService.SortedImages sortedImages = imageSortService.sortImagesIntoStructure(document); TextBlockFactory textBlockFactory = new TextBlockFactory(); Context context = new Context(new TableOfContents(), new HashMap<>(), new LinkedList<>(), new LinkedList<>(), textBlockFactory); @@ -121,13 +114,13 @@ public class DocumentGraphFactory { } - private List findTextBlocksWithSameClassificationAndAlignsY(AbstractTextContainer atc, List pageBlocks) { + private static List findTextBlocksWithSameClassificationAndAlignsY(AbstractTextContainer atc, List pageBlocks) { return pageBlocks.stream() .filter(abstractTextContainer -> !abstractTextContainer.equals(atc)) + .filter(abstractTextContainer -> abstractTextContainer.getPage() == atc.getPage()) .filter(abstractTextContainer -> abstractTextContainer instanceof TextBlock) .filter(abstractTextContainer -> abstractTextContainer.intersectsY(atc)) - .filter(abstractTextContainer -> abstractTextContainer.getPage() == atc.getPage()) .map(abstractTextContainer -> (TextBlock) abstractTextContainer) .toList(); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/factory/SearchTextWithTextPositionFactory.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/factory/SearchTextWithTextPositionFactory.java index a1fd414e..dcf77f8d 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/factory/SearchTextWithTextPositionFactory.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/factory/SearchTextWithTextPositionFactory.java @@ -7,13 +7,10 @@ import java.util.LinkedList; import java.util.List; import java.util.Objects; -import org.springframework.stereotype.Service; - import com.iqser.red.service.redaction.v1.server.parsing.model.RedTextPosition; import com.iqser.red.service.redaction.v1.server.parsing.model.TextDirection; import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence; -@Service public class SearchTextWithTextPositionFactory { public static final int HEIGHT_PADDING = 2; diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/FooterNode.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/FooterNode.java index e638ef4b..f76e8d11 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/FooterNode.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/FooterNode.java @@ -47,7 +47,7 @@ public class FooterNode implements SemanticNode { @Override public String toString() { - return tocId + ": " + NodeType.FOOTER + ": " + terminalTextBlock.getSearchText(); + return tocId + ": " + NodeType.FOOTER + ": " + terminalTextBlock.buildSummary(); } } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/HeaderNode.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/HeaderNode.java index 1be7cbba..938817f3 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/HeaderNode.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/HeaderNode.java @@ -47,7 +47,7 @@ public class HeaderNode implements SemanticNode { @Override public String toString() { - return tocId + ": " + NodeType.HEADER + ": " + terminalTextBlock.getSearchText(); + return tocId + ": " + NodeType.HEADER + ": " + terminalTextBlock.buildSummary(); } } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/HeadlineNode.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/HeadlineNode.java index 5da9bb05..5ac69e47 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/HeadlineNode.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/HeadlineNode.java @@ -47,7 +47,7 @@ public class HeadlineNode implements SemanticNode { @Override public String toString() { - return tocId + ": " + NodeType.HEADLINE + ": " + terminalTextBlock.toString(); + return tocId + ": " + NodeType.HEADLINE + ": " + terminalTextBlock.buildSummary(); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/entity/ImageNode.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/ImageNode.java similarity index 92% rename from redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/entity/ImageNode.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/ImageNode.java index 3f620fbe..6a4b9817 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/entity/ImageNode.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/ImageNode.java @@ -1,4 +1,4 @@ -package com.iqser.red.service.redaction.v1.server.document.graph.entity; +package com.iqser.red.service.redaction.v1.server.document.graph.nodes; import java.awt.geom.Rectangle2D; import java.util.Collections; @@ -9,9 +9,7 @@ import java.util.Map; import java.util.Set; import com.iqser.red.service.redaction.v1.server.document.graph.TableOfContents; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.NodeType; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.PageNode; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SemanticNode; +import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityNode; import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock; import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlockCollector; import com.iqser.red.service.redaction.v1.server.redaction.model.ImageType; diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/NodeType.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/NodeType.java index 0fbc0504..0da9d4e3 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/NodeType.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/NodeType.java @@ -1,52 +1,12 @@ package com.iqser.red.service.redaction.v1.server.document.graph.nodes; public enum NodeType { - SECTION { - public String toString() { - - return "Section"; - } - }, - HEADLINE { - public String toString() { - - return "Headline"; - } - }, - PARAGRAPH { - public String toString() { - - return "Paragraph"; - } - }, - TABLE { - public String toString() { - - return "Table"; - } - }, - TABLE_CELL { - public String toString() { - - return "Cell"; - } - }, - IMAGE { - public String toString() { - - return "Image"; - } - }, - HEADER { - public String toString() { - - return "Header"; - } - }, - FOOTER { - public String toString() { - - return "Footer"; - } - } + SECTION, + HEADLINE, + PARAGRAPH, + TABLE, + TABLE_CELL, + IMAGE, + HEADER, + FOOTER } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/PageNode.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/PageNode.java index ae7d698b..4d0459ba 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/PageNode.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/PageNode.java @@ -5,7 +5,6 @@ import java.util.List; import java.util.Set; import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityNode; -import com.iqser.red.service.redaction.v1.server.document.graph.entity.ImageNode; import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock; import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlockCollector; diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/ParagraphNode.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/ParagraphNode.java index b33144ed..6749e531 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/ParagraphNode.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/ParagraphNode.java @@ -45,7 +45,7 @@ public class ParagraphNode implements SemanticNode { @Override public String toString() { - return tocId + ": " + NodeType.PARAGRAPH + ": " + terminalTextBlock.toString(); + return tocId + ": " + NodeType.PARAGRAPH + ": " + terminalTextBlock.buildSummary(); } } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/SectionNode.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/SectionNode.java index e3769011..4b1f6ecd 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/SectionNode.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/SectionNode.java @@ -49,7 +49,7 @@ public class SectionNode implements SemanticNode { @Override public String toString() { - return tocId.toString() + ": " + NodeType.SECTION + ": " + buildTextBlock().getSearchText(); + return tocId.toString() + ": " + NodeType.SECTION + ": " + buildTextBlock().buildSummary(); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/SemanticNode.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/SemanticNode.java index 94ad501f..f3dd6ce8 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/SemanticNode.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/SemanticNode.java @@ -168,6 +168,16 @@ public interface SemanticNode { } + /** + * @param strings A List of Strings which the TextBlock might contain + * @return true, if this node's TextBlock contains any of the strings + */ + default boolean containsAnyString(List strings) { + + return strings.stream().anyMatch(this::containsString); + } + + /** * This function is used during insertion of EntityNodes into the graph, it checks if the boundary of the Entity intersects or even contains the Entity. * It sets the fields accordingly and recursively calls this function on all its children. @@ -221,7 +231,8 @@ public interface SemanticNode { /** - * If this Node is Terminal it will calculate the boundingBox of its TerminalTextBlock, otherwise it will calcuate the Union of the BoundingBoxes of all Children + * If this Node is Terminal it will calculate the boundingBox of its TerminalTextBlock, otherwise it will calculate the Union of the BoundingBoxes of all its Children. + * If called on the Document, it will return the cropbox of each page * * @return Rectangle2D fully encapsulating this Node for each page. */ @@ -237,6 +248,8 @@ public interface SemanticNode { /** + * TODO this does not yet work for sections spanning multiple columns + * * @param bBoxPerPage initial empty BoundingBox * @return The union of the BoundingBoxes of all children */ diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/TableCellNode.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/TableCellNode.java index 80de49cd..af632a57 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/TableCellNode.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/TableCellNode.java @@ -73,7 +73,7 @@ public class TableCellNode implements SemanticNode { @Override public String toString() { - return tocId + ": " + NodeType.TABLE_CELL + ": " + buildTextBlock().getSearchText(); + return tocId + ": " + NodeType.TABLE_CELL + ": " + buildTextBlock().buildSummary(); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/TableNode.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/TableNode.java index 099eb386..9351ff91 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/TableNode.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/TableNode.java @@ -67,7 +67,7 @@ public class TableNode implements SemanticNode { @Override public String toString() { - return tocId.toString() + ": " + NodeType.TABLE + ": " + buildTextBlock().getSearchText(); + return tocId.toString() + ": " + NodeType.TABLE + ": " + buildTextBlock().buildSummary(); } } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/textblock/AtomicTextBlock.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/textblock/AtomicTextBlock.java index 3023680c..3f0751e6 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/textblock/AtomicTextBlock.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/textblock/AtomicTextBlock.java @@ -112,7 +112,7 @@ public class AtomicTextBlock implements TextBlock { } - public List getEntityPositions(Boundary stringBoundary) { + public List getEntityPositionsPerPage(Boundary stringBoundary) { List positionsPerLine = stringBoundary.split(getLineBreaks().stream().map(lb -> lb + boundary.start()).filter(stringBoundary::contains).toList()) .stream() diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/textblock/ConcatenatedTextBlock.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/textblock/ConcatenatedTextBlock.java index 7e3de968..b8ce8729 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/textblock/ConcatenatedTextBlock.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/textblock/ConcatenatedTextBlock.java @@ -139,23 +139,23 @@ public class ConcatenatedTextBlock implements TextBlock { @Override - public List getEntityPositions(Boundary stringBoundary) { + public List getEntityPositionsPerPage(Boundary stringBoundary) { List textBlocks = getAllAtomicTextBlocksPartiallyInStringBoundary(stringBoundary); if (textBlocks.size() == 1) { - return textBlocks.get(0).getEntityPositions(stringBoundary); + return textBlocks.get(0).getEntityPositionsPerPage(stringBoundary); } AtomicTextBlock firstTextBlock = textBlocks.get(0); - List positions = new LinkedList<>(firstTextBlock.getEntityPositions(new Boundary(stringBoundary.start(), firstTextBlock.getBoundary().end()))); + List positions = new LinkedList<>(firstTextBlock.getEntityPositionsPerPage(new Boundary(stringBoundary.start(), firstTextBlock.getBoundary().end()))); for (AtomicTextBlock textBlock : textBlocks.subList(1, textBlocks.size() - 1)) { - positions.addAll(textBlock.getEntityPositions(textBlock.getBoundary())); + positions.addAll(textBlock.getEntityPositionsPerPage(textBlock.getBoundary())); } AtomicTextBlock lastTextBlock = textBlocks.get(textBlocks.size() - 1); - positions.addAll(lastTextBlock.getEntityPositions(new Boundary(lastTextBlock.getBoundary().start(), stringBoundary.end()))); + positions.addAll(lastTextBlock.getEntityPositionsPerPage(new Boundary(lastTextBlock.getBoundary().start(), stringBoundary.end()))); return mergeEntityPositionsWithSamePageNode(positions); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/textblock/TextBlock.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/textblock/TextBlock.java index c5707f68..b8e1b48c 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/textblock/TextBlock.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/textblock/TextBlock.java @@ -3,6 +3,8 @@ package com.iqser.red.service.redaction.v1.server.document.graph.textblock; import static java.lang.String.format; import java.awt.geom.Rectangle2D; +import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import java.util.Set; import java.util.stream.Collectors; @@ -27,6 +29,7 @@ public interface TextBlock extends CharSequence { int getPreviousLinebreak(int fromIndex); + List getLineBreaks(); @@ -36,16 +39,18 @@ public interface TextBlock extends CharSequence { List getPositions(Boundary stringBoundary); - List getEntityPositions(Boundary stringBoundary); + List getEntityPositionsPerPage(Boundary stringBoundary); int numberOfLines(); + default int indexOf(String searchTerm) { return indexOf(searchTerm, getBoundary().start()); } + default Set getPages() { return getAtomicTextBlocks().stream().map(AtomicTextBlock::getPage).collect(Collectors.toUnmodifiableSet()); @@ -89,6 +94,16 @@ public interface TextBlock extends CharSequence { } + default String buildSummary() { + + String[] words = getSearchText().split(" "); + int bound = Math.min(words.length, 4); + List list = new ArrayList<>(Arrays.asList(words).subList(0, bound)); + + return String.join(" ", list); + } + + @Override default CharSequence subSequence(int start, int end) { diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/mapper/DocumentDataMapper.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/mapper/DocumentDataMapper.java index e83abe48..eecf7183 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/mapper/DocumentDataMapper.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/mapper/DocumentDataMapper.java @@ -7,13 +7,14 @@ import java.util.Map; import org.springframework.stereotype.Service; +import com.iqser.red.service.redaction.v1.server.document.data.AtomicPositionBlockData; import com.iqser.red.service.redaction.v1.server.document.data.AtomicTextBlockData; import com.iqser.red.service.redaction.v1.server.document.data.DocumentData; import com.iqser.red.service.redaction.v1.server.document.data.PageData; import com.iqser.red.service.redaction.v1.server.document.data.TableOfContentsData; import com.iqser.red.service.redaction.v1.server.document.graph.DocumentGraph; import com.iqser.red.service.redaction.v1.server.document.graph.TableOfContents; -import com.iqser.red.service.redaction.v1.server.document.graph.entity.ImageNode; +import com.iqser.red.service.redaction.v1.server.document.graph.nodes.ImageNode; import com.iqser.red.service.redaction.v1.server.document.graph.nodes.PageNode; import com.iqser.red.service.redaction.v1.server.document.graph.nodes.TableCellNode; import com.iqser.red.service.redaction.v1.server.document.graph.nodes.TableNode; @@ -30,9 +31,21 @@ public class DocumentDataMapper { .distinct() .map(this::toAtomicTextBlockData) .toList(); + + List atomicPositionBlockData = documentGraph.streamTerminalTextBlocksInOrder() + .flatMap(textBlock -> textBlock.getAtomicTextBlocks().stream()) + .distinct() + .map(this::toAtomicPositionBlockData) + .toList(); + List pageData = documentGraph.getPages().stream().map(this::toPageData).toList(); TableOfContentsData tableOfContentsData = toTableOfContentsData(documentGraph.getTableOfContents()); - return DocumentData.builder().atomicTextBlocks(atomicTextBlockData).pages(pageData).tableOfContents(tableOfContentsData).build(); + return DocumentData.builder() + .atomicTextBlocks(atomicTextBlockData.toArray(new AtomicTextBlockData[0])) + .atomicPositionBlocks(atomicPositionBlockData.toArray(new AtomicPositionBlockData[0])) + .pages(pageData.toArray(new PageData[0])) + .tableOfContents(tableOfContentsData) + .build(); } @@ -65,7 +78,6 @@ public class DocumentDataMapper { .type(entry.type()) .atomicTextBlocks(atomicTextBlocks) .pages(entry.node().getPages().stream().map(PageNode::getNumber).map(Integer::longValue).toArray(Long[]::new)) - .numberOnPage(entry.node().getNumberOnPage()) .properties(properties) .build(); } @@ -77,9 +89,9 @@ public class DocumentDataMapper { } - private PageData toPageData(PageNode pageNode) { + private PageData toPageData(PageNode p) { - return PageData.builder().height(pageNode.getHeight()).width(pageNode.getWidth()).number(pageNode.getNumber()).build(); + return PageData.builder().rotation(p.getRotation()).height(p.getHeight()).width(p.getWidth()).number(p.getNumber()).build(); } @@ -89,11 +101,20 @@ public class DocumentDataMapper { .id(atomicTextBlock.getId()) .page(atomicTextBlock.getPage().getNumber().longValue()) .searchText(atomicTextBlock.getSearchText()) + .numberOnPage(atomicTextBlock.getNumberOnPage()) .start(atomicTextBlock.getBoundary().start()) .end(atomicTextBlock.getBoundary().end()) .lineBreaks(toPrimitiveIntArray(atomicTextBlock.getLineBreaks())) - .stringIdxToPositionIdx(toPrimitiveIntArray(atomicTextBlock.getStringIdxToPositionIdx())) + .build(); + } + + + private AtomicPositionBlockData toAtomicPositionBlockData(AtomicTextBlock atomicTextBlock) { + + return AtomicPositionBlockData.builder() + .id(atomicTextBlock.getId()) .positions(toPrimitiveFloatMatrix(atomicTextBlock.getPositions())) + .stringIdxToPositionIdx(toPrimitiveIntArray(atomicTextBlock.getStringIdxToPositionIdx())) .build(); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/mapper/DocumentGraphMapper.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/mapper/DocumentGraphMapper.java index 131a8034..8af164b1 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/mapper/DocumentGraphMapper.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/mapper/DocumentGraphMapper.java @@ -17,6 +17,7 @@ import org.apache.commons.lang3.NotImplementedException; import org.springframework.stereotype.Service; import com.google.common.primitives.Ints; +import com.iqser.red.service.redaction.v1.server.document.data.AtomicPositionBlockData; import com.iqser.red.service.redaction.v1.server.document.data.AtomicTextBlockData; import com.iqser.red.service.redaction.v1.server.document.data.DocumentData; import com.iqser.red.service.redaction.v1.server.document.data.PageData; @@ -24,10 +25,10 @@ import com.iqser.red.service.redaction.v1.server.document.data.TableOfContentsDa import com.iqser.red.service.redaction.v1.server.document.graph.Boundary; import com.iqser.red.service.redaction.v1.server.document.graph.DocumentGraph; import com.iqser.red.service.redaction.v1.server.document.graph.TableOfContents; -import com.iqser.red.service.redaction.v1.server.document.graph.entity.ImageNode; import com.iqser.red.service.redaction.v1.server.document.graph.nodes.FooterNode; import com.iqser.red.service.redaction.v1.server.document.graph.nodes.HeaderNode; import com.iqser.red.service.redaction.v1.server.document.graph.nodes.HeadlineNode; +import com.iqser.red.service.redaction.v1.server.document.graph.nodes.ImageNode; import com.iqser.red.service.redaction.v1.server.document.graph.nodes.NodeType; import com.iqser.red.service.redaction.v1.server.document.graph.nodes.PageNode; import com.iqser.red.service.redaction.v1.server.document.graph.nodes.ParagraphNode; @@ -45,14 +46,19 @@ public class DocumentGraphMapper { public DocumentGraph toDocumentGraph(DocumentData documentData) { - Context context = new Context(documentData, new TableOfContents(), new LinkedList<>(), new LinkedList<>(), documentData.getAtomicTextBlocks()); + Context context = new Context(documentData, + new TableOfContents(), + new LinkedList<>(), + new LinkedList<>(), + Arrays.stream(documentData.getAtomicTextBlocks()).toList(), + Arrays.stream(documentData.getAtomicPositionBlocks()).toList()); - context.pages.addAll(documentData.getPages().stream().map(this::buildPage).toList()); + context.pages.addAll(Arrays.stream(documentData.getPages()).map(this::buildPage).toList()); context.tableOfContents.setEntries(buildEntries(documentData.getTableOfContents().getEntries(), context)); DocumentGraph documentGraph = DocumentGraph.builder() - .numberOfPages(documentData.getPages().size()) + .numberOfPages(documentData.getPages().length) .pages(new HashSet<>(context.pages)) .tableOfContents(context.tableOfContents) .build(); @@ -82,7 +88,7 @@ public class DocumentGraphMapper { }; if (node.isTerminal()) { - TextBlock textBlock = toTextBlock(entryData.atomicTextBlocks(), context, node); + TextBlock textBlock = toTextBlock(entryData.atomicBlocks(), context, node); node.setTerminalTextBlock(textBlock); } List tocId = Arrays.stream(entryData.tocId()).boxed().toList(); @@ -109,7 +115,7 @@ public class DocumentGraphMapper { private static boolean isTerminal(TableOfContentsData.EntryData entryData) { - return entryData.atomicTextBlocks().length > 0; + return entryData.atomicBlocks().length > 0; } @@ -165,34 +171,42 @@ public class DocumentGraphMapper { private TextBlock toTextBlock(Long[] atomicTextBlockIds, Context context, SemanticNode parent) { return Arrays.stream(atomicTextBlockIds) - .map(atomicTextBlockId -> toAtomicTextBlock(context.atomicTextBlockData.get(toIntExact(atomicTextBlockId)), parent, context)) + .map(atomicTextBlockId -> toAtomicTextBlock(context.atomicTextBlockData.get(toIntExact(atomicTextBlockId)), + context.atomicPositionBlockData.get(toIntExact(atomicTextBlockId)), + parent, + context)) .collect(new TextBlockCollector()); } private PageNode buildPage(PageData p) { - return PageNode.builder().height(p.getHeight()).width(p.getWidth()).number(p.getNumber()).mainBody(new LinkedList<>()).build(); + return PageNode.builder().rotation(p.getRotation()).height(p.getHeight()).width(p.getWidth()).number(p.getNumber()).mainBody(new LinkedList<>()).build(); } - private AtomicTextBlock toAtomicTextBlock(AtomicTextBlockData atomicTextBlockData, SemanticNode parent, Context context) { + private AtomicTextBlock toAtomicTextBlock(AtomicTextBlockData atomicTextBlockData, AtomicPositionBlockData atomicPositionBlockData, SemanticNode parent, Context context) { return AtomicTextBlock.builder() .id(atomicTextBlockData.getId()) + .numberOnPage(atomicTextBlockData.getNumberOnPage()) .page(getPage(atomicTextBlockData.getPage(), context)) - .searchText(atomicTextBlockData.getSearchText()) .boundary(new Boundary(atomicTextBlockData.getStart(), atomicTextBlockData.getEnd())) + .searchText(atomicTextBlockData.getSearchText()) .lineBreaks(Ints.asList(atomicTextBlockData.getLineBreaks())) - .positions(Arrays.stream(atomicTextBlockData.getPositions()) - .map(floatArr -> (Rectangle2D) new Rectangle2D.Float(floatArr[0], floatArr[1], floatArr[2], floatArr[3])) - .toList()) - .stringIdxToPositionIdx(Ints.asList(atomicTextBlockData.getStringIdxToPositionIdx())) + .stringIdxToPositionIdx(Ints.asList(atomicPositionBlockData.getStringIdxToPositionIdx())) + .positions(toRectangle2DList(atomicPositionBlockData.getPositions())) .parent(parent) .build(); } + private static List toRectangle2DList(float[][] positions) { + + return Arrays.stream(positions).map(floatArr -> (Rectangle2D) new Rectangle2D.Float(floatArr[0], floatArr[1], floatArr[2], floatArr[3])).toList(); + } + + private PageNode getPage(Long pageIndex, Context context) { return context.pages.stream() @@ -202,7 +216,13 @@ public class DocumentGraphMapper { } - record Context(DocumentData documentData, TableOfContents tableOfContents, List pages, List sections, List atomicTextBlockData) { + record Context( + DocumentData documentData, + TableOfContents tableOfContents, + List pages, + List sections, + List atomicTextBlockData, + List atomicPositionBlockData) { } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/mapper/PropertiesMapper.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/mapper/PropertiesMapper.java index 3d4bf55c..4dfaa40d 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/mapper/PropertiesMapper.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/mapper/PropertiesMapper.java @@ -5,8 +5,8 @@ import static com.iqser.red.service.redaction.v1.server.document.graph.factory.R import java.util.HashMap; import java.util.Map; -import com.iqser.red.service.redaction.v1.server.document.graph.entity.ImageNode; import com.iqser.red.service.redaction.v1.server.document.graph.factory.RectangleTransformations; +import com.iqser.red.service.redaction.v1.server.document.graph.nodes.ImageNode; import com.iqser.red.service.redaction.v1.server.document.graph.nodes.TableCellNode; import com.iqser.red.service.redaction.v1.server.document.graph.nodes.TableNode; import com.iqser.red.service.redaction.v1.server.redaction.model.ImageType; diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/CharSequenceSearchUtils.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/CharSequenceSearchUtils.java index 391566ed..a3e74a80 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/CharSequenceSearchUtils.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/CharSequenceSearchUtils.java @@ -1,6 +1,6 @@ package com.iqser.red.service.redaction.v1.server.document.services; -import static com.iqser.red.service.redaction.v1.server.redaction.utils.SeparatorUtils.validateBoundaryIsSurroundedBySeparators; +import static com.iqser.red.service.redaction.v1.server.redaction.utils.SeparatorUtils.boundaryIsSurroundedBySeparators; import java.util.LinkedList; import java.util.List; @@ -43,7 +43,7 @@ public class CharSequenceSearchUtils { while (matcher.find()) { boundaries.add(new Boundary(matcher.start() + textBlock.getBoundary().start(), matcher.end() + textBlock.getBoundary().start())); } - return boundaries.stream().filter(boundary -> validateBoundaryIsSurroundedBySeparators(textBlock, boundary)).toList(); + return boundaries.stream().filter(boundary -> boundaryIsSurroundedBySeparators(textBlock, boundary)).toList(); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/EntityCreationService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/EntityCreationService.java index b38333d1..31c89306 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/EntityCreationService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/EntityCreationService.java @@ -1,7 +1,8 @@ package com.iqser.red.service.redaction.v1.server.document.services; import static com.iqser.red.service.redaction.v1.server.document.services.CharSequenceSearchUtils.findBoundariesByString; -import static com.iqser.red.service.redaction.v1.server.redaction.utils.SeparatorUtils.validateBoundaryIsSurroundedBySeparators; +import static com.iqser.red.service.redaction.v1.server.redaction.utils.SeparatorUtils.boundaryIsSurroundedBySeparators; +import static com.iqser.red.service.redaction.v1.server.redaction.utils.SeparatorUtils.isWhiteSpacesOrSeparatorsOnly; import java.util.Collections; import java.util.LinkedList; @@ -30,7 +31,7 @@ import lombok.extern.slf4j.Slf4j; @RequiredArgsConstructor public class EntityCreationService { - private final EntityTextEnrichmentService entityEnrichmentService; + private final EntityEnrichmentService entityEnrichmentService; public Set betweenStrings(String start, String stop, String type, EntityType entityType, SemanticNode node) { @@ -52,7 +53,7 @@ public class EntityCreationService { entityBoundaries.add(new Boundary(startBoundary.end(), optionalFirstStopBoundary.get().start())); } return entityBoundaries.stream() - .filter(boundary -> validateBoundaryIsSurroundedBySeparators(textBlock, boundary)) + .filter(boundary -> isValidEntityBoundary(textBlock, boundary)) .map(boundary -> byBoundary(boundary, type, entityType, node)) .collect(Collectors.toUnmodifiableSet()); } @@ -62,19 +63,32 @@ public class EntityCreationService { return searchImplementation.getBoundaries(node.buildTextBlock(), node.getBoundary()) .stream() - .filter(boundary -> validateBoundaryIsSurroundedBySeparators(node.buildTextBlock(), boundary)) + .filter(boundary -> isValidEntityBoundary(node.buildTextBlock(), boundary)) .map(bounds -> byBoundary(bounds, type, entityType, node)) .collect(Collectors.toUnmodifiableSet()); } + public Set lineAfterStrings(List strings, String type, EntityType entityType, SemanticNode node) { + + TextBlock textBlock = node.buildTextBlock(); + SearchImplementation searchImplementation = new SearchImplementation(strings, false); + return searchImplementation.getBoundaries(textBlock, node.getBoundary()) + .stream() + .map(boundary -> toLineAfterBoundary(textBlock, boundary)) + .filter(boundary -> isValidEntityBoundary(textBlock, boundary)) + .map(boundary -> byBoundary(boundary, type, entityType, node)) + .collect(Collectors.toUnmodifiableSet()); + } + + public Set lineAfterString(String string, String type, EntityType entityType, SemanticNode node) { TextBlock textBlock = node.buildTextBlock(); List boundaries = findBoundariesByString(string, textBlock); return boundaries.stream() .map(boundary -> toLineAfterBoundary(textBlock, boundary)) - .filter(boundary -> validateBoundaryIsSurroundedBySeparators(textBlock, boundary)) + .filter(boundary -> isValidEntityBoundary(textBlock, boundary)) .map(boundary -> byBoundary(boundary, type, entityType, node)) .collect(Collectors.toUnmodifiableSet()); } @@ -103,7 +117,7 @@ public class EntityCreationService { } - public EntityNode createMergedEntity(List entitiesToMerge, String type, EntityType entityType, SemanticNode node) { + public EntityNode byEntities(List entitiesToMerge, String type, EntityType entityType, SemanticNode node) { if (!allEntitiesIntersectAndHaveSameTypes(entitiesToMerge)) { throw new IllegalArgumentException("Provided entities can not be merged!"); @@ -121,6 +135,12 @@ public class EntityCreationService { } + public boolean isValidEntityBoundary(TextBlock textBlock, Boundary boundary) { + + return boundaryIsSurroundedBySeparators(textBlock, boundary) && !isWhiteSpacesOrSeparatorsOnly(textBlock, boundary); + } + + public void addEntityToGraph(EntityNode entity, TableOfContents tableOfContents) { try { diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/EntityTextEnrichmentService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/EntityEnrichmentService.java similarity index 69% rename from redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/EntityTextEnrichmentService.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/EntityEnrichmentService.java index 81aefc40..8e5141d8 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/EntityTextEnrichmentService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/EntityEnrichmentService.java @@ -1,11 +1,13 @@ package com.iqser.red.service.redaction.v1.server.document.services; import java.util.Arrays; +import java.util.Collection; import java.util.List; import java.util.Objects; import org.springframework.stereotype.Service; +import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine; import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityNode; import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock; import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings; @@ -14,17 +16,16 @@ import lombok.RequiredArgsConstructor; @Service @RequiredArgsConstructor -public class EntityTextEnrichmentService { +public class EntityEnrichmentService { private final RedactionServiceSettings redactionServiceSettings; - public EntityNode enrichEntity(EntityNode entity, TextBlock textBlock) { + public void enrichEntity(EntityNode entity, TextBlock textBlock) { entity.setValue(textBlock.subSequence(entity.getBoundary()).toString()); entity.setTextAfter(findTextAfter(entity.getBoundary().end(), textBlock)); entity.setTextBefore(findTextBefore(entity.getBoundary().start(), textBlock)); - return entity; } @@ -64,7 +65,7 @@ public class EntityTextEnrichmentService { } - private String concatWordsBefore(List words, boolean endWithSpace) { + private static String concatWordsBefore(List words, boolean endWithSpace) { StringBuilder sb = new StringBuilder(); @@ -77,7 +78,7 @@ public class EntityTextEnrichmentService { } - private String concatWordsAfter(List words, boolean startWithSpace) { + private static String concatWordsAfter(List words, boolean startWithSpace) { StringBuilder sb = new StringBuilder(); @@ -89,4 +90,27 @@ public class EntityTextEnrichmentService { return startWithSpace ? " " + result : result; } + + public static void setFields(EntityNode entityNode, int matchedRule, String redactionReason, String legalBasis, Engine engine) { + + if (entityNode.getMatchedRule() == -1 && matchedRule != -1) { + entityNode.setMatchedRule(matchedRule); + } + if (entityNode.getRedactionReason().equals("") && redactionReason != null) { + entityNode.setRedactionReason(redactionReason); + } + if (entityNode.getLegalBasis().equals("") && legalBasis != null) { + entityNode.setLegalBasis(legalBasis); + } + if (engine != null) { + entityNode.addEngine(engine); + } + } + + + public static void setFields(Collection entityNodes, int matchedRule, String redactionReason, String legalBasis, Engine engine) { + + entityNodes.forEach(entityNode -> setFields(entityNode, matchedRule, redactionReason, legalBasis, engine)); + } + } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/EntityFieldSetter.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/EntityFieldSetter.java deleted file mode 100644 index 40555733..00000000 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/EntityFieldSetter.java +++ /dev/null @@ -1,32 +0,0 @@ -package com.iqser.red.service.redaction.v1.server.document.services; - -import java.util.Collection; - -import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine; -import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityNode; - -public class EntityFieldSetter { - - public static void setFields(EntityNode entityNode, int matchedRule, String redactionReason, String legalBasis, Engine engine) { - - if (entityNode.getMatchedRule() == -1 && matchedRule != -1) { - entityNode.setMatchedRule(matchedRule); - } - if (entityNode.getRedactionReason().equals("") && redactionReason != null) { - entityNode.setRedactionReason(redactionReason); - } - if (entityNode.getLegalBasis().equals("") && legalBasis != null) { - entityNode.setLegalBasis(legalBasis); - } - if (engine != null) { - entityNode.addEngine(engine); - } - } - - - public static void setFields(Collection entityNodes, int matchedRule, String redactionReason, String legalBasis, Engine engine) { - - entityNodes.forEach(entityNode -> setFields(entityNode, matchedRule, redactionReason, legalBasis, engine)); - } - -} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Dictionary.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Dictionary.java index 574c5041..746a3a63 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Dictionary.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Dictionary.java @@ -92,7 +92,7 @@ public class Dictionary { if (localAccessMap.get(type).getLocalEntries() == null) { throw new IllegalArgumentException(format("DictionaryModel of type %s has no local Entries", type)); } - if (StringUtils.isEmpty(value) && value.length() < 3) { + if (StringUtils.isEmpty(value)) { throw new IllegalArgumentException(format("%s is not a valid dictionary entry", value)); } localAccessMap.get(type).getLocalEntries().add(value); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DroolsExecutionService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DroolsExecutionService.java index f388c625..5d768ac2 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DroolsExecutionService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DroolsExecutionService.java @@ -26,11 +26,15 @@ import org.kie.api.runtime.rule.QueryResults; import org.kie.api.runtime.rule.QueryResultsRow; import org.springframework.stereotype.Service; +import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute; import com.iqser.red.service.redaction.v1.server.client.RulesClient; +import com.iqser.red.service.redaction.v1.server.document.graph.DocumentGraph; +import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SectionNode; +import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SemanticNode; +import com.iqser.red.service.redaction.v1.server.document.services.EntityCreationService; import com.iqser.red.service.redaction.v1.server.exception.RulesValidationException; import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary; import com.iqser.red.service.redaction.v1.server.redaction.model.Entity; -import com.iqser.red.service.redaction.v1.server.redaction.model.Paragraph; import com.iqser.red.service.redaction.v1.server.redaction.model.Section; import io.micrometer.core.annotation.Timed; @@ -77,6 +81,8 @@ public class DroolsExecutionService { }); } + private final EntityCreationService entityCreationService; + public KieContainer getKieContainer(String dossierTemplateId) { @@ -90,18 +96,46 @@ public class DroolsExecutionService { @Timed("redactmanager_executeRules") - public List executeRules(KieContainer kieContainer, List
sections, List paragraphs, Dictionary dictionary) { + public List executeRules(KieContainer kieContainer, DocumentGraph document, Dictionary dictionary, List fileAttributes) { KieSession kieSession = kieContainer.newKieSession(); + kieSession.setGlobal("document", document); + kieSession.setGlobal("entityCreationService", entityCreationService); kieSession.setGlobal("dictionary", dictionary); - sections.forEach(kieSession::insert); - paragraphs.forEach(kieSession::insert); + + document.getEntities().forEach(kieSession::insert); + document.getTableOfContents().streamEntriesInOrder().forEach(entry -> kieSession.insert(entry.node())); + document.getPages().forEach(kieSession::insert); + fileAttributes.forEach(kieSession::insert); + + kieSession.getAgenda().getAgendaGroup("LOCAL_DICTIONARY_ADDS").setFocus(); kieSession.fireAllRules(); - List entities = getEntities(kieSession); - kieSession.dispose(); - return entities; + return getFileAttributes(kieSession); + } + + @Timed("redactmanager_executeRules") + public List executeRules(KieContainer kieContainer, + DocumentGraph document, + List sectionsToReanalyze, + Dictionary dictionary, + List fileAttributes) { + + KieSession kieSession = kieContainer.newKieSession(); + kieSession.setGlobal("document", document); + kieSession.setGlobal("entityCreationService", entityCreationService); + kieSession.setGlobal("dictionary", dictionary); + + document.getEntities().forEach(kieSession::insert); + sectionsToReanalyze.stream().flatMap(SemanticNode::streamAllSubNodes).forEach(kieSession::insert); + document.getPages().forEach(kieSession::insert); + fileAttributes.forEach(kieSession::insert); + + kieSession.getAgenda().getAgendaGroup("LOCAL_DICTIONARY_ADDS").setFocus(); + kieSession.fireAllRules(); + + return getFileAttributes(kieSession); } @@ -119,6 +153,17 @@ public class DroolsExecutionService { } + public List getFileAttributes(KieSession kieSession) { + + List fileAttributes = new LinkedList<>(); + QueryResults entitiesResult = kieSession.getQueryResults("getFileAttributes"); + for (QueryResultsRow resultsRow : entitiesResult) { + fileAttributes.add((FileAttribute) resultsRow.get("$result")); + } + return fileAttributes; + } + + public List getEntities(KieSession ks) { List entities = new LinkedList<>(); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java index 6f715bc6..0fc204da 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java @@ -52,7 +52,7 @@ public class RedactionLogCreatorService { // Duplicates can exist due table extraction columns over multiple rows. - for (EntityPosition entityPosition : entityNode.getEntityPositions()) { + for (EntityPosition entityPosition : entityNode.getEntityPositionsPerPage()) { RedactionLogEntry redactionLogEntry = createRedactionLogEntry(entityNode, dossierTemplateId); @@ -61,7 +61,6 @@ public class RedactionLogCreatorService { } processedIds.add(entityPosition.getId()); - redactionLogEntry.setId(entityPosition.getId()); List rectanglesPerLine = entityPosition.getRectanglePerLine() @@ -70,6 +69,7 @@ public class RedactionLogCreatorService { .toList(); redactionLogEntry.setPositions(rectanglesPerLine); + redactionLogEntities.add(redactionLogEntry); } return redactionLogEntities; @@ -79,7 +79,7 @@ public class RedactionLogCreatorService { private RedactionLogEntry createRedactionLogEntry(EntityNode entity, String dossierTemplateId) { Set referenceIds = new HashSet<>(); - entity.getReferences().forEach(ref -> ref.getEntityPositions().forEach(pos -> referenceIds.add(pos.getId()))); + entity.getReferences().forEach(ref -> ref.getEntityPositionsPerPage().forEach(pos -> referenceIds.add(pos.getId()))); return RedactionLogEntry.builder() .color(getColor(entity.getType(), dossierTemplateId, entity.isRedaction())) @@ -91,7 +91,7 @@ public class RedactionLogCreatorService { .isHint(isHint(entity.getType(), dossierTemplateId)) .isRecommendation(entity.getEntityType().equals(EntityType.RECOMMENDATION)) .isFalsePositive(entity.getEntityType().equals(EntityType.FALSE_POSITIVE) || entity.getEntityType().equals(EntityType.FALSE_RECOMMENDATION)) - .section(entity.getDeepestFullyContainingNode().getTocId().toString()) + .section(entity.getDeepestFullyContainingNode().toString()) .sectionNumber(entity.getDeepestFullyContainingNode().getTocId().get(0)) .matchedRule(entity.getMatchedRule()) .isDictionaryEntry(entity.isDictionaryEntry()) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/analyze/AnalyzeService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/analyze/AnalyzeService.java index b036e85a..b8eecf05 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/analyze/AnalyzeService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/analyze/AnalyzeService.java @@ -33,11 +33,15 @@ import com.iqser.red.service.redaction.v1.server.classification.model.Simplified import com.iqser.red.service.redaction.v1.server.classification.model.Text; import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient; import com.iqser.red.service.redaction.v1.server.client.model.NerEntities; +import com.iqser.red.service.redaction.v1.server.document.graph.DocumentGraph; +import com.iqser.red.service.redaction.v1.server.document.graph.factory.DocumentGraphFactory; +import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SectionNode; +import com.iqser.red.service.redaction.v1.server.document.mapper.DocumentDataMapper; +import com.iqser.red.service.redaction.v1.server.document.mapper.DocumentGraphMapper; import com.iqser.red.service.redaction.v1.server.exception.RedactionException; import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary; import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryIncrement; import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryVersion; -import com.iqser.red.service.redaction.v1.server.redaction.model.PageEntities; import com.iqser.red.service.redaction.v1.server.redaction.model.PdfImage; import com.iqser.red.service.redaction.v1.server.redaction.service.DictionaryService; import com.iqser.red.service.redaction.v1.server.redaction.service.DroolsExecutionService; @@ -81,6 +85,9 @@ public class AnalyzeService { ImageService imageService; ImportedRedactionService importedRedactionService; SectionFinder sectionFinder; + DocumentGraphFactory documentGraphFactory; + DocumentDataMapper documentDataMapper; + DocumentGraphMapper documentGraphMapper; FunctionTimerValues redactmanagerAnalyzePagewiseValues; @@ -109,6 +116,8 @@ public class AnalyzeService { throw new RedactionException(e); } + DocumentGraph documentGraph = documentGraphFactory.buildDocumentGraph(classifiedDoc); + List sectionTexts = sectionTextBuilderService.buildSectionText(classifiedDoc); sectionGridCreatorService.createSectionGrid(classifiedDoc, pageCount); @@ -123,8 +132,8 @@ public class AnalyzeService { sectionText.getSectionAreas().stream().map(SectionArea::getPage).collect(Collectors.toSet()), sectionText.getSectionAreas()))); - log.info("Store text, simplified text and section grid for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); - redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.TEXT, text); + log.info("Store document graph, text, simplified text, and section grid for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); + redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.TEXT, documentDataMapper.toDocumentData(documentGraph)); redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.SIMPLIFIED_TEXT, convert(text)); redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.SECTION_GRID, classifiedDoc.getSectionGrid()); @@ -144,11 +153,11 @@ public class AnalyzeService { long startTime = System.currentTimeMillis(); - var redactionLog = redactionStorageService.getRedactionLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId()); - var text = redactionStorageService.getText(analyzeRequest.getDossierId(), analyzeRequest.getFileId()); + RedactionLog redactionLog = redactionStorageService.getRedactionLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId()); + DocumentGraph documentGraph = documentGraphMapper.toDocumentGraph(redactionStorageService.getDocumentData(analyzeRequest.getDossierId(), analyzeRequest.getFileId())); // not yet ready for reanalysis - if (redactionLog == null || text == null || text.getNumberOfPages() == 0) { + if (redactionLog == null || documentGraph == null || documentGraph.getNumberOfPages() == 0) { return analyze(analyzeRequest); } @@ -156,28 +165,31 @@ public class AnalyzeService { new DictionaryVersion(redactionLog.getDictionaryVersion(), redactionLog.getDossierDictionaryVersion()), analyzeRequest.getDossierId()); - Set sectionsToReanalyse = analyzeRequest.getSectionsToReanalyse().isEmpty() // - ? sectionFinder.findSectionsToReanalyse(dictionaryIncrement, redactionLog, text, analyzeRequest) // + Set sectionsToReanalyseIds = analyzeRequest.getSectionsToReanalyse().isEmpty() // + ? sectionFinder.findSectionsToReanalyse(dictionaryIncrement, redactionLog, documentGraph, analyzeRequest) // : analyzeRequest.getSectionsToReanalyse(); - log.info("Should reanalyze {} sections for request: {}", sectionsToReanalyse.size(), analyzeRequest); + log.info("Should reanalyze {} sections for request: {}", sectionsToReanalyseIds.size(), analyzeRequest); - if (sectionsToReanalyse.isEmpty()) { - return finalizeAnalysis(analyzeRequest, startTime, redactionLog, text, dictionaryIncrement.getDictionaryVersion(), true, new HashSet<>()); + if (sectionsToReanalyseIds.isEmpty()) { + return finalizeAnalysis(analyzeRequest, startTime, redactionLog, documentGraph.getNumberOfPages(), dictionaryIncrement.getDictionaryVersion(), true, new HashSet<>()); } - Dictionary dictionary = dictionaryService.getDeepCopyDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId()); - List reanalysisSections = text.getSectionTexts() + List sectionsToReAnalyse = documentGraph.getMainSections() .stream() - .filter(sectionText -> sectionsToReanalyse.contains(sectionText.getSectionNumber())) + .filter(section -> sectionsToReanalyseIds.contains(section.getTocId().get(0))) .collect(Collectors.toList()); + + Dictionary dictionary = dictionaryService.getDeepCopyDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId()); + NerEntities nerEntities = redactionServiceSettings.isNerServiceEnabled() // ? redactionStorageService.getNerEntities(analyzeRequest.getDossierId(), analyzeRequest.getFileId()) // : new NerEntities(); + KieContainer kieContainer = droolsExecutionService.updateRules(analyzeRequest.getDossierTemplateId()); - PageEntities pageEntities = entityRedactionService.findEntities(dictionary, reanalysisSections, kieContainer, analyzeRequest, nerEntities); + Set addedFileAttributes = entityRedactionService.addRuleEntities(dictionary, documentGraph, sectionsToReAnalyse, kieContainer, analyzeRequest, nerEntities); - var newRedactionLogEntries = redactionLogCreatorService.createRedactionLog(pageEntities, text.getNumberOfPages(), analyzeRequest.getDossierTemplateId()); + List newRedactionLogEntries = redactionLogCreatorService.createRedactionLog(documentGraph.getEntities(), analyzeRequest.getDossierTemplateId()); var importedRedactionFilteredEntries = importedRedactionService.processImportedRedactions(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId(), @@ -185,10 +197,10 @@ public class AnalyzeService { newRedactionLogEntries, false); - redactionLog.getRedactionLogEntry().removeIf(entry -> sectionsToReanalyse.contains(entry.getSectionNumber()) && !entry.getType().equals(IMPORTED_REDACTION_TYPE)); + redactionLog.getRedactionLogEntry().removeIf(entry -> sectionsToReanalyseIds.contains(entry.getSectionNumber()) && !entry.getType().equals(IMPORTED_REDACTION_TYPE)); redactionLog.getRedactionLogEntry().addAll(importedRedactionFilteredEntries); - return finalizeAnalysis(analyzeRequest, startTime, redactionLog, text, dictionaryIncrement.getDictionaryVersion(), true, pageEntities.getAddedFileAttributes()); + return finalizeAnalysis(analyzeRequest, startTime, redactionLog, documentGraph.getNumberOfPages(), dictionaryIncrement.getDictionaryVersion(), true, addedFileAttributes); } @@ -196,7 +208,7 @@ public class AnalyzeService { public AnalyzeResult analyze(AnalyzeRequest analyzeRequest) { long startTime = System.currentTimeMillis(); - var text = redactionStorageService.getText(analyzeRequest.getDossierId(), analyzeRequest.getFileId()); + DocumentGraph documentGraph = documentGraphMapper.toDocumentGraph(redactionStorageService.getDocumentData(analyzeRequest.getDossierId(), analyzeRequest.getFileId())); NerEntities nerEntities; if (redactionServiceSettings.isNerServiceEnabled()) { @@ -209,12 +221,13 @@ public class AnalyzeService { long rulesVersion = droolsExecutionService.getRulesVersion(analyzeRequest.getDossierTemplateId()); Dictionary dictionary = dictionaryService.getDeepCopyDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId()); - PageEntities pageEntities = entityRedactionService.findEntities(dictionary, text.getSectionTexts(), kieContainer, analyzeRequest, nerEntities); + entityRedactionService.addDictionaryEntities(dictionary, documentGraph); + Set addedFileAttributes = entityRedactionService.addRuleEntities(dictionary, documentGraph, kieContainer, analyzeRequest, nerEntities); - List redactionLogEntries = redactionLogCreatorService.createRedactionLog(pageEntities, text.getNumberOfPages(), analyzeRequest.getDossierTemplateId()); + List redactionLogEntries = redactionLogCreatorService.createRedactionLog(documentGraph.getEntities(), analyzeRequest.getDossierTemplateId()); - var legalBasis = legalBasisClient.getLegalBasisMapping(analyzeRequest.getDossierTemplateId()); - var redactionLog = new RedactionLog(redactionServiceSettings.getAnalysisVersion(), + List legalBasis = legalBasisClient.getLegalBasisMapping(analyzeRequest.getDossierTemplateId()); + RedactionLog redactionLog = new RedactionLog(redactionServiceSettings.getAnalysisVersion(), analyzeRequest.getAnalysisNumber(), redactionLogEntries, convert(legalBasis), @@ -223,21 +236,21 @@ public class AnalyzeService { rulesVersion, legalBasisClient.getVersion(analyzeRequest.getDossierTemplateId())); - var importedRedactionFilteredEntries = importedRedactionService.processImportedRedactions(analyzeRequest.getDossierTemplateId(), + List importedRedactionFilteredEntries = importedRedactionService.processImportedRedactions(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId(), analyzeRequest.getFileId(), redactionLog.getRedactionLogEntry(), true); redactionLog.setRedactionLogEntry(importedRedactionFilteredEntries); - return finalizeAnalysis(analyzeRequest, startTime, redactionLog, text, dictionary.getVersion(), false, pageEntities.getAddedFileAttributes()); + return finalizeAnalysis(analyzeRequest, startTime, redactionLog, documentGraph.getNumberOfPages(), dictionary.getVersion(), false, addedFileAttributes); } private AnalyzeResult finalizeAnalysis(AnalyzeRequest analyzeRequest, long startTime, RedactionLog redactionLog, - Text text, + int numberOfPages, DictionaryVersion dictionaryVersion, boolean isReanalysis, Set addedFileAttributes) { @@ -255,13 +268,13 @@ public class AnalyzeService { long duration = System.currentTimeMillis() - startTime; - redactmanagerAnalyzePagewiseValues.increase(text.getNumberOfPages(), duration); + redactmanagerAnalyzePagewiseValues.increase(numberOfPages, duration); return AnalyzeResult.builder() .dossierId(analyzeRequest.getDossierId()) .fileId(analyzeRequest.getFileId()) .duration(duration) - .numberOfPages(text.getNumberOfPages()) + .numberOfPages(numberOfPages) .hasUpdates(redactionLogChange.isHasChanges()) .analysisVersion(redactionServiceSettings.getAnalysisVersion()) .analysisNumber(analyzeRequest.getAnalysisNumber()) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/analyze/SectionFinder.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/analyze/SectionFinder.java index a09bfda4..8ccb7cbd 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/analyze/SectionFinder.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/analyze/SectionFinder.java @@ -19,13 +19,12 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry; -import com.iqser.red.service.redaction.v1.server.classification.model.SectionText; -import com.iqser.red.service.redaction.v1.server.classification.model.Text; +import com.iqser.red.service.redaction.v1.server.document.graph.DocumentGraph; +import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SectionNode; import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryIncrement; import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryIncrementValue; import com.iqser.red.service.redaction.v1.server.redaction.model.Image; import com.iqser.red.service.redaction.v1.server.redaction.model.RedRectangle2D; -import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUtils; import com.iqser.red.service.redaction.v1.server.redaction.utils.SearchImplementation; import io.micrometer.core.annotation.Timed; @@ -39,8 +38,9 @@ import lombok.extern.slf4j.Slf4j; @RequiredArgsConstructor @FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE) class SectionFinder { + @Timed("redactmanager_findSectionsToReanalyse") - public Set findSectionsToReanalyse(DictionaryIncrement dictionaryIncrement, RedactionLog redactionLog, Text text, AnalyzeRequest analyzeRequest) { + public Set findSectionsToReanalyse(DictionaryIncrement dictionaryIncrement, RedactionLog redactionLog, DocumentGraph documentGraph, AnalyzeRequest analyzeRequest) { long start = System.currentTimeMillis(); Set relevantManuallyModifiedAnnotationIds = getRelevantManuallyModifiedAnnotationIds(analyzeRequest.getManualRedactions()); @@ -59,10 +59,10 @@ class SectionFinder { var dictionaryIncrementsSearch = new SearchImplementation(dictionaryIncrement.getValues().stream().map(DictionaryIncrementValue::getValue).collect(Collectors.toList()), true); - for (SectionText sectionText : text.getSectionTexts()) { + for (SectionNode section : documentGraph.getMainSections()) { - if (EntitySearchUtils.sectionContainsAny(sectionText.getText(), dictionaryIncrementsSearch)) { - sectionsToReanalyse.add(sectionText.getSectionNumber()); + if (dictionaryIncrementsSearch.atLeastOneMatches(section.buildTextBlock().getSearchText())) { + sectionsToReanalyse.add(section.getTocId().get(0)); } } @@ -72,6 +72,7 @@ class SectionFinder { return sectionsToReanalyse; } + private static Set getRelevantManuallyModifiedAnnotationIds(ManualRedactions manualRedactions) { if (manualRedactions == null) { @@ -85,6 +86,7 @@ class SectionFinder { manualRedactions.getForceRedactions().stream().map(ManualForceRedaction::getAnnotationId))))).collect(Collectors.toSet()); } + private static Image convert(RedactionLogEntry entry) { Rectangle position = entry.getPositions().get(0); @@ -98,4 +100,5 @@ class SectionFinder { .hasTransparency(entry.isImageHasTransparency()) .build(); } + } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/entityredaction/EntityRedactionService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/entityredaction/EntityRedactionService.java index f286f556..7f47d849 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/entityredaction/EntityRedactionService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/entityredaction/EntityRedactionService.java @@ -1,39 +1,22 @@ package com.iqser.red.service.redaction.v1.server.redaction.service.entityredaction; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.HashSet; import java.util.List; -import java.util.Map; import java.util.Set; import java.util.stream.Collectors; -import java.util.stream.Stream; import org.kie.api.runtime.KieContainer; import org.springframework.stereotype.Service; import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest; import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute; -import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus; -import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval; -import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualImageRecategorization; -import com.iqser.red.service.redaction.v1.server.classification.model.SectionText; import com.iqser.red.service.redaction.v1.server.client.model.NerEntities; +import com.iqser.red.service.redaction.v1.server.document.graph.DocumentGraph; +import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SectionNode; +import com.iqser.red.service.redaction.v1.server.document.services.EntityCreationService; import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary; -import com.iqser.red.service.redaction.v1.server.redaction.model.Entities; -import com.iqser.red.service.redaction.v1.server.redaction.model.Entity; -import com.iqser.red.service.redaction.v1.server.redaction.model.EntityPositionSequence; -import com.iqser.red.service.redaction.v1.server.redaction.model.FindEntitiesResult; -import com.iqser.red.service.redaction.v1.server.redaction.model.Image; -import com.iqser.red.service.redaction.v1.server.redaction.model.PageEntities; -import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText; -import com.iqser.red.service.redaction.v1.server.redaction.model.Section; -import com.iqser.red.service.redaction.v1.server.redaction.model.SectionSearchableTextPair; +import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryModel; +import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType; import com.iqser.red.service.redaction.v1.server.redaction.service.DroolsExecutionService; -import com.iqser.red.service.redaction.v1.server.redaction.service.SurroundingWordsService; -import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUtils; -import com.iqser.red.service.redaction.v1.server.redaction.utils.IdBuilder; -import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings; import lombok.AccessLevel; import lombok.RequiredArgsConstructor; @@ -47,283 +30,47 @@ import lombok.extern.slf4j.Slf4j; public class EntityRedactionService { DroolsExecutionService droolsExecutionService; - EntityFinder entityFinder; - RedactionServiceSettings redactionServiceSettings; + EntityCreationService entityCreationService; - public PageEntities findEntities(Dictionary dictionary, List sectionTexts, KieContainer kieContainer, AnalyzeRequest analyzeRequest, NerEntities nerEntities) { + public Set addRuleEntities(Dictionary dictionary, + DocumentGraph documentGraph, + KieContainer kieContainer, + AnalyzeRequest analyzeRequest, + NerEntities nerEntities) { - Map> imagesPerPage = new HashMap<>(); - long start = System.currentTimeMillis(); - FindEntitiesResult findEntitiesResult = findEntities(sectionTexts, dictionary, kieContainer, analyzeRequest, false, null, imagesPerPage, nerEntities); - System.out.printf("First iteration took %d ms and found %d entities\n", System.currentTimeMillis() - start, findEntitiesResult.getEntities().size()); - if (dictionary.hasLocalEntries() || !findEntitiesResult.getAddedFileAttributes().isEmpty()) { - - if (!findEntitiesResult.getAddedFileAttributes().isEmpty()) { - //AnalyzeRequest provides immutable list. - List mergedFileAttributes = new ArrayList<>(); - mergedFileAttributes.addAll(analyzeRequest.getFileAttributes()); - mergedFileAttributes.addAll(findEntitiesResult.getAddedFileAttributes()); - analyzeRequest.setFileAttributes(mergedFileAttributes); - } - long start1 = System.currentTimeMillis(); - Map> hintsPerSectionNumber = getHintsPerSection(findEntitiesResult.getEntities(), dictionary); - FindEntitiesResult foundByLocalEntitiesResult = findEntities(sectionTexts, - dictionary, - kieContainer, - analyzeRequest, - true, - hintsPerSectionNumber, - imagesPerPage, - nerEntities); - EntitySearchUtils.addEntitiesWithHigherRank(findEntitiesResult.getEntities(), foundByLocalEntitiesResult.getEntities(), dictionary); - EntitySearchUtils.removeEntitiesContainedInLarger(findEntitiesResult.getEntities()); - System.out.printf("Second iteration took %d ms\n", System.currentTimeMillis() - start1); - } - - Map> entitiesPerPage = convertToEntitiesPerPage(findEntitiesResult.getEntities()); - //EntitySearchUtils.removeEntitiesContainedInRedactedLogos(imagesPerPage, entitiesPerPage); - System.out.printf("Total time for extracting entities %d ms\n", System.currentTimeMillis() - start); - return new PageEntities(entitiesPerPage, imagesPerPage, findEntitiesResult.getAddedFileAttributes()); + List allFileAttributes = droolsExecutionService.executeRules(kieContainer, documentGraph, dictionary, analyzeRequest.getFileAttributes()); + return allFileAttributes.stream().filter(fileAttribute -> !analyzeRequest.getFileAttributes().contains(fileAttribute)).collect(Collectors.toUnmodifiableSet()); } - private FindEntitiesResult findEntities(List reanalysisSections, - Dictionary dictionary, - KieContainer kieContainer, - AnalyzeRequest analyzeRequest, - boolean local, - Map> hintsPerSectionNumber, - Map> imagesPerPage, - NerEntities nerEntities) { + public Set addRuleEntities(Dictionary dictionary, + DocumentGraph documentGraph, + List sectionsToReanalyze, + KieContainer kieContainer, + AnalyzeRequest analyzeRequest, + NerEntities nerEntities) { - long start = System.currentTimeMillis(); - List sectionSearchableTextPairs = extractSearchableTextPairs(reanalysisSections, + List allFileAttributes = droolsExecutionService.executeRules(kieContainer, + documentGraph, + sectionsToReanalyze, dictionary, - analyzeRequest, - local, - hintsPerSectionNumber, - nerEntities); - System.out.printf("Total Dict Search and insertion time %d ms\n", System.currentTimeMillis() - start); - Set addedFileAttributes = new HashSet<>(); - Set entities = new HashSet<>(); - - long start1 = System.currentTimeMillis(); - sectionSearchableTextPairs.forEach(sectionSearchableTextPair -> { - - if (!addedFileAttributes.isEmpty()) { - //Section.Builder provides immutable list. - List mergedFileAttributes = new ArrayList<>(); - mergedFileAttributes.addAll(sectionSearchableTextPair.getSection().getAddedFileAttributes()); - mergedFileAttributes.addAll(addedFileAttributes); - sectionSearchableTextPair.getSection().setFileAttributes(mergedFileAttributes); - } - if (sectionSearchableTextPair.getSection() == null) { - return; - } - Section analysedSection = droolsExecutionService.executeRules(kieContainer, sectionSearchableTextPair.getSection()); - - addedFileAttributes.addAll(analysedSection.getAddedFileAttributes()); - - EntitySearchUtils.removeEntitiesContainedInLarger(analysedSection.getEntities()); - - var entriesWithoutSurroundingText = analysedSection.getEntities() - .stream() - .filter(e -> e.getTextAfter() == null && e.getTextBefore() == null) - .collect(Collectors.toSet()); - - addSurroundingText(dictionary, sectionSearchableTextPair.getSearchableText(), sectionSearchableTextPair.getCellStarts(), entriesWithoutSurroundingText); - - entities.addAll(analysedSection.getEntities()); - - if (!local) { - for (Image image : analysedSection.getImages()) { - imagesPerPage.computeIfAbsent(image.getPage(), (a) -> new HashSet<>()).add(image); - } - addLocalValuesToDictionary(analysedSection, dictionary); - } - - }); - System.out.printf("rules took %d ms in total\n", System.currentTimeMillis() - start1); - - return FindEntitiesResult.builder().entities(entities).addedFileAttributes(addedFileAttributes).build(); + analyzeRequest.getFileAttributes()); + return allFileAttributes.stream().filter(fileAttribute -> !analyzeRequest.getFileAttributes().contains(fileAttribute)).collect(Collectors.toUnmodifiableSet()); } - private void addSurroundingText(Dictionary dictionary, SearchableText searchableText, List cellStarts, Set entriesWithoutSurroundingText) { + public void addDictionaryEntities(Dictionary dictionary, DocumentGraph documentGraph) { - if (cellStarts != null && !cellStarts.isEmpty()) { - SurroundingWordsService.addSurroundingText(entriesWithoutSurroundingText, - searchableText, - dictionary, - cellStarts, - redactionServiceSettings.getSurroundingWordsOffsetWindow(), - redactionServiceSettings.getNumberOfSurroundingWords()); - - } else { - SurroundingWordsService.addSurroundingText(entriesWithoutSurroundingText, - searchableText, - dictionary, - redactionServiceSettings.getSurroundingWordsOffsetWindow(), - redactionServiceSettings.getNumberOfSurroundingWords()); - } + dictionary.getDictionaryModels().forEach(dictionaryModel -> addDictionaryModelEntities(dictionaryModel, documentGraph)); } - private List extractSearchableTextPairs(List reanalysisSections, - Dictionary dictionary, - AnalyzeRequest analyzeRequest, - boolean local, - Map> hintsPerSectionNumber, - NerEntities nerEntities) { + private void addDictionaryModelEntities(DictionaryModel dictionaryModel, DocumentGraph documentGraph) { - return reanalysisSections.stream().map(reanalysisSection -> { - long start = System.currentTimeMillis(); - Entities entities = entityFinder.findEntities(reanalysisSection.getSearchableText(), - reanalysisSection.getHeadline(), - reanalysisSection.getSectionNumber(), - dictionary, - local, - nerEntities, - reanalysisSection.getCellStarts(), - analyzeRequest.getManualRedactions()); - addSurroundingText(dictionary, reanalysisSection.getSearchableText(), reanalysisSection.getCellStarts(), entities.getEntities()); - if (!local && analyzeRequest.getManualRedactions() != null) { - - var approvedForceRedactions = analyzeRequest.getManualRedactions() - .getForceRedactions() - .stream() - .filter(fr -> fr.getStatus() == AnnotationStatus.APPROVED) - .filter(fr -> fr.getRequestDate() != null) - .collect(Collectors.toList()); - // only approved id removals, that haven't been forced back afterwards - var idsToRemove = analyzeRequest.getManualRedactions() - .getIdsToRemove() - .stream() - .filter(idr -> idr.getStatus() == AnnotationStatus.APPROVED && !idr.isRemoveFromDictionary()) - .filter(idr -> idr.getRequestDate() != null) - .filter(idr -> approvedForceRedactions.stream() - .noneMatch(forceRedact -> forceRedact.getAnnotationId().equals(idr.getAnnotationId()) && forceRedact.getRequestDate() - .isAfter(idr.getRequestDate()))) - .map(IdRemoval::getAnnotationId) - .collect(Collectors.toSet()); - - if (reanalysisSection.getImages() != null && !reanalysisSection.getImages().isEmpty() && analyzeRequest.getManualRedactions().getImageRecategorization() != null) { - for (Image image : reanalysisSection.getImages()) { - String imageId = IdBuilder.buildId(image.getPosition(), image.getPage()); - for (ManualImageRecategorization imageRecategorization : analyzeRequest.getManualRedactions().getImageRecategorization()) { - if (imageRecategorization.getStatus().equals(AnnotationStatus.APPROVED) && imageRecategorization.getAnnotationId().equals(imageId)) { - image.setType(imageRecategorization.getType()); - } - } - if (idsToRemove.contains(imageId)) { - image.setIgnored(true); - } - } - } - - entities.getEntities().forEach(entity -> entity.getPositionSequences().forEach(ps -> { - if (idsToRemove.contains(ps.getId())) { - entity.setIgnored(true); - } - })); - } - log.debug("Section {}, Images: {}", reanalysisSection.getSectionNumber(), reanalysisSection.getImages()); - - return toSectionSearchableTextPair(dictionary, analyzeRequest, hintsPerSectionNumber, reanalysisSection, entities); - }).collect(Collectors.toList()); - } - - - private SectionSearchableTextPair toSectionSearchableTextPair(Dictionary dictionary, - AnalyzeRequest analyzeRequest, - Map> hintsPerSectionNumber, - SectionText reanalysisSection, - Entities entities) { - - return new SectionSearchableTextPair(Section.builder() - .isLocal(false) - .dictionaryTypes(dictionary.getTypes()) - .entities(hintsPerSectionNumber != null && hintsPerSectionNumber.containsKey(reanalysisSection.getSectionNumber()) ? Stream.concat(entities.getEntities().stream(), - hintsPerSectionNumber.get(reanalysisSection.getSectionNumber()).stream()).collect(Collectors.toSet()) : entities.getEntities()) - .nerEntities(entities.getNerEntities()) - .text(reanalysisSection.getSearchableText().getAsStringWithLinebreaks()) - .searchText(reanalysisSection.getSearchableText().toString()) - .headline(reanalysisSection.getHeadline()) - .sectionNumber(reanalysisSection.getSectionNumber()) - .tabularData(reanalysisSection.getTabularData()) - .searchableText(reanalysisSection.getSearchableText()) - .dictionary(dictionary) - .images(reanalysisSection.getImages()) - .sectionAreas(reanalysisSection.getSectionAreas()) - .fileAttributes(analyzeRequest.getFileAttributes()) - .manualRedactions(analyzeRequest.getManualRedactions()) - .isInTable(reanalysisSection.isTable()) - .build(), reanalysisSection.getSearchableText(), reanalysisSection.getCellStarts()); - } - - - private Map> convertToEntitiesPerPage(Set entities) { - - Map> entitiesPerPage = new HashMap<>(); - for (Entity entity : entities) { - Map> sequenceOnPage = new HashMap<>(); - for (EntityPositionSequence entityPositionSequence : entity.getPositionSequences()) { - sequenceOnPage.computeIfAbsent(entityPositionSequence.getPageNumber(), (x) -> new ArrayList<>()).add(entityPositionSequence); - } - - for (Map.Entry> entry : sequenceOnPage.entrySet()) { - entitiesPerPage.computeIfAbsent(entry.getKey(), (x) -> new ArrayList<>()) - .add(new Entity(entity.getWord(), - entity.getType(), - entity.isRedaction(), - entity.getRedactionReason(), - entry.getValue(), - entity.getHeadline(), - entity.getMatchedRule(), - entity.getSectionNumber(), - entity.getLegalBasis(), - entity.isDictionaryEntry(), - entity.getTextBefore(), - entity.getTextAfter(), - entity.getStart(), - entity.getEnd(), - entity.isDossierDictionaryEntry(), - entity.getEngines(), - entity.getReferences(), - entity.getEntityType())); - } - } - return entitiesPerPage; - } - - - private Map> getHintsPerSection(Set entities, Dictionary dictionary) { - - Map> hintsPerSectionNumber = new HashMap<>(); - entities.forEach(entity -> { - if (dictionary.isHint(entity.getType()) && entity.isDictionaryEntry()) { - hintsPerSectionNumber.computeIfAbsent(entity.getSectionNumber(), (x) -> new HashSet<>()).add(entity); - } - }); - return hintsPerSectionNumber; - } - - - private void addLocalValuesToDictionary(Section analysedSection, Dictionary dictionary) { - - analysedSection.getLocalDictionaryAdds().keySet().forEach(key -> analysedSection.getLocalDictionaryAdds().get(key).forEach(value -> { - - if (dictionary.getLocalAccessMap().get(key) == null) { - log.warn("Dictionary {} is null", key); - } - - if (dictionary.getLocalAccessMap().get(key).getLocalEntries() == null) { - log.warn("Dictionary {} localEntries is null", key); - } - - dictionary.getLocalAccessMap().get(key).getLocalEntries().add(value); - })); + entityCreationService.bySearchImplementation(dictionaryModel.getEntriesSearch(), dictionaryModel.getType(), EntityType.ENTITY, documentGraph); + entityCreationService.bySearchImplementation(dictionaryModel.getFalsePositiveSearch(), dictionaryModel.getType(), EntityType.FALSE_POSITIVE, documentGraph); + entityCreationService.bySearchImplementation(dictionaryModel.getFalseRecommendationsSearch(), dictionaryModel.getType(), EntityType.FALSE_RECOMMENDATION, documentGraph); } } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/SeparatorUtils.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/SeparatorUtils.java index e9a28a93..b97010a3 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/SeparatorUtils.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/SeparatorUtils.java @@ -4,6 +4,7 @@ import java.util.Set; import java.util.regex.Pattern; import com.iqser.red.service.redaction.v1.server.document.graph.Boundary; +import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock; import lombok.experimental.UtilityClass; import lombok.extern.slf4j.Slf4j; @@ -31,22 +32,38 @@ public final class SeparatorUtils { } + public static boolean isWhiteSpacesOrSeparatorsOnly(TextBlock textBlock, Boundary boundary) { - public static boolean validateBoundaryIsSurroundedBySeparators(CharSequence charSequence, Boundary boundary) { - - return validateStart(charSequence, boundary) && validateEnd(charSequence, boundary); + String stringWithoutWhiteSpace = textBlock.subSequence(boundary).toString().replace(" ", ""); + int numberOfSeparators = 0; + for (int i = 0; i < stringWithoutWhiteSpace.length(); i++) { + if (isSeparator(stringWithoutWhiteSpace.charAt(i))) { + numberOfSeparators++; + } + } + return stringWithoutWhiteSpace.length() == numberOfSeparators; } - private static boolean validateEnd(CharSequence charSequence, Boundary boundary) { + public static boolean boundaryIsSurroundedBySeparators(TextBlock textBlock, Boundary boundary) { - return boundary.end() == charSequence.length() || SeparatorUtils.isSeparator(charSequence.charAt(boundary.end())) || SeparatorUtils.isSeparator(charSequence.charAt(boundary.end() - 1)); + return validateStart(textBlock, boundary) && validateEnd(textBlock, boundary); } - private static boolean validateStart(CharSequence charSequence, Boundary boundary) { + private static boolean validateEnd(TextBlock textBlock, Boundary boundary) { - return boundary.start() == 0 || SeparatorUtils.isSeparator(charSequence.charAt(boundary.start() - 1)) || SeparatorUtils.isSeparator(charSequence.charAt(boundary.start())); + return boundary.end() == textBlock.getBoundary().end() ||// + SeparatorUtils.isSeparator(textBlock.charAt(boundary.end())) ||// + SeparatorUtils.isSeparator(textBlock.charAt(boundary.end() - 1)); + } + + + private static boolean validateStart(TextBlock textBlock, Boundary boundary) { + + return boundary.start() == textBlock.getBoundary().start() ||// + SeparatorUtils.isSeparator(textBlock.charAt(boundary.start() - 1)) ||// + SeparatorUtils.isSeparator(textBlock.charAt(boundary.start())); } } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/RedactionStorageService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/RedactionStorageService.java index 2bcbd9cd..e504d675 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/RedactionStorageService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/RedactionStorageService.java @@ -10,6 +10,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlo import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.SectionGrid; import com.iqser.red.service.redaction.v1.server.classification.model.Text; import com.iqser.red.service.redaction.v1.server.client.model.NerEntities; +import com.iqser.red.service.redaction.v1.server.document.data.DocumentData; import com.iqser.red.service.redaction.v1.server.exception.NotFoundException; import com.iqser.red.service.redaction.v1.server.multitenancy.TenantContext; import com.iqser.red.storage.commons.exception.StorageObjectDoesNotExist; @@ -55,7 +56,9 @@ public class RedactionStorageService { public ImportedRedactions getImportedRedactions(String dossierId, String fileId) { try { - return storageService.readJSONObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.IMPORTED_REDACTIONS), ImportedRedactions.class); + return storageService.readJSONObject(TenantContext.getTenantId(), + StorageIdUtils.getStorageId(dossierId, fileId, FileType.IMPORTED_REDACTIONS), + ImportedRedactions.class); } catch (StorageObjectDoesNotExist e) { log.debug("Imported redactions not available."); return null; @@ -88,6 +91,18 @@ public class RedactionStorageService { } + @Timed("redactmanager_getDocumentGraph") + public DocumentData getDocumentData(String dossierId, String fileId) { + + try { + return storageService.readJSONObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.TEXT), DocumentData.class); + } catch (StorageObjectDoesNotExist e) { + log.debug("Text not available."); + return null; + } + } + + @Timed("redactmanager_getNerEntities") public NerEntities getNerEntities(String dossierId, String fileId) { diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DocumentGraphIntegrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DocumentGraphIntegrationTest.java index a36b8333..844a5746 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DocumentGraphIntegrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DocumentGraphIntegrationTest.java @@ -1,6 +1,6 @@ package com.iqser.red.service.redaction.v1.server; -import static com.iqser.red.service.redaction.v1.server.redaction.utils.SeparatorUtils.validateBoundaryIsSurroundedBySeparators; +import static com.iqser.red.service.redaction.v1.server.redaction.utils.SeparatorUtils.boundaryIsSurroundedBySeparators; import static com.iqser.red.service.redaction.v1.server.utils.PdfDraw.drawRectangle2DList; import java.awt.Color; @@ -65,7 +65,7 @@ public class DocumentGraphIntegrationTest extends AbstractTestWithDictionaries { @SneakyThrows public void testDroolsOnDocumentGraph() { - String filename = "files/new/crafted document"; + String filename = "files/Metolachlor/S-Metolachlor_RAR_01_Volume_1_2018-09-06"; prepareStorage(filename + ".pdf"); ClassPathResource fileResource = new ClassPathResource(filename + ".pdf"); @@ -217,7 +217,7 @@ public class DocumentGraphIntegrationTest extends AbstractTestWithDictionaries { .stream() .filter(entityNode -> !entityNode.isRemoved()) .filter(EntityNode::isRedaction) - .flatMap(entityNode -> entityNode.getEntityPositions().stream()) + .flatMap(entityNode -> entityNode.getEntityPositionsPerPage().stream()) .filter(entityPosition -> entityPosition.getPageNode().equals(page)) .flatMap(entityPosition -> entityPosition.getRectanglePerLine().stream()) .toList(); @@ -231,7 +231,7 @@ public class DocumentGraphIntegrationTest extends AbstractTestWithDictionaries { .stream() .filter(entityNode -> !entityNode.isRemoved()) .filter(entityNode -> !entityNode.isRedaction()) - .flatMap(entityNode -> entityNode.getEntityPositions().stream()) + .flatMap(entityNode -> entityNode.getEntityPositionsPerPage().stream()) .filter(entityPosition -> entityPosition.getPageNode().equals(page)) .flatMap(entityPosition -> entityPosition.getRectanglePerLine().stream()) .toList(); @@ -255,7 +255,7 @@ public class DocumentGraphIntegrationTest extends AbstractTestWithDictionaries { TextBlock textBlock = documentGraph.getTextBlock(); searchImplementation.getBoundaries(textBlock, textBlock.getBoundary()) .stream() - .filter(boundary -> validateBoundaryIsSurroundedBySeparators(textBlock, boundary)) + .filter(boundary -> boundaryIsSurroundedBySeparators(textBlock, boundary)) .map(bounds -> EntityNode.initialEntityNode(bounds, type, entityType)) .forEach(foundEntities::add); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentGraphMappingTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentGraphMappingTest.java index e2f5395a..3cb8b888 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentGraphMappingTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentGraphMappingTest.java @@ -5,7 +5,11 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.core.io.ClassPathResource; import com.iqser.red.service.redaction.v1.server.AbstractTestWithDictionaries; +import com.iqser.red.service.redaction.v1.server.document.data.AtomicPositionBlockData; +import com.iqser.red.service.redaction.v1.server.document.data.AtomicTextBlockData; import com.iqser.red.service.redaction.v1.server.document.data.DocumentData; +import com.iqser.red.service.redaction.v1.server.document.data.PageData; +import com.iqser.red.service.redaction.v1.server.document.data.TableOfContentsData; import com.iqser.red.service.redaction.v1.server.document.graph.factory.DocumentGraphFactory; import com.iqser.red.service.redaction.v1.server.document.mapper.DocumentDataMapper; import com.iqser.red.service.redaction.v1.server.document.mapper.DocumentGraphMapper; @@ -33,7 +37,7 @@ public class DocumentGraphMappingTest extends AbstractTestWithDictionaries { @SneakyThrows public void testGraphMapping() { - String filename = "files/Metolachlor/S-Metolachlor_RAR_01_Volume_1_2018-09-06"; + String filename = "files/new/crafted document"; prepareStorage(filename + ".pdf"); ClassPathResource fileResource = new ClassPathResource(filename + ".pdf"); @@ -41,7 +45,18 @@ public class DocumentGraphMappingTest extends AbstractTestWithDictionaries { var classifiedDoc = segmentationService.parseDocument(TEST_DOSSIER_ID, TEST_FILE_ID, fileResource.getInputStream(), null); DocumentGraph document = documentGraphFactory.buildDocumentGraph(classifiedDoc); DocumentData documentData = documentDataMapper.toDocumentData(document); - storageService.storeJSONObject(TenantContext.getTenantId(), filename + ".json", documentData); + storageService.storeJSONObject(TenantContext.getTenantId(), filename + "_PAGES" + ".json", documentData.getPages()); + storageService.storeJSONObject(TenantContext.getTenantId(), filename + "_TEXT" + ".json", documentData.getAtomicTextBlocks()); + storageService.storeJSONObject(TenantContext.getTenantId(), filename + "_POSITIONS" + ".json", documentData.getAtomicPositionBlocks()); + storageService.storeJSONObject(TenantContext.getTenantId(), filename + "_STRUCTURE" + ".json", documentData.getTableOfContents()); + + PageData[] pageData = storageService.readJSONObject(TenantContext.getTenantId(), filename + "_PAGES" + ".json", PageData[].class); + AtomicTextBlockData[] atomicTextBlockData = storageService.readJSONObject(TenantContext.getTenantId(), filename + "_TEXT" + ".json", AtomicTextBlockData[].class); + AtomicPositionBlockData[] atomicPositionBlockData = storageService.readJSONObject(TenantContext.getTenantId(), + filename + "_POSITIONS" + ".json", + AtomicPositionBlockData[].class); + TableOfContentsData tableOfContentsData = storageService.readJSONObject(TenantContext.getTenantId(), filename + "_STRUCTURE" + ".json", TableOfContentsData.class); + DocumentGraph newDocumentGraph = documentGraphMapper.toDocumentGraph(documentData); assert document.toString().equals(newDocumentGraph.toString()); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/entity_rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/entity_rules.drl index 52b30e68..349ff4bb 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/entity_rules.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/entity_rules.drl @@ -2,7 +2,7 @@ package drools import static java.lang.String.format; import static com.iqser.red.service.redaction.v1.server.document.services.CharSequenceSearchUtils.anyMatch; -import static com.iqser.red.service.redaction.v1.server.document.services.EntityFieldSetter.setFields; +import static com.iqser.red.service.redaction.v1.server.document.services.EntityEnrichmentService.setFields; import java.util.List; @@ -131,6 +131,18 @@ rule "9: Add recommendation for Addresses in Test Animals sections" entities.forEach(entity -> insert(entity)); end +rule "12: Recommend CTL/BL laboratory that start with BL or CTL" + + when + $section : SectionNode(containsString("BL") || containsString("CT")) + then + Set entities = entityCreationService.byRegex("((\\b((([Cc]T(([1ILli\\/])| L|~P))|(BL))[\\. ]?([\\dA-Ziltphz~\\/.:!]| ?[\\(',][Ppi](\\(e)?|([\\(-?']\\/))+( ?[\\(\\/\\dA-Znasieg]+)?)\\b( ?\\/? ?\\d+)?)|(\\bCT[L1i]\\b))", "CBI_address", EntityType.RECOMMENDATION, $section); + setFields(entities, 12, "Syngenta Specific Laboratory keyword", null, Engine.RULE); + entities.forEach(entity -> insert(entity)); + entities.forEach(entity -> dictionary.addLocalDictionaryEntry("PII", entity.getValue(), false)); + end + + // --------------------------------------- PII rules ------------------------------------------------------------------- @@ -182,43 +194,106 @@ rule "13: Redact Emails by RegEx (Vertebrate study)" entities.forEach(entity -> insert(entity)); end -rule "14: Redact contact information (Non vertebrate study)" + +rule "14: Redact line after contact information (Non vertebrate study)" + agenda-group "LOCAL_DICTIONARY_ADDS" when FileAttribute(label == "Vertebrate Study" , value.toLowerCase() != "yes") - $section: SectionNode() + $string: String() from List.of("Contact point:", + "Contact:", + "Alternative contact:", + "European contact:", + "No:", + "Contact:", + "Tel.:", + "Tel:", + "Telephone number:", + "Telephone No:", + "Telephone:", + "Phone No:", + "Phone:", + "Fax number:", + "Fax:", + "E-mail:", + "Email:", + "e-mail:", + "E-mail address:") + $section: SectionNode(containsString($string)) then + Set entities = new HashSet<>(); - entities.addAll(entityCreationService.lineAfterString("Contact point:", "PII", EntityType.ENTITY, $section)); - entities.addAll(entityCreationService.lineAfterString("Contact:", "PII", EntityType.ENTITY, $section)); - entities.addAll(entityCreationService.lineAfterString("Alternative contact:", "PII", EntityType.ENTITY, $section)); - entities.addAll(entityCreationService.lineAfterString("European contact:", "PII", EntityType.ENTITY, $section)); - entities.addAll(entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section)); - entities.addAll(entityCreationService.betweenStrings("Contact:", "Tel.:", "PII", EntityType.ENTITY, $section)); - setFields(entities, 14, "Found by rule based keywords", null, Engine.RULE); + entities.addAll(entityCreationService.lineAfterString($string, "PII", EntityType.ENTITY, $section)); + setFields(entities, 14, "Found by contacts keyword", null, Engine.RULE); entities.forEach(entity -> insert(entity)); entities.forEach(entity -> dictionary.addLocalDictionaryEntry("PII", entity.getValue(), false)); end -rule "15: Redact contact information (Vertebrate study)" + +rule "15: Redact line after contact information (Vertebrate study)" + agenda-group "LOCAL_DICTIONARY_ADDS" when FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "yes") - $section: SectionNode() + $string: String() from List.of("Contact point:", + "Contact:", + "Alternative contact:", + "European contact:", + "No:", + "Contact:", + "Tel.:", + "Tel:", + "Telephone number:", + "Telephone No:", + "Telephone:", + "Phone No:", + "Phone:", + "Fax number:", + "Fax:", + "E-mail:", + "Email:", + "e-mail:", + "E-mail address:") + $section: SectionNode(containsString($string)) then - Set entities = new HashSet<>(); - entities.addAll(entityCreationService.lineAfterString("Contact point:", "PII", EntityType.ENTITY, $section)); - entities.addAll(entityCreationService.lineAfterString("Contact:", "PII", EntityType.ENTITY, $section)); - entities.addAll(entityCreationService.lineAfterString("Alternative contact:", "PII", EntityType.ENTITY, $section)); - entities.addAll(entityCreationService.lineAfterString("European contact:", "PII", EntityType.ENTITY, $section)); - entities.addAll(entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section)); - entities.addAll(entityCreationService.betweenStrings("Contact:", "Tel.:", "PII", EntityType.ENTITY, $section)); - setFields(entities, 15, "Found by contacts keywords", null, Engine.RULE); + Set entities = entityCreationService.lineAfterString($string, "PII", EntityType.ENTITY, $section); + setFields(entities, 15, "Found by contacts keyword", null, Engine.RULE); entities.forEach(entity -> insert(entity)); entities.forEach(entity -> dictionary.addLocalDictionaryEntry("PII", entity.getValue(), false)); end -rule "16: Redact Phone and Fax by RegEx" + +rule "16: redact line between contact keywords" + agenda-group "LOCAL_DICTIONARY_ADDS" + + when + FileAttribute(label == "Vertebrate Study" , value.toLowerCase() != "yes") + $section: SectionNode((containsString("No:") && containsString("Fax")) || (containsString("Contact:") && containsString("Tel"))) + then + Set entities = new HashSet<>(); + entities.addAll(entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section)); + entities.addAll(entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section)); + setFields(entities, 16, "Found between contacts keyword", null, Engine.RULE); + entities.forEach(entity -> insert(entity)); + entities.forEach(entity -> dictionary.addLocalDictionaryEntry("PII", entity.getValue(), false)); + end + +rule "17: redact line between contact keywords" + agenda-group "LOCAL_DICTIONARY_ADDS" + + when + FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "yes") + $section: SectionNode((containsString("No:") && containsString("Fax")) || (containsString("Contact:") && containsString("Tel"))) + then + Set entities = new HashSet<>(); + entities.addAll(entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section)); + entities.addAll(entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section)); + setFields(entities, 17, "Found between contacts keyword", null, Engine.RULE); + entities.forEach(entity -> insert(entity)); + entities.forEach(entity -> dictionary.addLocalDictionaryEntry("PII", entity.getValue(), false)); + end + +rule "18: Redact Phone and Fax by RegEx" when $section: SectionNode(containsString("Contact") || @@ -232,6 +307,6 @@ rule "16: Redact Phone and Fax by RegEx" containsString("Fer")) then Set entities = entityCreationService.byRegex("\\b(contact|telephone|phone|fax|tel|ter|mobile|fel|fer)[a-zA-Z\\s]{0,10}[:.\\s]{0,3}([\\+\\d\\(][\\s\\d\\(\\)\\-\\/\\.]{4,100}\\d)\\b", "PII", EntityType.ENTITY, $section); - setFields(entities, 16, "Found by Phone and Fax regex", null, Engine.RULE); + setFields(entities, 18, "Found by Phone and Fax regex", null, Engine.RULE); entities.forEach(entity -> insert(entity)); end diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/merge_entity_rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/merge_entity_rules.drl index 39b8a3f9..6dcd77b5 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/merge_entity_rules.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/merge_entity_rules.drl @@ -22,6 +22,9 @@ global DocumentGraph document global EntityCreationService entityCreationService global Dictionary dictionary +query "getFileAttributes" + $fileAttribute: FileAttribute() + end rule "merge intersecting Entities of same type" salience 100 @@ -32,7 +35,7 @@ rule "merge intersecting Entities of same type" then $first.removeFromGraph(); $second.removeFromGraph(); - EntityNode mergedEntity = entityCreationService.createMergedEntity(List.of($first, $second), $type, $entityType, document); + EntityNode mergedEntity = entityCreationService.byEntities(List.of($first, $second), $type, $entityType, document); insert(mergedEntity); retract($first); retract($second); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl index 9dd68653..495479fe 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl @@ -1,367 +1,390 @@ package drools -import com.iqser.red.service.redaction.v1.server.redaction.model.Section - -global Section section +import static java.lang.String.format; +import static com.iqser.red.service.redaction.v1.server.document.services.CharSequenceSearchUtils.anyMatch; +import static com.iqser.red.service.redaction.v1.server.document.services.EntityEnrichmentService.setFields; -// --------------------------------------- AI rules ------------------------------------------------------------------- +import java.util.List; +import java.util.LinkedList; +import java.util.HashSet; -rule "0: Add CBI_author from ai" - when - Section(aiMatchesType("CBI_author")) - then - section.addAiEntities("CBI_author", "CBI_author"); +import com.iqser.red.service.redaction.v1.server.document.graph.* +import com.iqser.red.service.redaction.v1.server.document.graph.nodes.* +import com.iqser.red.service.redaction.v1.server.document.graph.entity.* +import com.iqser.red.service.redaction.v1.server.document.graph.textblock.* +import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType; +import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute; +import java.util.Set +import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine +import com.iqser.red.service.redaction.v1.server.document.services.EntityCreationService; +import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary; +import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryModel; + +global DocumentGraph document +global EntityCreationService entityCreationService +global Dictionary dictionary + + +query "getFileAttributes" + $fileAttribute: FileAttribute() end -rule "0: Combine address parts from ai to CBI_address (org is mandatory)" +rule "merge intersecting Entities of same type" + salience 100 + when - Section(aiMatchesType("ORG")) + $first: EntityNode($type: type, $entityType: entityType, !resized, !skipRemoveEntitiesContainedInLarger) + $second: EntityNode(intersects($first), type == $type, entityType == $entityType, this != $first, !resized, !skipRemoveEntitiesContainedInLarger) then - section.combineAiTypes("ORG", "STREET,POSTAL,COUNTRY,CARDINAL,CITY,STATE", 20, "CBI_address", 3, false); + $first.removeFromGraph(); + $second.removeFromGraph(); + EntityNode mergedEntity = entityCreationService.byEntities(List.of($first, $second), $type, $entityType, document); + insert(mergedEntity); + retract($first); + retract($second); end -rule "0: Combine address parts from ai to CBI_address (street is mandatory)" +rule "remove Entity of type ENTITY when contained by FALSE_POSITIVE" + salience 100 + when - Section(aiMatchesType("STREET")) + $falsePositive: EntityNode($type: type, entityType == EntityType.FALSE_POSITIVE) + entity: EntityNode(containedBy($falsePositive), type == $type, entityType == EntityType.ENTITY, !resized, !skipRemoveEntitiesContainedInLarger) then - section.combineAiTypes("STREET", "ORG,POSTAL,COUNTRY,CARDINAL,CITY,STATE", 20, "CBI_address", 3, false); + entity.removeFromGraph(); + retract(entity); end -rule "0: Combine address parts from ai to CBI_address (city is mandatory)" +rule "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION" + salience 100 + when - Section(aiMatchesType("CITY")) + $falseRecommendation: EntityNode($type: type, entityType == EntityType.FALSE_RECOMMENDATION) + $recommendation: EntityNode(containedBy($falseRecommendation), type == $type, entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger) then - section.combineAiTypes("CITY", "ORG,STREET,POSTAL,COUNTRY,CARDINAL,STATE", 20, "CBI_address", 3, false); + $recommendation.removeFromGraph(); + retract($recommendation); + end + +rule "remove Entity of type RECOMMENDATION when contained by ENTITY" + salience 100 + + when + $entity: EntityNode($type: type, entityType == EntityType.ENTITY) + $recommendation: EntityNode(containedBy($entity), type == $type, entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger) + then + $recommendation.removeFromGraph(); + retract($recommendation); + end + +rule "remove Entity of lower rank, when equal boundaries and entityType" + salience 100 + + when + $higherRank: EntityNode($type: type, $entityType: entityType, $boundary: boundary) + $lowerRank: EntityNode($boundary == boundary, type != $type, entityType == $entityType, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type)) + then + $lowerRank.removeFromGraph(); + retract($lowerRank); + end + +rule "run local dictionary search" + agenda-group "LOCAL_DICTIONARY_ADDS" + salience -999 + + when + DictionaryModel(!localEntries.isEmpty(), $type: type, $searchImplementation: localSearch) from dictionary.getDictionaryModels() + then + Set entityNodes = entityCreationService.bySearchImplementation($searchImplementation, $type, EntityType.RECOMMENDATION, document); + System.out.printf("local dictionary search found %d entities\n", entityNodes.size()); + entityNodes.forEach(entityNode -> insert(entityNode)); end // --------------------------------------- CBI rules ------------------------------------------------------------------- rule "1: Redact CBI Authors (Non vertebrate study)" + no-loop true + when - Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesType("CBI_author")) + FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "yes") + $entity: EntityNode(type == "CBI_author", entityType == EntityType.ENTITY) then - section.redact("CBI_author", 1, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); + $entity.setRedaction(true); + setFields($entity, 1, "Address found", "Article 39(e)(2) of Regulation (EC) No 178/2002", null); + update($entity) end rule "2: Redact CBI Authors (Vertebrate study)" + no-loop true + when - Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesType("CBI_author")) + FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "no") + $entity: EntityNode(type == "CBI_author", entityType == EntityType.ENTITY) then - section.redact("CBI_author", 2, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); + $entity.setRedaction(true); + setFields($entity, 2, "Address found", "Article 39(e)(2) of Regulation (EC) No 178/2002", null); + update($entity) end +rule "3: Don't redact CBI Address (Non vertebrate study)" + no-loop true -rule "3: Redact not CBI Address (Non vertebrate study)" when - Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesType("CBI_address")) + FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "no") + $entity: EntityNode(type == "CBI_address", entityType == EntityType.ENTITY) then - section.redactNot("CBI_address", 3, "Address found for non vertebrate study"); - section.ignoreRecommendations("CBI_address"); + setFields($entity, 3, "Address found", "Article 39(e)(2) of Regulation (EC) No 178/2002", null); + update($entity) end rule "4: Redact CBI Address (Vertebrate study)" + no-loop true + when - Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesType("CBI_address")) + FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "yes") + $entity: EntityNode(type == "CBI_address", entityType == EntityType.ENTITY) then - section.redact("CBI_address", 4, "Address found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); + $entity.setRedaction(true); + setFields($entity, 4, "Address found", "Article 39(e)(2) of Regulation (EC) No 178/2002", null); + update($entity) + end + +rule "5: Add FALSE_POSITIVE Entity for genitive CBI_author" + + when + $entity: EntityNode(type == "CBI_author", anyMatch(textAfter, "['’’'ʼˈ´`‘′ʻ’']s"), redaction) + then + EntityNode entity = entityCreationService.byBoundary($entity.getBoundary(), "CBI_author", EntityType.FALSE_POSITIVE, document); + setFields($entity, 5, "Genitive Author", null, Engine.RULE); + insert(entity); end -rule "5: Do not redact genitive CBI_author" +rule "6: Create Entity from Author(s) cells in Tables with Author(s) header and add Authorname as Recommendation" + agenda-group "LOCAL_DICTIONARY_ADDS" + when - Section(matchesType("CBI_author")) + authorCell: TableCellNode(!header, (hasHeader("Author(s)") || hasHeader("Author")), hasText()) then - section.expandToFalsePositiveByRegEx("CBI_author", "['’’'ʼˈ´`‘′ʻ’']s", false, 0); + EntityNode entity = entityCreationService.bySemanticNode(authorCell, "CBI_author", EntityType.ENTITY); + setFields(entity, 6, "Header Author(s) found", null, Engine.RULE); + insert(entity); + dictionary.addLocalDictionaryEntry("CBI_author", entity.getValue(), true); + end + +rule "7: Add CBI_author with \"et al.\" Regex" + agenda-group "LOCAL_DICTIONARY_ADDS" + + when + $section: SectionNode(containsString("et al.")) + then + Set entities = entityCreationService.byRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, $section); + setFields(entities, 7, "Found by \"et al.\" regex", null, Engine.RULE); + entities.forEach(entity -> insert(entity)); + entities.forEach(entity -> dictionary.addLocalDictionaryEntry("CBI_author", entity.getValue(), false)); + end + +rule "8: Add recommendation for Addresses in Test Organism sections" + + when + FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "yes") + $section: SectionNode(containsString("Species") && containsString("Source") && !containsString("Species:") && !containsString("Source:")) + then + Set entities = entityCreationService.lineAfterString("Source", "CBI_address", EntityType.RECOMMENDATION, $section); + setFields(entities, 8, "Line after \"Source\" in Test Organism Section", null, Engine.RULE); + entities.forEach(entity -> insert(entity)); end -rule "6: Redact Author(s) cells in Tables with Author(s) header (Non vertebrate study)" +rule "9: Add recommendation for Addresses in Test Animals sections" + when - Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && hasTableHeader("Author(s)") && !hasTableHeader("Vertebrate study Y/N")) + FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "yes") + $section: SectionNode(containsString("Species:") && containsString("Source:")) then - section.redactCell("Author(s)", 6, "CBI_author", false, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); + Set entities = entityCreationService.lineAfterString("Source:", "CBI_address", EntityType.RECOMMENDATION, $section); + setFields(entities, 9, "Line after \"Source:\" in Test Organism Section", null, Engine.RULE); + entities.forEach(entity -> insert(entity)); end -rule "7: Redact Author(s) cells in Tables with Author(s) header (Vertebrate study)" +rule "12: Recommend CTL/BL laboratory that start with BL or CTL" + when - Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && hasTableHeader("Author(s)") && !hasTableHeader("Vertebrate study Y/N")) + $section : SectionNode(containsString("BL") || containsString("CT")) then - section.redactCell("Author(s)", 7, "CBI_author", false, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - end - - -rule "8: Redact Author cells in Tables with Author header (Non vertebrate study)" - when - Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && hasTableHeader("Author") && !hasTableHeader("Vertebrate study Y/N")) - then - section.redactCell("Author", 8, "CBI_author", false, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - end - -rule "9: Redact Author cells in Tables with Author header (Vertebrate study)" - when - Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && hasTableHeader("Author") && !hasTableHeader("Vertebrate study Y/N")) - then - section.redactCell("Author", 9, "CBI_author", false, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - end - - -rule "10: Redact and recommand Authors in Tables with Vertebrate study Y/N header (Non vertebrate study)" - when - Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && (rowEquals("Vertebrate study Y/N", "Y") || rowEquals("Vertebrate study Y/N", "Yes") || rowEquals("Vertebrate study Y/N", "N") || rowEquals("Vertebrate study Y/N", "No"))) - then - section.redactCell("Author(s)", 10, "CBI_author", true, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - end - -rule "11: Redact and recommand Authors in Tables with Vertebrate study Y/N header (Vertebrate study)" - when - Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && (rowEquals("Vertebrate study Y/N", "Y") || rowEquals("Vertebrate study Y/N", "Yes") || rowEquals("Vertebrate study Y/N", "N") || rowEquals("Vertebrate study Y/N", "No"))) - then - section.redactCell("Author(s)", 11, "CBI_author", true, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - end - -rule "14: Redact and add recommendation for et al. author (Non vertebrate study)" - when - Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("et al")) - then - section.redactAndRecommendByRegEx("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", false, 1, "CBI_author", 14, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - end - -rule "15: Redact and add recommendation for et al. author (Vertebrate study)" - when - Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("et al")) - then - section.redactAndRecommendByRegEx("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", false, 1, "CBI_author", 15, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - end - - -rule "16: Add recommendation for Addresses in Test Organism sections" - when - Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("Species:") && searchText.contains("Source:")) - then - section.recommendLineAfter("Source:", "CBI_address"); - end - -rule "17: Add recommendation for Addresses in Test Animals sections" - when - Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("Species") && searchText.contains("Source")) - then - section.recommendLineAfter("Source", "CBI_address"); - end - - -rule "18: Do not redact Names and Addresses if Published Information found" - when - Section(matchesType("published_information")) - then - section.redactNotAndReference("CBI_author","published_information", 18, "Published Information found"); - section.redactNotAndReference("CBI_address","published_information", 18, "Published Information found"); + Set entities = entityCreationService.byRegex("((\\b((([Cc]T(([1ILli\\/])| L|~P))|(BL))[\\. ]?([\\dA-Ziltphz~\\/.:!]| ?[\\(',][Ppi](\\(e)?|([\\(-?']\\/))+( ?[\\(\\/\\dA-Znasieg]+)?)\\b( ?\\/? ?\\d+)?)|(\\bCT[L1i]\\b))", "CBI_address", EntityType.RECOMMENDATION, $section); + setFields(entities, 12, "Syngenta Specific Laboratory keyword", null, Engine.RULE); + entities.forEach(entity -> insert(entity)); + entities.forEach(entity -> dictionary.addLocalDictionaryEntry("PII", entity.getValue(), false)); end // --------------------------------------- PII rules ------------------------------------------------------------------- -rule "19: Redacted PII Personal Identification Information (Non vertebrate study)" +rule "10: Redacted PII Personal Identification Information (Non vertebrate study)" + no-loop true + when - Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesType("PII")) + FileAttribute(label == "Vertebrate Study" , value.toLowerCase() != "yes") + $entity: EntityNode(type == "PII", entityType == EntityType.ENTITY) then - section.redact("PII", 19, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - end - -rule "20: Redacted PII Personal Identification Information (Vertebrate study)" - when - Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesType("PII")) - then - section.redact("PII", 20, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - end - - -rule "21: Redact Emails by RegEx (Non vertebrate study)" - when - Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("@")) - then - section.redactByRegEx("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", true, 1, "PII", 21, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - end - -rule "22: Redact Emails by RegEx (Vertebrate study)" - when - Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("@")) - then - section.redactByRegEx("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", true, 1, "PII", 22, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - end - - -rule "23: Redact contact information (Non vertebrate study)" - when - Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && (text.contains("Contact point:") - || text.contains("Contact:") - || text.contains("Alternative contact:") - || (text.contains("No:") && text.contains("Fax")) - || (text.contains("Contact:") && text.contains("Tel.:")) - || text.contains("European contact:") - )) - then - section.redactLineAfter("Contact point:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - section.redactLineAfter("Contact:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - section.redactLineAfter("Alternative contact:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - section.redactBetween("No:", "Fax", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - section.redactBetween("Contact:", "Tel.:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - section.redactLineAfter("European contact:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - end - -rule "24: Redact contact information (Vertebrate study)" - when - Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && (text.contains("Contact point:") - || text.contains("Contact:") - || text.contains("Alternative contact:") - || (text.contains("No:") && text.contains("Fax")) - || (text.contains("Contact:") && text.contains("Tel.:")) - || text.contains("European contact:") - )) - then - section.redactLineAfter("Contact point:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - section.redactLineAfter("Contact:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - section.redactLineAfter("Alternative contact:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - section.redactBetween("No:", "Fax", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - section.redactBetween("Contact:", "Tel.:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - section.redactLineAfter("European contact:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - end - -rule "25: Redact Phone and Fax by RegEx (Non vertebrate study)" - when - Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && ( - text.contains("Contact") - || text.contains("Telephone") - || text.contains("Phone") - || text.contains("Fax") - || text.contains("Tel") - || text.contains("Ter") - || text.contains("Mobile") - || text.contains("Fel") - || text.contains("Fer") - )) - then - section.redactByRegEx("\\b(contact|telephone|phone|fax|tel|ter|mobile|fel|fer)[a-zA-Z\\s]{0,10}[:.\\s]{0,3}([\\+\\d\\(][\\s\\d\\(\\)\\-\\/\\.]{4,100}\\d)\\b", true, 2, "PII", 25, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - end - -rule "26: Redact Phone and Fax by RegEx (Vertebrate study)" - when - Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && ( - text.contains("Contact") - || text.contains("Telephone") - || text.contains("Phone") - || text.contains("Fax") - || text.contains("Tel") - || text.contains("Ter") - || text.contains("Mobile") - || text.contains("Fel") - || text.contains("Fer") - )) - then - section.redactByRegEx("\\b(contact|telephone|phone|fax|tel|ter|mobile|fel|fer)[a-zA-Z\\s]{0,10}[:.\\s]{0,3}([\\+\\d\\(][\\s\\d\\(\\)\\-\\/\\.]{4,100}\\d)\\b", true, 2, "PII", 26, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - end - -rule "27: Redact AUTHOR(S) (Non vertebrate study)" - when - Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") - && searchText.contains("AUTHOR(S):") - && searchText.contains("COMPLETION DATE:") - && !searchText.contains("STUDY COMPLETION DATE:") - ) - then - section.redactLinesBetween("AUTHOR(S):", "COMPLETION DATE:", "PII", 27, true, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - end - -rule "28: Redact AUTHOR(S) (Vertebrate study)" - when - Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") - && searchText.contains("AUTHOR(S):") - && searchText.contains("COMPLETION DATE:") - && !searchText.contains("STUDY COMPLETION DATE:") - ) - then - section.redactLinesBetween("AUTHOR(S):", "COMPLETION DATE:", "PII", 28, true, "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - end - - -rule "29: Redact AUTHOR(S) (Non vertebrate study)" - when - Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") - && searchText.contains("AUTHOR(S):") - && searchText.contains("STUDY COMPLETION DATE:") - ) - then - section.redactLinesBetween("AUTHOR(S):", "STUDY COMPLETION DATE:", "PII", 29, true, "AUTHOR(S) was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - end - -rule "30: Redact AUTHOR(S) (Vertebrate study)" - when - Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") - && searchText.contains("AUTHOR(S):") - && searchText.contains("STUDY COMPLETION DATE:") - ) - then - section.redactLinesBetween("AUTHOR(S):", "STUDY COMPLETION DATE:", "PII", 30, true, "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - end - - -rule "31: Redact PERFORMING LABORATORY (Non vertebrate study)" - when - Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") - && searchText.contains("PERFORMING LABORATORY:") - ) - then - section.redactBetween("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", 31, true, "PERFORMING LABORATORY was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - section.redactNot("CBI_address", 31, "Performing laboratory found for non vertebrate study"); - end - -rule "32: Redact PERFORMING LABORATORY (Vertebrate study)" - when - Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") - && searchText.contains("PERFORMING LABORATORY:")) - then - section.redactBetween("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", 32, true, "PERFORMING LABORATORY was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - end - -rule "33: Redact study director abbreviation" - when - Section((searchText.contains("KATH") || searchText.contains("BECH") || searchText.contains("KML"))) - then - section.redactWordPartByRegEx("((KATH)|(BECH)|(KML)) ?(\\d{4})", true, 0, 1, "PII", 34, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - end - - -// --------------------------------------- other rules ------------------------------------------------------------------- - -rule "34: Purity Hint" - when - Section(searchText.toLowerCase().contains("purity")) - then - section.addHintAnnotationByRegEx("(purity ?( of|\\(.{1,20}\\))?( ?:)?) .{0,5}[\\d\\.]+( .{0,4}\\.)? ?%", true, 1, "hint_only"); + $entity.setRedaction(true); + setFields($entity, 10, "Personal Information found", "Article 39(e)(2) of Regulation (EC) No 178/2002", null); + update($entity) end -rule "35: Redact signatures (Non vertebrate study)" +rule "11: Redacted PII Personal Identification Information (Vertebrate study)" + no-loop true + when - Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesImageType("signature")) + FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "yes") + $entity: EntityNode(type == "PII", entityType == EntityType.ENTITY) then - section.redactImage("signature", 35, "Signature found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); + $entity.setRedaction(true); + setFields($entity, 11, "Personal Information found", "Article 39(e)(2) of Regulation (EC) No 178/2002", null); + update($entity) end -rule "36: Redact signatures (Vertebrate study)" +rule "12: Redact Emails by RegEx (Non vertebrate study)" + when - Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesImageType("signature")) + FileAttribute(label == "Vertebrate Study" , value.toLowerCase() != "yes") + $section: SectionNode(containsString("@")) then - section.redactImage("signature", 36, "Signature found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); + Set entities = entityCreationService.byRegex("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", "PII", EntityType.ENTITY, $section); + setFields(entities, 12, "Found by email regex", null, Engine.RULE); + entities.forEach(entity -> insert(entity)); + end + +rule "13: Redact Emails by RegEx (Vertebrate study)" + + when + FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "yes") + $section: SectionNode(containsString("@")) + then + Set entities = entityCreationService.byRegex("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", "PII", EntityType.ENTITY, $section); + setFields(entities, 13, "Found by email regex", null, Engine.RULE); + entities.forEach(entity -> insert(entity)); end -rule "43: Redact Logos (Vertebrate study)" +rule "14: Redact line after contact information (Non vertebrate study)" + agenda-group "LOCAL_DICTIONARY_ADDS" + when - Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesImageType("logo")) + FileAttribute(label == "Vertebrate Study" , value.toLowerCase() != "yes") + $string: String() from List.of("Contact point:", + "Contact:", + "Alternative contact:", + "European contact:", + "No:", + "Contact:", + "Tel.:", + "Tel:", + "Telephone number:", + "Telephone No:", + "Telephone:", + "Phone No:", + "Phone:", + "Fax number:", + "Fax:", + "E-mail:", + "Email:", + "e-mail:", + "E-mail address:") + $section: SectionNode(containsString($string)) then - section.redactImage("logo", 43, "Logo found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); + + Set entities = new HashSet<>(); + entities.addAll(entityCreationService.lineAfterString($string, "PII", EntityType.ENTITY, $section)); + setFields(entities, 14, "Found by contacts keyword", null, Engine.RULE); + entities.forEach(entity -> insert(entity)); + entities.forEach(entity -> dictionary.addLocalDictionaryEntry("PII", entity.getValue(), false)); + end + + +rule "15: Redact line after contact information (Vertebrate study)" + agenda-group "LOCAL_DICTIONARY_ADDS" + + when + FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "yes") + $string: String() from List.of("Contact point:", + "Contact:", + "Alternative contact:", + "European contact:", + "No:", + "Contact:", + "Tel.:", + "Tel:", + "Telephone number:", + "Telephone No:", + "Telephone:", + "Phone No:", + "Phone:", + "Fax number:", + "Fax:", + "E-mail:", + "Email:", + "e-mail:", + "E-mail address:") + $section: SectionNode(containsString($string)) + then + Set entities = entityCreationService.lineAfterString($string, "PII", EntityType.ENTITY, $section); + setFields(entities, 15, "Found by contacts keyword", null, Engine.RULE); + entities.forEach(entity -> insert(entity)); + entities.forEach(entity -> dictionary.addLocalDictionaryEntry("PII", entity.getValue(), false)); + end + + +rule "16: redact line between contact keywords" + agenda-group "LOCAL_DICTIONARY_ADDS" + + when + FileAttribute(label == "Vertebrate Study" , value.toLowerCase() != "yes") + $section: SectionNode((containsString("No:") && containsString("Fax")) || (containsString("Contact:") && containsString("Tel"))) + then + Set entities = new HashSet<>(); + entities.addAll(entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section)); + entities.addAll(entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section)); + setFields(entities, 16, "Found between contacts keyword", null, Engine.RULE); + entities.forEach(entity -> insert(entity)); + entities.forEach(entity -> dictionary.addLocalDictionaryEntry("PII", entity.getValue(), false)); + end + +rule "17: redact line between contact keywords" + agenda-group "LOCAL_DICTIONARY_ADDS" + + when + FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "yes") + $section: SectionNode((containsString("No:") && containsString("Fax")) || (containsString("Contact:") && containsString("Tel"))) + then + Set entities = new HashSet<>(); + entities.addAll(entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section)); + entities.addAll(entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section)); + setFields(entities, 17, "Found between contacts keyword", null, Engine.RULE); + entities.forEach(entity -> insert(entity)); + entities.forEach(entity -> dictionary.addLocalDictionaryEntry("PII", entity.getValue(), false)); + end + +rule "18: Redact Phone and Fax by RegEx" + + when + $section: SectionNode(containsString("Contact") || + containsString("Telephone") || + containsString("Phone") || + containsString("Fax") || + containsString("Tel") || + containsString("Ter") || + containsString("Mobile") || + containsString("Fel") || + containsString("Fer")) + then + Set entities = entityCreationService.byRegex("\\b(contact|telephone|phone|fax|tel|ter|mobile|fel|fer)[a-zA-Z\\s]{0,10}[:.\\s]{0,3}([\\+\\d\\(][\\s\\d\\(\\)\\-\\/\\.]{4,100}\\d)\\b", "PII", EntityType.ENTITY, $section); + setFields(entities, 18, "Found by Phone and Fax regex", null, Engine.RULE); + entities.forEach(entity -> insert(entity)); end diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules2.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules2.drl deleted file mode 100644 index 3c498213..00000000 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules2.drl +++ /dev/null @@ -1 +0,0 @@ -rules.drl \ No newline at end of file diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/testRules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/testRules.drl index 97c7ac3f..2be006f7 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/testRules.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/testRules.drl @@ -1,431 +1,390 @@ package drools -import com.iqser.red.service.redaction.v1.server.redaction.model.Section - -global Section section +import static java.lang.String.format; +import static com.iqser.red.service.redaction.v1.server.document.services.CharSequenceSearchUtils.anyMatch; +import static com.iqser.red.service.redaction.v1.server.document.services.EntityEnrichmentService.setFields; -// --------------------------------------- AI rules ------------------------------------------------------------------- +import java.util.List; +import java.util.LinkedList; +import java.util.HashSet; -rule "0: Add CBI_author from ai" - when - Section(aiMatchesType("CBI_author")) - then - section.addAiEntities("CBI_author", "CBI_author",dictionary); +import com.iqser.red.service.redaction.v1.server.document.graph.* +import com.iqser.red.service.redaction.v1.server.document.graph.nodes.* +import com.iqser.red.service.redaction.v1.server.document.graph.entity.* +import com.iqser.red.service.redaction.v1.server.document.graph.textblock.* +import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType; +import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute; +import java.util.Set +import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine +import com.iqser.red.service.redaction.v1.server.document.services.EntityCreationService; +import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary; +import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryModel; + +global DocumentGraph document +global EntityCreationService entityCreationService +global Dictionary dictionary + + +query "getFileAttributes" + $fileAttribute: FileAttribute() end -rule "0: Combine address parts from ai to CBI_address (org is mandatory)" +rule "merge intersecting Entities of same type" + salience 100 + when - Section(aiMatchesType("ORG")) + $first: EntityNode($type: type, $entityType: entityType, !resized, !skipRemoveEntitiesContainedInLarger) + $second: EntityNode(intersects($first), type == $type, entityType == $entityType, this != $first, !resized, !skipRemoveEntitiesContainedInLarger) then - section.combineAiTypes("ORG", "STREET,POSTAL,COUNTRY,CARDINAL,CITY,STATE", 20, "CBI_address", 3, false, dictionary); + $first.removeFromGraph(); + $second.removeFromGraph(); + EntityNode mergedEntity = entityCreationService.byEntities(List.of($first, $second), $type, $entityType, document); + insert(mergedEntity); + retract($first); + retract($second); end -rule "0: Combine address parts from ai to CBI_address (street is mandatory)" +rule "remove Entity of type ENTITY when contained by FALSE_POSITIVE" + salience 100 + when - Section(aiMatchesType("STREET")) + $falsePositive: EntityNode($type: type, entityType == EntityType.FALSE_POSITIVE) + $entity: EntityNode(containedBy($falsePositive), type == $type, entityType == EntityType.ENTITY, !resized, !skipRemoveEntitiesContainedInLarger) then - section.combineAiTypes("STREET", "ORG,POSTAL,COUNTRY,CARDINAL,CITY,STATE", 20, "CBI_address", 3, false, dictionary); + $entity.removeFromGraph(); + retract($entity); end -rule "0: Combine address parts from ai to CBI_address (city is mandatory)" +rule "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION" + salience 100 + when - Section(aiMatchesType("CITY")) + $falseRecommendation: EntityNode($type: type, entityType == EntityType.FALSE_RECOMMENDATION) + $recommendation: EntityNode(containedBy($falseRecommendation), type == $type, entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger) then - section.combineAiTypes("CITY", "ORG,STREET,POSTAL,COUNTRY,CARDINAL,STATE", 20, "CBI_address", 3, false, dictionary); + $recommendation.removeFromGraph(); + retract($recommendation); + end + +rule "remove Entity of type RECOMMENDATION when contained by ENTITY" + salience 100 + + when + $entity: EntityNode($type: type, entityType == EntityType.ENTITY) + $recommendation: EntityNode(containedBy($entity), type == $type, entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger) + then + $recommendation.removeFromGraph(); + retract($recommendation); + end + +rule "remove Entity of lower rank, when equal boundaries and entityType" + salience 100 + + when + $higherRank: EntityNode($type: type, $entityType: entityType, $boundary: boundary) + $lowerRank: EntityNode($boundary == boundary, type != $type, entityType == $entityType, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type)) + then + $lowerRank.removeFromGraph(); + retract($lowerRank); + end + +rule "run local dictionary search" + agenda-group "LOCAL_DICTIONARY_ADDS" + salience -999 + + when + DictionaryModel(!localEntries.isEmpty(), $type: type, $searchImplementation: localSearch) from dictionary.getDictionaryModels() + then + Set entityNodes = entityCreationService.bySearchImplementation($searchImplementation, $type, EntityType.RECOMMENDATION, document); + System.out.printf("local dictionary search found %d entities\n", entityNodes.size()); + entityNodes.forEach(entityNode -> insert(entityNode)); end // --------------------------------------- CBI rules ------------------------------------------------------------------- rule "1: Redact CBI Authors (Non vertebrate study)" + no-loop true + when - Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesType("CBI_author")) + FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "yes") + $entity: EntityNode(type == "CBI_author", entityType == EntityType.ENTITY) then - section.redact("CBI_author", 1, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); + $entity.setRedaction(true); + setFields($entity, 1, "Address found", "Article 39(e)(2) of Regulation (EC) No 178/2002", null); + update($entity) end rule "2: Redact CBI Authors (Vertebrate study)" + no-loop true + when - Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesType("CBI_author")) + FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "no") + $entity: EntityNode(type == "CBI_author", entityType == EntityType.ENTITY) then - section.redact("CBI_author", 2, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); + $entity.setRedaction(true); + setFields($entity, 2, "Address found", "Article 39(e)(2) of Regulation (EC) No 178/2002", null); + update($entity) end +rule "3: Don't redact CBI Address (Non vertebrate study)" + no-loop true -rule "3: Redact not CBI Address (Non vertebrate study)" when - Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesType("CBI_address")) + FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "no") + $entity: EntityNode(type == "CBI_address", entityType == EntityType.ENTITY) then - section.redactNot("CBI_address", 3, "Address found for non vertebrate study"); - section.ignoreRecommendations("CBI_address"); + setFields($entity, 3, "Address found", "Article 39(e)(2) of Regulation (EC) No 178/2002", null); + update($entity) end rule "4: Redact CBI Address (Vertebrate study)" + no-loop true + when - Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesType("CBI_address")) + FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "yes") + $entity: EntityNode(type == "CBI_address", entityType == EntityType.ENTITY) then - section.redact("CBI_address", 4, "Address found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); + $entity.setRedaction(true); + setFields($entity, 4, "Address found", "Article 39(e)(2) of Regulation (EC) No 178/2002", null); + update($entity) + end + +rule "5: Add FALSE_POSITIVE Entity for genitive CBI_author" + + when + $entity: EntityNode(type == "CBI_author", anyMatch(textAfter, "['’’'ʼˈ´`‘′ʻ’']s"), redaction) + then + EntityNode entity = entityCreationService.byBoundary($entity.getBoundary(), "CBI_author", EntityType.FALSE_POSITIVE, document); + setFields($entity, 5, "Genitive Author", null, Engine.RULE); + insert(entity); end -rule "5: Do not redact genitive CBI_author" +rule "6: Create Entity from Author(s) cells in Tables with Author(s) header and add Authorname as Recommendation" + agenda-group "LOCAL_DICTIONARY_ADDS" + when - Section(matchesType("CBI_author")) + authorCell: TableCellNode(!header, (hasHeader("Author(s)") || hasHeader("Author")), hasText()) then - section.expandToFalsePositiveByRegEx("CBI_author", "['’’'ʼˈ´`‘′ʻ’']s", false, 0, dictionary); + EntityNode entity = entityCreationService.bySemanticNode(authorCell, "CBI_author", EntityType.ENTITY); + setFields(entity, 6, "Header Author(s) found", null, Engine.RULE); + insert(entity); + dictionary.addLocalDictionaryEntry("CBI_author", entity.getValue(), true); + end + +rule "7: Add CBI_author with \"et al.\" Regex" + agenda-group "LOCAL_DICTIONARY_ADDS" + + when + $section: SectionNode(containsString("et al.")) + then + Set entities = entityCreationService.byRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, $section); + setFields(entities, 7, "Found by \"et al.\" regex", null, Engine.RULE); + entities.forEach(entity -> insert(entity)); + entities.forEach(entity -> dictionary.addLocalDictionaryEntry("CBI_author", entity.getValue(), false)); + end + +rule "8: Add recommendation for Addresses in Test Organism sections" + + when + FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "yes") + $section: SectionNode(containsString("Species") && containsString("Source") && !containsString("Species:") && !containsString("Source:")) + then + Set entities = entityCreationService.lineAfterString("Source", "CBI_address", EntityType.RECOMMENDATION, $section); + setFields(entities, 8, "Line after \"Source\" in Test Organism Section", null, Engine.RULE); + entities.forEach(entity -> insert(entity)); end -rule "6: Redact Author(s) cells in Tables with Author(s) header (Non vertebrate study)" +rule "9: Add recommendation for Addresses in Test Animals sections" + when - Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && hasTableHeader("Author(s)") && !hasTableHeader("Vertebrate study Y/N")) + FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "yes") + $section: SectionNode(containsString("Species:") && containsString("Source:")) then - section.redactCell("Author(s)", 6, "CBI_author", false, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary); + Set entities = entityCreationService.lineAfterString("Source:", "CBI_address", EntityType.RECOMMENDATION, $section); + setFields(entities, 9, "Line after \"Source:\" in Test Organism Section", null, Engine.RULE); + entities.forEach(entity -> insert(entity)); end -rule "7: Redact Author(s) cells in Tables with Author(s) header (Vertebrate study)" - when - Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && hasTableHeader("Author(s)") && !hasTableHeader("Vertebrate study Y/N")) - then - section.redactCell("Author(s)", 7, "CBI_author", false, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary); - end - - -rule "8: Redact Author cells in Tables with Author header (Non vertebrate study)" - when - Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && hasTableHeader("Author") && !hasTableHeader("Vertebrate study Y/N")) - then - section.redactCell("Author", 8, "CBI_author", false, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary); - end - -rule "9: Redact Author cells in Tables with Author header (Vertebrate study)" - when - Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && hasTableHeader("Author") && !hasTableHeader("Vertebrate study Y/N")) - then - section.redactCell("Author", 9, "CBI_author", false, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary); - end - - -rule "10: Redact and recommand Authors in Tables with Vertebrate study Y/N header (Non vertebrate study)" - when - Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && (rowEquals("Vertebrate study Y/N", "Y") || rowEquals("Vertebrate study Y/N", "Yes") || rowEquals("Vertebrate study Y/N", "N") || rowEquals("Vertebrate study Y/N", "No"))) - then - section.redactCell("Author(s)", 10, "CBI_author", true, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary); - end - -rule "11: Redact and recommand Authors in Tables with Vertebrate study Y/N header (Vertebrate study)" - when - Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && (rowEquals("Vertebrate study Y/N", "Y") || rowEquals("Vertebrate study Y/N", "Yes") || rowEquals("Vertebrate study Y/N", "N") || rowEquals("Vertebrate study Y/N", "No"))) - then - section.redactCell("Author(s)", 11, "CBI_author", true, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary); - end - -/* Syngenta specific laboratory rule */ rule "12: Recommend CTL/BL laboratory that start with BL or CTL" + when - Section(searchText.contains("CT") || searchText.contains("BL")) + $section : SectionNode(containsString("BL") || containsString("CT")) then - section.addRecommendationByRegEx("((\\b((([Cc]T(([1ILli\\/])| L|~P))|(BL))[\\. ]?([\\dA-Ziltphz~\\/.:!]| ?[\\(',][Ppi](\\(e)?|([\\(-?']\\/))+( ?[\\(\\/\\dA-Znasieg]+)?)\\b( ?\\/? ?\\d+)?)|(\\bCT[L1i]\\b))", true, 0, "CBI_address"); - end - -rule "14: Redact and add recommendation for et al. author (Non vertebrate study)" - when - Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("et al")) - then - section.redactAndRecommendByRegEx("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", false, 1, "CBI_author", 14, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary); - end - -rule "15: Redact and add recommendation for et al. author (Vertebrate study)" - when - Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("et al")) - then - section.redactAndRecommendByRegEx("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", false, 1, "CBI_author", 15, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary); - end - - -rule "16: Add recommendation for Addresses in Test Organism sections" - when - Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("Species:") && searchText.contains("Source:")) - then - section.recommendLineAfter("Source:", "CBI_address"); - end - -rule "17: Add recommendation for Addresses in Test Animals sections" - when - Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("Species") && searchText.contains("Source")) - then - section.recommendLineAfter("Source", "CBI_address"); - end - - -rule "18: Do not redact Names and Addresses if Published Information found" - when - Section(matchesType("published_information")) - then - section.redactNotAndReference("CBI_author","published_information", 18, "Published Information found"); - section.redactNotAndReference("CBI_address","published_information", 18, "Published Information found"); + Set entities = entityCreationService.byRegex("((\\b((([Cc]T(([1ILli\\/])| L|~P))|(BL))[\\. ]?([\\dA-Ziltphz~\\/.:!]| ?[\\(',][Ppi](\\(e)?|([\\(-?']\\/))+( ?[\\(\\/\\dA-Znasieg]+)?)\\b( ?\\/? ?\\d+)?)|(\\bCT[L1i]\\b))", "CBI_address", EntityType.RECOMMENDATION, $section); + setFields(entities, 12, "Syngenta Specific Laboratory keyword", null, Engine.RULE); + entities.forEach(entity -> insert(entity)); + entities.forEach(entity -> dictionary.addLocalDictionaryEntry("PII", entity.getValue(), false)); end // --------------------------------------- PII rules ------------------------------------------------------------------- -rule "19: Redacted PII Personal Identification Information (Non vertebrate study)" - when - Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesType("PII")) - then - section.redact("PII", 19, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - end +rule "10: Redacted PII Personal Identification Information (Non vertebrate study)" + no-loop true -rule "20: Redacted PII Personal Identification Information (Vertebrate study)" when - Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesType("PII")) + FileAttribute(label == "Vertebrate Study" , value.toLowerCase() != "yes") + $entity: EntityNode(type == "PII", entityType == EntityType.ENTITY) then - section.redact("PII", 20, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); + $entity.setRedaction(true); + setFields($entity, 10, "Personal Information found", "Article 39(e)(2) of Regulation (EC) No 178/2002", null); + update($entity) end -rule "21: Redact Emails by RegEx (Non vertebrate study)" + +rule "11: Redacted PII Personal Identification Information (Vertebrate study)" + no-loop true + when - Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("@")) + FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "yes") + $entity: EntityNode(type == "PII", entityType == EntityType.ENTITY) then - section.redactByRegEx("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", true, 1, "PII", 21, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary); + $entity.setRedaction(true); + setFields($entity, 11, "Personal Information found", "Article 39(e)(2) of Regulation (EC) No 178/2002", null); + update($entity) end -rule "22: Redact Emails by RegEx (Vertebrate study)" +rule "12: Redact Emails by RegEx (Non vertebrate study)" + when - Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("@")) + FileAttribute(label == "Vertebrate Study" , value.toLowerCase() != "yes") + $section: SectionNode(containsString("@")) then - section.redactByRegEx("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", true, 1, "PII", 22, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary); + Set entities = entityCreationService.byRegex("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", "PII", EntityType.ENTITY, $section); + setFields(entities, 12, "Found by email regex", null, Engine.RULE); + entities.forEach(entity -> insert(entity)); + end + +rule "13: Redact Emails by RegEx (Vertebrate study)" + + when + FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "yes") + $section: SectionNode(containsString("@")) + then + Set entities = entityCreationService.byRegex("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", "PII", EntityType.ENTITY, $section); + setFields(entities, 13, "Found by email regex", null, Engine.RULE); + entities.forEach(entity -> insert(entity)); end -rule "23: Redact contact information (Non vertebrate study)" - when - Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && (text.contains("Contact point:") - || text.contains("Phone:") - || text.contains("Fax:") - || text.contains("Tel.:") - || text.contains("Tel:") - || text.contains("E-mail:") - || text.contains("Email:") - || text.contains("e-mail:") - || text.contains("E-mail address:") - || text.contains("Contact:") - || text.contains("Alternative contact:") - || text.contains("Telephone number:") - || text.contains("Telephone No:") - || text.contains("Fax number:") - || text.contains("Telephone:") - || text.contains("Phone No.") - || (text.contains("No:") && text.contains("Fax")) - || (text.contains("Contact:") && text.contains("Tel.:")) - || text.contains("European contact:") - )) - then - section.redactLineAfter("Contact point:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary); - section.redactLineAfter("Phone:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary); - section.redactLineAfter("Fax:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary); - section.redactLineAfter("Tel.:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary); - section.redactLineAfter("Tel:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary); - section.redactLineAfter("E-mail:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary); - section.redactLineAfter("Email:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary); - section.redactLineAfter("e-mail:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary); - section.redactLineAfter("E-mail address:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary); - section.redactLineAfter("Contact:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary); - section.redactLineAfter("Alternative contact:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary); - section.redactLineAfter("Telephone number:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary); - section.redactLineAfter("Telephone No:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary); - section.redactLineAfter("Fax number:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary); - section.redactLineAfter("Telephone:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary); - section.redactLineAfter("Phone No.", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary); - section.redactBetween("No:", "Fax", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary); - section.redactBetween("Contact:", "Tel.:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary); - section.redactLineAfter("European contact:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary); - end +rule "14: Redact line after contact information (Non vertebrate study)" + agenda-group "LOCAL_DICTIONARY_ADDS" -rule "24: Redact contact information (Vertebrate study)" when - Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && (text.contains("Contact point:") - || text.contains("Phone:") - || text.contains("Fax:") - || text.contains("Tel.:") - || text.contains("Tel:") - || text.contains("E-mail:") - || text.contains("Email:") - || text.contains("e-mail:") - || text.contains("E-mail address:") - || text.contains("Contact:") - || text.contains("Alternative contact:") - || text.contains("Telephone number:") - || text.contains("Telephone No:") - || text.contains("Fax number:") - || text.contains("Telephone:") - || text.contains("Phone No.") - || (text.contains("No:") && text.contains("Fax")) - || (text.contains("Contact:") && text.contains("Tel.:")) - || text.contains("European contact:") - )) + FileAttribute(label == "Vertebrate Study" , value.toLowerCase() != "yes") + $string: String() from List.of("Contact point:", + "Contact:", + "Alternative contact:", + "European contact:", + "No:", + "Contact:", + "Tel.:", + "Tel:", + "Telephone number:", + "Telephone No:", + "Telephone:", + "Phone No:", + "Phone:", + "Fax number:", + "Fax:", + "E-mail:", + "Email:", + "e-mail:", + "E-mail address:") + $section: SectionNode(containsString($string)) then - section.redactLineAfter("Contact point:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary); - section.redactLineAfter("Phone:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary); - section.redactLineAfter("Fax:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary); - section.redactLineAfter("Tel.:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary); - section.redactLineAfter("Tel:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary); - section.redactLineAfter("E-mail:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary); - section.redactLineAfter("Email:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary); - section.redactLineAfter("e-mail:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary); - section.redactLineAfter("E-mail address:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary); - section.redactLineAfter("Contact:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary); - section.redactLineAfter("Alternative contact:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary); - section.redactLineAfter("Telephone number:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary); - section.redactLineAfter("Telephone No:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary); - section.redactLineAfter("Fax number:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary); - section.redactLineAfter("Telephone:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary); - section.redactLineAfter("Phone No.", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary); - section.redactBetween("No:", "Fax", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary); - section.redactBetween("Contact:", "Tel.:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary); - section.redactLineAfter("European contact:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary); - end - -rule "25: Redact Phone and Fax by RegEx (Non vertebrate study)" - when - Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && ( - text.contains("Telephone") - || text.contains("Phone") - || text.contains("Ph.") - || text.contains("Fax") - || text.contains("Tel") - || text.contains("Ter") - || text.contains("Cell") - || text.contains("Mobile") - || text.contains("Fel") - || text.contains("Fer") - )) - then - section.redactByRegEx("\\b(telephone|phone|fax|tel|ter|cell|mobile|fel|fer)[:.\\s]{0,3}((\\(?\\+?[0-9])(\\(?[0-9\\/.\\-\\s]+\\)?)*([0-9]+\\)?))\\b", true, 2, "PII", 25, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary); - end - -rule "26: Redact Phone and Fax by RegEx (Vertebrate study)" - when - Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && ( - text.contains("Telephone") - || text.contains("Phone") - || text.contains("Ph.") - || text.contains("Fax") - || text.contains("Tel") - || text.contains("Ter") - || text.contains("Cell") - || text.contains("Mobile") - || text.contains("Fel") - || text.contains("Fer") - )) - then - section.redactByRegEx("\\b(telephone|phone|fax|tel|ter|cell|mobile|fel|fer)[:.\\s]{0,3}((\\(?\\+?[0-9])(\\(?[0-9\\/.\\-\\s]+\\)?)*([0-9]+\\)?))\\b", true, 2, "PII", 26, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary); + Set entities = new HashSet<>(); + entities.addAll(entityCreationService.lineAfterString($string, "PII", EntityType.ENTITY, $section)); + setFields(entities, 14, "Found by contacts keyword", null, Engine.RULE); + entities.forEach(entity -> insert(entity)); + entities.forEach(entity -> dictionary.addLocalDictionaryEntry("PII", entity.getValue(), false)); end -rule "27: Redact AUTHOR(S) (Non vertebrate study)" - when - Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") - && searchText.contains("AUTHOR(S):") - && searchText.contains("COMPLETION DATE:") - && !searchText.contains("STUDY COMPLETION DATE:") - ) - then - section.redactLinesBetween("AUTHOR(S):", "COMPLETION DATE:", "PII", 27, true, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary); - end +rule "15: Redact line after contact information (Vertebrate study)" + agenda-group "LOCAL_DICTIONARY_ADDS" -rule "28: Redact AUTHOR(S) (Vertebrate study)" when - Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") - && searchText.contains("AUTHOR(S):") - && searchText.contains("COMPLETION DATE:") - && !searchText.contains("STUDY COMPLETION DATE:") - ) + FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "yes") + $string: String() from List.of("Contact point:", + "Contact:", + "Alternative contact:", + "European contact:", + "No:", + "Contact:", + "Tel.:", + "Tel:", + "Telephone number:", + "Telephone No:", + "Telephone:", + "Phone No:", + "Phone:", + "Fax number:", + "Fax:", + "E-mail:", + "Email:", + "e-mail:", + "E-mail address:") + $section: SectionNode(containsString($string)) then - section.redactLinesBetween("AUTHOR(S):", "COMPLETION DATE:", "PII", 28, true, "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary); + Set entities = entityCreationService.lineAfterString($string, "PII", EntityType.ENTITY, $section); + setFields(entities, 15, "Found by contacts keyword", null, Engine.RULE); + entities.forEach(entity -> insert(entity)); + entities.forEach(entity -> dictionary.addLocalDictionaryEntry("PII", entity.getValue(), false)); end -rule "29: Redact AUTHOR(S) (Non vertebrate study)" +rule "16: redact line between contact keywords" + agenda-group "LOCAL_DICTIONARY_ADDS" + when - Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") - && searchText.contains("AUTHOR(S):") - && searchText.contains("STUDY COMPLETION DATE:") - ) + FileAttribute(label == "Vertebrate Study" , value.toLowerCase() != "yes") + $section: SectionNode((containsString("No:") && containsString("Fax")) || (containsString("Contact:") && containsString("Tel"))) then - section.redactLinesBetween("AUTHOR(S):", "STUDY COMPLETION DATE:", "PII", 29, true, "AUTHOR(S) was found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary); + Set entities = new HashSet<>(); + entities.addAll(entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section)); + entities.addAll(entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section)); + setFields(entities, 16, "Found between contacts keyword", null, Engine.RULE); + entities.forEach(entity -> insert(entity)); + entities.forEach(entity -> dictionary.addLocalDictionaryEntry("PII", entity.getValue(), false)); end -rule "30: Redact AUTHOR(S) (Vertebrate study)" +rule "17: redact line between contact keywords" + agenda-group "LOCAL_DICTIONARY_ADDS" + when - Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") - && searchText.contains("AUTHOR(S):") - && searchText.contains("STUDY COMPLETION DATE:") - ) + FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "yes") + $section: SectionNode((containsString("No:") && containsString("Fax")) || (containsString("Contact:") && containsString("Tel"))) then - section.redactLinesBetween("AUTHOR(S):", "STUDY COMPLETION DATE:", "PII", 30, true, "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary); + Set entities = new HashSet<>(); + entities.addAll(entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section)); + entities.addAll(entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section)); + setFields(entities, 17, "Found between contacts keyword", null, Engine.RULE); + entities.forEach(entity -> insert(entity)); + entities.forEach(entity -> dictionary.addLocalDictionaryEntry("PII", entity.getValue(), false)); end +rule "18: Redact Phone and Fax by RegEx" -rule "31: Redact PERFORMING LABORATORY (Non vertebrate study)" when - Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") - && searchText.contains("PERFORMING LABORATORY:") - ) + $section: SectionNode(containsString("Contact") || + containsString("Telephone") || + containsString("Phone") || + containsString("Fax") || + containsString("Tel") || + containsString("Ter") || + containsString("Mobile") || + containsString("Fel") || + containsString("Fer")) then - section.redactBetween("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", 31, true, "PERFORMING LABORATORY was found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary); - section.redactNot("CBI_address", 31, "Performing laboratory found for non vertebrate study"); - end - -rule "32: Redact PERFORMING LABORATORY (Vertebrate study)" - when - Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") - && searchText.contains("PERFORMING LABORATORY:")) - then - section.redactBetween("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", 32, true, "PERFORMING LABORATORY was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary); - end - - -// --------------------------------------- other rules ------------------------------------------------------------------- - -rule "33: Purity Hint" - when - Section(searchText.toLowerCase().contains("purity")) - then - section.addHintAnnotationByRegEx("(purity ?( of|\\(.{1,20}\\))?( ?:)?) .{0,5}[\\d\\.]+( .{0,4}\\.)? ?%", true, 1, "hint_only",dictionary); - end - - -rule "34: Ignore dossier_redaction entries if confidentiality is not 'confidential'" - when - Section(!fileAttributeByLabelEqualsIgnoreCase("Confidentiality","confidential") && matchesType("dossier_redaction")); - then - section.ignore("dossier_redaction"); - end - - -rule "35: Redact signatures (Non vertebrate study)" - when - Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesImageType("signature")) - then - section.redactImage("signature", 35, "Signature found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - end - -rule "36: Redact signatures (Vertebrate study)" - when - Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesImageType("signature")) - then - section.redactImage("signature", 36, "Signature found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - end - - -rule "43: Redact Logos (Vertebrate study)" - when - Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesImageType("logo")) - then - section.redactImage("logo", 43, "Logo found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); + Set entities = entityCreationService.byRegex("\\b(contact|telephone|phone|fax|tel|ter|mobile|fel|fer)[a-zA-Z\\s]{0,10}[:.\\s]{0,3}([\\+\\d\\(][\\s\\d\\(\\)\\-\\/\\.]{4,100}\\d)\\b", "PII", EntityType.ENTITY, $section); + setFields(entities, 18, "Found by Phone and Fax regex", null, Engine.RULE); + entities.forEach(entity -> insert(entity)); end diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/test_rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/test_rules.drl deleted file mode 100644 index 526a95a3..00000000 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/test_rules.drl +++ /dev/null @@ -1,35 +0,0 @@ -package drools - -import static java.lang.String.format; - -import java.util.List; -import java.util.LinkedList; -import java.util.HashSet; - -import com.iqser.red.service.redaction.v1.server.document.graph.* -import com.iqser.red.service.redaction.v1.server.redaction.model.*; -import com.iqser.red.service.redaction.v1.model.FileAttribute; -import com.iqser.red.service.redaction.v1.model.Engine; -import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntry; -import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUtils; -import java.util.Set; - -global DocumentGraph document - -rule "dummy" - when - - then - System.out.println("This actually works"); - end - -rule "1: Redact CBI_author" - when - //FileAttribute(label == "Vertebrate Study" && (value == "Yes")) - $entity: EntityNode(type == "CBI_author") - then - System.out.println("Entity found"); - modify($entity){ - $entity.setRedact(true) - } - end