From d04a576397e6411598328dbca1892aa878b3940e Mon Sep 17 00:00:00 2001 From: Kilian Schuettler Date: Mon, 20 Mar 2023 12:51:47 +0100 Subject: [PATCH] RED-6368: Include Tables and Images into Prototype Document Structure *rebased to master and included tenant capabilities *added separator validation to entity search *added EntityPosition class to bridge the gap to the redactionLogEntry --- .../mapper}/DocumentDataMapper.java | 2 +- .../mapper}/DocumentGraphMapper.java | 6 +- .../v1/server/document/graph/Boundary.java | 37 +++- .../server/document/graph/DocumentGraph.java | 2 +- .../document/graph/entity/EntityNode.java | 166 ++++++++++++++++++ .../document/graph/entity/EntityPosition.java | 36 ++++ .../graph/{nodes => entity}/ImageNode.java | 6 +- .../graph/factory/DocumentGraphFactory.java | 52 ++---- .../factory/RectangleTransformations.java | 62 ++++++- .../SearchTextWithTextPositionFactory.java | 5 +- .../document/graph/nodes/EntityNode.java | 127 -------------- .../document/graph/nodes/FooterNode.java | 12 +- .../document/graph/nodes/HeaderNode.java | 11 +- .../document/graph/nodes/HeadlineNode.java | 4 +- .../server/document/graph/nodes/PageNode.java | 7 +- .../document/graph/nodes/ParagraphNode.java | 4 +- .../document/graph/nodes/SectionNode.java | 3 +- .../document/graph/nodes/SemanticNode.java | 39 +++- .../document/graph/nodes/TableCellNode.java | 18 +- .../document/graph/nodes/TableNode.java | 3 +- .../graph/textblock/AtomicTextBlock.java | 50 +++++- .../textblock/ConcatenatedTextBlock.java | 42 ++++- .../document/graph/textblock/TextBlock.java | 17 +- .../services/EntityCreationService.java | 53 +++++- ....java => EntityTextEnrichmentService.java} | 15 +- .../document/services/ImageSortService.java | 40 ++--- .../document/services/RegexMatcher.java | 7 +- .../v1/server/redaction/model/Entity.java | 18 +- .../v1/server/redaction/model/Section.java | 1 - .../model/SectionSearchableTextPair.java | 16 ++ .../service/DroolsExecutionService.java | 12 ++ .../service/RedactionLogCreatorService.java | 19 ++ .../EntityRedactionService.java | 55 +++--- .../redaction/utils/EntitySearchUtils.java | 20 ++- .../v1/server/redaction/utils/IdBuilder.java | 7 + .../redaction/utils/SearchImplementation.java | 4 +- .../server/AbstractTestWithDictionaries.java | 78 ++++---- .../server/DocumentGraphIntegrationTest.java | 50 ++++-- .../server/document/graph/BoundaryTest.java | 26 ++- .../graph/DocumentGraphMappingTest.java | 7 +- .../document/graph/DocumentGraphTest.java | 17 +- .../utils/EntitySearchUtilsTest.java | 50 +++--- .../resources/drools/merge_entity_rules.drl | 56 ++++++ 43 files changed, 848 insertions(+), 414 deletions(-) rename redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/{services => data/mapper}/DocumentDataMapper.java (98%) rename redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/{services => data/mapper}/DocumentGraphMapper.java (96%) create mode 100644 redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/entity/EntityNode.java create mode 100644 redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/entity/EntityPosition.java rename redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/{nodes => entity}/ImageNode.java (90%) delete mode 100644 redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/EntityNode.java rename redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/{EntityEnrichmentService.java => EntityTextEnrichmentService.java} (95%) create mode 100644 redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/SectionSearchableTextPair.java create mode 100644 redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/merge_entity_rules.drl diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/DocumentDataMapper.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/data/mapper/DocumentDataMapper.java similarity index 98% rename from redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/DocumentDataMapper.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/data/mapper/DocumentDataMapper.java index 7a540f51..66755991 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/DocumentDataMapper.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/data/mapper/DocumentDataMapper.java @@ -1,4 +1,4 @@ -package com.iqser.red.service.redaction.v1.server.document.services; +package com.iqser.red.service.redaction.v1.server.document.data.mapper; import java.awt.geom.Rectangle2D; import java.util.List; diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/DocumentGraphMapper.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/data/mapper/DocumentGraphMapper.java similarity index 96% rename from redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/DocumentGraphMapper.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/data/mapper/DocumentGraphMapper.java index ea8f63dd..209387d1 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/DocumentGraphMapper.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/data/mapper/DocumentGraphMapper.java @@ -1,4 +1,4 @@ -package com.iqser.red.service.redaction.v1.server.document.services; +package com.iqser.red.service.redaction.v1.server.document.data.mapper; import static java.lang.Math.toIntExact; import static java.lang.String.format; @@ -71,7 +71,7 @@ public class DocumentGraphMapper { private void buildSection(TableOfContentsData.EntryData entryData, List currentTocId, Context context) { - SectionNode section = SectionNode.builder().entities(new HashSet<>()).pages(new HashSet<>()).tableOfContents(context.tableOfContents()).build(); + SectionNode section = SectionNode.builder().entities(new HashSet<>()).tableOfContents(context.tableOfContents()).build(); context.sections().add(section); @@ -90,7 +90,7 @@ public class DocumentGraphMapper { Set pages = Arrays.stream(entryData.pages()).map(pageNumber -> getPage(pageNumber, context)).collect(Collectors.toSet()); SectionNode parentSection = (SectionNode) context.tableOfContents().getEntryById(currentTocId).node(); - ParagraphNode paragraph = ParagraphNode.builder().pages(pages).build(); + ParagraphNode paragraph = ParagraphNode.builder().build(); TextBlock textBlock = toTextBlock(entryData.atomicTextBlocks(), context, paragraph); paragraph.setTerminalTextBlock(textBlock); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/Boundary.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/Boundary.java index 3d14b3d6..56b9ed70 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/Boundary.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/Boundary.java @@ -2,6 +2,9 @@ package com.iqser.red.service.redaction.v1.server.document.graph; import static java.lang.String.format; +import java.util.LinkedList; +import java.util.List; + import lombok.Setter; @Setter @@ -47,14 +50,14 @@ public class Boundary implements Comparable { public boolean containedBy(Boundary boundary) { - return boundary.start() <= start && end <= boundary.end(); + return boundary.contains(this); } public boolean contains(int start, int end) { if (start > end) { - throw new UnsupportedOperationException("start > end"); + throw new IllegalArgumentException(format("start: %d > end: %d", start, end)); } return this.start <= start && end <= this.end; } @@ -63,7 +66,7 @@ public class Boundary implements Comparable { public boolean containedBy(int start, int end) { if (start > end) { - throw new UnsupportedOperationException("start > end"); + throw new IllegalArgumentException(format("start: %d > end: %d", start, end)); } return start <= this.start && this.end <= end; } @@ -81,6 +84,22 @@ public class Boundary implements Comparable { } + public List split(List splitIndices) { + + if(splitIndices.stream().anyMatch(idx -> !this.contains(idx))) { + throw new IndexOutOfBoundsException(format("%s splitting indices are out of range for %s", splitIndices.stream().filter(idx -> !this.contains(idx)).toList(), this)); + } + List splitBoundaries = new LinkedList<>(); + int previousIndex = start; + for (int splitIndex : splitIndices) { + splitBoundaries.add(new Boundary(previousIndex, splitIndex)); + previousIndex = splitIndex; + } + splitBoundaries.add(new Boundary(previousIndex, end)); + return splitBoundaries; + } + + @Override public String toString() { @@ -100,4 +119,16 @@ public class Boundary implements Comparable { return 0; } + @Override + public int hashCode() { + + return toString().hashCode(); + } + + @Override + public boolean equals(Object object) { + + return hashCode() == object.hashCode(); + } + } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentGraph.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentGraph.java index 3f342cd4..e386673b 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentGraph.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentGraph.java @@ -7,7 +7,7 @@ import java.util.stream.Stream; import com.iqser.red.service.redaction.v1.server.document.graph.nodes.PageNode; import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SemanticNode; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.EntityNode; +import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityNode; import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SectionNode; import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock; import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlockCollector; diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/entity/EntityNode.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/entity/EntityNode.java new file mode 100644 index 00000000..d9d06d66 --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/entity/EntityNode.java @@ -0,0 +1,166 @@ +package com.iqser.red.service.redaction.v1.server.document.graph.entity; + +import java.nio.charset.StandardCharsets; +import java.util.HashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Set; + +import com.google.common.hash.Hashing; +import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine; +import com.iqser.red.service.redaction.v1.server.document.graph.Boundary; +import com.iqser.red.service.redaction.v1.server.document.graph.nodes.PageNode; +import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SemanticNode; +import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType; + +import lombok.AccessLevel; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.experimental.FieldDefaults; + +@Data +@Builder +@AllArgsConstructor +@FieldDefaults(level = AccessLevel.PRIVATE) +public class EntityNode { + + public static EntityNode initialEntityNode(Boundary boundary, String type, EntityType entityType) { + + return EntityNode.builder() + .type(type) + .entityType(entityType) + .boundary(boundary) + .redaction(false) + .falsePositive(false) + .removed(false) + .ignored(false) + .resized(false) + .skipRemoveEntitiesContainedInLarger(false) + .dictionaryEntry(false) + .dossierDictionaryEntry(false) + .engines(new HashSet<>()) + .references(new HashSet<>()) + .matchedRule(-1) + .redactionReason("") + .legalBasis("") + .build(); + } + + + // initial values + final Boundary boundary; + final String type; + final EntityType entityType; + + // empty defaults + boolean redaction; + boolean falsePositive; + boolean removed; + boolean ignored; + boolean resized; + boolean skipRemoveEntitiesContainedInLarger; + boolean dictionaryEntry; + boolean dossierDictionaryEntry; + Set engines; + Set references; + int matchedRule; + String redactionReason; + String legalBasis; + + // inferred on graph insertion + String value; + CharSequence textBefore; + CharSequence textAfter; + @Builder.Default + Set pages = new HashSet<>(); + List entityPositions; + @Builder.Default + List intersectingNodes = new LinkedList<>(); + SemanticNode deepestFullyContainingNode; + + + public void addIntersectingNode(SemanticNode containingNode) { + + intersectingNodes.add(containingNode); + } + + + public void removeFromGraph() { + + getIntersectingNodes().forEach(node -> node.getEntities().remove(this)); + deepestFullyContainingNode = null; + getPages().forEach(page -> page.getEntities().remove(this)); + setRemoved(true); + } + + + public List getEntityPositions() { + + if (entityPositions == null || entityPositions.isEmpty()) { + entityPositions = deepestFullyContainingNode.buildTextBlock().getEntityPositions(boundary); + } + + return entityPositions; + } + + + public boolean containedBy(EntityNode entityNode) { + + return this.boundary.containedBy(entityNode.getBoundary()); + } + + + public boolean contains(EntityNode entityNode) { + + return this.boundary.contains(entityNode.getBoundary()); + } + + + public void addEngine(Engine engine) { + + engines.add(engine); + } + + public void addEngines(Set engines) { + + this.engines.addAll(engines); + } + + + @Override + public String toString() { + + StringBuilder sb = new StringBuilder(); + sb.append("Entity[\""); + sb.append(value); + sb.append("\", "); + sb.append(boundary); + sb.append(", pages["); + pages.forEach(page -> { + sb.append(page.getNumber()); + sb.append(", "); + }); + sb.delete(sb.length() - 2, sb.length()); + sb.append("], type = \""); + sb.append(type); + sb.append("\", EntityType."); + sb.append(entityType); + sb.append("]"); + return sb.toString(); + } + + + @Override + public int hashCode() { + + return Hashing.murmur3_128().hashString(toString(), StandardCharsets.UTF_8).hashCode(); + } + + @Override + public boolean equals(Object o) { + + return o.hashCode() == hashCode(); + } + +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/entity/EntityPosition.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/entity/EntityPosition.java new file mode 100644 index 00000000..29aa1cea --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/entity/EntityPosition.java @@ -0,0 +1,36 @@ +package com.iqser.red.service.redaction.v1.server.document.graph.entity; + +import java.awt.geom.Rectangle2D; +import java.nio.charset.StandardCharsets; + +import com.google.common.hash.Hashing; +import com.iqser.red.service.redaction.v1.server.document.graph.nodes.PageNode; + +import lombok.AccessLevel; +import lombok.Builder; +import lombok.Data; +import lombok.experimental.FieldDefaults; + +@Data +@Builder +@FieldDefaults(level = AccessLevel.PRIVATE) +public class EntityPosition { + + PageNode pageNode; + Rectangle2D position; + + + public String getId() { + + return String.valueOf(hashCode()); + } + + + @Override + public int hashCode() { + + String string = String.valueOf(pageNode.getNumber()) + position.getX() + position.getY() + position.getWidth() + position.getHeight(); + return Hashing.murmur3_128().hashString(string, StandardCharsets.UTF_8).hashCode(); + } + +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/ImageNode.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/entity/ImageNode.java similarity index 90% rename from redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/ImageNode.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/entity/ImageNode.java index ec4f4e33..f939eac1 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/ImageNode.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/entity/ImageNode.java @@ -1,4 +1,4 @@ -package com.iqser.red.service.redaction.v1.server.document.graph.nodes; +package com.iqser.red.service.redaction.v1.server.document.graph.entity; import java.awt.geom.Rectangle2D; import java.util.Collections; @@ -10,6 +10,9 @@ import java.util.Set; import com.iqser.red.service.redaction.v1.server.document.graph.PageElement; import com.iqser.red.service.redaction.v1.server.document.graph.TableOfContents; +import com.iqser.red.service.redaction.v1.server.document.graph.nodes.NodeType; +import com.iqser.red.service.redaction.v1.server.document.graph.nodes.PageNode; +import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SemanticNode; import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock; import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlockCollector; import com.iqser.red.service.redaction.v1.server.redaction.model.ImageType; @@ -77,6 +80,7 @@ public class ImageNode extends PageElement implements SemanticNode { return tocId + ": " + NodeType.IMAGE + ": " + imageType.toString() + " " + position; } + @Override public Map getBBox() { diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/factory/DocumentGraphFactory.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/factory/DocumentGraphFactory.java index cfdb1960..e31ed12a 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/factory/DocumentGraphFactory.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/factory/DocumentGraphFactory.java @@ -34,7 +34,7 @@ import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SemanticNo import com.iqser.red.service.redaction.v1.server.document.graph.nodes.FooterNode; import com.iqser.red.service.redaction.v1.server.document.graph.nodes.HeaderNode; import com.iqser.red.service.redaction.v1.server.document.graph.nodes.HeadlineNode; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.ImageNode; +import com.iqser.red.service.redaction.v1.server.document.graph.entity.ImageNode; import com.iqser.red.service.redaction.v1.server.document.graph.nodes.NodeType; import com.iqser.red.service.redaction.v1.server.document.graph.nodes.ParagraphNode; import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SectionNode; @@ -91,7 +91,7 @@ public class DocumentGraphFactory { private void addSection(SemanticNode parentNode, List pageBlocks, List images, Context context) { Map> blocksPerPage = pageBlocks.stream().collect(groupingBy(AbstractTextContainer::getPage)); - SectionNode sectionNode = SectionNode.builder().entities(new HashSet<>()).pages(new HashSet<>()).tableOfContents(context.tableOfContents()).build(); + SectionNode sectionNode = SectionNode.builder().entities(new HashSet<>()).tableOfContents(context.tableOfContents()).build(); context.sections().add(sectionNode); blocksPerPage.keySet().forEach(pageNumber -> addSectionNodeToPageNode(context, sectionNode, pageNumber)); @@ -129,6 +129,7 @@ public class DocumentGraphFactory { private List findTextBlocksWithSameClassificationAndAlignsY(AbstractTextContainer atc, List pageBlocks) { return pageBlocks.stream() + .filter(abstractTextContainer -> !abstractTextContainer.equals(atc)) .filter(abstractTextContainer -> abstractTextContainer instanceof TextBlock) .filter(abstractTextContainer -> abstractTextContainer.intersectsY(atc)) .filter(abstractTextContainer -> abstractTextContainer.getPage() == atc.getPage()) @@ -137,12 +138,9 @@ public class DocumentGraphFactory { } - - private void addSectionNodeToPageNode(Context context, SectionNode sectionNode, Integer pageNumber) { PageNode page = getPage(pageNumber, context); - sectionNode.getPages().add(page); page.getMainBody().add(sectionNode); } @@ -150,12 +148,7 @@ public class DocumentGraphFactory { private void addTable(SemanticNode parentNode, Table table, Context context) { PageNode page = getPage(table.getPage(), context); - TableNode tableNode = TableNode.builder() - .tableOfContents(context.tableOfContents()) - .pages(Collections.singleton(page)) - .numberOfCols(table.getColCount()) - .numberOfRows(table.getRowCount()) - .build(); + TableNode tableNode = TableNode.builder().tableOfContents(context.tableOfContents()).numberOfCols(table.getColCount()).numberOfRows(table.getRowCount()).build(); if (!page.getMainBody().contains(parentNode)) { parentNode.getPages().add(page); @@ -166,32 +159,32 @@ public class DocumentGraphFactory { List tocId = context.tableOfContents().createNewChildEntryAndReturnId(parentNode.getTocId(), NodeType.TABLE, tableNode); tableNode.setTocId(tocId); - addTableCells(table.getRows(), tableNode, context); + addTableCells(table.getRows(), tableNode, context, table.getPage()); } - private void addTableCells(List> rows, SemanticNode parentNode, Context context) { + private void addTableCells(List> rows, SemanticNode parentNode, Context context, int pageNumber) { for (int rowIndex = 0; rowIndex < rows.size(); rowIndex++) { for (int colIndex = 0; colIndex < rows.get(rowIndex).size(); colIndex++) { - addTableCell(rows.get(rowIndex).get(colIndex), rowIndex, colIndex, parentNode, context); + addTableCell(rows.get(rowIndex).get(colIndex), rowIndex, colIndex, parentNode, pageNumber, context); } } } - private void addTableCell(Cell cell, int rowIndex, int colIndex, SemanticNode parentNode, Context context) { + private void addTableCell(Cell cell, int rowIndex, int colIndex, SemanticNode parentNode, int pageNumber, Context context) { + + PageNode page = getPage(pageNumber, context); + cell.getTextBlocks().stream().filter(tb -> tb.getPage() == 0).forEach(tb -> tb.setPage(pageNumber)); - PageNode page = parentNode.getFirstPage(); TableCellNode tableCellNode = TableCellNode.builder() .tableOfContents(context.tableOfContents()) .row(rowIndex) .col(colIndex) .header(cell.isHeaderCell()) - .pages(new HashSet<>()) .bBox(cell.getBounds2D()) .build(); - tableCellNode.getPages().add(page); page.getMainBody().add(tableCellNode); com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock textBlock; @@ -230,22 +223,15 @@ public class DocumentGraphFactory { private void addParagraphOrHeadline(SemanticNode parentNode, TextBlock originalTextBlock, Context context, List textBlocksToMerge) { - PageNode page; - if (originalTextBlock.getPage() == 0) { - page = parentNode.getFirstPage(); - } else { - page = getPage(originalTextBlock.getPage(), context); - } + PageNode page = getPage(originalTextBlock.getPage(), context); + SemanticNode node; if (StringUtils.startsWith(originalTextBlock.getClassification(), "H")) { - node = HeadlineNode.builder().pages(Collections.singleton(page)).tableOfContents(context.tableOfContents()).build(); + node = HeadlineNode.builder().tableOfContents(context.tableOfContents()).build(); } else { - node = ParagraphNode.builder().pages(Collections.singleton(page)).tableOfContents(context.tableOfContents()).build(); + node = ParagraphNode.builder().tableOfContents(context.tableOfContents()).build(); } - if (!page.getMainBody().contains(parentNode)) { - parentNode.getPages().add(page); - } page.getMainBody().add(node); List textBlocks = new LinkedList<>(textBlocksToMerge); @@ -317,7 +303,7 @@ public class DocumentGraphFactory { private void addFooter(List textBlocks, Context context) { PageNode page = getPage(textBlocks.get(0).getPage(), context); - FooterNode footer = FooterNode.builder().page(page).tableOfContents(context.tableOfContents()).build(); + FooterNode footer = FooterNode.builder().tableOfContents(context.tableOfContents()).build(); AtomicTextBlock textBlock = context.textBlockFactory.buildAtomicTextBlock(mergeAndSortTextPositionSequenceByYThenX(textBlocks), footer, context, page); List tocId = context.tableOfContents().createNewEntryAndReturnId(FOOTER, footer); footer.setTocId(tocId); @@ -329,7 +315,7 @@ public class DocumentGraphFactory { public void addHeader(List textBlocks, Context context) { PageNode page = getPage(textBlocks.get(0).getPage(), context); - HeaderNode header = HeaderNode.builder().page(page).tableOfContents(context.tableOfContents()).build(); + HeaderNode header = HeaderNode.builder().tableOfContents(context.tableOfContents()).build(); AtomicTextBlock textBlock = context.textBlockFactory.buildAtomicTextBlock(mergeAndSortTextPositionSequenceByYThenX(textBlocks), header, context, 0, page); List tocId = context.tableOfContents().createNewEntryAndReturnId(HEADER, header); header.setTocId(tocId); @@ -341,7 +327,7 @@ public class DocumentGraphFactory { private void addEmptyFooter(int pageIndex, Context context) { PageNode page = getPage(pageIndex, context); - FooterNode footer = FooterNode.builder().page(page).tableOfContents(context.tableOfContents()).build(); + FooterNode footer = FooterNode.builder().tableOfContents(context.tableOfContents()).build(); AtomicTextBlock textBlock = context.textBlockFactory.emptyTextBlock(footer, context, page); List tocId = context.tableOfContents().createNewEntryAndReturnId(FOOTER, footer); footer.setTocId(tocId); @@ -353,7 +339,7 @@ public class DocumentGraphFactory { private void addEmptyHeader(int pageIndex, Context context) { PageNode page = getPage(pageIndex, context); - HeaderNode header = HeaderNode.builder().page(page).tableOfContents(context.tableOfContents()).build(); + HeaderNode header = HeaderNode.builder().tableOfContents(context.tableOfContents()).build(); AtomicTextBlock textBlock = context.textBlockFactory.emptyTextBlock(header, context, 0, page); List tocId = context.tableOfContents().createNewEntryAndReturnId(HEADER, header); header.setTocId(tocId); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/factory/RectangleTransformations.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/factory/RectangleTransformations.java index 05b91df7..9353d7ab 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/factory/RectangleTransformations.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/factory/RectangleTransformations.java @@ -1,7 +1,14 @@ package com.iqser.red.service.redaction.v1.server.document.graph.factory; +import java.awt.geom.Area; import java.awt.geom.Rectangle2D; import java.util.List; +import java.util.Set; +import java.util.function.BiConsumer; +import java.util.function.BinaryOperator; +import java.util.function.Function; +import java.util.function.Supplier; +import java.util.stream.Collector; import com.iqser.red.service.redaction.v1.server.document.graph.textblock.AtomicTextBlock; import com.iqser.red.service.redaction.v1.server.redaction.model.RedRectangle2D; @@ -17,16 +24,19 @@ public class RectangleTransformations { public static Rectangle2D bBoxUnionAbstractTextContainer(List abstractTextContainers) { - return abstractTextContainers.stream().map(RectangleTransformations::toRectangle2D).reduce((a, b) -> a.createUnion(b).getBounds2D()).orElseThrow(); + return abstractTextContainers.stream().map(RectangleTransformations::toRectangle2D).collect(new Rectangle2DUnion()); } public static Rectangle2D bBoxUnionAtomicTextBlock(List atomicTextBlocks) { - return atomicTextBlocks.stream() - .flatMap(atomicTextBlock -> atomicTextBlock.getPositions().stream()) - .reduce((a, b) -> a.createUnion(b).getBounds2D()) - .orElse(new Rectangle2D.Double()); + return atomicTextBlocks.stream().flatMap(atomicTextBlock -> atomicTextBlock.getPositions().stream()).collect(new Rectangle2DUnion()); + } + + + public static Rectangle2D rectangleUnion(List rectangle2DList) { + + return rectangle2DList.stream().collect(new Rectangle2DUnion()); } @@ -41,4 +51,46 @@ public class RectangleTransformations { return new Rectangle2D.Double(redRectangle2D.getX(), redRectangle2D.getY(), redRectangle2D.getWidth(), redRectangle2D.getHeight()); } + + private static class Rectangle2DUnion implements Collector { + + @Override + public Supplier supplier() { + + return Area::new; + } + + + @Override + public BiConsumer accumulator() { + + return (a, b) -> a.add(new Area(b)); + } + + + @Override + public BinaryOperator combiner() { + + return (area1, area2) -> { + area1.add(area2); + return area1; + }; + } + + + @Override + public Function finisher() { + + return Area::getBounds2D; + } + + + @Override + public Set characteristics() { + + return Set.of(Characteristics.CONCURRENT, Characteristics.UNORDERED); + } + + } + } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/factory/SearchTextWithTextPositionFactory.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/factory/SearchTextWithTextPositionFactory.java index be94dbc5..63e02bec 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/factory/SearchTextWithTextPositionFactory.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/factory/SearchTextWithTextPositionFactory.java @@ -52,10 +52,9 @@ public class SearchTextWithTextPositionFactory { stringIdxToPositionIdx = stringIdxToPositionIdx.subList(0, lastHyphenIdx); stringIdx = lastHyphenIdx; lastHyphenIdx = -3; - - } else { - lineBreaksStringIdx.add(stringIdx); } + lineBreaksStringIdx.add(stringIdx); + } if (!isRepeatedWhitespace(currentTextPosition.getUnicode(), previousTextPosition.getUnicode())) { diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/EntityNode.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/EntityNode.java deleted file mode 100644 index c39165a7..00000000 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/EntityNode.java +++ /dev/null @@ -1,127 +0,0 @@ -package com.iqser.red.service.redaction.v1.server.document.graph.nodes; - -import java.awt.geom.Rectangle2D; -import java.nio.charset.StandardCharsets; -import java.util.HashSet; -import java.util.LinkedList; -import java.util.List; -import java.util.Set; - -import com.google.common.hash.Hashing; -import com.iqser.red.service.redaction.v1.model.Engine; -import com.iqser.red.service.redaction.v1.server.document.graph.Boundary; -import com.iqser.red.service.redaction.v1.server.redaction.model.Entity; -import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType; - -import lombok.AccessLevel; -import lombok.AllArgsConstructor; -import lombok.Builder; -import lombok.Data; -import lombok.experimental.FieldDefaults; - -@Data -@Builder -@AllArgsConstructor -@FieldDefaults(level = AccessLevel.PRIVATE) -public class EntityNode { - - public static EntityNode initialEntityNode(Boundary boundary, String type, EntityType entityType) { - - return EntityNode.builder().type(type).entityType(entityType).boundary(boundary).build(); - } - - - // initial values - Boundary boundary; - String type; - EntityType entityType; - - @Builder.Default - boolean redaction = false; - @Builder.Default - boolean falsePositive = false; - @Builder.Default - boolean removed = false; - @Builder.Default - boolean ignored = false; - @Builder.Default - boolean resized = false; - @Builder.Default - boolean skipRemoveEntitiesContainedInLarger = false; - @Builder.Default - boolean isDictionaryEntry = false; - @Builder.Default - Set engines = new HashSet<>(); - @Builder.Default - Set references = new HashSet<>(); - @Builder.Default - int matchedRule = -1; - @Builder.Default - String redactionReason = ""; - @Builder.Default - String legalBasis = ""; - - // inferred on graph insertion - String value; - CharSequence textBefore; - CharSequence textAfter; - @Builder.Default - Set pages = new HashSet<>(); - List positions; - @Builder.Default - List intersectingNodes = new LinkedList<>(); - SemanticNode deepestFullyContainingNode; - - - public void addIntersectingNode(SemanticNode containingNode) { - - intersectingNodes.add(containingNode); - } - - - public void removeFromGraph() { - - getIntersectingNodes().forEach(node -> node.getEntities().remove(this)); - deepestFullyContainingNode = null; - getPages().forEach(page -> page.getEntities().remove(this)); - setRemoved(true); - } - - - @Override - public String toString() { - - StringBuilder sb = new StringBuilder(); - sb.append("Entity ["); - sb.append(value); - sb.append(", "); - sb.append(boundary); - sb.append(", Pages["); - pages.forEach(page -> { - sb.append(page.getNumber()); - sb.append(", "); - }); - sb.delete(sb.length() - 2, sb.length()); - sb.append("], containingNode "); - sb.append(deepestFullyContainingNode); - return sb.toString(); - } - - - @Override - public int hashCode() { - - StringBuilder sb = new StringBuilder(); - sb.append("Entity ["); - sb.append(value); - sb.append(", "); - sb.append(boundary); - sb.append(", Pages["); - pages.forEach(page -> { - sb.append(page.getNumber()); - sb.append(", "); - }); - return Hashing.murmur3_128().hashString(sb.toString(), StandardCharsets.UTF_8).hashCode(); - } - -} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/FooterNode.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/FooterNode.java index 1395a8c9..e1ddcb2c 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/FooterNode.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/FooterNode.java @@ -1,12 +1,12 @@ package com.iqser.red.service.redaction.v1.server.document.graph.nodes; -import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Set; import com.iqser.red.service.redaction.v1.server.document.graph.PageElement; import com.iqser.red.service.redaction.v1.server.document.graph.TableOfContents; +import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityNode; import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock; import lombok.AccessLevel; @@ -30,9 +30,6 @@ public class FooterNode extends PageElement implements SemanticNode { @Builder.Default boolean terminal = true; - @EqualsAndHashCode.Exclude - PageNode page; - @EqualsAndHashCode.Exclude TableOfContents tableOfContents; @@ -48,13 +45,6 @@ public class FooterNode extends PageElement implements SemanticNode { } - @Override - public Set getPages() { - - return Collections.singleton(page); - } - - @Override public String toString() { diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/HeaderNode.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/HeaderNode.java index 063597ee..e257e8b9 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/HeaderNode.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/HeaderNode.java @@ -1,12 +1,12 @@ package com.iqser.red.service.redaction.v1.server.document.graph.nodes; -import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Set; import com.iqser.red.service.redaction.v1.server.document.graph.PageElement; import com.iqser.red.service.redaction.v1.server.document.graph.TableOfContents; +import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityNode; import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock; import lombok.AccessLevel; @@ -30,9 +30,6 @@ public class HeaderNode extends PageElement implements SemanticNode { @Builder.Default boolean terminal = true; - @EqualsAndHashCode.Exclude - PageNode page; - @EqualsAndHashCode.Exclude TableOfContents tableOfContents; @@ -47,12 +44,6 @@ public class HeaderNode extends PageElement implements SemanticNode { return terminalTextBlock; } - @Override - public Set getPages() { - - return Collections.singleton(page); - } - @Override public String toString() { diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/HeadlineNode.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/HeadlineNode.java index 0b25b583..8bf787fd 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/HeadlineNode.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/HeadlineNode.java @@ -6,6 +6,7 @@ import java.util.Set; import com.iqser.red.service.redaction.v1.server.document.graph.PageElement; import com.iqser.red.service.redaction.v1.server.document.graph.TableOfContents; +import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityNode; import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock; import lombok.AccessLevel; @@ -32,9 +33,6 @@ public class HeadlineNode extends PageElement implements SemanticNode { @EqualsAndHashCode.Exclude TableOfContents tableOfContents; - @EqualsAndHashCode.Exclude - Set pages; - @Builder.Default @EqualsAndHashCode.Exclude Set entities = new HashSet<>(); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/PageNode.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/PageNode.java index be04426a..ae7d698b 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/PageNode.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/PageNode.java @@ -4,11 +4,8 @@ import java.util.HashSet; import java.util.List; import java.util.Set; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SemanticNode; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.EntityNode; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.FooterNode; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.HeaderNode; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.ImageNode; +import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityNode; +import com.iqser.red.service.redaction.v1.server.document.graph.entity.ImageNode; import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock; import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlockCollector; diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/ParagraphNode.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/ParagraphNode.java index 21a2d12d..01d86ec6 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/ParagraphNode.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/ParagraphNode.java @@ -6,6 +6,7 @@ import java.util.Set; import com.iqser.red.service.redaction.v1.server.document.graph.PageElement; import com.iqser.red.service.redaction.v1.server.document.graph.TableOfContents; +import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityNode; import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock; import lombok.AccessLevel; @@ -30,9 +31,6 @@ public class ParagraphNode extends PageElement implements SemanticNode { @EqualsAndHashCode.Exclude TableOfContents tableOfContents; - @EqualsAndHashCode.Exclude - Set pages; - @Builder.Default @EqualsAndHashCode.Exclude Set entities = new HashSet<>(); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/SectionNode.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/SectionNode.java index 573f1e19..4f67dabc 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/SectionNode.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/SectionNode.java @@ -6,6 +6,7 @@ import java.util.Set; import com.iqser.red.service.redaction.v1.server.document.graph.PageElement; import com.iqser.red.service.redaction.v1.server.document.graph.TableOfContents; +import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityNode; import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock; import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlockCollector; import com.iqser.red.service.redaction.v1.server.exception.NotFoundException; @@ -30,8 +31,6 @@ public class SectionNode extends PageElement implements SemanticNode { TextBlock textBlock; @EqualsAndHashCode.Exclude TableOfContents tableOfContents; - @EqualsAndHashCode.Exclude - Set pages; @Builder.Default @EqualsAndHashCode.Exclude diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/SemanticNode.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/SemanticNode.java index b276b8b0..cdb82874 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/SemanticNode.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/SemanticNode.java @@ -10,7 +10,9 @@ import java.util.Map; import java.util.Set; import java.util.stream.Stream; +import com.iqser.red.service.redaction.v1.server.document.graph.Boundary; import com.iqser.red.service.redaction.v1.server.document.graph.TableOfContents; +import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityNode; import com.iqser.red.service.redaction.v1.server.document.graph.factory.RectangleTransformations; import com.iqser.red.service.redaction.v1.server.document.graph.textblock.AtomicTextBlock; import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock; @@ -28,14 +30,17 @@ public interface SemanticNode { /** * Any Node maintains its own Set of Entities. - * This Set contains all Entities, whose boundary intersects the boundary of the entity. + * This Set contains all Entities whose boundary intersects the boundary of this node. * * @return Set of all Entities associated with this Node */ Set getEntities(); - Set getPages(); + default Set getPages() { + + return buildTextBlock().getPages(); + } default PageNode getFirstPage() { @@ -117,6 +122,12 @@ public interface SemanticNode { } + default boolean containsString(String string) { + + return buildTextBlock().getSearchText().contains(string); + } + + default void addThisToEntityIfIntersects(EntityNode entity) { TextBlock textBlock = buildTextBlock(); @@ -148,15 +159,33 @@ public interface SemanticNode { Map bBoxPerPage = new HashMap<>(); if (isTerminal()) { - Map> atomicTextBlockPerPage = buildTextBlock().getAtomicTextBlocks().stream().collect(groupingBy(AtomicTextBlock::getPage)); - atomicTextBlockPerPage.forEach((page, atbs) -> bBoxPerPage.put(page, RectangleTransformations.bBoxUnionAtomicTextBlock(atbs))); - return bBoxPerPage; + return getBBoxFromTerminalTextBlock(bBoxPerPage); } + return getBBoxFromChildren(bBoxPerPage); + } + + + default Boundary getBoundary() { + + return buildTextBlock().getBoundary(); + } + + + private Map getBBoxFromChildren(Map bBoxPerPage) { + return streamChildren().map(SemanticNode::getBBox).reduce((map1, map2) -> { map1.forEach((page, rectangle) -> map2.merge(page, rectangle, (rect1, rect2) -> rect1.createUnion(rect2).getBounds2D())); return map2; }).orElse(bBoxPerPage); } + + private Map getBBoxFromTerminalTextBlock(Map bBoxPerPage) { + + Map> atomicTextBlockPerPage = buildTextBlock().getAtomicTextBlocks().stream().collect(groupingBy(AtomicTextBlock::getPage)); + atomicTextBlockPerPage.forEach((page, atbs) -> bBoxPerPage.put(page, RectangleTransformations.bBoxUnionAtomicTextBlock(atbs))); + return bBoxPerPage; + } + } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/TableCellNode.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/TableCellNode.java index 8b2233e3..9e9bdc2e 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/TableCellNode.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/TableCellNode.java @@ -10,6 +10,7 @@ import java.util.stream.Stream; import com.iqser.red.service.redaction.v1.server.document.graph.PageElement; import com.iqser.red.service.redaction.v1.server.document.graph.TableOfContents; +import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityNode; import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock; import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlockCollector; @@ -42,9 +43,6 @@ public class TableCellNode extends PageElement implements SemanticNode { @EqualsAndHashCode.Exclude TableOfContents tableOfContents; - @EqualsAndHashCode.Exclude - Set pages; - @Builder.Default @EqualsAndHashCode.Exclude Set entities = new HashSet<>(); @@ -54,10 +52,11 @@ public class TableCellNode extends PageElement implements SemanticNode { public Map getBBox() { Map bBoxPerPage = new HashMap<>(); - pages.forEach(page -> bBoxPerPage.put(page, bBox)); + getPages().forEach(page -> bBoxPerPage.put(page, bBox)); return bBoxPerPage; } + @Override public TextBlock buildTextBlock() { @@ -67,13 +66,22 @@ public class TableCellNode extends PageElement implements SemanticNode { return textBlock; } + @Override public String toString() { return tocId + ": " + NodeType.TABLE_CELL + ": " + buildTextBlock().getSearchText(); } - public Stream getHeaders() { + + public boolean hasHeader(String headerString) { + + return getHeaders().anyMatch(header -> header.buildTextBlock().getSearchText().contains(headerString)); + } + + + private Stream getHeaders() { + TableNode tableNode = (TableNode) getParent(); return tableNode.streamHeadersForCell(row, col); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/TableNode.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/TableNode.java index c371f4c3..2c5a2ed6 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/TableNode.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/TableNode.java @@ -7,6 +7,7 @@ import java.util.stream.Stream; import com.iqser.red.service.redaction.v1.server.document.graph.PageElement; import com.iqser.red.service.redaction.v1.server.document.graph.TableOfContents; +import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityNode; import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock; import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlockCollector; @@ -30,8 +31,6 @@ public class TableNode extends PageElement implements SemanticNode { Integer numberOfCols; TextBlock textBlock; - @EqualsAndHashCode.Exclude - Set pages; @Builder.Default @EqualsAndHashCode.Exclude diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/textblock/AtomicTextBlock.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/textblock/AtomicTextBlock.java index d99c9a9c..6e4fa403 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/textblock/AtomicTextBlock.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/textblock/AtomicTextBlock.java @@ -3,11 +3,14 @@ package com.iqser.red.service.redaction.v1.server.document.graph.textblock; import static java.lang.String.format; import java.awt.geom.Rectangle2D; +import java.util.LinkedList; import java.util.List; import com.iqser.red.service.redaction.v1.server.document.graph.Boundary; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SemanticNode; +import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityPosition; +import com.iqser.red.service.redaction.v1.server.document.graph.factory.RectangleTransformations; import com.iqser.red.service.redaction.v1.server.document.graph.nodes.PageNode; +import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SemanticNode; import lombok.AccessLevel; import lombok.AllArgsConstructor; @@ -42,7 +45,21 @@ public class AtomicTextBlock implements TextBlock { @Override public int numberOfLines() { - return lineBreaks.size(); + return lineBreaks.size() + 1; + } + + + public CharSequence getLine(int lineNumber) { + + if (lineNumber >= numberOfLines() || lineNumber < 0) { + throw new IndexOutOfBoundsException(format("line %d out of range for AtomicTextBlock with %d lines", lineNumber, numberOfLines())); + } + if (lineNumber == 0) { + return subSequence(boundary.start(), lineBreaks.get(0) + boundary.start()); + } else if (lineNumber == numberOfLines() - 1) { + return subSequence(lineBreaks.get(lineBreaks.size() - 1) + boundary.start(), boundary.end()); + } + return subSequence(lineBreaks.get(lineNumber - 1) + boundary.start(), lineBreaks.get(lineNumber) + boundary.start()); } @@ -81,17 +98,34 @@ public class AtomicTextBlock implements TextBlock { @Override - public List getPositions(Boundary boundary) { + public List getPositions(Boundary stringBoundary) { - if (!containsBoundary(boundary)) { - throw new IndexOutOfBoundsException(format("%s is out of bounds for %s", boundary, this.boundary)); + if (!containsBoundary(stringBoundary)) { + throw new IndexOutOfBoundsException(format("%s is out of bounds for %s", stringBoundary, this.boundary)); } - if (boundary.end() == this.boundary.end()) { - return positions.subList(stringIdxToPositionIdx.get(boundary.start() - this.boundary.start()), positions.size()); + if (stringBoundary.end() == this.boundary.end()) { + return positions.subList(stringIdxToPositionIdx.get(stringBoundary.start() - this.boundary.start()), positions.size()); } - return positions.subList(stringIdxToPositionIdx.get(boundary.start() - this.boundary.start()), stringIdxToPositionIdx.get(boundary.end() - this.boundary.start())); + return positions.subList(stringIdxToPositionIdx.get(stringBoundary.start() - this.boundary.start()), + stringIdxToPositionIdx.get(stringBoundary.end() - this.boundary.start())); + } + + + public List getEntityPositions(Boundary stringBoundary) { + + List positionsPerLine = stringBoundary.split(getLineBreaks().stream().map(lb -> lb + boundary.start()).filter(stringBoundary::contains).toList()) + .stream() + .map(this::getPositions) + .map(RectangleTransformations::rectangleUnion) + .toList(); + + List entityPositions = new LinkedList<>(); + for (Rectangle2D position : positionsPerLine) { + entityPositions.add(EntityPosition.builder().position(position).pageNode(page).build()); + } + return entityPositions; } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/textblock/ConcatenatedTextBlock.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/textblock/ConcatenatedTextBlock.java index 5d01507d..0360b62d 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/textblock/ConcatenatedTextBlock.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/textblock/ConcatenatedTextBlock.java @@ -8,6 +8,7 @@ import java.util.List; import java.util.function.Supplier; import com.iqser.red.service.redaction.v1.server.document.graph.Boundary; +import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityPosition; import lombok.AccessLevel; import lombok.Data; @@ -57,7 +58,7 @@ public class ConcatenatedTextBlock implements TextBlock, Supplier getAllAtomicTextBlocksPartiallyInStringIdxRange(Boundary boundary) { + private List getAllAtomicTextBlocksPartiallyInStringBoundary(Boundary boundary) { return atomicTextBlocks.stream().filter(tb -> tb.getBoundary().intersects(boundary)).toList(); } @@ -95,6 +96,12 @@ public class ConcatenatedTextBlock implements TextBlock, Supplier getLineBreaks() { + + return getAtomicTextBlocks().stream().flatMap(atomicTextBlock -> atomicTextBlock.getLineBreaks().stream()).toList(); + } + @Override public Rectangle2D getPosition(int stringIdx) { @@ -104,23 +111,46 @@ public class ConcatenatedTextBlock implements TextBlock, Supplier getPositions(Boundary boundary) { + public List getPositions(Boundary stringBoundary) { - List textBlocks = getAllAtomicTextBlocksPartiallyInStringIdxRange(boundary); + List textBlocks = getAllAtomicTextBlocksPartiallyInStringBoundary(stringBoundary); if (textBlocks.size() == 1) { - return textBlocks.get(0).getPositions(boundary); + return textBlocks.get(0).getPositions(stringBoundary); } AtomicTextBlock firstTextBlock = textBlocks.get(0); - List positions = new LinkedList<>(firstTextBlock.getPositions(new Boundary(boundary.start(), firstTextBlock.getBoundary().end()))); + List positions = new LinkedList<>(firstTextBlock.getPositions(new Boundary(stringBoundary.start(), firstTextBlock.getBoundary().end()))); for (AtomicTextBlock textBlock : textBlocks.subList(1, textBlocks.size() - 1)) { positions.addAll(textBlock.getPositions()); } var lastTextBlock = textBlocks.get(textBlocks.size() - 1); - positions.addAll(lastTextBlock.getPositions(new Boundary(lastTextBlock.getBoundary().start(), boundary.end()))); + positions.addAll(lastTextBlock.getPositions(new Boundary(lastTextBlock.getBoundary().start(), stringBoundary.end()))); + + return positions; + } + + + @Override + public List getEntityPositions(Boundary stringBoundary) { + + List textBlocks = getAllAtomicTextBlocksPartiallyInStringBoundary(stringBoundary); + + if (textBlocks.size() == 1) { + return textBlocks.get(0).getEntityPositions(stringBoundary); + } + + AtomicTextBlock firstTextBlock = textBlocks.get(0); + List positions = new LinkedList<>(firstTextBlock.getEntityPositions(new Boundary(stringBoundary.start(), firstTextBlock.getBoundary().end()))); + + for (AtomicTextBlock textBlock : textBlocks.subList(1, textBlocks.size() - 1)) { + positions.addAll(textBlock.getEntityPositions(textBlock.getBoundary())); + } + + AtomicTextBlock lastTextBlock = textBlocks.get(textBlocks.size() - 1); + positions.addAll(lastTextBlock.getEntityPositions(new Boundary(lastTextBlock.getBoundary().start(), stringBoundary.end()))); return positions; } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/textblock/TextBlock.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/textblock/TextBlock.java index 0a880d63..c9229bd8 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/textblock/TextBlock.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/textblock/TextBlock.java @@ -4,8 +4,12 @@ import static java.lang.String.format; import java.awt.geom.Rectangle2D; import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; import com.iqser.red.service.redaction.v1.server.document.graph.Boundary; +import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityPosition; +import com.iqser.red.service.redaction.v1.server.document.graph.nodes.PageNode; public interface TextBlock extends CharSequence { @@ -23,21 +27,30 @@ public interface TextBlock extends CharSequence { int getPreviousLinebreak(int fromIndex); + List getLineBreaks(); + Rectangle2D getPosition(int stringIdx); - List getPositions(Boundary range); + List getPositions(Boundary stringBoundary); + + + List getEntityPositions(Boundary stringBoundary); int numberOfLines(); - default int indexOf(String searchTerm) { return indexOf(searchTerm, getBoundary().start()); } + default Set getPages() { + + return getAtomicTextBlocks().stream().map(AtomicTextBlock::getPage).collect(Collectors.toUnmodifiableSet()); + } + default int indexOf(String searchTerm, int startOffset) { diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/EntityCreationService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/EntityCreationService.java index 30e09bcc..fea63003 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/EntityCreationService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/EntityCreationService.java @@ -2,19 +2,22 @@ package com.iqser.red.service.redaction.v1.server.document.services; import static java.lang.String.format; +import java.util.List; import java.util.Set; +import java.util.stream.Collectors; import org.springframework.stereotype.Service; import com.iqser.red.service.redaction.v1.server.document.graph.Boundary; import com.iqser.red.service.redaction.v1.server.document.graph.DocumentGraph; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.PageNode; import com.iqser.red.service.redaction.v1.server.document.graph.TableOfContents; +import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityNode; +import com.iqser.red.service.redaction.v1.server.document.graph.nodes.PageNode; import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SemanticNode; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.EntityNode; import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock; import com.iqser.red.service.redaction.v1.server.exception.NotFoundException; import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType; +import com.iqser.red.service.redaction.v1.server.redaction.utils.SearchImplementation; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; @@ -24,10 +27,29 @@ import lombok.extern.slf4j.Slf4j; @RequiredArgsConstructor public class EntityCreationService { - private final EntityEnrichmentService entityEnrichmentService; + private final EntityTextEnrichmentService entityEnrichmentService; - public EntityNode createAndAddEntity(Boundary boundary, String type, EntityType entityType, DocumentGraph documentGraph) { + public Set createEntitiesByLineAfterString(String string, SemanticNode node, String type, EntityType entityType, DocumentGraph documentGraph) { + + TextBlock textBlock = node.buildTextBlock(); + SearchImplementation searchImplementation = new SearchImplementation(List.of(string), true); + List boundaries = searchImplementation.getBoundaries(textBlock, node.getBoundary()); + return boundaries.stream() + .map(boundary -> toLineAfterBoundary(textBlock, boundary)) + .map(boundary -> createEntityByBoundary(boundary, type, entityType, documentGraph)) + .collect(Collectors.toUnmodifiableSet()); + } + + + public Set createEntitiesByRegex(String regexPattern, String type, EntityType entityType, DocumentGraph documentGraph) { + + List boundaries = RegexMatcher.findBoundaries(regexPattern, documentGraph.buildTextBlock()); + return boundaries.stream().map(boundary -> createEntityByBoundary(boundary, type, entityType, documentGraph)).collect(Collectors.toUnmodifiableSet()); + } + + + public EntityNode createEntityByBoundary(Boundary boundary, String type, EntityType entityType, DocumentGraph documentGraph) { EntityNode entity = EntityNode.initialEntityNode(boundary, type, entityType); addEntityToGraph(entity, documentGraph); @@ -35,7 +57,21 @@ public class EntityCreationService { } - public EntityNode createAndAddEntity(String entityName, int startOffset, String type, EntityType entityType, DocumentGraph documentGraph) { + public EntityNode createEntityByEntity(EntityNode entity, String type, EntityType entityType, DocumentGraph documentGraph) { + + return createEntityByBoundary(entity.getBoundary(), type, entityType, documentGraph); + } + + + public EntityNode createEntityBySemanticNode(SemanticNode node, String type, EntityType entityType, DocumentGraph documentGraph) { + + Boundary boundary = node.buildTextBlock().getBoundary(); + Boundary nodeBoundaryWithoutTrailingWhitespace = new Boundary(boundary.start(), boundary.end() - 1); + return createEntityByBoundary(nodeBoundaryWithoutTrailingWhitespace, type, entityType, documentGraph); + } + + + public Set createEntitiesByString(String entityName, int startOffset, String type, EntityType entityType, DocumentGraph documentGraph) { int start = documentGraph.buildTextBlock().indexOf(entityName, startOffset); if (start == -1) { @@ -43,7 +79,7 @@ public class EntityCreationService { } EntityNode entity = EntityNode.initialEntityNode(new Boundary(start, start + entityName.length()), type, entityType); addEntityToGraph(entity, documentGraph); - return entity; + return Set.of(entity); } @@ -87,4 +123,9 @@ public class EntityCreationService { entity.getIntersectingNodes().forEach(node -> node.getEntities().add(entity)); } + private static Boundary toLineAfterBoundary(TextBlock textBlock, Boundary boundary) { + + return new Boundary(boundary.end(), textBlock.getNextLinebreak(boundary.end())); + } + } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/EntityEnrichmentService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/EntityTextEnrichmentService.java similarity index 95% rename from redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/EntityEnrichmentService.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/EntityTextEnrichmentService.java index 8e6717c8..81aefc40 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/EntityEnrichmentService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/EntityTextEnrichmentService.java @@ -6,7 +6,7 @@ import java.util.Objects; import org.springframework.stereotype.Service; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.EntityNode; +import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityNode; import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock; import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings; @@ -14,16 +14,16 @@ import lombok.RequiredArgsConstructor; @Service @RequiredArgsConstructor -public class EntityEnrichmentService { +public class EntityTextEnrichmentService { private final RedactionServiceSettings redactionServiceSettings; + public EntityNode enrichEntity(EntityNode entity, TextBlock textBlock) { - entity.setPositions(textBlock.getPositions(entity.getBoundary())); + entity.setValue(textBlock.subSequence(entity.getBoundary()).toString()); entity.setTextAfter(findTextAfter(entity.getBoundary().end(), textBlock)); entity.setTextBefore(findTextBefore(entity.getBoundary().start(), textBlock)); - entity.setValue(textBlock.subSequence(entity.getBoundary()).toString()); return entity; } @@ -43,8 +43,6 @@ public class EntityEnrichmentService { } - - private CharSequence findTextBefore(int index, TextBlock textBlock) { int offsetBefore = Math.max(index - redactionServiceSettings.getSurroundingWordsOffsetWindow(), textBlock.getBoundary().start()); @@ -65,11 +63,11 @@ public class EntityEnrichmentService { return Arrays.stream(textAfter.split(" ")).filter(word -> !Objects.equals("", word)).toList(); } + private String concatWordsBefore(List words, boolean endWithSpace) { StringBuilder sb = new StringBuilder(); - for (String word : words) { sb.append(word).append(" "); } @@ -78,11 +76,12 @@ public class EntityEnrichmentService { return endWithSpace ? result + " " : result; } + private String concatWordsAfter(List words, boolean startWithSpace) { StringBuilder sb = new StringBuilder(); - for (String word: words) { + for (String word : words) { sb.append(word).append(" "); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/ImageSortService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/ImageSortService.java index 00e36092..4980b089 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/ImageSortService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/ImageSortService.java @@ -1,8 +1,9 @@ package com.iqser.red.service.redaction.v1.server.document.services; +import static com.iqser.red.service.redaction.v1.server.document.graph.factory.RectangleTransformations.bBoxUnionAbstractTextContainer; +import static com.iqser.red.service.redaction.v1.server.document.graph.factory.RectangleTransformations.toRectangle2D; import static java.util.stream.Collectors.groupingBy; -import java.awt.geom.Rectangle2D; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -63,9 +64,6 @@ public class ImageSortService { return; } } - if (sectionContainingTextContainer.isPresent()) { - - } } @@ -89,21 +87,24 @@ public class ImageSortService { private static Optional getContainingCell(Table table, PdfImage image) { - return table.getRows().stream().flatMap(List::stream).filter(cell -> cell.contains(toRectangle2D(image))).findFirst(); + return table.getRows().stream().flatMap(List::stream).filter(cell -> cell.contains(toRectangle2D(image.getPosition()))).findFirst(); } private List
getContainedSections(PdfImage image, List
sectionsOnPage) { return sectionsOnPage.stream() - .filter(section -> toRectangle2D(image).contains(bBoxUnion(section.getPageBlocks().stream().filter(block -> block.getPage() == image.getPage()).toList()))) + .filter(section -> toRectangle2D(image.getPosition()).contains(bBoxUnionAbstractTextContainer(section.getPageBlocks() + .stream() + .filter(block -> block.getPage() == image.getPage()) + .toList()))) .toList(); } private List getContainedTextContainers(PdfImage image, List textContainersOnPage) { - return textContainersOnPage.stream().filter(textContainer -> toRectangle2D(image).contains(toRectangle2D(textContainer))).toList(); + return textContainersOnPage.stream().filter(textContainer -> toRectangle2D(image.getPosition()).contains(toRectangle2D(textContainer))).toList(); } @@ -111,33 +112,14 @@ public class ImageSortService { return sectionsOnPage.stream()// .filter(section -> // - bBoxUnion(section.getPageBlocks().stream().filter(block -> block.getPage() == image.getPage()).toList())// - .contains(toRectangle2D(image))).findFirst(); + bBoxUnionAbstractTextContainer(section.getPageBlocks().stream().filter(block -> block.getPage() == image.getPage()).toList())// + .contains(toRectangle2D(image.getPosition()))).findFirst(); } private Optional getContainingTextContainer(PdfImage image, List textContainersOnPage) { - return textContainersOnPage.stream().filter(textContainer -> toRectangle2D(textContainer).contains(toRectangle2D(image))).findFirst(); - } - - - private static Rectangle2D toRectangle2D(AbstractTextContainer container) { - - return new Rectangle2D.Double(container.getMinX(), container.getMinY(), container.getWidth(), container.getHeight()); - } - - - private static Rectangle2D toRectangle2D(PdfImage image) { - - var redRectangle2D = image.getPosition(); - return new Rectangle2D.Double(redRectangle2D.getX(), redRectangle2D.getY(), redRectangle2D.getWidth(), redRectangle2D.getHeight()); - } - - - private static Rectangle2D bBoxUnion(List blocks) { - - return blocks.stream().map(ImageSortService::toRectangle2D).reduce((a, b) -> a.createUnion(b).getBounds2D()).orElse(new Rectangle2D.Double()); + return textContainersOnPage.stream().filter(textContainer -> toRectangle2D(textContainer).contains(toRectangle2D(image.getPosition()))).findFirst(); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/RegexMatcher.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/RegexMatcher.java index de35c4a0..173338e8 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/RegexMatcher.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/RegexMatcher.java @@ -3,6 +3,7 @@ package com.iqser.red.service.redaction.v1.server.document.services; import java.util.LinkedList; import java.util.List; import java.util.regex.Matcher; +import java.util.regex.Pattern; import com.iqser.red.service.redaction.v1.server.document.graph.Boundary; import com.iqser.red.service.redaction.v1.server.redaction.utils.Patterns; @@ -11,21 +12,21 @@ public class RegexMatcher { public static boolean anyMatch(CharSequence searchText, String regexPattern) { - var pattern = Patterns.getCompiledPattern(regexPattern, false); + Pattern pattern = Patterns.getCompiledPattern(regexPattern, false); return pattern.matcher(searchText).find(); } public static Boundary findFirstBoundary(String regexPattern, CharSequence searchText) { - var pattern = Patterns.getCompiledPattern(regexPattern, false); + Pattern pattern = Patterns.getCompiledPattern(regexPattern, false); Matcher matcher = pattern.matcher(searchText); return new Boundary(matcher.start(), matcher.end()); } public static List findBoundaries(String regexPattern, CharSequence searchText) { - var pattern = Patterns.getCompiledPattern(regexPattern, false); + Pattern pattern = Patterns.getCompiledPattern(regexPattern, false); Matcher matcher = pattern.matcher(searchText); List boundaries = new LinkedList<>(); while (matcher.find()) { diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Entity.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Entity.java index e73aec45..e297bc45 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Entity.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Entity.java @@ -1,15 +1,19 @@ package com.iqser.red.service.redaction.v1.server.redaction.model; -import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine; -import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence; - -import lombok.*; - import java.util.ArrayList; import java.util.HashSet; import java.util.List; import java.util.Set; +import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine; +import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; + @Data @Builder @NoArgsConstructor @@ -66,7 +70,6 @@ public class Entity implements ReasonHolder { String headline, int matchedRule, int sectionNumber, - int paragraphNumber, String legalBasis, boolean isDictionaryEntry, String textBefore, @@ -86,7 +89,6 @@ public class Entity implements ReasonHolder { this.headline = headline; this.matchedRule = matchedRule; this.sectionNumber = sectionNumber; - this.paragraphNumber = paragraphNumber; this.legalBasis = legalBasis; this.isDictionaryEntry = isDictionaryEntry; this.textBefore = textBefore; @@ -106,7 +108,6 @@ public class Entity implements ReasonHolder { Integer end, String headline, int sectionNumber, - int paragraphNumber, boolean isDictionaryEntry, boolean isDossierDictionaryEntry, Engine engine, @@ -118,7 +119,6 @@ public class Entity implements ReasonHolder { this.end = end; this.headline = headline; this.sectionNumber = sectionNumber; - this.paragraphNumber = paragraphNumber; this.isDictionaryEntry = isDictionaryEntry; this.isDossierDictionaryEntry = isDossierDictionaryEntry; this.engines.add(engine); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java index d27f2ff8..27aa9cfc 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java @@ -1363,7 +1363,6 @@ public class Section { value.getRowSpanStart() + word.length(), headline, sectionNumber, - -1, false, false, Engine.RULE, diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/SectionSearchableTextPair.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/SectionSearchableTextPair.java new file mode 100644 index 00000000..d164e7b9 --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/SectionSearchableTextPair.java @@ -0,0 +1,16 @@ +package com.iqser.red.service.redaction.v1.server.redaction.model; + +import java.util.List; + +import lombok.AllArgsConstructor; +import lombok.Data; + +@Data +@AllArgsConstructor +public class SectionSearchableTextPair { + + private Section section; + private SearchableText searchableText; + private List cellStarts; + +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DroolsExecutionService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DroolsExecutionService.java index 32e073f0..e5456ac4 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DroolsExecutionService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DroolsExecutionService.java @@ -104,6 +104,18 @@ public class DroolsExecutionService { } + @Timed("redactmanager_executeRules") + public Section executeRules(KieContainer kieContainer, Section section) { + + KieSession kieSession = kieContainer.newKieSession(); + kieSession.insert(section); + kieSession.fireAllRules(); + kieSession.dispose(); + + return section; + + } + public List getEntities(KieSession ks) { List entities = new LinkedList<>(); QueryResults entitiesResult = ks.getQueryResults("getEntities"); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java index f83c07b5..747d0baf 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java @@ -1,6 +1,7 @@ package com.iqser.red.service.redaction.v1.server.redaction.service; import java.util.ArrayList; +import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -12,6 +13,8 @@ import org.springframework.stereotype.Service; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Point; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry; +import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityNode; +import com.iqser.red.service.redaction.v1.server.document.graph.nodes.PageNode; import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence; import com.iqser.red.service.redaction.v1.server.redaction.model.Entity; import com.iqser.red.service.redaction.v1.server.redaction.model.EntityPositionSequence; @@ -32,6 +35,22 @@ public class RedactionLogCreatorService { private final DictionaryService dictionaryService; + public List createRedactionLog(Map> entityNodesPerPageNode, int numberOfPages, String dossierTemplateId) { + + List entries = new ArrayList<>(); + entityNodesPerPageNode.forEach(((pageNode, entityNodes) -> entries.addAll(toRedactionLogEntries(pageNode, entityNodes, dossierTemplateId)))); + + + return entries; + } + + + private Set toRedactionLogEntries(PageNode pageNode, Set entityNodes, String dossierTemplateId) { + + return Collections.emptySet(); + } + + @Timed("redactmanager_createRedactionLog") public List createRedactionLog(PageEntities pageEntities, int numberOfPages, String dossierTemplateId) { diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/entityredaction/EntityRedactionService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/entityredaction/EntityRedactionService.java index 6c945bef..c0c9cd4a 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/entityredaction/EntityRedactionService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/entityredaction/EntityRedactionService.java @@ -26,12 +26,14 @@ import com.iqser.red.service.redaction.v1.server.redaction.model.EntityPositionS import com.iqser.red.service.redaction.v1.server.redaction.model.FindEntitiesResult; import com.iqser.red.service.redaction.v1.server.redaction.model.Image; import com.iqser.red.service.redaction.v1.server.redaction.model.PageEntities; +import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText; import com.iqser.red.service.redaction.v1.server.redaction.model.Section; import com.iqser.red.service.redaction.v1.server.redaction.model.SectionSearchableTextPair; import com.iqser.red.service.redaction.v1.server.redaction.service.DroolsExecutionService; import com.iqser.red.service.redaction.v1.server.redaction.service.SurroundingWordsService; import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUtils; import com.iqser.red.service.redaction.v1.server.redaction.utils.IdBuilder; +import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings; import lombok.AccessLevel; import lombok.RequiredArgsConstructor; @@ -45,8 +47,8 @@ import lombok.extern.slf4j.Slf4j; public class EntityRedactionService { DroolsExecutionService droolsExecutionService; - SurroundingWordsService surroundingWordsService; EntityFinder entityFinder; + RedactionServiceSettings redactionServiceSettings; public PageEntities findEntities(Dictionary dictionary, List sectionTexts, KieContainer kieContainer, AnalyzeRequest analyzeRequest, NerEntities nerEntities) { @@ -78,20 +80,20 @@ public class EntityRedactionService { } Map> entitiesPerPage = convertToEntitiesPerPage(findEntitiesResult.getEntities()); - EntitySearchUtils.removeEntitiesContainedInRedactedLogos(imagesPerPage, entitiesPerPage); + //EntitySearchUtils.removeEntitiesContainedInRedactedLogos(imagesPerPage, entitiesPerPage); return new PageEntities(entitiesPerPage, imagesPerPage, findEntitiesResult.getAddedFileAttributes()); } - public FindEntitiesResult findEntities(List reanalysisSections, - Dictionary dictionary, - KieContainer kieContainer, - AnalyzeRequest analyzeRequest, - boolean local, - Map> hintsPerSectionNumber, - Map> imagesPerPage, - NerEntities nerEntities) { + private FindEntitiesResult findEntities(List reanalysisSections, + Dictionary dictionary, + KieContainer kieContainer, + AnalyzeRequest analyzeRequest, + boolean local, + Map> hintsPerSectionNumber, + Map> imagesPerPage, + NerEntities nerEntities) { List sectionSearchableTextPairs = extractSearchableTextPairs(reanalysisSections, dictionary, @@ -123,14 +125,7 @@ public class EntityRedactionService { .filter(e -> e.getTextAfter() == null && e.getTextBefore() == null) .collect(Collectors.toSet()); - if (sectionSearchableTextPair.getCellStarts() != null && !sectionSearchableTextPair.getCellStarts().isEmpty()) { - surroundingWordsService.addSurroundingText(entriesWithoutSurroundingText, - sectionSearchableTextPair.getSearchableText(), - dictionary, - sectionSearchableTextPair.getCellStarts()); - } else { - surroundingWordsService.addSurroundingText(entriesWithoutSurroundingText, sectionSearchableTextPair.getSearchableText(), dictionary); - } + addSurroundingText(dictionary, sectionSearchableTextPair.getSearchableText(), sectionSearchableTextPair.getCellStarts(), entriesWithoutSurroundingText); entities.addAll(analysedSection.getEntities()); @@ -147,6 +142,24 @@ public class EntityRedactionService { } + private void addSurroundingText(Dictionary dictionary, SearchableText searchableText, List cellStarts, Set entriesWithoutSurroundingText) { + + if (cellStarts != null && !cellStarts.isEmpty()) { + SurroundingWordsService.addSurroundingText(entriesWithoutSurroundingText, + searchableText, dictionary, + cellStarts, + redactionServiceSettings.getSurroundingWordsOffsetWindow(), + redactionServiceSettings.getNumberOfSurroundingWords()); + + } else { + SurroundingWordsService.addSurroundingText(entriesWithoutSurroundingText, + searchableText, dictionary, + redactionServiceSettings.getSurroundingWordsOffsetWindow(), + redactionServiceSettings.getNumberOfSurroundingWords()); + } + } + + private List extractSearchableTextPairs(List reanalysisSections, Dictionary dictionary, AnalyzeRequest analyzeRequest, @@ -165,11 +178,7 @@ public class EntityRedactionService { reanalysisSection.getCellStarts(), analyzeRequest.getManualRedactions()); - if (reanalysisSection.getCellStarts() != null && !reanalysisSection.getCellStarts().isEmpty()) { - surroundingWordsService.addSurroundingText(entities.getEntities(), reanalysisSection.getSearchableText(), dictionary, reanalysisSection.getCellStarts()); - } else { - surroundingWordsService.addSurroundingText(entities.getEntities(), reanalysisSection.getSearchableText(), dictionary); - } + addSurroundingText(dictionary, reanalysisSection.getSearchableText(), reanalysisSection.getCellStarts(), entities.getEntities()); if (!local && analyzeRequest.getManualRedactions() != null) { diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/EntitySearchUtils.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/EntitySearchUtils.java index 33b64706..7d428de6 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/EntitySearchUtils.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/EntitySearchUtils.java @@ -63,7 +63,7 @@ public final class EntitySearchUtils { Set entities = new HashSet<>(); - searchImplementation.getMatches(inputString).forEach(match -> validateAndAddEntity(entities, findEntityDetails, inputString, match.getStartIndex(), match.getEndIndex())); + searchImplementation.getMatches(inputString).forEach(match -> validateAndAddEntity(entities, findEntityDetails, inputString, match.startIndex(), match.endIndex())); return entities; } @@ -190,7 +190,7 @@ public final class EntitySearchUtils { public void removeEntitiesContainedInLarger(Set entities) { List wordsToRemove = new ArrayList<>(); - for (Entity word : entities) { + for (Entity outer : entities) { for (Entity inner : entities) { // // skip cross-type false positives // if(word.getEntityType() == EntityType.FALSE_POSITIVE && inner.getEntityType() == EntityType.ENTITY && !inner.getType().equals(word.getType())){ @@ -201,17 +201,19 @@ public final class EntitySearchUtils { // continue; // } - if ((inner.isSkipRemoveEntitiesContainedInLarger() || word.isSkipRemoveEntitiesContainedInLarger()) && !inner.getType().equals(word.getType())) { + if ((inner.isSkipRemoveEntitiesContainedInLarger() || outer.isSkipRemoveEntitiesContainedInLarger()) && !inner.getType().equals(outer.getType())) { continue; } - if (inner.getWord().length() < word.getWord() - .length() && inner.getStart() >= word.getStart() && inner.getEnd() <= word.getEnd() && word != inner && word.getSectionNumber() == inner.getSectionNumber()) { - if (word.getEntityType().equals(EntityType.RECOMMENDATION) && inner.getEntityType().equals(EntityType.ENTITY)) { - wordsToRemove.add(word); - } else if (!(inner.getEntityType() == EntityType.FALSE_RECOMMENDATION && word.getEntityType() == EntityType.ENTITY || inner.getEntityType() == EntityType.ENTITY && word.getEntityType() == EntityType.FALSE_RECOMMENDATION)) { + if (inner.getWord().length() < outer.getWord() + .length() && inner.getStart() >= outer.getStart() && inner.getEnd() <= outer.getEnd() && outer != inner && outer.getSectionNumber() == inner.getSectionNumber()) { + if (outer.getEntityType().equals(EntityType.RECOMMENDATION) && inner.getEntityType().equals(EntityType.ENTITY)) { + wordsToRemove.add(outer); + } else if (!( + inner.getEntityType() == EntityType.FALSE_RECOMMENDATION && outer.getEntityType() == EntityType.ENTITY || + inner.getEntityType() == EntityType.ENTITY && outer.getEntityType() == EntityType.FALSE_RECOMMENDATION)) { if (inner.isResized()) { - wordsToRemove.add(word); + wordsToRemove.add(outer); } else { wordsToRemove.add(inner); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/IdBuilder.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/IdBuilder.java index 13ed4f39..246ea964 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/IdBuilder.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/IdBuilder.java @@ -1,5 +1,6 @@ package com.iqser.red.service.redaction.v1.server.redaction.utils; +import java.awt.geom.Rectangle2D; import java.nio.charset.StandardCharsets; import java.util.List; @@ -27,6 +28,12 @@ public final class IdBuilder { } + public String buildId(Rectangle2D rectangle2D, int page) { + + return hashFunction.hashString("x" + rectangle2D.getX() + "y" + rectangle2D.getY() + "h" + rectangle2D.getHeight() + "w" + rectangle2D.getWidth() + "p" + page, + StandardCharsets.UTF_8).toString(); + } + public String buildId(RedRectangle2D rectangle2D, int page) { return hashFunction.hashString("x" + rectangle2D.getX() + "y" + rectangle2D.getY() + "h" + rectangle2D.getHeight() + "w" + rectangle2D.getWidth() + "p" + page, diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/SearchImplementation.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/SearchImplementation.java index b7fa1d84..e5a87eed 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/SearchImplementation.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/SearchImplementation.java @@ -78,7 +78,7 @@ public class SearchImplementation { } - public List getMatches(CharSequence text) { + public List getBoundaries(CharSequence text) { if (this.values.isEmpty()) { return new ArrayList<>(); @@ -91,7 +91,7 @@ public class SearchImplementation { } - public List getMatches(CharSequence text, Boundary region) { + public List getBoundaries(CharSequence text, Boundary region) { if (this.values.isEmpty()) { return new ArrayList<>(); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AbstractTestWithDictionaries.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AbstractTestWithDictionaries.java index d77998a6..86a3be53 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AbstractTestWithDictionaries.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AbstractTestWithDictionaries.java @@ -43,14 +43,15 @@ import org.springframework.context.annotation.Primary; import org.springframework.core.io.ClassPathResource; import org.springframework.test.context.junit.jupiter.SpringExtension; -import com.iqser.red.service.persistence.service.v1.api.model.common.JSONPrimitive; -import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.configuration.Colors; -import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType; -import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntry; -import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.Type; -import com.iqser.red.service.redaction.v1.model.AnalyzeRequest; +import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest; +import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.configuration.Colors; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntry; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type; import com.iqser.red.service.redaction.v1.server.client.DictionaryClient; import com.iqser.red.service.redaction.v1.server.client.RulesClient; +import com.iqser.red.service.redaction.v1.server.multitenancy.TenantContext; import com.iqser.red.service.redaction.v1.server.redaction.utils.ResourceLoader; import com.iqser.red.service.redaction.v1.server.redaction.utils.TextNormalizationUtilities; import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService; @@ -67,6 +68,7 @@ public class AbstractTestWithDictionaries { protected static final String RULES = loadFromClassPath("drools/rules.drl"); protected static final String RULES_PATH = "drools/rules.drl"; protected static final String ENTITY_RULES_PATH = "drools/entity_rules.drl"; + protected static final String MERGE_ENTITY_RULES_PATH = "drools/merge_entity_rules.drl"; private static final String VERTEBRATE = "vertebrate"; private static final String ADDRESS = "CBI_address"; private static final String AUTHOR = "CBI_author"; @@ -118,6 +120,8 @@ public class AbstractTestWithDictionaries { @BeforeEach public void stubClients() { + TenantContext.setTenantId("redaction"); + when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L); when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(JSONPrimitive.of(RULES)); @@ -146,25 +150,6 @@ public class AbstractTestWithDictionaries { } - private static String loadFromClassPath(String path) { - - URL resource = ResourceLoader.class.getClassLoader().getResource(path); - if (resource == null) { - throw new IllegalArgumentException("could not load classpath resource: drools/rules.drl"); - } - try (BufferedReader br = new BufferedReader(new InputStreamReader(resource.openStream(), StandardCharsets.UTF_8))) { - StringBuilder sb = new StringBuilder(); - String str; - while ((str = br.readLine()) != null) { - sb.append(str).append("\n"); - } - return sb.toString(); - } catch (IOException e) { - throw new IllegalArgumentException("could not load classpath resource: " + path, e); - } - } - - private void mockDictionaryCalls(Long version) { when(dictionaryClient.getDictionaryForType(VERTEBRATE + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer) invocation -> getDictionaryResponse(VERTEBRATE, @@ -205,6 +190,25 @@ public class AbstractTestWithDictionaries { } + private static String loadFromClassPath(String path) { + + URL resource = ResourceLoader.class.getClassLoader().getResource(path); + if (resource == null) { + throw new IllegalArgumentException("could not load classpath resource: drools/rules.drl"); + } + try (BufferedReader br = new BufferedReader(new InputStreamReader(resource.openStream(), StandardCharsets.UTF_8))) { + StringBuilder sb = new StringBuilder(); + String str; + while ((str = br.readLine()) != null) { + sb.append(str).append("\n"); + } + return sb.toString(); + } catch (IOException e) { + throw new IllegalArgumentException("could not load classpath resource: " + path, e); + } + } + + private String cleanDictionaryEntry(String entry) { return TextNormalizationUtilities.removeHyphenLineBreaks(entry).replaceAll("\\n", " "); @@ -357,7 +361,9 @@ public class AbstractTestWithDictionaries { private void loadNerForTest() { ClassPathResource responseJson = new ClassPathResource("files/ner_response.json"); - storageService.storeObject(RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.NER_ENTITIES), responseJson.getInputStream()); + storageService.storeObject(TenantContext.getTenantId(), + RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.NER_ENTITIES), + responseJson.getInputStream()); } @@ -379,6 +385,7 @@ public class AbstractTestWithDictionaries { .collect(Collectors.toList()); } + protected List getPathsRecursively(File path) { List result = new ArrayList<>(); @@ -396,6 +403,7 @@ public class AbstractTestWithDictionaries { } + protected void loadOnlyDictionaryForSimpleFile() { dictionary.clear(); @@ -478,9 +486,13 @@ public class AbstractTestWithDictionaries { .lastProcessed(OffsetDateTime.now()) .build(); - storageService.storeObject(RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.IMAGE_INFO), imageInfoStream); - storageService.storeObject(RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.TABLES), cvServiceResponseFileStream); - storageService.storeObject(RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.ORIGIN), fileStream); + storageService.storeObject(TenantContext.getTenantId(), + RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.IMAGE_INFO), + imageInfoStream); + storageService.storeObject(TenantContext.getTenantId(), + RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.TABLES), + cvServiceResponseFileStream); + storageService.storeObject(TenantContext.getTenantId(), RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.ORIGIN), fileStream); return request; } @@ -505,24 +517,24 @@ public class AbstractTestWithDictionaries { KieServices kieServices = KieServices.Factory.get(); KieFileSystem kieFileSystem = kieServices.newKieFileSystem(); - //kieFileSystem.write(ResourceFactory.newClassPathResource(RULES_PATH, "UTF-8")); + kieFileSystem.write(ResourceFactory.newClassPathResource(MERGE_ENTITY_RULES_PATH, "UTF-8")); kieFileSystem.write(ResourceFactory.newClassPathResource(ENTITY_RULES_PATH, "UTF-8")); KieRepository kieRepository = kieServices.getRepository(); kieRepository.addKieModule(new KieModule() { public ReleaseId getReleaseId() { + return kieRepository.getDefaultReleaseId(); } }); - KieBuilder kieBuilder = kieServices - .newKieBuilder(kieFileSystem) - .buildAll(); + KieBuilder kieBuilder = kieServices.newKieBuilder(kieFileSystem).buildAll(); return kieServices.newKieContainer(kieRepository.getDefaultReleaseId()); } + @Bean @Primary public StorageService inmemoryStorage() { diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DocumentGraphIntegrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DocumentGraphIntegrationTest.java index a7808274..015c82f0 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DocumentGraphIntegrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DocumentGraphIntegrationTest.java @@ -17,10 +17,12 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.core.io.ClassPathResource; -import com.iqser.red.service.redaction.v1.model.FileAttribute; +import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute; +import com.iqser.red.service.redaction.v1.server.document.graph.Boundary; import com.iqser.red.service.redaction.v1.server.document.graph.DocumentGraph; +import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityNode; +import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityPosition; import com.iqser.red.service.redaction.v1.server.document.graph.factory.DocumentGraphFactory; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.EntityNode; import com.iqser.red.service.redaction.v1.server.document.graph.nodes.PageNode; import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SectionNode; import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SemanticNode; @@ -29,7 +31,9 @@ import com.iqser.red.service.redaction.v1.server.document.services.EntityCreatio import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary; import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType; import com.iqser.red.service.redaction.v1.server.redaction.service.DictionaryService; +import com.iqser.red.service.redaction.v1.server.redaction.service.RedactionLogCreatorService; import com.iqser.red.service.redaction.v1.server.redaction.utils.SearchImplementation; +import com.iqser.red.service.redaction.v1.server.redaction.utils.SeparatorUtils; import com.iqser.red.service.redaction.v1.server.segmentation.PdfSegmentationService; import com.iqser.red.service.redaction.v1.server.utils.PdfDraw; @@ -49,13 +53,14 @@ public class DocumentGraphIntegrationTest extends AbstractTestWithDictionaries { @Autowired private EntityCreationService entityCreationService; + @Autowired + private RedactionLogCreatorService redactionLogCreatorService; + @Qualifier("kieContainer") @Autowired private KieContainer kieContainer; - - @Test @SneakyThrows public void testDroolsOnDocumentGraph() { @@ -80,16 +85,22 @@ public class DocumentGraphIntegrationTest extends AbstractTestWithDictionaries { } KieSession kieSession = kieContainer.newKieSession(); + entityCreationService.createEntitiesByString("Michael N.", 0, "CBI_author", EntityType.FALSE_POSITIVE, document)// + .forEach(kieSession::insert); kieSession.setGlobal("document", document); + kieSession.setGlobal("entityCreationService", entityCreationService); document.getEntities().forEach(kieSession::insert); document.getTableOfContents().streamEntriesInOrder().forEach(entry -> kieSession.insert(entry.node())); document.getPages().forEach(kieSession::insert); + long start = System.currentTimeMillis(); kieSession.insert(FileAttribute.builder().label("Vertebrate Study").value("Yes").build()); kieSession.fireAllRules(); + System.out.printf("Firing all rules took %d ms\n", System.currentTimeMillis() - start); + //List redactionLogEntries = redactionLogCreatorService.createRedactionLog(); drawAllEntities(filename, fileResource, document); } @@ -168,7 +179,7 @@ public class DocumentGraphIntegrationTest extends AbstractTestWithDictionaries { var searchTime = System.currentTimeMillis() - searchStart; totalSearchTime += searchTime; - var insertStart = System.currentTimeMillis(); + var insertStart = System.currentTimeMillis(); DocumentGraph finalDocument = document; foundEntities.forEach(entity -> entityCreationService.addEntityToGraph(entity, finalDocument)); var insertTime = System.currentTimeMillis() - insertStart; @@ -196,9 +207,10 @@ public class DocumentGraphIntegrationTest extends AbstractTestWithDictionaries { for (PageNode page : document.getPages()) { List entityPositionsOnPage = page.getEntities() .stream() + .filter(entityNode -> !entityNode.isRemoved()) .filter(EntityNode::isRedaction) - .map(EntityNode::getPositions) - .flatMap(List::stream) + .flatMap(entityNode -> entityNode.getEntityPositions().stream()) + .map(EntityPosition::getPosition) .toList(); PdfDraw.Options options = PdfDraw.Options.builder().strokeColor(Color.BLACK).stroke(true).build(); @@ -208,9 +220,10 @@ public class DocumentGraphIntegrationTest extends AbstractTestWithDictionaries { for (PageNode page : document.getPages()) { List entityPositionsOnPage = page.getEntities() .stream() + .filter(entityNode -> !entityNode.isRemoved()) .filter(e -> !e.isRedaction()) - .map(EntityNode::getPositions) - .flatMap(List::stream) + .flatMap(entityNode -> entityNode.getEntityPositions().stream()) + .map(EntityPosition::getPosition) .toList(); PdfDraw.Options options = PdfDraw.Options.builder().strokeColor(Color.BLUE).stroke(true).build(); @@ -229,10 +242,19 @@ public class DocumentGraphIntegrationTest extends AbstractTestWithDictionaries { List foundEntities, String type) { - TextBlock textBlock = documentGraph.getTextBlock(); - searchImplementation.getMatches(textBlock, textBlock.getBoundary()) - .stream() - .map(bounds -> EntityNode.initialEntityNode(bounds, type, entityType)) - .forEach(foundEntities::add); + TextBlock textBlock = documentGraph.getTextBlock(); + searchImplementation.getBoundaries(textBlock, textBlock.getBoundary()) + .stream() + .filter(boundary -> validateSeparators(textBlock, boundary)) + .map(bounds -> EntityNode.initialEntityNode(bounds, type, entityType)) + .forEach(foundEntities::add); } + + + private static boolean validateSeparators(TextBlock textBlock, Boundary boundary) { + + return (boundary.start() == 0 || SeparatorUtils.isSeparator(textBlock.charAt(boundary.start() - 1)) || SeparatorUtils.isSeparator(textBlock.charAt(boundary.start()))) && (boundary.end() == textBlock.length() || SeparatorUtils.isSeparator( + textBlock.charAt(boundary.end())) || SeparatorUtils.isSeparator(textBlock.charAt(boundary.end() - 1))); + } + } diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/BoundaryTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/BoundaryTest.java index e61a7122..65a575da 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/BoundaryTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/BoundaryTest.java @@ -1,6 +1,12 @@ package com.iqser.red.service.redaction.v1.server.document.graph; -import static org.junit.jupiter.api.Assertions.*; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.Collections; +import java.util.List; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -8,14 +14,18 @@ import org.junit.jupiter.api.Test; class BoundaryTest { Boundary startBoundary; + + @BeforeEach void setUp() { + startBoundary = new Boundary(10, 100); } @Test void testContains() { + assertTrue(startBoundary.contains(11)); assertTrue(startBoundary.contains(50)); assertFalse(startBoundary.contains(9)); @@ -34,6 +44,7 @@ class BoundaryTest { @Test void testIntersects() { + assertTrue(startBoundary.intersects(new Boundary(1, 11))); assertTrue(startBoundary.intersects(new Boundary(11, 12))); assertTrue(startBoundary.intersects(new Boundary(11, 100))); @@ -41,4 +52,17 @@ class BoundaryTest { assertTrue(startBoundary.intersects(new Boundary(99, 101))); } + + @Test + void testSplit() { + + assertEquals(4, startBoundary.split(List.of(12, 40, 90)).size()); + assertEquals(List.of(new Boundary(10, 12), new Boundary(12, 40), new Boundary(40, 90), new Boundary(90, 100)), startBoundary.split(List.of(12, 40, 90))); + assertEquals(List.of(new Boundary(10, 40), new Boundary(40, 100)), startBoundary.split(List.of(40))); + assertEquals(1, startBoundary.split(Collections.emptyList()).size()); + assertThrows(IndexOutOfBoundsException.class, () -> startBoundary.split(Collections.singletonList(0))); + assertThrows(IndexOutOfBoundsException.class, () -> startBoundary.split(Collections.singletonList(100))); + assertThrows(IndexOutOfBoundsException.class, () -> startBoundary.split(List.of(12, 40, 100))); + } + } \ No newline at end of file diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentGraphMappingTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentGraphMappingTest.java index 887d430b..bb915854 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentGraphMappingTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentGraphMappingTest.java @@ -6,9 +6,10 @@ import org.springframework.core.io.ClassPathResource; import com.iqser.red.service.redaction.v1.server.AbstractTestWithDictionaries; import com.iqser.red.service.redaction.v1.server.document.data.DocumentData; -import com.iqser.red.service.redaction.v1.server.document.services.DocumentDataMapper; +import com.iqser.red.service.redaction.v1.server.document.data.mapper.DocumentDataMapper; import com.iqser.red.service.redaction.v1.server.document.graph.factory.DocumentGraphFactory; -import com.iqser.red.service.redaction.v1.server.document.services.DocumentGraphMapper; +import com.iqser.red.service.redaction.v1.server.document.data.mapper.DocumentGraphMapper; +import com.iqser.red.service.redaction.v1.server.multitenancy.TenantContext; import com.iqser.red.service.redaction.v1.server.segmentation.PdfSegmentationService; import lombok.SneakyThrows; @@ -40,7 +41,7 @@ public class DocumentGraphMappingTest extends AbstractTestWithDictionaries { var classifiedDoc = segmentationService.parseDocument(TEST_DOSSIER_ID, TEST_FILE_ID, fileResource.getInputStream(), null); DocumentGraph document = documentGraphFactory.buildDocumentGraph(classifiedDoc); DocumentData documentData = documentDataMapper.toDocumentData(document); - storageService.storeJSONObject(filename + ".json", documentData); + storageService.storeJSONObject(TenantContext.getTenantId(), filename + ".json", documentData); DocumentGraph newDocumentGraph = documentGraphMapper.toDocumentGraph(documentData); assert document.toString().equals(newDocumentGraph.toString()); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentGraphTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentGraphTest.java index 7aa05a20..1514b524 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentGraphTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentGraphTest.java @@ -16,7 +16,7 @@ import com.iqser.red.service.redaction.v1.server.AbstractTestWithDictionaries; import com.iqser.red.service.redaction.v1.server.classification.model.Document; import com.iqser.red.service.redaction.v1.server.document.graph.nodes.PageNode; import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SemanticNode; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.EntityNode; +import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityNode; import com.iqser.red.service.redaction.v1.server.document.graph.nodes.HeadlineNode; import com.iqser.red.service.redaction.v1.server.document.graph.nodes.NodeType; import com.iqser.red.service.redaction.v1.server.document.graph.nodes.ParagraphNode; @@ -56,7 +56,7 @@ public class DocumentGraphTest extends AbstractTestWithDictionaries { assert start != -1; Boundary boundary = new Boundary(start, start + searchTerm.length()); - EntityNode entityNode = entityCreationService.createAndAddEntity(boundary, "CBI_author", EntityType.ENTITY, documentGraph); + EntityNode entityNode = entityCreationService.createEntityByBoundary(boundary, "CBI_author", EntityType.ENTITY, documentGraph); assertEquals("Expand to Hint ", entityNode.getTextBefore()); assertEquals("’s Donut ←", entityNode.getTextAfter()); @@ -80,7 +80,7 @@ public class DocumentGraphTest extends AbstractTestWithDictionaries { assert start != -1; Boundary boundary = new Boundary(start, start + searchTerm.length()); - EntityNode entityNode = entityCreationService.createAndAddEntity(boundary, "CBI_author", EntityType.ENTITY, documentGraph); + EntityNode entityNode = entityCreationService.createEntityByBoundary(boundary, "CBI_author", EntityType.ENTITY, documentGraph); assertEquals("", entityNode.getTextBefore()); assertEquals(" Purity Hint", entityNode.getTextAfter()); @@ -103,7 +103,7 @@ public class DocumentGraphTest extends AbstractTestWithDictionaries { assert start != -1; Boundary boundary = new Boundary(start, start + searchTerm.length()); - EntityNode entityNode = entityCreationService.createAndAddEntity(boundary, "CBI_author", EntityType.ENTITY, documentGraph); + EntityNode entityNode = entityCreationService.createEntityByBoundary(boundary, "CBI_author", EntityType.ENTITY, documentGraph); assertEquals("", entityNode.getTextBefore()); assertEquals("", entityNode.getTextAfter()); @@ -189,7 +189,7 @@ public class DocumentGraphTest extends AbstractTestWithDictionaries { assert start != -1; Boundary boundary = new Boundary(start, start + searchTerm.length()); - EntityNode entityNode = entityCreationService.createAndAddEntity(boundary, "CBI_author", EntityType.ENTITY, documentGraph); + EntityNode entityNode = entityCreationService.createEntityByBoundary(boundary, "CBI_author", EntityType.ENTITY, documentGraph); assertEquals("except Cranberry; Vegetable, ", entityNode.getTextBefore()); assertEquals(", Group 9;", entityNode.getTextAfter()); @@ -214,7 +214,7 @@ public class DocumentGraphTest extends AbstractTestWithDictionaries { assert start != -1; Boundary boundary = new Boundary(start, start + searchTerm.length()); - EntityNode entityNode = entityCreationService.createAndAddEntity(boundary, "CBI_author", EntityType.ENTITY, documentGraph); + EntityNode entityNode = entityCreationService.createEntityByBoundary(boundary, "CBI_author", EntityType.ENTITY, documentGraph); assertEquals("2.6.1 Summary of ", entityNode.getTextBefore()); assertEquals(" and excretion in", entityNode.getTextAfter()); @@ -238,7 +238,7 @@ public class DocumentGraphTest extends AbstractTestWithDictionaries { assert start != -1; Boundary boundary = new Boundary(start, start + searchTerm.length()); - EntityNode entityNode = entityCreationService.createAndAddEntity(boundary, "CBI_author", EntityType.ENTITY, documentGraph); + EntityNode entityNode = entityCreationService.createEntityByBoundary(boundary, "CBI_author", EntityType.ENTITY, documentGraph); assertEquals("2-[(2-(1-hydroxy-ethyl)-6methyl-phenyl-amino]propan-1-ol (", entityNode.getTextBefore()); assertEquals(" of metabolite of", entityNode.getTextAfter()); @@ -293,14 +293,13 @@ public class DocumentGraphTest extends AbstractTestWithDictionaries { assert start != -1; Boundary boundary = new Boundary(start, start + searchTerm.length()); - EntityNode entityNode = entityCreationService.createAndAddEntity(boundary, "CBI_author", EntityType.ENTITY, documentGraph); + EntityNode entityNode = entityCreationService.createEntityByBoundary(boundary, "CBI_author", EntityType.ENTITY, documentGraph); PageNode pageNode = documentGraph.getPages().get(pageNumber - 1); assertEquals(entityNode.getValue(), searchTerm); assertTrue(pageNode.getEntities().contains(entityNode)); assertTrue(documentGraph.getPages().stream().filter(page -> page != pageNode).noneMatch(page -> page.getEntities().contains(entityNode))); assertTrue(entityNode.getPages().contains(pageNode)); - assertTrue(entityNode.getPositions().size() >= searchTerm.replace(" ", "").length()); assertSameOffsetInAllIntersectingNodes(searchTerm, start, entityNode); assertTrue(entityNode.getIntersectingNodes().stream().allMatch(node -> node.getEntities().contains(entityNode))); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/utils/EntitySearchUtilsTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/utils/EntitySearchUtilsTest.java index e01636ec..6752c4bb 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/utils/EntitySearchUtilsTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/utils/EntitySearchUtilsTest.java @@ -17,8 +17,8 @@ public class EntitySearchUtilsTest { public void testNestedEntitiesRemoval() { Set entities = new HashSet<>(); - Entity nested = new Entity("nested", "fake type", 10, 16, "fake headline", 0, 0, false, false, Engine.RULE, EntityType.ENTITY); - Entity nesting = new Entity("nesting nested", "fake type", 2, 16, "fake headline", 0, 0, false, false, Engine.RULE, EntityType.ENTITY); + Entity nested = new Entity("nested", "fake type", 10, 16, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY); + Entity nesting = new Entity("nesting nested", "fake type", 2, 16, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY); entities.add(nested); entities.add(nesting); EntitySearchUtils.removeEntitiesContainedInLarger(entities); @@ -40,14 +40,14 @@ public class EntitySearchUtilsTest { // Arrange Set existingEntities = new HashSet<>(); - Entity existingEntity1 = new Entity("Batman", "fake type", 0, 5, "fake headline", 0, 0, false, false, Engine.RULE, EntityType.ENTITY); - Entity existingEntity2 = new Entity("Superman", "fake type", 10, 17, "fake headline", 0, 0, false, false, Engine.RULE, EntityType.ENTITY); + Entity existingEntity1 = new Entity("Batman", "fake type", 0, 5, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY); + Entity existingEntity2 = new Entity("Superman", "fake type", 10, 17, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY); existingEntities.add(existingEntity1); existingEntities.add(existingEntity2); Set foundEntities = new HashSet<>(); - Entity foundEntities1 = new Entity("Batman X.", "fake type", 0, 8, "fake headline", 0, 0, false, false, Engine.RULE, EntityType.ENTITY); - Entity foundEntities2 = new Entity("Superman Y.", "fake type", 10, 20, "fake headline", 0, 0, false, false, Engine.RULE, EntityType.ENTITY); + Entity foundEntities1 = new Entity("Batman X.", "fake type", 0, 8, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY); + Entity foundEntities2 = new Entity("Superman Y.", "fake type", 10, 20, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY); foundEntities.add(foundEntities1); foundEntities.add(foundEntities2); @@ -73,14 +73,14 @@ public class EntitySearchUtilsTest { // Arrange Set existingEntities = new HashSet<>(); - Entity existingEntity1 = new Entity("Batman X.", "fake type", 0, 8, "fake headline", 0, 0, false, false, Engine.RULE, EntityType.ENTITY); - Entity existingEntity2 = new Entity("Superman", "fake type", 10, 17, "fake headline", 0, 0, false, false, Engine.RULE, EntityType.ENTITY); + Entity existingEntity1 = new Entity("Batman X.", "fake type", 0, 8, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY); + Entity existingEntity2 = new Entity("Superman", "fake type", 10, 17, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY); existingEntities.add(existingEntity1); existingEntities.add(existingEntity2); Set foundEntities = new HashSet<>(); - Entity foundEntities1 = new Entity("Batman X.", "fake type", 0, 8, "fake headline", 0, 0,false, false, Engine.RULE, EntityType.ENTITY); - Entity foundEntities2 = new Entity("X. Superman Y.", "fake type", 7, 20, "fake headline", 0, 0, false, false, Engine.RULE, EntityType.ENTITY); + Entity foundEntities1 = new Entity("Batman X.", "fake type", 0, 8, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY); + Entity foundEntities2 = new Entity("X. Superman Y.", "fake type", 7, 20, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY); foundEntities.add(foundEntities1); foundEntities.add(foundEntities2); @@ -105,14 +105,14 @@ public class EntitySearchUtilsTest { // Arrange Set existingEntities = new HashSet<>(); - Entity existingEntity1 = new Entity("Batman X.", "fake type", 0, 8, "fake headline", 0, 0, false, false, Engine.RULE, EntityType.ENTITY); - Entity existingEntity2 = new Entity("Superman", "fake type", 10, 17, "fake headline", 0, 0,false, false, Engine.RULE, EntityType.ENTITY); + Entity existingEntity1 = new Entity("Batman X.", "fake type", 0, 8, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY); + Entity existingEntity2 = new Entity("Superman", "fake type", 10, 17, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY); existingEntities.add(existingEntity1); existingEntities.add(existingEntity2); Set foundEntities = new HashSet<>(); - Entity foundEntities1 = new Entity("Batman X.", "fake type", 0, 8, "fake headline", 0, 0,false, false, Engine.RULE, EntityType.ENTITY); - Entity foundEntities2 = new Entity("X. Superman", "fake type", 7, 17, "fake headline", 0, 0,false, false, Engine.RULE, EntityType.ENTITY); + Entity foundEntities1 = new Entity("Batman X.", "fake type", 0, 8, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY); + Entity foundEntities2 = new Entity("X. Superman", "fake type", 7, 17, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY); foundEntities.add(foundEntities1); foundEntities.add(foundEntities2); @@ -137,15 +137,15 @@ public class EntitySearchUtilsTest { // Arrange Set existingEntities = new HashSet<>(); - Entity existingEntity1 = new Entity("X. Superman", "fake type", 7, 17, "fake headline", 0, 0,false, false, Engine.RULE, EntityType.ENTITY); - Entity existingEntity2 = new Entity("Batman", "fake type", 0, 5, "fake headline", 0, 0,false, false, Engine.RULE, EntityType.ENTITY); + Entity existingEntity1 = new Entity("X. Superman", "fake type", 7, 17, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY); + Entity existingEntity2 = new Entity("Batman", "fake type", 0, 5, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY); existingEntities.add(existingEntity1); existingEntities.add(existingEntity2); Set foundEntities = new HashSet<>(); - Entity foundEntities1 = new Entity("Batman X.", "fake type", 0, 8, "fake headline", 0, 0,false, false, Engine.RULE, EntityType.ENTITY); - Entity foundEntities2 = new Entity("Superman", "fake type", 10, 17, "fake headline", 0, 0,false, false, Engine.RULE, EntityType.ENTITY); - Entity foundEntities3 = new Entity("Superman Y.", "fake type", 10, 20, "fake headline", 0, 0,false, false, Engine.RULE, EntityType.ENTITY); + Entity foundEntities1 = new Entity("Batman X.", "fake type", 0, 8, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY); + Entity foundEntities2 = new Entity("Superman", "fake type", 10, 17, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY); + Entity foundEntities3 = new Entity("Superman Y.", "fake type", 10, 20, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY); foundEntities.add(foundEntities1); foundEntities.add(foundEntities2); foundEntities.add(foundEntities3); @@ -171,16 +171,16 @@ public class EntitySearchUtilsTest { // Arrange Set existingEntities = new HashSet<>(); - Entity existingEntity1 = new Entity("X. Superman", "fake type", 7, 17, "fake headline", 0, 0,false, false, Engine.RULE, EntityType.ENTITY); - Entity existingEntity2 = new Entity("Batman", "fake type", 0, 5, "fake headline", 0, 0,false, false, Engine.RULE, EntityType.ENTITY); + Entity existingEntity1 = new Entity("X. Superman", "fake type", 7, 17, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY); + Entity existingEntity2 = new Entity("Batman", "fake type", 0, 5, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY); existingEntities.add(existingEntity1); existingEntities.add(existingEntity2); Set foundEntities = new HashSet<>(); - Entity foundEntitiesOverlap1 = new Entity("Batman X. Superman Y.", "fake type", 0, 17, "fake headline", 0, 0,false, false, Engine.RULE, EntityType.ENTITY); - Entity foundEntitiesOverlap2 = new Entity("Superman Y.", "fake type", 10, 20, "fake headline", 0, 0,false, false, Engine.RULE, EntityType.ENTITY); - Entity foundEntitiesSubset1 = new Entity("Batman X. Superman", "fake type", 0, 17, "fake headline", 0, 0,false, false, Engine.RULE, EntityType.ENTITY); - Entity foundEntitiesSubset2 = new Entity("Superman", "fake type", 10, 17, "fake headline", 0, 0,false, false, Engine.RULE, EntityType.ENTITY); + Entity foundEntitiesOverlap1 = new Entity("Batman X. Superman Y.", "fake type", 0, 17, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY); + Entity foundEntitiesOverlap2 = new Entity("Superman Y.", "fake type", 10, 20, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY); + Entity foundEntitiesSubset1 = new Entity("Batman X. Superman", "fake type", 0, 17, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY); + Entity foundEntitiesSubset2 = new Entity("Superman", "fake type", 10, 17, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY); foundEntities.add(foundEntitiesOverlap1); foundEntities.add(foundEntitiesOverlap2); foundEntities.add(foundEntitiesSubset1); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/merge_entity_rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/merge_entity_rules.drl new file mode 100644 index 00000000..080af875 --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/merge_entity_rules.drl @@ -0,0 +1,56 @@ +package drools + +import static java.lang.String.format; +import static com.iqser.red.service.redaction.v1.server.document.services.RegexMatcher.anyMatch; + + +import java.util.List; +import java.util.LinkedList; +import java.util.HashSet; + +import com.iqser.red.service.redaction.v1.server.document.graph.* +import com.iqser.red.service.redaction.v1.server.document.graph.nodes.* +import com.iqser.red.service.redaction.v1.server.document.graph.entity.* +import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType; +import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute; +import java.util.Set; + +global DocumentGraph document + +rule "remove contained entity" + + when + $largerEntity: EntityNode() + $smallerEntity: EntityNode(this != $largerEntity, containedBy($largerEntity)) + then + $smallerEntity.removeFromGraph(); + $smallerEntity.setRemoved(true); + $smallerEntity.setSkipRemoveEntitiesContainedInLarger(true); + System.out.printf("%s contained by %s\n", $smallerEntity, $largerEntity); + delete($smallerEntity); + end + + + + +rule "1: Redact CBI_author" + + when + FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "yes") + entity: EntityNode(type == "CBI_author") + then + entity.setRedaction(true); + update(entity) + end + +rule "2: do not redact genitive CBI_author" + + when + entity: EntityNode(type == "CBI_author", anyMatch(textAfter, "['’’'ʼˈ´`‘′ʻ’']s"), redaction == true) + then + entity.setRedaction(false); + entity.setEntityType(EntityType.FALSE_POSITIVE); + update(entity) + end + +