From 7879bcd909212347ff127f21216c73dc4c41bfaa Mon Sep 17 00:00:00 2001 From: Kilian Schuettler Date: Thu, 2 Mar 2023 18:44:52 +0100 Subject: [PATCH] RED-6093: Prototype document structure *refactored Nodes *added some tests *wip --- .../document/graph/AtomicTextBlockEntity.java | 97 ------ .../v1/server/document/graph/Boundary.java | 84 +++++ .../server/document/graph/DocumentEntity.java | 49 --- .../server/document/graph/DocumentGraph.java | 90 ++++++ .../v1/server/document/graph/Entity2.java | 14 - .../server/document/graph/FooterEntity.java | 47 --- .../server/document/graph/HeaderEntity.java | 47 --- .../v1/server/document/graph/Node.java | 18 -- .../v1/server/document/graph/PageEntity.java | 37 --- .../document/graph/ParagraphEntity.java | 54 ---- .../server/document/graph/SectionEntity.java | 56 ---- .../document/graph/TableCellEntity.java | 44 --- .../v1/server/document/graph/TableEntity.java | 53 ---- .../document/graph/TableOfContents.java | 15 +- .../document/graph/TextBlockCollector.java | 53 ---- .../document/graph/TextBlockEntity.java | 166 ---------- .../document/graph/nodes/DocumentNode.java | 202 ++++++++++++ .../document/graph/nodes/EntityNode.java | 84 +++++ .../document/graph/nodes/FooterNode.java | 71 +++++ .../document/graph/nodes/HeaderNode.java | 72 +++++ .../server/document/graph/nodes/PageNode.java | 80 +++++ .../document/graph/nodes/ParagraphNode.java | 70 +++++ .../document/graph/nodes/SectionNode.java | 107 +++++++ .../document/graph/nodes/TableCellNode.java | 57 ++++ .../document/graph/nodes/TableNode.java | 87 ++++++ .../graph/textblock/AtomicTextBlock.java | 134 ++++++++ .../textblock/ConcatenatedTextBlock.java | 155 ++++++++++ .../document/graph/textblock/TextBlock.java | 59 ++++ .../graph/textblock/TextBlockCollector.java | 51 +++ .../services/DocumentGraphFactory.java | 290 ++++++++++++------ .../services/EntityEnrichmentUtility.java | 31 ++ .../document/services/RangeComparators.java | 38 +++ .../document/services/RegexMatcher.java | 37 +++ .../SearchTextWithTextPositionFactory.java | 3 +- .../redaction/utils/SearchImplementation.java | 26 +- .../server/AbstractTestWithDictionaries.java | 29 +- .../server/DocumentGraphIntegrationTest.java | 222 ++++++++++++-- .../server/document/graph/BoundaryTest.java | 44 +++ .../redaction/v1/server/utils/PdfDraw.java | 27 ++ .../test/resources/drools/entity_rules.drl | 43 +++ .../src/test/resources/drools/test_rules.drl | 29 ++ 41 files changed, 2114 insertions(+), 858 deletions(-) delete mode 100644 redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/AtomicTextBlockEntity.java create mode 100644 redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/Boundary.java delete mode 100644 redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentEntity.java create mode 100644 redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentGraph.java delete mode 100644 redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/Entity2.java delete mode 100644 redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/FooterEntity.java delete mode 100644 redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/HeaderEntity.java delete mode 100644 redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/Node.java delete mode 100644 redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/PageEntity.java delete mode 100644 redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/ParagraphEntity.java delete mode 100644 redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/SectionEntity.java delete mode 100644 redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/TableCellEntity.java delete mode 100644 redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/TableEntity.java delete mode 100644 redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/TextBlockCollector.java delete mode 100644 redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/TextBlockEntity.java create mode 100644 redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/DocumentNode.java create mode 100644 redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/EntityNode.java create mode 100644 redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/FooterNode.java create mode 100644 redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/HeaderNode.java create mode 100644 redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/PageNode.java create mode 100644 redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/ParagraphNode.java create mode 100644 redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/SectionNode.java create mode 100644 redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/TableCellNode.java create mode 100644 redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/TableNode.java create mode 100644 redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/textblock/AtomicTextBlock.java create mode 100644 redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/textblock/ConcatenatedTextBlock.java create mode 100644 redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/textblock/TextBlock.java create mode 100644 redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/textblock/TextBlockCollector.java create mode 100644 redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/EntityEnrichmentUtility.java create mode 100644 redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/RangeComparators.java create mode 100644 redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/RegexMatcher.java create mode 100644 redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/BoundaryTest.java create mode 100644 redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/utils/PdfDraw.java create mode 100644 redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/entity_rules.drl create mode 100644 redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/test_rules.drl diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/AtomicTextBlockEntity.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/AtomicTextBlockEntity.java deleted file mode 100644 index b3f33664..00000000 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/AtomicTextBlockEntity.java +++ /dev/null @@ -1,97 +0,0 @@ -package com.iqser.red.service.redaction.v1.server.document.graph; - -import java.util.List; - -import com.iqser.red.service.redaction.v1.server.redaction.model.RedRectangle2D; - -import lombok.AccessLevel; -import lombok.AllArgsConstructor; -import lombok.Builder; -import lombok.Data; -import lombok.experimental.FieldDefaults; - -@Data -@Builder -@AllArgsConstructor -@FieldDefaults(level = AccessLevel.PRIVATE) -public class AtomicTextBlockEntity implements CharSequence { - - Integer id; - //string coords - Integer offset; - String searchText; - List lineBreaks; - - //position coords - List stringIdxToPositionIdx; - List positions; - - Node parent; - - - public int indexOf(String searchTerm) { - - int pos = searchText.indexOf(searchTerm); - return pos == -1 ? -1 : pos + offset; - } - - - public int numberOfLines() { - - return lineBreaks.size(); - } - - - public int getNextLinebreak(int startIndex) { - - return lineBreaks.stream()// - .filter(linebreak -> linebreak > startIndex) // - .findFirst() // - .orElse(searchText.length()) + offset; - } - - - public RedRectangle2D getPosition(int stringIdx) { - - return positions.get(stringIdxToPositionIdx.get(stringIdx - offset)); - } - - - public List getPositions(int startStringIdx, int endStringIdx) { - - return positions.subList(startStringIdx - offset, endStringIdx - offset); - } - - - @Override - public int length() { - - return searchText.length(); - } - - - @Override - public char charAt(int index) { - - return searchText.charAt(index - offset); - } - - - @Override - public CharSequence subSequence(int start, int end) { - - throw new UnsupportedOperationException("AtomicTextBlockEntity can not be sliced!"); - } - - - public String getFirstLine() { - - return searchText.substring(0, getNextLinebreak(0) - offset); - } - - @Override - public String toString() { - return searchText; - } - -} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/Boundary.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/Boundary.java new file mode 100644 index 00000000..f59b8b4d --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/Boundary.java @@ -0,0 +1,84 @@ +package com.iqser.red.service.redaction.v1.server.document.graph; + +import static java.lang.String.format; + +import lombok.Setter; + +@Setter +public class Boundary { + + private int start; + private int end; + + + public Boundary(int start, int end) { + + assert start <= end; + this.start = start; + this.end = end; + } + + public int length() { + return end - start; + } + + + public int start() { + + return start; + } + + + public int end() { + + return end; + } + + + public boolean contains(Boundary boundary) { + + return start <= boundary.start() && boundary.end() <= end; + } + + + public boolean containedBy(Boundary boundary) { + + return boundary.start() <= start && end <= boundary.end(); + } + + + public boolean contains(int start, int end) { + + if (start > end) { + throw new UnsupportedOperationException("start > end"); + } + return this.start <= start && end <= this.end; + } + + + public boolean containedBy(int start, int end) { + + if (start > end) { + throw new UnsupportedOperationException("start > end"); + } + return start <= this.start && this.end <= end; + } + + + public boolean contains(int index) { + + return start <= index && index < end; + } + + + public boolean intersects(Boundary boundary) { + + return contains(boundary.start()) || contains(boundary.end()); + } + + @Override + public String toString() { + return format("Range [%d|%d)", start, end); + } + +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentEntity.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentEntity.java deleted file mode 100644 index f35b0109..00000000 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentEntity.java +++ /dev/null @@ -1,49 +0,0 @@ -package com.iqser.red.service.redaction.v1.server.document.graph; - -import java.util.List; -import java.util.stream.Stream; - -import lombok.AccessLevel; -import lombok.AllArgsConstructor; -import lombok.Builder; -import lombok.Data; -import lombok.experimental.FieldDefaults; - -@Data -@Builder -@AllArgsConstructor -@FieldDefaults(level = AccessLevel.PRIVATE) -public class DocumentEntity implements Node { - - List sections; - List pages; - TableOfContents tableOfContents; - Integer numberOfPages; - - - @Override - public TextBlockEntity buildTextBlock() { - - return streamAllNodes() - .filter(Node::hasTextBlock) - .map(Node::getTextBlock) - .collect(TextBlockCollector.collect()); - } - - - private Stream streamAllNodes() { - - return Stream.concat(Stream.concat(// - tableOfContents.streamEntriesInOrder().map(TableOfContents.Entry::node), // - pages.stream().map(PageEntity::getHeader)), // - pages.stream().map(PageEntity::getFooter)); // - } - - - @Override - public String toString() { - - return tableOfContents.toString(); - } - -} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentGraph.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentGraph.java new file mode 100644 index 00000000..d08b9cac --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentGraph.java @@ -0,0 +1,90 @@ +package com.iqser.red.service.redaction.v1.server.document.graph; + +import static com.iqser.red.service.redaction.v1.server.document.services.EntityEnrichmentUtility.enrichEntity; + +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import com.iqser.red.service.redaction.v1.server.document.graph.nodes.DocumentNode; +import com.iqser.red.service.redaction.v1.server.document.graph.nodes.EntityNode; +import com.iqser.red.service.redaction.v1.server.document.graph.nodes.PageNode; +import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SectionNode; +import com.iqser.red.service.redaction.v1.server.document.graph.textblock.ConcatenatedTextBlock; +import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock; +import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlockCollector; +import com.iqser.red.service.redaction.v1.server.exception.NotFoundException; +import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType; + +import lombok.AccessLevel; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.experimental.FieldDefaults; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +@Data +@Builder +@AllArgsConstructor +@FieldDefaults(level = AccessLevel.PRIVATE) +public class DocumentGraph { + + List sections; + List pages; + TableOfContents tableOfContents; + Integer numberOfPages; + TextBlock text; + + + public ConcatenatedTextBlock buildTextBlock() { + + return streamAllNodes().filter(DocumentNode::isTerminal).map(DocumentNode::getAtomicTextBlock).collect(new TextBlockCollector()); + } + + + public EntityNode createAndAddEntity(Boundary boundary, String type, EntityType entityType) { + + EntityNode entity = EntityNode.initialEntityNode(boundary, type, entityType); + addEntityToGraphAndSetFields(entity); + return entity; + } + + + public void addEntityToGraphAndSetFields(EntityNode entity) { + + try { + boolean inserted = streamAllNodes().anyMatch(node -> node.addEntityAndSetFieldsIfStartIndexContained(entity)); + } catch (NotFoundException e) { + enrichEntity(entity, text); + log.warn("Entity \"{}\" with {} is in between two main sections and will be removed!", entity.getValue(), entity.getBoundary()); + entity.removeFromGraph(); + } + } + + + + public Set getEntities() { + + return streamAllNodes().filter(DocumentNode::isTerminal).map(DocumentNode::getEntities).flatMap(List::stream).collect(Collectors.toSet()); + } + + + private Stream streamAllNodes() { + + return Stream.concat(// + tableOfContents.streamEntriesInOrder().map(TableOfContents.Entry::node), // + Stream.concat(// + pages.stream().map(PageNode::getHeader), // + pages.stream().map(PageNode::getFooter))); // + } + + + @Override + public String toString() { + + return text.toString(); + } + +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/Entity2.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/Entity2.java deleted file mode 100644 index a5ce23c9..00000000 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/Entity2.java +++ /dev/null @@ -1,14 +0,0 @@ -package com.iqser.red.service.redaction.v1.server.document.graph; - -import java.awt.geom.Rectangle2D; -import java.util.List; - -public class Entity2 { - - String uid; - String value; - List positions; - Boolean redact; - - List neighbours; -} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/FooterEntity.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/FooterEntity.java deleted file mode 100644 index 6a3e9ce2..00000000 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/FooterEntity.java +++ /dev/null @@ -1,47 +0,0 @@ -package com.iqser.red.service.redaction.v1.server.document.graph; - -import java.util.List; - -import lombok.AccessLevel; -import lombok.AllArgsConstructor; -import lombok.Builder; -import lombok.Data; -import lombok.experimental.FieldDefaults; - -@Data -@Builder -@AllArgsConstructor -@FieldDefaults(level = AccessLevel.PRIVATE) -public class FooterEntity implements Node { - - PageEntity page; - TextBlockEntity textBlock; - List entities; - - - @Override - public TextBlockEntity buildTextBlock() { - - return getTextBlock(); - } - - - @Override - public boolean hasTextBlock() { - - return true; - } - - - @Override - public TextBlockEntity getTextBlock() { - - return textBlock; - } - - @Override - public String toString() { - return textBlock.toString(); - } - -} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/HeaderEntity.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/HeaderEntity.java deleted file mode 100644 index 7f07fc42..00000000 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/HeaderEntity.java +++ /dev/null @@ -1,47 +0,0 @@ -package com.iqser.red.service.redaction.v1.server.document.graph; - -import java.util.List; - -import lombok.AccessLevel; -import lombok.AllArgsConstructor; -import lombok.Builder; -import lombok.Data; -import lombok.experimental.FieldDefaults; - -@Data -@Builder -@AllArgsConstructor -@FieldDefaults(level = AccessLevel.PRIVATE) -public class HeaderEntity implements Node { - - PageEntity page; - TextBlockEntity textBlock; - List entities; - - - @Override - public TextBlockEntity buildTextBlock() { - - return getTextBlock(); - } - - - @Override - public boolean hasTextBlock() { - - return true; - } - - - @Override - public TextBlockEntity getTextBlock() { - - return textBlock; - } - - @Override - public String toString() { - return textBlock.toString(); - } - -} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/Node.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/Node.java deleted file mode 100644 index 631dabfa..00000000 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/Node.java +++ /dev/null @@ -1,18 +0,0 @@ -package com.iqser.red.service.redaction.v1.server.document.graph; - -public interface Node { - TextBlockEntity buildTextBlock(); - - - default boolean hasTextBlock() { - - return false; - } - - - default TextBlockEntity getTextBlock() { - - throw new UnsupportedOperationException("Only terminal Nodes have direct access to TextBlocks"); - } - -} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/PageEntity.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/PageEntity.java deleted file mode 100644 index 62c154a8..00000000 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/PageEntity.java +++ /dev/null @@ -1,37 +0,0 @@ -package com.iqser.red.service.redaction.v1.server.document.graph; - -import java.util.List; - -import lombok.AccessLevel; -import lombok.AllArgsConstructor; -import lombok.Builder; -import lombok.Data; -import lombok.experimental.FieldDefaults; - -@Data -@Builder -@AllArgsConstructor -@FieldDefaults(level = AccessLevel.PRIVATE) -public class PageEntity implements Node { - - Integer number; - Integer height; - Integer width; - List sections; - List paragraphs; - HeaderEntity header; - FooterEntity footer; - - - @Override - public TextBlockEntity buildTextBlock() { - - return null; - } - - @Override - public String toString() { - return number.toString(); - } - -} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/ParagraphEntity.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/ParagraphEntity.java deleted file mode 100644 index 2fad58b0..00000000 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/ParagraphEntity.java +++ /dev/null @@ -1,54 +0,0 @@ -package com.iqser.red.service.redaction.v1.server.document.graph; - -import java.util.List; - -import lombok.AccessLevel; -import lombok.AllArgsConstructor; -import lombok.Builder; -import lombok.Data; -import lombok.experimental.FieldDefaults; - -@Data -@Builder -@AllArgsConstructor -@FieldDefaults(level = AccessLevel.PRIVATE) -public class ParagraphEntity implements Node { - - String tocId; - Integer id; - Integer numberOnPage; - Integer numberInSection; - - //foreign key - SectionEntity parentSection; - PageEntity page; - TextBlockEntity textBlock; - List entities; - - - @Override - public TextBlockEntity buildTextBlock() { - - return getTextBlock(); - } - - - @Override - public boolean hasTextBlock() { - - return true; - } - - - @Override - public TextBlockEntity getTextBlock() { - - return textBlock; - } - - - @Override - public String toString() { - return textBlock.toString(); - } -} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/SectionEntity.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/SectionEntity.java deleted file mode 100644 index 398e81a0..00000000 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/SectionEntity.java +++ /dev/null @@ -1,56 +0,0 @@ -package com.iqser.red.service.redaction.v1.server.document.graph; - -import java.util.List; - -import lombok.AccessLevel; -import lombok.AllArgsConstructor; -import lombok.Builder; -import lombok.Data; -import lombok.experimental.FieldDefaults; - -@Data -@Builder -@AllArgsConstructor -@FieldDefaults(level = AccessLevel.PRIVATE) -public class SectionEntity implements Node { - - Integer id; - TableOfContents tableOfContents; - String tocId; - Node parent; - //foreign key - TextBlockEntity headline; - List subSections; - List paragraphs; - List pages; - List entities; - - - @Override - public String toString() { - - return tocId + ": " + headline.toString(); - } - - - @Override - public TextBlockEntity buildTextBlock() { - - return tableOfContents.streamSubEntriesInOrder(tocId).map(TableOfContents.Entry::node).map(Node::getTextBlock).collect(TextBlockCollector.collect()); - } - - - @Override - public boolean hasTextBlock() { - - return true; - } - - - @Override - public TextBlockEntity getTextBlock() { - - return headline; - } - -} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/TableCellEntity.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/TableCellEntity.java deleted file mode 100644 index 50b69f94..00000000 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/TableCellEntity.java +++ /dev/null @@ -1,44 +0,0 @@ -package com.iqser.red.service.redaction.v1.server.document.graph; - -import java.util.List; - -import lombok.AccessLevel; -import lombok.AllArgsConstructor; -import lombok.Builder; -import lombok.Data; -import lombok.experimental.FieldDefaults; - -@Data -@Builder -@AllArgsConstructor -@FieldDefaults(level = AccessLevel.PRIVATE) -public class TableCellEntity implements Node { - - //foreign key - Integer table; - TextBlockEntity textBlock; - List entities; - - - @Override - public TextBlockEntity buildTextBlock() { - - return textBlock; - } - - - @Override - public boolean hasTextBlock() { - - return true; - } - - - @Override - public TextBlockEntity getTextBlock() { - - return textBlock; - } - - -} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/TableEntity.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/TableEntity.java deleted file mode 100644 index 1d7c4ad2..00000000 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/TableEntity.java +++ /dev/null @@ -1,53 +0,0 @@ -package com.iqser.red.service.redaction.v1.server.document.graph; - -import java.util.List; -import java.util.stream.Stream; - -import lombok.AccessLevel; -import lombok.AllArgsConstructor; -import lombok.Builder; -import lombok.Data; -import lombok.experimental.FieldDefaults; - -@Data -@Builder -@AllArgsConstructor -@FieldDefaults(level = AccessLevel.PRIVATE) -public class TableEntity implements Node { - - Integer id; - Integer numberOfRows; - Integer numberOfCols; - List> tableCells; - - // graph connection - SectionEntity section; - PageEntity page; - List entities; - - - private Stream streamTableCells() { - - return tableCells.stream().flatMap(List::stream); - } - - - private Stream streamTableRow(int row) { - - return tableCells.get(row).stream(); - } - - - private Stream streamTableCol(int col) { - - return tableCells.stream().map(row -> row.get(col)); - } - - - @Override - public TextBlockEntity buildTextBlock() { - - return streamTableCells().map(TableCellEntity::buildTextBlock).collect(TextBlockCollector.collect()); - } - -} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/TableOfContents.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/TableOfContents.java index 30e5d5c1..9a0dece8 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/TableOfContents.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/TableOfContents.java @@ -2,6 +2,7 @@ package com.iqser.red.service.redaction.v1.server.document.graph; import static java.lang.String.format; +import java.nio.charset.StandardCharsets; import java.util.Arrays; import java.util.LinkedList; import java.util.List; @@ -9,6 +10,9 @@ import java.util.stream.Stream; import javax.management.openmbean.InvalidKeyException; +import com.google.common.hash.Hashing; +import com.iqser.red.service.redaction.v1.server.document.graph.nodes.DocumentNode; + import lombok.Data; @Data @@ -28,7 +32,7 @@ public class TableOfContents { } - public String createNewEntryAndReturnId(String keyword, String summary, Node node) { + public String createNewEntryAndReturnId(String keyword, String summary, DocumentNode node) { String id = String.format("%d", entries.size()); entries.add(new Entry(keyword, id, summary, new LinkedList<>(), node)); @@ -36,7 +40,7 @@ public class TableOfContents { } - public String createNewChildEntryAndReturnId(String parentId, String keyword, String summary, Node node) { + public String createNewChildEntryAndReturnId(String parentId, String keyword, String summary, DocumentNode node) { Entry parent = getEntryById(parentId); String childId = parentId + String.format(".%d", parent.children().size()); @@ -96,7 +100,7 @@ public class TableOfContents { } - public record Entry(String type, String id, String summary, List children, Node node) { + public record Entry(String type, String id, String summary, List children, DocumentNode node) { @Override public String toString() { @@ -104,6 +108,9 @@ public class TableOfContents { return id + ": " + type + ".: " + summary; } + @Override + public int hashCode() { + return Hashing.murmur3_32_fixed().hashString(type + id + summary + children.hashCode(), StandardCharsets.UTF_8).hashCode(); + } } - } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/TextBlockCollector.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/TextBlockCollector.java deleted file mode 100644 index dad82cf2..00000000 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/TextBlockCollector.java +++ /dev/null @@ -1,53 +0,0 @@ -package com.iqser.red.service.redaction.v1.server.document.graph; - -import java.util.Collections; -import java.util.Set; -import java.util.function.BiConsumer; -import java.util.function.BinaryOperator; -import java.util.function.Function; -import java.util.function.Supplier; -import java.util.stream.Collector; - -public class TextBlockCollector implements Collector { - - public static Collector collect() { - - return new TextBlockCollector(); - } - - - @Override - public Supplier supplier() { - - return new TextBlockEntity(Collections.emptyList()); - } - - - @Override - public BiConsumer accumulator() { - - return TextBlockEntity::concat; - } - - - @Override - public BinaryOperator combiner() { - - return TextBlockEntity::concat; - } - - - @Override - public Function finisher() { - - return Function.identity(); - } - - - @Override - public Set characteristics() { - - return Set.of(Characteristics.IDENTITY_FINISH, Characteristics.CONCURRENT); - } - -} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/TextBlockEntity.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/TextBlockEntity.java deleted file mode 100644 index 02f482e4..00000000 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/TextBlockEntity.java +++ /dev/null @@ -1,166 +0,0 @@ -package com.iqser.red.service.redaction.v1.server.document.graph; - -import java.awt.geom.Rectangle2D; -import java.util.LinkedList; -import java.util.List; -import java.util.function.Supplier; - -import com.iqser.red.service.redaction.v1.server.redaction.model.RedRectangle2D; - -import lombok.AccessLevel; -import lombok.Data; -import lombok.experimental.FieldDefaults; - -@Data -@FieldDefaults(level = AccessLevel.PRIVATE) -public class TextBlockEntity implements CharSequence, Supplier { - - List atomicTextBlocks; - StringBuilder searchTextBuilder; - - //string coords - Integer offset; - - - public TextBlockEntity(List atomicTextBlocks) { - - this.atomicTextBlocks = new LinkedList<>(); - this.searchTextBuilder = new StringBuilder(); - if (atomicTextBlocks.isEmpty()) { - offset = -1; - return; - } - var firstTextBlock = atomicTextBlocks.get(0); - this.atomicTextBlocks.add(firstTextBlock); - this.offset = firstTextBlock.getOffset(); - this.searchTextBuilder.append(firstTextBlock.getSearchText()); - - atomicTextBlocks.subList(1, atomicTextBlocks.size()).forEach(this::concat); - } - - - public TextBlockEntity concat(TextBlockEntity textBlock) { - - if (this.atomicTextBlocks.isEmpty() && this.offset == -1) { - this.offset = textBlock.getOffset(); - } else if (textBlock.getOffset() != offset + length()) { - throw new UnsupportedOperationException("Can only concat consecutive TextBlocks"); - } - - this.searchTextBuilder.append(textBlock.getSearchTextBuilder()); - this.atomicTextBlocks.addAll(textBlock.getAtomicTextBlocks()); - return this; - } - - - public void concat(AtomicTextBlockEntity atomicTextBlock) { - - if (this.atomicTextBlocks.isEmpty() && this.offset == -1) { - this.offset = atomicTextBlock.getOffset(); - } else if (atomicTextBlock.getOffset() != offset + length()) { - throw new UnsupportedOperationException("Can only concat consecutive TextBlocks"); - } - - this.searchTextBuilder.append(atomicTextBlock.getSearchText()); - this.atomicTextBlocks.add(atomicTextBlock); - } - - - public int indexOf(String searchTerm) { - - int pos = this.searchTextBuilder.indexOf(searchTerm); - return pos == -1 ? -1 : pos + offset; - } - - - public int numberOfLines() { - - return atomicTextBlocks.stream().map(AtomicTextBlockEntity::getLineBreaks).mapToInt(List::size).sum(); - } - - - public int getNextLinebreak(int startIndex) { - - return getAtomicTextBlockByStringIndex(startIndex).getNextLinebreak(startIndex); - } - - - public RedRectangle2D getPositions(int stringIdx) { - - return getAtomicTextBlockByStringIndex(stringIdx).getPosition(stringIdx); - } - - - public List getPositions(int startStringIdx, int endStringIdx) { - - List textBlocks = getAllAtomicTextBlocksPartiallyInStringIdxRange(startStringIdx, endStringIdx); - - if (textBlocks.size() == 1) { - return textBlocks.get(0).getPositions(startStringIdx, endStringIdx); - } - - var firstTextBlock = textBlocks.get(0); - List positions = new LinkedList<>(firstTextBlock.getPositions(startStringIdx, firstTextBlock.getOffset() + firstTextBlock.length())); - - for (var textBlock : textBlocks.subList(1, textBlocks.size() - 1)) { - positions.addAll(textBlock.getPositions()); - } - - var lastTextBlock = textBlocks.get(textBlocks.size() - 1); - positions.addAll(lastTextBlock.getPositions(lastTextBlock.getOffset(), endStringIdx)); - - return positions; - } - - - private AtomicTextBlockEntity getAtomicTextBlockByStringIndex(int stringIdx) { - - return atomicTextBlocks.stream().filter(textBlock -> (textBlock.getOffset() + textBlock.length()) > stringIdx).findFirst().orElseThrow(IndexOutOfBoundsException::new); - } - - - private List getAllAtomicTextBlocksPartiallyInStringIdxRange(int startStringIdx, int endStringIdx) { - - if (offset > endStringIdx || offset + length() <= startStringIdx) { - throw new IndexOutOfBoundsException(); - } - - return atomicTextBlocks.stream().filter(tb -> tb.getOffset() <= endStringIdx && tb.getOffset() + tb.length() > startStringIdx).toList(); - } - - - public String getSearchText() { - - return searchTextBuilder.toString(); - } - - - @Override - public int length() { - - return this.searchTextBuilder.length(); - } - - - @Override - public char charAt(int index) { - - return searchTextBuilder.charAt(index); - } - - - @Override - public CharSequence subSequence(int start, int end) { - - throw new UnsupportedOperationException("AtomicTextBlockEntity can not be sliced!"); - } - - - @Override - public TextBlockEntity get() { - - return this; - } - -} - diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/DocumentNode.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/DocumentNode.java new file mode 100644 index 00000000..004b9a1d --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/DocumentNode.java @@ -0,0 +1,202 @@ +package com.iqser.red.service.redaction.v1.server.document.graph.nodes; + +import static com.iqser.red.service.redaction.v1.server.document.services.EntityEnrichmentUtility.enrichEntity; +import static java.lang.String.format; + +import java.util.List; +import java.util.stream.Stream; + +import com.iqser.red.service.redaction.v1.server.document.graph.Boundary; +import com.iqser.red.service.redaction.v1.server.document.graph.textblock.AtomicTextBlock; +import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock; +import com.iqser.red.service.redaction.v1.server.exception.NotFoundException; +import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType; + +public interface DocumentNode { + + /** + * Searches all Nodes located underneath this Node in the TableOfContents and concatenates their AtomicTextBlocks into a single TextBlockEntity. + * So, for a Section all AtomicTextBlocks of Subsections, Paragraphs, and Tables are concatenated into a single TextBlockEntity + * + * @return TextBlock containing all AtomicTextBlocks that are located under this Node. + */ + TextBlock buildTextBlock(); + + + /** + * Any Node maintains its own List of Entities. + * This List contains all Entities, whose first index is located in any of the AtomicTextBlocks underneath this Node. + * + * @return List of all Entities associated with this Node + */ + List getEntities(); + + + /** + * Returns the PageNode associated with this Node. + * If the node has more than one PageNode associated, it returns the PageNode with the lowest number. + * For example a section might span multiple pages, it then returns the page where the section starts. + * + * @return PageNode representing the first page on which the Node is located in the document + */ + PageNode getPage(); + + + /** + * Any Node except the First level of Sections, Header, Footer, and Pages have a direct Parent. + * For example a Paragraph has a parent Section, a Table Cell has a parent Table, etc... + * hasParent() may be used to check whether a parent is present. + * + * @return Node that represents the Parent or null, if no parent is present. + */ + DocumentNode getParent(); + + + Stream streamAllSubNodes(); + + + /** + * By default, a parent is always present, this needs to be overwritten for Headers, Footers, Pages, and Sections. + * + * @return boolean, indicating whether a parent is present. + */ + default boolean hasParent() { + + return true; + } + + + /** + * by default a Node does not have direct access to an AtomicTextBlock + * + * @return boolean, indicating if a Node has direct access to an AtomicTextBlock + */ + default boolean isTerminal() { + + return false; + } + + + /** + * by default a Node does not have direct access to an AtomicTextBlock, this method throws a UnsupportedOperationException if not overridden. + * + * @return AtomicTextBlock + */ + default AtomicTextBlock getAtomicTextBlock() { + + throw new UnsupportedOperationException("Only terminal Nodes have access to AtomicTextBlocks!"); + } + + + /** + * creates an EntityNode with only initial values set, inserts it into the subgraph contained by this node and sets the inferrable fields. + * Throws NotFoundException and removes the entity if the provided boundary could not be found in the subgraph. + * + * @param boundary start and end indices in String coordinates of the entity to be created + * @param type type of the entity to be created + * @param entityType entityType of the entity to be created + * @return the newly created and inserted EntityNode with all fields set. + */ + default EntityNode createAndAddEntity(Boundary boundary, String type, EntityType entityType) { + + EntityNode entity = EntityNode.initialEntityNode(boundary, type, entityType); + addEntityToNodeAndSetFields(entity); + return entity; + } + + + /** + * searches for the first terminal Node containing the start index of the entity to be inserted. + * Catches NotFoundException to remove the EntityNode from the graph, then rethrows it + * + * @param entity newly created EntityNode with only initial values set + */ + default void addEntityToNodeAndSetFields(EntityNode entity) { + + try { + streamAllSubNodes().anyMatch(node -> node.addEntityAndSetFieldsIfStartIndexContained(entity)); + } catch (NotFoundException e) { + entity.removeFromGraph(); + throw new RuntimeException(e); + } + } + + + /** + * If this Node's AtomicTextBlock contains the start index of the entity, the entity's position is read from the AtomicTextBlock. + * If the position can not be fully read from the AtomicTextBlock, it recursively looks in the TextBlocks of the parents until all positions are found. + * Further, the function throws NotFoundException if no parent contains all positions. + * This occurs, when the Entity is in between Nodes that do not share a parent, e.g. main sections. + * Finally, the function adds the Entity to its own list of Entities and to every parents' list recursively. + * + * @param entity The entity to be added to the graph + * @return true, if the entity has been added successfully. + * false, if the entity's start index is not contained or the Node doesn't have an AtomicTextBlock + */ + default boolean addEntityAndSetFieldsIfStartIndexContained(EntityNode entity) { + + if (!isTerminal()) { + return false; + } + + AtomicTextBlock atomicTextBlock = getAtomicTextBlock(); + if (atomicTextBlock.containsIndex(entity.getBoundary().start())) { + + entity.addContainingNode(this); + + getEntities().add(entity); + + addEntityToPage(entity); + addEntityToParents(entity); + setFields(entity, atomicTextBlock); + return true; + } + + return false; + } + + + private void addEntityToPage(EntityNode entity) { + + getPage().getEntities().add(entity); + entity.setPage(getPage()); + } + + + private void setFields(EntityNode entity, AtomicTextBlock atomicTextBlock) { + + if (atomicTextBlock.containsRange(entity.getBoundary())) { + enrichEntity(entity, atomicTextBlock); + } else { + this.setFieldsFromParents(this, entity); + } + } + + + private void addEntityToParents(EntityNode entity) { + + DocumentNode node = this; + while (node.hasParent()) { + node = node.getParent(); + node.getEntities().add(entity); + entity.addContainingNode(node); + } + } + + + private void setFieldsFromParents(DocumentNode node, EntityNode entity) { + + if (node.hasParent()) { + DocumentNode parent = node.getParent(); + TextBlock textBlock = parent.buildTextBlock(); + if (textBlock.containsRange(entity.getBoundary())) { + enrichEntity(entity, textBlock); + return; + } else { + setFieldsFromParents(parent, entity); + } + } + throw new NotFoundException(format("Position could not be found for Entity %s", entity.toString())); + } + +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/EntityNode.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/EntityNode.java new file mode 100644 index 00000000..ca99c08e --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/EntityNode.java @@ -0,0 +1,84 @@ +package com.iqser.red.service.redaction.v1.server.document.graph.nodes; + +import java.awt.geom.Rectangle2D; +import java.nio.charset.StandardCharsets; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import com.google.common.hash.Hashing; +import com.iqser.red.service.redaction.v1.server.document.graph.Boundary; +import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType; + +import lombok.AccessLevel; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.experimental.FieldDefaults; + +@Data +@Builder +@AllArgsConstructor +@FieldDefaults(level = AccessLevel.PRIVATE) +public class EntityNode { + + public static EntityNode initialEntityNode(Boundary boundary, String type, EntityType entityType) { + + return EntityNode.builder().type(type).entityType(entityType).boundary(boundary).build(); + } + + + // initial values + Boundary boundary; + String type; + EntityType entityType; + + @Builder.Default + boolean redact = false; + @Builder.Default + boolean recommend = false; + @Builder.Default + boolean removed = false; + + // inferrable from graph + int uniqueId; + String value; + CharSequence textBefore; + CharSequence textAfter; + PageNode page; + List positions; + @Builder.Default + Set containingNodes = new HashSet<>(); + + + public void addContainingNode(DocumentNode containingNode) { + + containingNodes.add(containingNode); + } + + + public void removeFromGraph() { + + getContainingNodes().forEach(node -> node.getEntities().remove(this)); + getPage().getEntities().remove(this); + setRemoved(true); + } + + + @Override + public int hashCode() { + + var sb = new StringBuilder(); + sb.append(value); + sb.append(boundary.start()); + sb.append(page.getNumber()); + positions.forEach(r -> { + sb.append(r.getMinX()); + sb.append(r.getMinY()); + sb.append(r.getWidth()); + sb.append(r.getHeight()); + }); + return Hashing.murmur3_128().hashString(sb.toString(), StandardCharsets.UTF_8).hashCode(); + } + +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/FooterNode.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/FooterNode.java new file mode 100644 index 00000000..ba360307 --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/FooterNode.java @@ -0,0 +1,71 @@ +package com.iqser.red.service.redaction.v1.server.document.graph.nodes; + +import java.util.LinkedList; +import java.util.List; +import java.util.stream.Stream; + +import com.iqser.red.service.redaction.v1.server.document.graph.textblock.AtomicTextBlock; + +import lombok.AccessLevel; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.experimental.FieldDefaults; + +@Data +@Builder +@AllArgsConstructor +@FieldDefaults(level = AccessLevel.PRIVATE) +public class FooterNode implements DocumentNode { + + @EqualsAndHashCode.Exclude + PageNode page; + AtomicTextBlock atomicTextBlock; + + @Builder.Default + @EqualsAndHashCode.Exclude + List entities = new LinkedList<>(); + + + @Override + public AtomicTextBlock buildTextBlock() { + + return atomicTextBlock; + } + + @Override + public DocumentNode getParent() { + + throw new UnsupportedOperationException("Footer has no Parent, use getPage() instead"); + } + + + @Override + public Stream streamAllSubNodes() { + + return Stream.of(this); + } + + + @Override + public boolean hasParent() { + + return false; + } + + + @Override + public boolean isTerminal() { + + return true; + } + + + @Override + public String toString() { + + return atomicTextBlock.toString(); + } + +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/HeaderNode.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/HeaderNode.java new file mode 100644 index 00000000..0c0d7ea8 --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/HeaderNode.java @@ -0,0 +1,72 @@ +package com.iqser.red.service.redaction.v1.server.document.graph.nodes; + +import java.util.LinkedList; +import java.util.List; +import java.util.stream.Stream; + +import com.iqser.red.service.redaction.v1.server.document.graph.textblock.AtomicTextBlock; + +import lombok.AccessLevel; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.experimental.FieldDefaults; + +@Data +@Builder +@AllArgsConstructor +@FieldDefaults(level = AccessLevel.PRIVATE) +public class HeaderNode implements DocumentNode { + + @EqualsAndHashCode.Exclude + PageNode page; + AtomicTextBlock atomicTextBlock; + + @Builder.Default + @EqualsAndHashCode.Exclude + List entities = new LinkedList<>(); + + + @Override + public AtomicTextBlock buildTextBlock() { + + return atomicTextBlock; + } + + + @Override + public DocumentNode getParent() { + + throw new UnsupportedOperationException("Header has no Parent, use getPage() instead"); + } + + + @Override + public Stream streamAllSubNodes() { + + return Stream.of(this); + } + + + @Override + public boolean hasParent() { + + return false; + } + + + @Override + public boolean isTerminal() { + + return true; + } + + + @Override + public String toString() { + + return atomicTextBlock.toString(); + } + +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/PageNode.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/PageNode.java new file mode 100644 index 00000000..e575dc43 --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/PageNode.java @@ -0,0 +1,80 @@ +package com.iqser.red.service.redaction.v1.server.document.graph.nodes; + +import java.util.LinkedList; +import java.util.List; +import java.util.stream.Stream; + +import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlockCollector; +import com.iqser.red.service.redaction.v1.server.document.graph.textblock.ConcatenatedTextBlock; + +import lombok.AccessLevel; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.experimental.FieldDefaults; + +@Data +@Builder +@AllArgsConstructor +@FieldDefaults(level = AccessLevel.PRIVATE) +public class PageNode implements DocumentNode { + + Integer number; + Integer height; + Integer width; + List mainBody; + HeaderNode header; + FooterNode footer; + + @Builder.Default + @EqualsAndHashCode.Exclude + List entities = new LinkedList<>(); + + + @Override + public ConcatenatedTextBlock buildTextBlock() { + + return mainBody.stream().filter(DocumentNode::isTerminal).map(DocumentNode::getAtomicTextBlock).collect(new TextBlockCollector()); + } + + + @Override + public DocumentNode getParent() { + + throw new UnsupportedOperationException("A Page has no parent!"); + } + + + @Override + public Stream streamAllSubNodes() { + + return Stream.concat(// + mainBody.stream(), // + Stream.concat(// + Stream.of(header), // + Stream.of(footer))); // + } + + + @Override + public PageNode getPage() { + + return this; + } + + + @Override + public boolean hasParent() { + + return false; + } + + + @Override + public String toString() { + + return number.toString(); + } + +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/ParagraphNode.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/ParagraphNode.java new file mode 100644 index 00000000..c6f243ae --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/ParagraphNode.java @@ -0,0 +1,70 @@ +package com.iqser.red.service.redaction.v1.server.document.graph.nodes; + +import java.util.LinkedList; +import java.util.List; +import java.util.stream.Stream; + +import com.iqser.red.service.redaction.v1.server.document.graph.textblock.AtomicTextBlock; + +import lombok.AccessLevel; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.experimental.FieldDefaults; + +@Data +@Builder +@AllArgsConstructor +@FieldDefaults(level = AccessLevel.PRIVATE) +public class ParagraphNode implements DocumentNode { + + String tocId; + Integer id; + Integer numberOnPage; + Integer numberInSection; + + @EqualsAndHashCode.Exclude + SectionNode parentSection; + @EqualsAndHashCode.Exclude + PageNode page; + AtomicTextBlock atomicTextBlock; + + @Builder.Default + @EqualsAndHashCode.Exclude + List entities = new LinkedList<>(); + + + @Override + public AtomicTextBlock buildTextBlock() { + + return atomicTextBlock; + } + + + @Override + public DocumentNode getParent() { + + return parentSection; + } + + + @Override + public boolean isTerminal() { + + return true; + } + + + @Override + public String toString() { + + return tocId + ": " + atomicTextBlock.toString(); + } + + @Override + public Stream streamAllSubNodes() { + + return Stream.of(this); + } +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/SectionNode.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/SectionNode.java new file mode 100644 index 00000000..dec1b62e --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/SectionNode.java @@ -0,0 +1,107 @@ +package com.iqser.red.service.redaction.v1.server.document.graph.nodes; + +import java.util.LinkedList; +import java.util.List; +import java.util.stream.Stream; + +import com.iqser.red.service.redaction.v1.server.document.graph.TableOfContents; +import com.iqser.red.service.redaction.v1.server.document.graph.textblock.AtomicTextBlock; +import com.iqser.red.service.redaction.v1.server.document.graph.textblock.ConcatenatedTextBlock; +import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlockCollector; + +import lombok.AccessLevel; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.experimental.FieldDefaults; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +@Data +@Builder +@AllArgsConstructor +@FieldDefaults(level = AccessLevel.PRIVATE) +public class SectionNode implements DocumentNode { + + String tocId; + Integer id; + @EqualsAndHashCode.Exclude + TableOfContents tableOfContents; + @EqualsAndHashCode.Exclude + DocumentNode parentSection; + + + AtomicTextBlock headline; + + List subSections; + List paragraphs; + List tables; + @EqualsAndHashCode.Exclude + List pages; + + @Builder.Default + @EqualsAndHashCode.Exclude + List entities = new LinkedList<>(); + + + @Override + public String toString() { + + return tocId + ": " + headline.toString(); + } + + + @Override + public ConcatenatedTextBlock buildTextBlock() { + + return streamAllSubNodes().map(DocumentNode::getAtomicTextBlock).collect(new TextBlockCollector()); + } + + + @Override + public DocumentNode getParent() { + + if (hasParent()) { + return parentSection; + } else { + throw new UnsupportedOperationException("This section has no parent Section!"); + } + } + + + @Override + public Stream streamAllSubNodes() { + + return tableOfContents.streamSubEntriesInOrder(tocId).map(TableOfContents.Entry::node); + } + + + @Override + public boolean hasParent() { + + return parentSection != null; + } + + + @Override + public boolean isTerminal() { + + return true; + } + + + @Override + public AtomicTextBlock getAtomicTextBlock() { + + return headline; + } + + + @Override + public PageNode getPage() { + + return pages.get(0); + } + +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/TableCellNode.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/TableCellNode.java new file mode 100644 index 00000000..55553077 --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/TableCellNode.java @@ -0,0 +1,57 @@ +package com.iqser.red.service.redaction.v1.server.document.graph.nodes; + +import java.util.LinkedList; +import java.util.List; +import java.util.stream.Stream; + +import com.iqser.red.service.redaction.v1.server.document.graph.textblock.AtomicTextBlock; + +import lombok.AccessLevel; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.experimental.FieldDefaults; + +@Data +@Builder +@AllArgsConstructor +@FieldDefaults(level = AccessLevel.PRIVATE) +public class TableCellNode implements DocumentNode { + + @EqualsAndHashCode.Exclude + TableNode parentTable; + AtomicTextBlock atomicTextBlock; + PageNode page; + + @Builder.Default + @EqualsAndHashCode.Exclude + List entities = new LinkedList<>(); + + + @Override + public AtomicTextBlock buildTextBlock() { + + return atomicTextBlock; + } + + + @Override + public DocumentNode getParent() { + + return parentTable; + } + + + @Override + public boolean isTerminal() { + + return true; + } + + @Override + public Stream streamAllSubNodes() { + + return Stream.of(this); + } +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/TableNode.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/TableNode.java new file mode 100644 index 00000000..b03b5ee5 --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/TableNode.java @@ -0,0 +1,87 @@ +package com.iqser.red.service.redaction.v1.server.document.graph.nodes; + +import java.util.LinkedList; +import java.util.List; +import java.util.function.Function; +import java.util.stream.Stream; + +import com.iqser.red.service.redaction.v1.server.document.graph.TableOfContents; +import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlockCollector; +import com.iqser.red.service.redaction.v1.server.document.graph.textblock.ConcatenatedTextBlock; + +import lombok.AccessLevel; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.experimental.FieldDefaults; + +@Data +@Builder +@AllArgsConstructor +@FieldDefaults(level = AccessLevel.PRIVATE) +public class TableNode implements DocumentNode { + + Integer id; + String tocId; + Integer numberOfRows; + Integer numberOfCols; + List tableHeaders; + List> tableCells; + TableOfContents tableOfContents; + + @EqualsAndHashCode.Exclude + SectionNode parentSection; + @EqualsAndHashCode.Exclude + List pages; + + @Builder.Default + @EqualsAndHashCode.Exclude + List entities = new LinkedList<>(); + + + private Stream streamTableCells() { + + return tableCells.stream().flatMap(List::stream); + } + + + private Stream streamTableRow(int row) { + + return tableCells.get(row).stream(); + } + + + private Stream streamTableCol(int col) { + + return tableCells.stream().map(row -> row.get(col)); + } + + + @Override + public ConcatenatedTextBlock buildTextBlock() { + + return streamTableCells().map(TableCellNode::getAtomicTextBlock).collect(new TextBlockCollector()); + } + + @Override + public Stream streamAllSubNodes() { + + return streamTableCells().map(Function.identity()); + } + + + @Override + public DocumentNode getParent() { + + return parentSection; + } + + + @Override + public PageNode getPage() { + + return pages.get(0); + } + +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/textblock/AtomicTextBlock.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/textblock/AtomicTextBlock.java new file mode 100644 index 00000000..d846081d --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/textblock/AtomicTextBlock.java @@ -0,0 +1,134 @@ +package com.iqser.red.service.redaction.v1.server.document.graph.textblock; + +import static java.lang.String.format; + +import java.awt.geom.Rectangle2D; +import java.util.List; + +import com.iqser.red.service.redaction.v1.server.document.graph.Boundary; +import com.iqser.red.service.redaction.v1.server.document.graph.nodes.DocumentNode; + +import lombok.AccessLevel; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.experimental.FieldDefaults; + +@Data +@Builder +@AllArgsConstructor +@FieldDefaults(level = AccessLevel.PRIVATE) +public class AtomicTextBlock implements TextBlock { + + Integer id; + + //string coordinates + Boundary boundary; + String searchText; + List lineBreaks; + + //position coordinates + List stringIdxToPositionIdx; + List positions; + + @EqualsAndHashCode.Exclude + DocumentNode parent; + + + public int indexOf(String searchTerm) { + + int pos = searchText.indexOf(searchTerm); + return pos == -1 ? -1 : pos + boundary.start(); + } + + + public int numberOfLines() { + + return lineBreaks.size(); + } + + + @Override + public List getAtomicTextBlocks() { + + return List.of(this); + } + + + public int getNextLinebreak(int fromIndex) { + + return lineBreaks.stream()// + .filter(linebreak -> linebreak > fromIndex) // + .findFirst() // + .orElse(searchText.length()) + boundary.start(); + } + + + public int getPreviousLinebreak(int fromIndex) { + + return lineBreaks.stream()// + .filter(linebreak -> linebreak <= fromIndex)// + .reduce((a, b) -> b)// + .orElse(0) + boundary.start(); + } + + + public Rectangle2D getPosition(int stringIdx) { + + return positions.get(stringIdxToPositionIdx.get(stringIdx - boundary.start())); + } + + + public List getPositions(Boundary range) { + + if (!containsRange(range)) { + throw new IndexOutOfBoundsException(format("String index Range [%d|%d) is out of boundary for String index [%d|%d)", + range.start(), + range.end(), + range.start(), + range.end())); + } + + if (range.end() == this.boundary.end()) { + return positions.subList(stringIdxToPositionIdx.get(range.start() - this.boundary.start()), positions.size()); + } + + return positions.subList(stringIdxToPositionIdx.get(range.start() - this.boundary.start()), stringIdxToPositionIdx.get(range.end() - this.boundary.start())); + } + + + @Override + public int length() { + + return searchText.length(); + } + + + @Override + public char charAt(int index) { + + return searchText.charAt(index - boundary.start()); + } + + + @Override + public CharSequence subSequence(int start, int end) { + + return searchText.substring(start - boundary.start(), end - boundary.start()); + } + + + public String getFirstLine() { + + return searchText.substring(0, getNextLinebreak(0) - boundary.start()); + } + + + @Override + public String toString() { + + return searchText; + } + +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/textblock/ConcatenatedTextBlock.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/textblock/ConcatenatedTextBlock.java new file mode 100644 index 00000000..20f36a2b --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/textblock/ConcatenatedTextBlock.java @@ -0,0 +1,155 @@ +package com.iqser.red.service.redaction.v1.server.document.graph.textblock; + +import static java.lang.String.format; + +import java.awt.geom.Rectangle2D; +import java.util.LinkedList; +import java.util.List; +import java.util.function.Supplier; + +import com.iqser.red.service.redaction.v1.server.document.graph.Boundary; + +import lombok.AccessLevel; +import lombok.Data; +import lombok.experimental.FieldDefaults; + +@Data +@FieldDefaults(level = AccessLevel.PRIVATE) +public class ConcatenatedTextBlock implements TextBlock, Supplier { + + List atomicTextBlocks; + StringBuilder searchText; + Boundary boundary; + + + public ConcatenatedTextBlock(List atomicTextBlocks) { + + this.atomicTextBlocks = new LinkedList<>(); + this.searchText = new StringBuilder(); + if (atomicTextBlocks.isEmpty()) { + boundary = new Boundary(-1, -1); + return; + } + var firstTextBlock = atomicTextBlocks.get(0); + this.atomicTextBlocks.add(firstTextBlock); + this.searchText.append(firstTextBlock.getSearchText()); + boundary = new Boundary(firstTextBlock.getBoundary().start(), firstTextBlock.getBoundary().end()); + + atomicTextBlocks.subList(1, atomicTextBlocks.size()).forEach(this::concat); + } + + + public ConcatenatedTextBlock(AtomicTextBlock atomicTextBlocks) { + + new ConcatenatedTextBlock(List.of(atomicTextBlocks)); + } + + + public ConcatenatedTextBlock concat(TextBlock textBlock) { + + if (this.atomicTextBlocks.isEmpty()) { + boundary.setStart(textBlock.getBoundary().start()); + boundary.setEnd(textBlock.getBoundary().end()); + } else if (boundary.end() != textBlock.getBoundary().start()) { + throw new UnsupportedOperationException(format("Can only concat consecutive TextBlocks, trying to concat %s and %s", textBlock.getBoundary(), boundary)); + } + this.searchText.append(textBlock.getSearchText()); + this.atomicTextBlocks.addAll(textBlock.getAtomicTextBlocks()); + boundary.setEnd(textBlock.getBoundary().end()); + return this; + } + + + public int indexOf(String searchTerm) { + + int pos = this.searchText.indexOf(searchTerm); + return pos == -1 ? -1 : pos + boundary.start(); + } + + + public int numberOfLines() { + + return atomicTextBlocks.stream().map(AtomicTextBlock::getLineBreaks).mapToInt(List::size).sum(); + } + + + public int getNextLinebreak(int fromIndex) { + + return getAtomicTextBlockByStringIndex(fromIndex).getNextLinebreak(fromIndex); + } + + + public int getPreviousLinebreak(int fromIndex) { + + return getAtomicTextBlockByStringIndex(fromIndex).getPreviousLinebreak(fromIndex); + } + + + public Rectangle2D getPosition(int stringIdx) { + + return getAtomicTextBlockByStringIndex(stringIdx).getPosition(stringIdx); + } + + + public List getPositions(Boundary boundary) { + + List textBlocks = getAllAtomicTextBlocksPartiallyInStringIdxRange(boundary); + + if (textBlocks.size() == 1) { + return textBlocks.get(0).getPositions(boundary); + } + + AtomicTextBlock firstTextBlock = textBlocks.get(0); + List positions = new LinkedList<>(firstTextBlock.getPositions(new Boundary(boundary.start(), firstTextBlock.getBoundary().end()))); + + for (AtomicTextBlock textBlock : textBlocks.subList(1, textBlocks.size() - 1)) { + positions.addAll(textBlock.getPositions()); + } + + var lastTextBlock = textBlocks.get(textBlocks.size() - 1); + positions.addAll(lastTextBlock.getPositions(new Boundary(lastTextBlock.getBoundary().start(), boundary.end()))); + + return positions; + } + + + private AtomicTextBlock getAtomicTextBlockByStringIndex(int stringIdx) { + + return atomicTextBlocks.stream().filter(textBlock -> (textBlock.getBoundary().end()) > stringIdx).findFirst().orElseThrow(IndexOutOfBoundsException::new); + } + + + private List getAllAtomicTextBlocksPartiallyInStringIdxRange(Boundary boundary) { + + return atomicTextBlocks.stream().filter(tb -> tb.getBoundary().intersects(boundary)).toList(); + } + + + @Override + public int length() { + + return this.searchText.length(); + } + + + @Override + public char charAt(int index) { + + return searchText.charAt(index - boundary.start()); + } + + + @Override + public CharSequence subSequence(int start, int end) { + + return searchText.subSequence(start - boundary.start(), end - boundary.start()); + } + + + @Override + public ConcatenatedTextBlock get() { + + return this; + } + +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/textblock/TextBlock.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/textblock/TextBlock.java new file mode 100644 index 00000000..66400aa3 --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/textblock/TextBlock.java @@ -0,0 +1,59 @@ +package com.iqser.red.service.redaction.v1.server.document.graph.textblock; + +import static java.lang.String.format; + +import java.awt.geom.Rectangle2D; +import java.util.List; + +import com.iqser.red.service.redaction.v1.server.document.graph.Boundary; + +public interface TextBlock extends CharSequence { + + CharSequence getSearchText(); + + + List getAtomicTextBlocks(); + + + Boundary getBoundary(); + + + int getNextLinebreak(int fromIndex); + + + int getPreviousLinebreak(int fromIndex); + + + Rectangle2D getPosition(int stringIdx); + + + List getPositions(Boundary range); + + + int numberOfLines(); + + + int indexOf(String searchTerm); + + + default boolean containsRange(Boundary range) { + + if (range.end() < range.start()) { + throw new IllegalArgumentException(format("Invalid range [%d|%d), StartIndex must be smaller than EndIndex", range.start(), range.end())); + } + return getBoundary().contains(range); + } + + + default boolean containsIndex(int stringIndex) { + + return getBoundary().contains(stringIndex); + } + + + default CharSequence subSequence(Boundary boundary) { + + return subSequence(boundary.start(), boundary.end()); + } + +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/textblock/TextBlockCollector.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/textblock/TextBlockCollector.java new file mode 100644 index 00000000..27320c89 --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/textblock/TextBlockCollector.java @@ -0,0 +1,51 @@ +package com.iqser.red.service.redaction.v1.server.document.graph.textblock; + +import java.util.Collections; +import java.util.Set; +import java.util.function.BiConsumer; +import java.util.function.BinaryOperator; +import java.util.function.Function; +import java.util.function.Supplier; +import java.util.stream.Collector; + +import lombok.NoArgsConstructor; + +@NoArgsConstructor +public class TextBlockCollector implements Collector { + + + @Override + public Supplier supplier() { + + return new ConcatenatedTextBlock(Collections.emptyList()); + } + + + @Override + public BiConsumer accumulator() { + + return ConcatenatedTextBlock::concat; + } + + + @Override + public BinaryOperator combiner() { + + return ConcatenatedTextBlock::concat; + } + + + @Override + public Function finisher() { + + return Function.identity(); + } + + + @Override + public Set characteristics() { + + return Set.of(Characteristics.IDENTITY_FINISH, Characteristics.CONCURRENT); + } + +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/DocumentGraphFactory.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/DocumentGraphFactory.java index 737f7f58..27d78f6a 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/DocumentGraphFactory.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/DocumentGraphFactory.java @@ -1,11 +1,17 @@ package com.iqser.red.service.redaction.v1.server.document.services; +import static java.lang.String.format; + +import java.awt.geom.Rectangle2D; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.Comparator; import java.util.LinkedList; import java.util.List; +import java.util.Map; import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Collectors; import org.springframework.stereotype.Service; @@ -15,19 +21,24 @@ import com.iqser.red.service.redaction.v1.server.classification.model.Header; import com.iqser.red.service.redaction.v1.server.classification.model.Page; import com.iqser.red.service.redaction.v1.server.classification.model.Section; import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock; -import com.iqser.red.service.redaction.v1.server.document.graph.AtomicTextBlockEntity; -import com.iqser.red.service.redaction.v1.server.document.graph.DocumentEntity; -import com.iqser.red.service.redaction.v1.server.document.graph.FooterEntity; -import com.iqser.red.service.redaction.v1.server.document.graph.HeaderEntity; -import com.iqser.red.service.redaction.v1.server.document.graph.Node; -import com.iqser.red.service.redaction.v1.server.document.graph.PageEntity; -import com.iqser.red.service.redaction.v1.server.document.graph.ParagraphEntity; -import com.iqser.red.service.redaction.v1.server.document.graph.SectionEntity; +import com.iqser.red.service.redaction.v1.server.document.graph.Boundary; +import com.iqser.red.service.redaction.v1.server.document.graph.textblock.AtomicTextBlock; +import com.iqser.red.service.redaction.v1.server.document.graph.DocumentGraph; import com.iqser.red.service.redaction.v1.server.document.graph.TableOfContents; -import com.iqser.red.service.redaction.v1.server.document.graph.TextBlockEntity; +import com.iqser.red.service.redaction.v1.server.document.graph.nodes.FooterNode; +import com.iqser.red.service.redaction.v1.server.document.graph.nodes.HeaderNode; +import com.iqser.red.service.redaction.v1.server.document.graph.nodes.DocumentNode; +import com.iqser.red.service.redaction.v1.server.document.graph.nodes.PageNode; +import com.iqser.red.service.redaction.v1.server.document.graph.nodes.ParagraphNode; +import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SectionNode; +import com.iqser.red.service.redaction.v1.server.document.graph.nodes.TableNode; +import com.iqser.red.service.redaction.v1.server.exception.NotFoundException; +import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence; +import com.iqser.red.service.redaction.v1.server.redaction.model.RedRectangle2D; import com.iqser.red.service.redaction.v1.server.redaction.model.SearchTextWithTextPositionModel; import com.iqser.red.service.redaction.v1.server.redaction.service.SearchTextWithTextPositionFactory; import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer; +import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table; import lombok.RequiredArgsConstructor; @@ -37,96 +48,205 @@ public class DocumentGraphFactory { private final SearchTextWithTextPositionFactory searchTextWithTextPositionFactory; - private Context context; + public DocumentGraph buildDocumentGraph(Document document) { - public DocumentEntity buildEntityFromDocument(Document document) { - - context = new Context(new TableOfContents(), new LinkedList<>(), new LinkedList<>(), new LinkedList<>(), new LinkedList<>(), new AtomicInteger(0), new AtomicInteger(0)); + Context context = new Context(new TableOfContents(), + new LinkedList<>(), + new LinkedList<>(), + new LinkedList<>(), + new LinkedList<>(), + new AtomicInteger(0), + new AtomicInteger(0)); context.pages.addAll(document.getPages().stream().map(this::buildPage).toList()); // is tracked by Table of Contents - for (int sectionIdx = 0; sectionIdx < document.getSections().size(); sectionIdx++) { - addSection(document.getSections().get(sectionIdx), sectionIdx); - } + addSections(document, context); // not tracked by Table of Contents - document.getHeaders().stream().map(Header::getTextBlocks).flatMap(List::stream).forEach(this::addHeader); - document.getFooters().stream().map(Footer::getTextBlocks).flatMap(List::stream).forEach(this::addFooter); - - return DocumentEntity.builder().numberOfPages(context.pages.size()).pages(context.pages).sections(context.sections).tableOfContents(context.tableOfContents).build(); + addHeaderAndFooterToEachPage(document, context); + DocumentGraph documentGraph = DocumentGraph.builder() + .numberOfPages(context.pages.size()) + .pages(context.pages) + .sections(context.sections) + .tableOfContents(context.tableOfContents) + .build(); + documentGraph.setText(documentGraph.buildTextBlock()); + return documentGraph; } - private void addSection(Section section, int sectionIdx) { + private void addSections(Document document, Context context) { - SectionEntity sectionEntity = SectionEntity.builder() + for (int sectionIdx = 0; sectionIdx < document.getSections().size(); sectionIdx++) { + addSection(document.getSections().get(sectionIdx), sectionIdx, context); + } + } + + + private void addSection(Section section, int sectionIdx, Context context) { + + SectionNode sectionEntity = SectionNode.builder() .id(sectionIdx) .entities(new LinkedList<>()) .pages(new LinkedList<>()) .paragraphs(new LinkedList<>()) - .tableOfContents(context.tableOfContents()) + .tables(new LinkedList<>()) .subSections(new LinkedList<>()) + .tableOfContents(context.tableOfContents()) .build(); + context.sections().add(sectionEntity); - if (section.getPageBlocks().get(0) instanceof TextBlock) { - sectionEntity.setHeadline(new TextBlockEntity(List.of(buildAtomicTextBlock((TextBlock) section.getPageBlocks().get(0), sectionEntity)))); - section.getPageBlocks().remove(0); + List pageBlocks = new ArrayList<>(section.getPageBlocks()); + if (pageBlocks.get(0) instanceof TextBlock) { + sectionEntity.setHeadline(buildAtomicTextBlock(((TextBlock) pageBlocks.get(0)).getSequences(), sectionEntity, context)); + pageBlocks.remove(0); + sectionEntity.getPages().add(getPage(section.getPageBlocks().get(0).getPage(), context)); } else { - sectionEntity.setHeadline(emptyTextBlock(sectionEntity)); + sectionEntity.setHeadline(emptyTextBlock(sectionEntity, context)); } - String sectionId = context.tableOfContents.createNewEntryAndReturnId(TableOfContents.SECTION, - buildSummary(sectionEntity.getHeadline().getAtomicTextBlocks().get(0)), - sectionEntity); + String sectionId = context.tableOfContents.createNewEntryAndReturnId(TableOfContents.SECTION, buildSummary(sectionEntity.getHeadline()), sectionEntity); sectionEntity.setTocId(sectionId); int paragraphIdx = 0; - for (AbstractTextContainer abstractTextContainer : section.getPageBlocks()) { + int tableIdx = 0; + for (AbstractTextContainer abstractTextContainer : pageBlocks) { if (abstractTextContainer instanceof TextBlock) { - addParagraph(sectionEntity, (TextBlock) abstractTextContainer, paragraphIdx); + addParagraph(sectionEntity, (TextBlock) abstractTextContainer, paragraphIdx, context); paragraphIdx++; + } else if (abstractTextContainer instanceof Table) { + addTable(sectionEntity, (Table) abstractTextContainer, tableIdx, context); + tableIdx++; + } } } - private TextBlockEntity emptyTextBlock(Node parent) { + private void addTable(SectionNode sectionEntity, Table table, int tableIdx, Context context) { + + PageNode page = getPage(table.getPage(), context); + TableNode tableEntity = TableNode.builder().id(tableIdx).tableOfContents(context.tableOfContents()).pages(new LinkedList<>()).parentSection(sectionEntity).build(); + sectionEntity.getTables().add(tableEntity); + page.getMainBody().add(tableEntity); + + if (!page.getMainBody().contains(sectionEntity)) { + sectionEntity.getPages().add(page); + page.getMainBody().add(sectionEntity); + } - return new TextBlockEntity(List.of(AtomicTextBlockEntity.builder() - .id(context.textBlockIdx.getAndIncrement()) - .offset(context.stringOffset.get()) - .searchText("") - .lineBreaks(Collections.emptyList()) - .stringIdxToPositionIdx(Collections.emptyList()) - .positions(Collections.emptyList()) - .parent(parent) - .build())); } - private void addParagraph(SectionEntity sectionEntity, TextBlock originalTextBlock, int idx) { - - PageEntity page = getPage(originalTextBlock.getPage()); - ParagraphEntity paragraph = ParagraphEntity.builder().id(idx).numberOnPage(originalTextBlock.getIndexOnPage()).page(page).parentSection(sectionEntity).build(); + private void addParagraph(SectionNode sectionEntity, TextBlock originalTextBlock, int paragraphIdx, Context context) { + PageNode page = getPage(originalTextBlock.getPage(), context); + ParagraphNode paragraph = ParagraphNode.builder().id(paragraphIdx).numberOnPage(originalTextBlock.getIndexOnPage()).page(page).parentSection(sectionEntity).build(); sectionEntity.getParagraphs().add(paragraph); - page.getParagraphs().add(paragraph); + page.getMainBody().add(paragraph); - if (!page.getSections().contains(sectionEntity)) { - page.getSections().add(sectionEntity); + if (!page.getMainBody().contains(sectionEntity)) { + sectionEntity.getPages().add(page); + page.getMainBody().add(sectionEntity); } - var textBlock = buildAtomicTextBlock(originalTextBlock, paragraph); - paragraph.setTextBlock(new TextBlockEntity(List.of(textBlock))); + var textBlock = buildAtomicTextBlock(originalTextBlock.getSequences(), paragraph, context); + paragraph.setAtomicTextBlock(textBlock); String tocId = context.tableOfContents.createNewChildEntryAndReturnId(sectionEntity.getTocId(), TableOfContents.PARAGRAPH, buildSummary(textBlock), paragraph); paragraph.setTocId(tocId); } - private static String buildSummary(AtomicTextBlockEntity textBlock) { + private void addHeaderAndFooterToEachPage(Document document, Context context) { + + Map> headers = document.getHeaders() + .stream() + .map(Header::getTextBlocks) + .flatMap(List::stream) + .collect(Collectors.groupingBy(AbstractTextContainer::getPage, Collectors.toList())); + + Map> footers = document.getFooters() + .stream() + .map(Footer::getTextBlocks) + .flatMap(List::stream) + .collect(Collectors.groupingBy(AbstractTextContainer::getPage, Collectors.toList())); + + for (int pageIndex = 1; pageIndex <= document.getPages().size(); pageIndex++) { + if (headers.containsKey(pageIndex)) { + addHeader(headers.get(pageIndex), context); + } else { + addEmptyHeader(pageIndex, context); + } + } + + for (int pageIndex = 1; pageIndex <= document.getPages().size(); pageIndex++) { + if (footers.containsKey(pageIndex)) { + addFooter(footers.get(pageIndex), context); + } else { + addEmptyFooter(pageIndex, context); + } + } + } + + + private void addFooter(List textBlocks, Context context) { + + PageNode page = getPage(textBlocks.get(0).getPage(), context); + FooterNode footer = FooterNode.builder().page(page).build(); + AtomicTextBlock atomicTextBlock = buildAtomicTextBlock(mergeAndSortTextPositionSequences(textBlocks), footer, context); + footer.setAtomicTextBlock(atomicTextBlock); + page.setFooter(footer); + } + + + public void addHeader(List textBlocks, Context context) { + + PageNode page = getPage(textBlocks.get(0).getPage(), context); + HeaderNode header = HeaderNode.builder().page(page).build(); + AtomicTextBlock atomicTextBlock = buildAtomicTextBlock(mergeAndSortTextPositionSequences(textBlocks), header, context); + header.setAtomicTextBlock(atomicTextBlock); + page.setHeader(header); + } + + + private void addEmptyFooter(int pageIndex, Context context) { + + PageNode page = getPage(pageIndex, context); + FooterNode footer = FooterNode.builder().page(page).build(); + AtomicTextBlock atomicTextBlock = emptyTextBlock(footer, context); + footer.setAtomicTextBlock(atomicTextBlock); + page.setFooter(footer); + } + + + private void addEmptyHeader(int pageIndex, Context context) { + + PageNode page = getPage(pageIndex, context); + HeaderNode header = HeaderNode.builder().page(page).build(); + AtomicTextBlock atomicTextBlock = emptyTextBlock(header, context); + header.setAtomicTextBlock(atomicTextBlock); + page.setHeader(header); + } + + + private AtomicTextBlock emptyTextBlock(DocumentNode parent, Context context) { + + return AtomicTextBlock.builder() + .id(context.textBlockIdx.getAndIncrement()) + .boundary(new Boundary(context.stringOffset.get(), context.stringOffset.get())) + .searchText("") + .lineBreaks(Collections.emptyList()) + .stringIdxToPositionIdx(Collections.emptyList()) + .positions(Collections.emptyList()) + .parent(parent) + .build(); + } + + + private static String buildSummary(AtomicTextBlock textBlock) { if (textBlock == null) { return " probably a table"; @@ -140,67 +260,63 @@ public class DocumentGraphFactory { } - private PageEntity buildPage(Page p) { + private PageNode buildPage(Page p) { - return PageEntity.builder() - .height((int) p.getPageHeight()) - .width((int) p.getPageWidth()) - .number(p.getPageNumber()) - .paragraphs(new LinkedList<>()) - .sections(new LinkedList<>()) - .build(); + return PageNode.builder().height((int) p.getPageHeight()).width((int) p.getPageWidth()).number(p.getPageNumber()).mainBody(new LinkedList<>()).build(); } - private void addFooter(TextBlock textBlock) { + private List mergeAndSortTextPositionSequences(List textBlocks) { - PageEntity page = getPage(textBlock.getPage()); - FooterEntity footer = FooterEntity.builder().page(page).build(); - AtomicTextBlockEntity textBlockEntity = buildAtomicTextBlock(textBlock, footer); - footer.setTextBlock(new TextBlockEntity(List.of(textBlockEntity))); - page.setFooter(footer); + Comparator sortByX = (sequence1, sequence2) -> (int) (sequence1.getTextPositions().get(0).getPosition()[0] - sequence2.getTextPositions() + .get(0) + .getPosition()[0]); + Comparator sortByY = (sequence1, sequence2) -> (int) (sequence1.getTextPositions().get(0).getPosition()[1] - sequence2.getTextPositions() + .get(0) + .getPosition()[1]); + + return textBlocks.stream().map(TextBlock::getSequences).flatMap(List::stream).sorted(sortByX.thenComparing(sortByY)).toList(); } - public void addHeader(TextBlock textBlock) { + private AtomicTextBlock buildAtomicTextBlock(List sequences, DocumentNode parent, Context context) { - PageEntity page = getPage(textBlock.getPage()); - HeaderEntity header = HeaderEntity.builder().page(page).build(); - AtomicTextBlockEntity textBlockEntity = buildAtomicTextBlock(textBlock, header); - header.setTextBlock(new TextBlockEntity(List.of(textBlockEntity))); - page.setHeader(header); - } - - - public AtomicTextBlockEntity buildAtomicTextBlock(TextBlock textBlock, Node parent) { - - SearchTextWithTextPositionModel searchTextWithTextPositionModel = searchTextWithTextPositionFactory.buildSearchTextToTextPositionModel(textBlock.getSequences()); + SearchTextWithTextPositionModel searchTextWithTextPositionModel = searchTextWithTextPositionFactory.buildSearchTextToTextPositionModel(sequences); int offset = context.stringOffset().getAndAdd(searchTextWithTextPositionModel.getSearchText().length()); - return AtomicTextBlockEntity.builder() + return AtomicTextBlock.builder() .id(context.textBlockIdx.getAndIncrement()) .parent(parent) .searchText(searchTextWithTextPositionModel.getSearchText()) .lineBreaks(searchTextWithTextPositionModel.getLineBreaks()) - .positions(searchTextWithTextPositionModel.getPositions()) + .positions(toRectangle2D(searchTextWithTextPositionModel.getPositions())) .stringIdxToPositionIdx(searchTextWithTextPositionModel.getStringCoordsToPositionCoords()) - .offset(offset) + .boundary(new Boundary(offset, offset + searchTextWithTextPositionModel.getSearchText().length())) .build(); } - public PageEntity getPage(int i) { + private List toRectangle2D(List positions) { - return context.pages.stream().filter(page -> page.getNumber() == i).findFirst().orElseThrow(NoSuchFieldError::new); + return positions.stream().map(r -> (Rectangle2D) new Rectangle2D.Double(r.getX(), r.getY(), r.getWidth(), r.getHeight())).toList(); + } + + + private PageNode getPage(int pageIndex, Context context) { + + return context.pages.stream() + .filter(page -> page.getNumber() == pageIndex) + .findFirst() + .orElseThrow(() -> new NotFoundException(format("Page with number %d not found", pageIndex))); } record Context( TableOfContents tableOfContents, - List pages, - List sections, - List headers, - List footers, + List pages, + List sections, + List headers, + List footers, AtomicInteger stringOffset, AtomicInteger textBlockIdx) { diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/EntityEnrichmentUtility.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/EntityEnrichmentUtility.java new file mode 100644 index 00000000..a33bf92c --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/EntityEnrichmentUtility.java @@ -0,0 +1,31 @@ +package com.iqser.red.service.redaction.v1.server.document.services; + +import com.iqser.red.service.redaction.v1.server.document.graph.nodes.EntityNode; +import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock; + +public class EntityEnrichmentUtility { + + public static EntityNode enrichEntity(EntityNode entity, TextBlock textBlock) { + + entity.setPositions(textBlock.getPositions(entity.getBoundary())); + entity.setTextAfter(findTextAfter(entity.getBoundary().end(), textBlock)); + entity.setTextBefore(findTextBefore(entity.getBoundary().start(), textBlock)); + entity.setValue(textBlock.subSequence(entity.getBoundary()).toString()); + return entity; + } + + + private static CharSequence findTextAfter(int index, TextBlock textBlock) { + + int nextLineBreak = textBlock.getNextLinebreak(index); + return textBlock.subSequence(index, nextLineBreak); + } + + + private static CharSequence findTextBefore(int index, TextBlock textBlock) { + + int previousLinebreak = textBlock.getPreviousLinebreak(index); + return textBlock.subSequence(previousLinebreak, index); + } + +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/RangeComparators.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/RangeComparators.java new file mode 100644 index 00000000..52a44734 --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/RangeComparators.java @@ -0,0 +1,38 @@ +package com.iqser.red.service.redaction.v1.server.document.services; + +import java.util.Comparator; + +import com.iqser.red.service.redaction.v1.server.document.graph.Boundary; + +public class RangeComparators { + + public static Comparator contained() { + + return (range1, range2) -> { + if (contained(range1, range2)) { + return -1; + } else if (contained(range2, range1)) { + return 1; + } else { + return 0; + } + }; + } + + + /** + * @param range1 A Range + * @param range2 Also Range + * @return true, if range1 contains range2 + * false, otherwise + */ + public static boolean contained(Boundary range1, Boundary range2) { + + return range1.start() <= range2.start() && range2.end() <= range1.end(); + } + + public static boolean contained(Boundary range, int index) { + + return range.start() <= index && index < range.end(); + } +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/RegexMatcher.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/RegexMatcher.java new file mode 100644 index 00000000..73ac24f9 --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/RegexMatcher.java @@ -0,0 +1,37 @@ +package com.iqser.red.service.redaction.v1.server.document.services; + +import java.util.LinkedList; +import java.util.List; +import java.util.regex.Matcher; + +import com.iqser.red.service.redaction.v1.server.document.graph.Boundary; +import com.iqser.red.service.redaction.v1.server.redaction.utils.Patterns; + +public class RegexMatcher { + + public static boolean anyMatch(String regexPattern, CharSequence searchText) { + + var pattern = Patterns.getCompiledPattern(regexPattern, false); + return pattern.matcher(searchText).find(); + } + + + public static Boundary findFirstBoundary(String regexPattern, CharSequence searchText) { + + var pattern = Patterns.getCompiledPattern(regexPattern, false); + Matcher matcher = pattern.matcher(searchText); + return new Boundary(matcher.start(), matcher.end()); + } + + public static List findBoundaries(String regexPattern, CharSequence searchText) { + + var pattern = Patterns.getCompiledPattern(regexPattern, false); + Matcher matcher = pattern.matcher(searchText); + List boundaries = new LinkedList<>(); + while (matcher.find()) { + boundaries.add(new Boundary(matcher.start(), matcher.end())); + } + return boundaries; + } + +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/SearchTextWithTextPositionFactory.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/SearchTextWithTextPositionFactory.java index d567f293..85766ac9 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/SearchTextWithTextPositionFactory.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/SearchTextWithTextPositionFactory.java @@ -1,6 +1,5 @@ package com.iqser.red.service.redaction.v1.server.redaction.service; -import java.awt.geom.Rectangle2D; import java.util.LinkedList; import java.util.List; import java.util.Objects; @@ -33,7 +32,7 @@ public class SearchTextWithTextPositionFactory { currentUnicode = word.getTextPositions().get(i).getUnicode(); if (isLineBreak(currentUnicode)) { - lineBreaksStringIdx.add(stringIdx); + lineBreaksStringIdx.add(stringIdx + 1); } else if (!isRepeatedWhitespace(currentUnicode, previousUnicode) && // !isHyphenLinebreak(currentUnicode)) { diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/SearchImplementation.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/SearchImplementation.java index 336d4fa8..4a2b1e7f 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/SearchImplementation.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/SearchImplementation.java @@ -9,6 +9,8 @@ import java.util.stream.Collectors; import org.ahocorasick.trie.Trie; +import com.iqser.red.service.redaction.v1.server.document.graph.Boundary; + import lombok.Data; @Data @@ -75,15 +77,30 @@ public class SearchImplementation { } } - public List getMatches(CharSequence text) { + + public List getMatches(CharSequence text) { + if (this.values.isEmpty()) { return new ArrayList<>(); } if (this.pattern != null) { - - return this.pattern.matcher(text).results().map(r -> new MatchPosition(r.start(), r.end())).collect(Collectors.toList()); + return this.pattern.matcher(text).results().map(r -> new Boundary(r.start(), r.end())).collect(Collectors.toList()); } else { - return this.trie.parseText(text).stream().map(r -> new MatchPosition(r.getStart(), r.getEnd() + 1)).collect(Collectors.toList()); + return this.trie.parseText(text).stream().map(r -> new Boundary(r.getStart(), r.getEnd() + 1)).collect(Collectors.toList()); + } + } + + + public List getMatches(CharSequence text, int startOffset) { + + if (this.values.isEmpty()) { + return new ArrayList<>(); + } + CharSequence subSequence = text.subSequence(startOffset, startOffset + text.length()); + if (this.pattern != null) { + return this.pattern.matcher(subSequence).results().map(r -> new Boundary(r.start() + startOffset, r.end() + startOffset)).collect(Collectors.toList()); + } else { + return this.trie.parseText(subSequence).stream().map(r -> new Boundary(r.getStart() + startOffset, r.getEnd() + startOffset + 1)).collect(Collectors.toList()); } } @@ -104,6 +121,7 @@ public class SearchImplementation { } } + public record MatchPosition(int startIndex, int endIndex) { } diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AbstractTestWithDictionaries.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AbstractTestWithDictionaries.java index afbcb225..ef784d1c 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AbstractTestWithDictionaries.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AbstractTestWithDictionaries.java @@ -3,7 +3,6 @@ package com.iqser.red.service.redaction.v1.server; import static org.mockito.Mockito.when; import java.io.BufferedReader; -import java.io.ByteArrayInputStream; import java.io.File; import java.io.IOException; import java.io.InputStream; @@ -27,7 +26,10 @@ import org.kie.api.KieServices; import org.kie.api.builder.KieBuilder; import org.kie.api.builder.KieFileSystem; import org.kie.api.builder.KieModule; +import org.kie.api.builder.KieRepository; +import org.kie.api.builder.ReleaseId; import org.kie.api.runtime.KieContainer; +import org.kie.internal.io.ResourceFactory; import org.mockito.stubbing.Answer; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.autoconfigure.EnableAutoConfiguration; @@ -62,7 +64,9 @@ import lombok.SneakyThrows; @Import(AbstractTestWithDictionaries.TestConfiguration.class) public class AbstractTestWithDictionaries { - private static final String RULES = loadFromClassPath("drools/rules.drl"); + protected static final String RULES = loadFromClassPath("drools/rules.drl"); + protected static final String RULES_PATH = "drools/rules.drl"; + protected static final String ENTITY_RULES_PATH = "drools/entity_rules.drl"; private static final String VERTEBRATE = "vertebrate"; private static final String ADDRESS = "CBI_address"; private static final String AUTHOR = "CBI_author"; @@ -499,13 +503,22 @@ public class AbstractTestWithDictionaries { KieServices kieServices = KieServices.Factory.get(); KieFileSystem kieFileSystem = kieServices.newKieFileSystem(); - InputStream input = new ByteArrayInputStream(RULES.getBytes(StandardCharsets.UTF_8)); - kieFileSystem.write("src/test/resources/drools/rules.drl", kieServices.getResources().newInputStreamResource(input)); - KieBuilder kieBuilder = kieServices.newKieBuilder(kieFileSystem); - kieBuilder.buildAll(); - KieModule kieModule = kieBuilder.getKieModule(); + //kieFileSystem.write(ResourceFactory.newClassPathResource(RULES_PATH, "UTF-8")); + kieFileSystem.write(ResourceFactory.newClassPathResource(ENTITY_RULES_PATH, "UTF-8")); - return kieServices.newKieContainer(kieModule.getReleaseId()); + KieRepository kieRepository = kieServices.getRepository(); + + kieRepository.addKieModule(new KieModule() { + public ReleaseId getReleaseId() { + return kieRepository.getDefaultReleaseId(); + } + }); + + KieBuilder kieBuilder = kieServices + .newKieBuilder(kieFileSystem) + .buildAll(); + + return kieServices.newKieContainer(kieRepository.getDefaultReleaseId()); } @Bean diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DocumentGraphIntegrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DocumentGraphIntegrationTest.java index 6e83951b..9e131d65 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DocumentGraphIntegrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DocumentGraphIntegrationTest.java @@ -1,15 +1,36 @@ package com.iqser.red.service.redaction.v1.server; -import java.util.List; +import static com.iqser.red.service.redaction.v1.server.utils.PdfDraw.drawRectangle2DList; +import java.awt.Color; +import java.awt.Shape; +import java.awt.geom.AffineTransform; +import java.awt.geom.Rectangle2D; +import java.io.File; +import java.io.IOException; +import java.util.HashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Set; + +import org.apache.pdfbox.pdmodel.PDDocument; import org.junit.jupiter.api.Test; +import org.kie.api.runtime.KieContainer; +import org.kie.api.runtime.KieSession; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.core.io.ClassPathResource; -import com.iqser.red.service.redaction.v1.server.document.graph.DocumentEntity; -import com.iqser.red.service.redaction.v1.server.document.graph.TextBlockEntity; +import com.iqser.red.service.redaction.v1.model.FileAttribute; +import com.iqser.red.service.redaction.v1.server.document.graph.DocumentGraph; +import com.iqser.red.service.redaction.v1.server.document.graph.nodes.EntityNode; +import com.iqser.red.service.redaction.v1.server.document.graph.nodes.PageNode; +import com.iqser.red.service.redaction.v1.server.document.graph.nodes.ParagraphNode; +import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SectionNode; +import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock; import com.iqser.red.service.redaction.v1.server.document.services.DocumentGraphFactory; import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary; +import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType; import com.iqser.red.service.redaction.v1.server.redaction.service.DictionaryService; import com.iqser.red.service.redaction.v1.server.redaction.utils.SearchImplementation; import com.iqser.red.service.redaction.v1.server.segmentation.PdfSegmentationService; @@ -27,10 +48,14 @@ public class DocumentGraphIntegrationTest extends AbstractTestWithDictionaries { @Autowired private DictionaryService dictionaryService; + @Qualifier("kieContainer") + @Autowired + private KieContainer kieContainer; + @Test @SneakyThrows - public void testBuildDocumentEntity() { + public void testDroolsOnDocumentGraph() { String filename = "files/new/crafted document"; @@ -38,28 +63,189 @@ public class DocumentGraphIntegrationTest extends AbstractTestWithDictionaries { ClassPathResource fileResource = new ClassPathResource(filename + ".pdf"); var classifiedDoc = segmentationService.parseDocument(TEST_DOSSIER_ID, TEST_FILE_ID, fileResource.getInputStream(), null); - DocumentEntity document = documentGraphFactory.buildEntityFromDocument(classifiedDoc); + DocumentGraph document = documentGraphFactory.buildDocumentGraph(classifiedDoc); - var start = System.currentTimeMillis(); - - TextBlockEntity textBlock = document.buildTextBlock(); dictionaryService.updateDictionary(TEST_DOSSIER_TEMPLATE_ID, TEST_DOSSIER_ID); Dictionary dictionary = dictionaryService.getDeepCopyDictionary(TEST_DOSSIER_TEMPLATE_ID, TEST_DOSSIER_ID); + List foundEntities = new LinkedList<>(); for (var model : dictionary.getDictionaryModels()) { - List entries = model.getEntriesSearch().getMatches(textBlock); - List falsePositives = model.getFalsePositiveSearch().getMatches(textBlock); - List falseRecommendations = model.getFalseRecommendationsSearch().getMatches(textBlock); - - if (!entries.isEmpty()) { - var pos = entries.get(0); - System.out.println(textBlock.getPositions(pos.startIndex(), pos.endIndex())); - } - + findEntitiesWithSearchImplementationAndAddToGraph(document, model.getEntriesSearch(), EntityType.ENTITY, foundEntities, model.getType()); + findEntitiesWithSearchImplementationAndAddToGraph(document, model.getFalsePositiveSearch(), EntityType.FALSE_POSITIVE, foundEntities, model.getType()); + findEntitiesWithSearchImplementationAndAddToGraph(document, model.getFalseRecommendationsSearch(), EntityType.FALSE_RECOMMENDATION, foundEntities, model.getType()); } - System.out.printf("Search took %fs", ((float) (System.currentTimeMillis() - start)) / 1000); + KieSession kieSession = kieContainer.newKieSession(); + kieSession.setGlobal("document", document); + document.getEntities().forEach(kieSession::insert); + document.getSections().forEach(kieSession::insert); + document.getPages().forEach(kieSession::insert); + document.getSections().forEach(sec -> sec.getParagraphs().forEach(kieSession::insert)); + kieSession.insert(FileAttribute.builder().label("Vertebrate Study").value("Yes").build()); + kieSession.fireAllRules(); + drawAllEntities(filename, fileResource, document); } + + @Test + @SneakyThrows + public void testBuildTextBlockPerformance() { + + int n = 10000; + String filename = "files/Metolachlor/S-Metolachlor_RAR_01_Volume_1_2018-09-06"; + + prepareStorage(filename + ".pdf"); + ClassPathResource fileResource = new ClassPathResource(filename + ".pdf"); + + var classifiedDoc = segmentationService.parseDocument(TEST_DOSSIER_ID, TEST_FILE_ID, fileResource.getInputStream(), null); + DocumentGraph document = documentGraphFactory.buildDocumentGraph(classifiedDoc); + var start = System.currentTimeMillis(); + for (int i = 0; i < n; i++) { + TextBlock textBlock = document.getText(); + } + float durationMillis = ((float) (System.currentTimeMillis() - start)); + System.out.printf("%d calls of buildTextBlock() on document took %f s, average is %f ms\n", n, durationMillis / 1000, durationMillis / n); + + SectionNode section = document.getSections().get(8); + start = System.currentTimeMillis(); + for (int i = 0; i < n; i++) { + section.buildTextBlock(); + } + durationMillis = ((float) (System.currentTimeMillis() - start)); + System.out.printf("%d calls of buildTextBlock() on section took %f s, average is %f ms\n", n, durationMillis / 1000, durationMillis / n); + + ParagraphNode paragraph = document.getSections().get(8).getParagraphs().get(1); + start = System.currentTimeMillis(); + for (int i = 0; i < n; i++) { + paragraph.buildTextBlock(); + } + durationMillis = ((float) (System.currentTimeMillis() - start)); + System.out.printf("%d calls of buildTextBlock() on paragraph took %f s, average is %f ms\n", n, durationMillis / 1000, durationMillis / n); + } + + + @Test + @SneakyThrows + public void testDictionarySearchOnDocumentGraph() { + + String filename = "files/Metolachlor/S-Metolachlor_RAR_01_Volume_1_2018-09-06"; + + prepareStorage(filename + ".pdf"); + ClassPathResource fileResource = new ClassPathResource(filename + ".pdf"); + + var classifiedDoc = segmentationService.parseDocument(TEST_DOSSIER_ID, TEST_FILE_ID, fileResource.getInputStream(), null); + DocumentGraph document = documentGraphFactory.buildDocumentGraph(classifiedDoc); + + dictionaryService.updateDictionary(TEST_DOSSIER_TEMPLATE_ID, TEST_DOSSIER_ID); + Dictionary dictionary = dictionaryService.getDeepCopyDictionary(TEST_DOSSIER_TEMPLATE_ID, TEST_DOSSIER_ID); + + int numberOfSearches = 25; + float totalTime = 0; + float totalGraphTime = 0; + List foundEntities = new LinkedList<>(); + for (int i = 0; i < numberOfSearches; i++) { + + var graphStart = System.currentTimeMillis(); + document = documentGraphFactory.buildDocumentGraph(classifiedDoc); + var graphTime = ((float) (System.currentTimeMillis() - graphStart)) / 1000; + totalGraphTime += graphTime; + var start = System.currentTimeMillis(); + foundEntities = new LinkedList<>(); + + for (var model : dictionary.getDictionaryModels()) { + findEntitiesWithSearchImplementationAndAddToGraph(document, model.getEntriesSearch(), EntityType.ENTITY, foundEntities, model.getType()); + findEntitiesWithSearchImplementationAndAddToGraph(document, model.getFalsePositiveSearch(), EntityType.FALSE_POSITIVE, foundEntities, model.getType()); + findEntitiesWithSearchImplementationAndAddToGraph(document, model.getFalseRecommendationsSearch(), EntityType.FALSE_RECOMMENDATION, foundEntities, model.getType()); + } + var time = ((float) (System.currentTimeMillis() - start)) / 1000; + totalTime += time; + System.out.printf("%d Search %fs; Graph construction %fs \n", i, time, graphTime); + } + + System.out.printf("%d Searches took %f s, average %f s\n", numberOfSearches, totalTime, totalTime / numberOfSearches); + System.out.printf("%d Graph constructions took %f s, average %f s\n", numberOfSearches, totalGraphTime, totalGraphTime / numberOfSearches); + Set distinctFoundEntities = new HashSet<>(foundEntities); + System.out.printf("Found %d entities and saved %d\n", distinctFoundEntities.size(), document.getEntities().size()); + //assert document.getEntities().size() == distinctFoundEntities.size(); + + drawAllEntities(filename, fileResource, document); + } + + + private static void drawAllEntities(String filename, ClassPathResource fileResource, DocumentGraph document) throws IOException { + + var tmpFileName = "/tmp/" + filename.split("/")[2] + "_ENTITY_BBOX.pdf"; + try (var fileStream = fileResource.getInputStream()) { + PDDocument pdDocument = PDDocument.load(fileStream); + + for (PageNode page : document.getPages()) { + AffineTransform mirrorY = new AffineTransform(1, 0, 0, -1, 0, page.getHeight() + 6); + List entityPositionsOnPage = page.getEntities() + .stream() + .filter(EntityNode::isRedact) + .map(EntityNode::getPositions) + .flatMap(List::stream) + .map(mirrorY::createTransformedShape) + .map(Shape::getBounds2D) + .toList(); + + drawRectangle2DList(pdDocument, page.getNumber(), entityPositionsOnPage, Color.BLACK); + } + + for (PageNode page : document.getPages()) { + AffineTransform mirrorY = new AffineTransform(1, 0, 0, -1, 0, page.getHeight() + 6); + List entityPositionsOnPage = page.getEntities() + .stream() + .filter(e -> !e.isRedact()) + .map(EntityNode::getPositions) + .flatMap(List::stream) + .map(mirrorY::createTransformedShape) + .map(Shape::getBounds2D) + .toList(); + + drawRectangle2DList(pdDocument, page.getNumber(), entityPositionsOnPage, Color.BLUE); + } + File outputFile = new File(tmpFileName); + pdDocument.save(outputFile); + pdDocument.close(); + } + } + + + private void findEntitiesWithSearchImplementationAndAddToGraph(DocumentGraph documentGraph, + SearchImplementation searchImplementation, + EntityType entityType, + List foundEntities, + String type) { + + documentGraph.getSections().forEach( section -> { + TextBlock textBlock = section.buildTextBlock(); + searchImplementation.getMatches(textBlock, textBlock.getBoundary().start()) + .stream() + .map(bounds -> section.createAndAddEntity(bounds, type, entityType)) + .forEach(foundEntities::add); + }); + documentGraph.getPages().forEach( page -> { + TextBlock textBlock = page.getHeader().buildTextBlock(); + searchImplementation.getMatches(textBlock, textBlock.getBoundary().start()) + .stream() + .map(bounds -> page.getHeader().createAndAddEntity(bounds, type, entityType)) + .forEach(foundEntities::add); + }); + documentGraph.getPages().forEach( page -> { + TextBlock textBlock = page.getFooter().buildTextBlock(); + searchImplementation.getMatches(textBlock, textBlock.getBoundary().start()) + .stream() + .map(bounds -> page.getFooter().createAndAddEntity(bounds, type, entityType)) + .forEach(foundEntities::add); + }); + /* + searchImplementation.getMatches(documentGraph.getText()) + .stream() + .map(pos -> documentGraph.createAndAddEntity(new Boundary(pos.startIndex(), pos.endIndex()), type, entityType)) + .forEach(foundEntities::add); + + */ + } } diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/BoundaryTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/BoundaryTest.java new file mode 100644 index 00000000..e61a7122 --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/BoundaryTest.java @@ -0,0 +1,44 @@ +package com.iqser.red.service.redaction.v1.server.document.graph; + +import static org.junit.jupiter.api.Assertions.*; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +class BoundaryTest { + + Boundary startBoundary; + @BeforeEach + void setUp() { + startBoundary = new Boundary(10, 100); + } + + + @Test + void testContains() { + assertTrue(startBoundary.contains(11)); + assertTrue(startBoundary.contains(50)); + assertFalse(startBoundary.contains(9)); + assertFalse(startBoundary.contains(100)); + assertFalse(startBoundary.contains(150)); + assertFalse(startBoundary.contains(-123)); + assertTrue(startBoundary.contains(new Boundary(11, 99))); + assertTrue(startBoundary.contains(new Boundary(10, 100))); + assertTrue(startBoundary.contains(new Boundary(11, 11))); + assertFalse(startBoundary.contains(9, 100)); + assertTrue(startBoundary.contains(100, 100)); + assertFalse(startBoundary.contains(100, 101)); + assertFalse(startBoundary.contains(150, 151)); + } + + + @Test + void testIntersects() { + assertTrue(startBoundary.intersects(new Boundary(1, 11))); + assertTrue(startBoundary.intersects(new Boundary(11, 12))); + assertTrue(startBoundary.intersects(new Boundary(11, 100))); + assertFalse(startBoundary.intersects(new Boundary(100, 101))); + assertTrue(startBoundary.intersects(new Boundary(99, 101))); + } + +} \ No newline at end of file diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/utils/PdfDraw.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/utils/PdfDraw.java new file mode 100644 index 00000000..f5115f84 --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/utils/PdfDraw.java @@ -0,0 +1,27 @@ +package com.iqser.red.service.redaction.v1.server.utils; + +import java.awt.Color; +import java.awt.geom.Rectangle2D; +import java.util.List; + +import org.apache.pdfbox.pdmodel.PDDocument; +import org.apache.pdfbox.pdmodel.PDPageContentStream; + +import lombok.SneakyThrows; +import lombok.experimental.UtilityClass; + +@UtilityClass +public class PdfDraw { + + @SneakyThrows + public static void drawRectangle2DList(PDDocument document, int pageNumber, List rectCollection, Color color) { + var pdPage = document.getPage(pageNumber - 1); + var contentStream = new PDPageContentStream(document, pdPage, PDPageContentStream.AppendMode.APPEND, true); + contentStream.setStrokingColor(color); + for (var r : rectCollection) { + contentStream.addRect((float) r.getMinX(), (float) r.getMinY(), (float) r.getWidth(), (float) r.getHeight()); + contentStream.stroke(); + } + contentStream.close(); + } +} \ No newline at end of file diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/entity_rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/entity_rules.drl new file mode 100644 index 00000000..d178e7f4 --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/entity_rules.drl @@ -0,0 +1,43 @@ +package drools + +import static java.lang.String.format; +import static com.iqser.red.service.redaction.v1.server.document.services.RegexMatcher.anyMatch; + + +import java.util.List; +import java.util.LinkedList; +import java.util.HashSet; + +import com.iqser.red.service.redaction.v1.server.document.graph.* +import com.iqser.red.service.redaction.v1.server.document.graph.nodes.* +import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType; +import com.iqser.red.service.redaction.v1.model.FileAttribute; +import com.iqser.red.service.redaction.v1.model.Engine; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntry; +import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUtils; +import java.util.Set; + +global DocumentGraph document + + +rule "1: Redact CBI_author" + + when + FileAttribute(label == "Vertebrate Study" && (value.toLowerCase() == "yes")) + entity: EntityNode(type == "CBI_author") + then + entity.setRedact(true); + update(entity) + end + +rule "2: do not redact genitive CBI_author" + + when + entity: EntityNode(type == "CBI_author", anyMatch("['’’'ʼˈ´`‘′ʻ’']s", textAfter), redact == true) + then + entity.setRedact(false); + entity.setEntityType(EntityType.FALSE_POSITIVE); + update(entity) + end + + diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/test_rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/test_rules.drl new file mode 100644 index 00000000..939af4b9 --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/test_rules.drl @@ -0,0 +1,29 @@ +package drools + +import static java.lang.String.format; + +import java.util.List; +import java.util.LinkedList; +import java.util.HashSet; + +import com.iqser.red.service.redaction.v1.server.document.graph.* +import com.iqser.red.service.redaction.v1.server.redaction.model.*; +import com.iqser.red.service.redaction.v1.model.FileAttribute; +import com.iqser.red.service.redaction.v1.model.Engine; +import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntry; +import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUtils; +import java.util.Set; + +global DocumentGraph document; + + + +rule "1: Redact CBI_author" + when + FileAttribute(label == "Vertebrate Study" && (value == "Yes" || value == "Y" || value == "y")) + $entity: Entity2(type == "CBI_author") + then + modify($entity){ + $entity.setRedact(true) + } + end