RED-6009: Document Tree Structure

* added method getNodeType() to SemanticNodes
* moved DocumentDataMapper to constructors
* moved a few other methods to named constructors
* other minor refactors

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/model/AbstractPageBlock.java

@@ -33,6 +33,12 @@ public abstract class AbstractPageBlock {
     public abstract String getText();
 
 
+    public boolean isHeadline() {
+
+        return this instanceof ClassificationTextBlock && this.getClassification() != null && this.getClassification().startsWith("H");
+    }
+
+
     public boolean containsBlock(ClassificationTextBlock other) {
 
         return this.minX <= other.getPdfMinX() && this.maxX >= other.getPdfMaxX() && this.minY >= other.getPdfMinY() && this.maxY <= other.getPdfMaxY();

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/model/text/SearchableText.java

@@ -36,10 +36,14 @@ public class SearchableText {
 
         StringBuilder sb = new StringBuilder();
         for (TextPositionSequence word : sequences) {
-            sb.append(word.toString());
+            sb.append(word);
             sb.append(' ');
         }
-        return TextNormalizationUtilities.removeHyphenLineBreaks(sb.toString()).replaceAll("\n", " ").replaceAll(" {2}", " ");
+        String text = sb.toString();
+        text = TextNormalizationUtilities.removeHyphenLineBreaks(text);
+        text = TextNormalizationUtilities.removeLineBreaks(text);
+        text = TextNormalizationUtilities.removeRepeatingWhitespaces(text);
+        return text;
     }
 
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/data/AtomicPositionBlockData.java

@@ -27,21 +27,11 @@ public class AtomicPositionBlockData {
         return AtomicPositionBlockData.builder()
                 .id(atomicTextBlock.getId())
                 .positions(toPrimitiveFloatMatrix(atomicTextBlock.getPositions()))
-                .stringIdxToPositionIdx(toPrimitiveIntArray(atomicTextBlock.getStringIdxToPositionIdx()))
+                .stringIdxToPositionIdx(atomicTextBlock.getStringIdxToPositionIdx().stream().mapToInt(Integer::intValue).toArray())
                 .build();
     }
 
 
-    private static int[] toPrimitiveIntArray(List<Integer> list) {
-
-        int[] array = new int[list.size()];
-        for (int i = 0; i < list.size(); i++) {
-            array[i] = list.get(i);
-        }
-        return array;
-    }
-
-
     private static float[][] toPrimitiveFloatMatrix(List<Rectangle2D> positions) {
 
         float[][] positionMatrix = new float[positions.size()][];

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/data/AtomicTextBlockData.java

@@ -1,7 +1,5 @@
 package com.iqser.red.service.redaction.v1.server.layoutparsing.document.data;
 
-import java.util.List;
-
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.AtomicTextBlock;
 
 import lombok.AccessLevel;
@@ -34,18 +32,8 @@ public class AtomicTextBlockData {
                 .numberOnPage(atomicTextBlock.getNumberOnPage())
                 .start(atomicTextBlock.getBoundary().start())
                 .end(atomicTextBlock.getBoundary().end())
-                .lineBreaks(toPrimitiveIntArray(atomicTextBlock.getLineBreaks()))
+                .lineBreaks(atomicTextBlock.getLineBreaks().stream().mapToInt(Integer::intValue).toArray())
                 .build();
     }
 
-
-    private static int[] toPrimitiveIntArray(List<Integer> list) {
-
-        int[] array = new int[list.size()];
-        for (int i = 0; i < list.size(); i++) {
-            array[i] = list.get(i);
-        }
-        return array;
-    }
-
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/data/DocumentData.java

@@ -1,5 +1,7 @@
 package com.iqser.red.service.redaction.v1.server.layoutparsing.document.data;
 
+import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.Document;
+
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
 import lombok.Builder;
@@ -17,4 +19,25 @@ public class DocumentData {
     AtomicPositionBlockData[] atomicPositionBlocks;
     DocumentTreeData documentTreeData;
 
+
+    public static DocumentData fromDocument(Document document) {
+
+        var atomicPositionBlocks = document.streamTerminalTextBlocksInOrder()
+                .flatMap(textBlock -> textBlock.getAtomicTextBlocks().stream())
+                .distinct()
+                .map(AtomicPositionBlockData::fromAtomicTextBlock)
+                .toArray(AtomicPositionBlockData[]::new);
+
+        var atomicTextBlocks = document.streamTerminalTextBlocksInOrder()
+                .flatMap(textBlock -> textBlock.getAtomicTextBlocks().stream())
+                .distinct()
+                .map(AtomicTextBlockData::fromAtomicTextBlock)
+                .toArray(AtomicTextBlockData[]::new);
+
+        var pages = document.getPages().stream().map(PageData::fromPage).toArray(PageData[]::new);
+
+        var documentTreeData = new DocumentTreeData(DocumentTreeData.EntryData.fromEntry(document.getDocumentTree().getRoot()));
+        return new DocumentData(pages, atomicTextBlocks, atomicPositionBlocks, documentTreeData);
+    }
+
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/data/DocumentTreeData.java

@@ -1,10 +1,19 @@
 package com.iqser.red.service.redaction.v1.server.layoutparsing.document.data;
 
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.stream.Stream;
 
+import com.iqser.red.service.redaction.v1.server.layoutparsing.document.data.mapper.PropertiesMapper;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.DocumentTree;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.Image;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.NodeType;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.Page;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.Section;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.Table;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.TableCell;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.AtomicTextBlock;
 
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
@@ -63,12 +72,40 @@ public class DocumentTreeData {
 
         NodeType type;
         int[] treeId;
-        Long[] atomicBlocks;
+        Long[] atomicBlockIds;
-        Long[] pages;
+        Long[] pageNumbers;
         Map<String, String> properties;
         List<EntryData> subEntries;
 
 
+        public static EntryData fromEntry(DocumentTree.Entry entry) {
+
+            Long[] atomicBlockIds = toAtomicTextBlockIds(entry);
+
+            Map<String, String> properties = switch (entry.getType()) {
+                case TABLE -> PropertiesMapper.buildTableProperties((Table) entry.getNode());
+                case TABLE_CELL -> PropertiesMapper.buildTableCellProperties((TableCell) entry.getNode());
+                case IMAGE -> PropertiesMapper.buildImageProperties((Image) entry.getNode());
+                case SECTION -> PropertiesMapper.buildSectionProperties((Section) entry.getNode());
+                default -> new HashMap<>();
+            };
+            var treeId = entry.getTreeId().stream().mapToInt(Integer::intValue).toArray();
+            var pageNumbers = entry.getNode().getPages().stream().map(Page::getNumber).map(Integer::longValue).toArray(Long[]::new);
+            var subEntries = entry.getChildren().stream().map(EntryData::fromEntry).toList();
+            return new EntryData(entry.getType(), treeId, atomicBlockIds, pageNumbers, properties, subEntries);
+        }
+
+
+        private static Long[] toAtomicTextBlockIds(DocumentTree.Entry entry) {
+
+            if (entry.getNode().isLeaf()) {
+                return entry.getNode().getLeafTextBlock().getAtomicTextBlocks().stream().map(AtomicTextBlock::getId).toArray(Long[]::new);
+            } else {
+                return new Long[]{};
+            }
+        }
+
+
         @Override
         public String toString() {
 
@@ -83,7 +120,7 @@ public class DocumentTreeData {
 
             sb.append(type);
             sb.append(" atbs = ");
-            sb.append(atomicBlocks.length);
+            sb.append(atomicBlockIds.length);
 
             return sb.toString();
         }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/data/PageData.java

@@ -1,5 +1,7 @@
 package com.iqser.red.service.redaction.v1.server.layoutparsing.document.data;
 
+import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.Page;
+
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
 import lombok.Builder;
@@ -17,4 +19,10 @@ public class PageData {
     int width;
     int rotation;
 
+
+    public static PageData fromPage(Page page) {
+
+        return new PageData(page.getNumber(), page.getHeight(), page.getWidth(), page.getRotation());
+    }
+
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/data/mapper/DocumentDataMapper.java

@@ -1,108 +0,0 @@
-package com.iqser.red.service.redaction.v1.server.layoutparsing.document.data.mapper;
-
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import com.iqser.red.service.redaction.v1.server.layoutparsing.document.data.AtomicPositionBlockData;
-import com.iqser.red.service.redaction.v1.server.layoutparsing.document.data.AtomicTextBlockData;
-import com.iqser.red.service.redaction.v1.server.layoutparsing.document.data.DocumentData;
-import com.iqser.red.service.redaction.v1.server.layoutparsing.document.data.DocumentTreeData;
-import com.iqser.red.service.redaction.v1.server.layoutparsing.document.data.PageData;
-import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.DocumentTree;
-import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.Document;
-import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.Image;
-import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.Page;
-import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.Section;
-import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.Table;
-import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.TableCell;
-import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.AtomicTextBlock;
-import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.TextBlock;
-
-import lombok.experimental.UtilityClass;
-
-@UtilityClass
-public class DocumentDataMapper {
-
-    public DocumentData toDocumentData(Document document) {
-
-        List<AtomicTextBlockData> atomicTextBlockData = document.streamTerminalTextBlocksInOrder()
-                .flatMap(textBlock -> textBlock.getAtomicTextBlocks().stream())
-                .distinct()
-                .map(AtomicTextBlockData::fromAtomicTextBlock)
-                .toList();
-
-        List<AtomicPositionBlockData> atomicPositionBlockData = document.streamTerminalTextBlocksInOrder()
-                .flatMap(textBlock -> textBlock.getAtomicTextBlocks().stream())
-                .distinct()
-                .map(AtomicPositionBlockData::fromAtomicTextBlock)
-                .toList();
-
-        List<PageData> pageData = document.getPages().stream().map(DocumentDataMapper::toPageData).toList();
-        DocumentTreeData documentTreeData = toTableOfContentsData(document.getDocumentTree());
-        return DocumentData.builder()
-                .atomicTextBlocks(atomicTextBlockData.toArray(new AtomicTextBlockData[0]))
-                .atomicPositionBlocks(atomicPositionBlockData.toArray(new AtomicPositionBlockData[0]))
-                .pages(pageData.toArray(new PageData[0]))
-                .documentTreeData(documentTreeData)
-                .build();
-    }
-
-
-    private DocumentTreeData toTableOfContentsData(DocumentTree documentTree) {
-
-        return new DocumentTreeData(toEntryData(documentTree.getRoot()));
-    }
-
-
-    private DocumentTreeData.EntryData toEntryData(DocumentTree.Entry entry) {
-
-        Long[] atomicTextBlocks;
-
-        if (entry.getNode().isLeaf()) {
-            atomicTextBlocks = toAtomicTextBlockIds(entry.getNode().getLeafTextBlock());
-        } else {
-            atomicTextBlocks = new Long[]{};
-        }
-
-        Map<String, String> properties = switch (entry.getType()) {
-            case TABLE -> PropertiesMapper.buildTableProperties((Table) entry.getNode());
-            case TABLE_CELL -> PropertiesMapper.buildTableCellProperties((TableCell) entry.getNode());
-            case IMAGE -> PropertiesMapper.buildImageProperties((Image) entry.getNode());
-            case SECTION -> PropertiesMapper.buildSectionProperties((Section) entry.getNode());
-            default -> new HashMap<>();
-        };
-
-        return DocumentTreeData.EntryData.builder()
-                .treeId(toPrimitiveIntArray(entry.getTreeId()))
-                .subEntries(entry.getChildren().stream().map(DocumentDataMapper::toEntryData).toList())
-                .type(entry.getType())
-                .atomicBlocks(atomicTextBlocks)
-                .pages(entry.getNode().getPages().stream().map(Page::getNumber).map(Integer::longValue).toArray(Long[]::new))
-                .properties(properties)
-                .build();
-    }
-
-
-    private Long[] toAtomicTextBlockIds(TextBlock textBlock) {
-
-        return textBlock.getAtomicTextBlocks().stream().map(AtomicTextBlock::getId).toArray(Long[]::new);
-    }
-
-
-    private PageData toPageData(Page p) {
-
-        return PageData.builder().rotation(p.getRotation()).height(p.getHeight()).width(p.getWidth()).number(p.getNumber()).build();
-    }
-
-
-    private static int[] toPrimitiveIntArray(List<Integer> list) {
-
-        int[] array = new int[list.size()];
-        for (int i = 0; i < list.size(); i++) {
-            array[i] = list.get(i);
-        }
-        return array;
-    }
-
-}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/data/mapper/DocumentGraphMapper.java

@@ -65,7 +65,7 @@ public class DocumentGraphMapper {
         for (DocumentTreeData.EntryData entryData : entries) {
 
             boolean terminal = isTerminal(entryData);
-            List<Page> pages = Arrays.stream(entryData.getPages()).map(pageNumber -> getPage(pageNumber, context)).toList();
+            List<Page> pages = Arrays.stream(entryData.getPageNumbers()).map(pageNumber -> getPage(pageNumber, context)).toList();
 
             SemanticNode node = switch (entryData.getType()) {
                 case SECTION -> buildSection(context, entryData.getProperties());
@@ -75,12 +75,12 @@ public class DocumentGraphMapper {
                 case FOOTER -> buildFooter(context, terminal);
                 case TABLE -> buildTable(context, entryData.getProperties());
                 case TABLE_CELL -> buildTableCell(context, entryData.getProperties(), terminal);
-                case IMAGE -> buildImage(context, entryData.getProperties(), entryData.getPages());
+                case IMAGE -> buildImage(context, entryData.getProperties(), entryData.getPageNumbers());
                 default -> throw new UnsupportedOperationException("Not yet implemented for type " + entryData.getType());
             };
 
             if (node.isLeaf()) {
-                TextBlock textBlock = toTextBlock(entryData.getAtomicBlocks(), context, node);
+                TextBlock textBlock = toTextBlock(entryData.getAtomicBlockIds(), context, node);
                 node.setLeafTextBlock(textBlock);
             }
             List<Integer> treeId = Arrays.stream(entryData.getTreeId()).boxed().toList();
@@ -107,7 +107,7 @@ public class DocumentGraphMapper {
 
     private static boolean isTerminal(DocumentTreeData.EntryData entryData) {
 
-        return entryData.getAtomicBlocks().length > 0;
+        return entryData.getAtomicBlockIds().length > 0;
     }
 
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/factory/DocumentGraphFactory.java

@@ -11,7 +11,6 @@ import java.util.List;
 import java.util.Map;
 import java.util.NoSuchElementException;
 import java.util.Set;
-import java.util.concurrent.atomic.AtomicInteger;
 
 import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.AbstractPageBlock;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.ClassificationDocument;
@@ -36,6 +35,7 @@ import com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.Te
 import com.iqser.red.service.redaction.v1.server.redaction.utils.IdBuilder;
 
 import lombok.AccessLevel;
+import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Getter;
 import lombok.experimental.FieldDefaults;
@@ -46,11 +46,10 @@ public class DocumentGraphFactory {
 
     public Document buildDocumentGraph(ClassificationDocument document) {
 
-        TextBlockFactory textBlockFactory = new TextBlockFactory();
         Document documentGraph = new Document();
-        Context context = new Context(new DocumentTree(documentGraph), new HashMap<>(), new LinkedList<>(), new LinkedList<>(), textBlockFactory);
+        Context context = new Context(documentGraph);
 
-        document.getPages().stream().map(DocumentGraphFactory::buildPage).forEach(page -> context.getPages().put(page, new AtomicInteger(1)));
+        document.getPages().forEach(context::buildAndAddPageWithCounter);
         document.getSections().stream().flatMap(section -> section.getImages().stream()).forEach(image -> context.getImages().add(image));
         addSections(document, context);
         addHeaderAndFooterToEachPage(document, context);
@@ -69,12 +68,6 @@ public class DocumentGraphFactory {
     }
 
 
-    public static boolean pageBlockIsHeadline(AbstractPageBlock abstractPageBlock) {
-
-        return abstractPageBlock instanceof ClassificationTextBlock && abstractPageBlock.getClassification() != null && abstractPageBlock.getClassification().startsWith("H");
-    }
-
-
     public void addParagraphOrHeadline(GenericSemanticNode parentNode,
                                        ClassificationTextBlock originalTextBlock,
                                        Context context,
@@ -83,7 +76,7 @@ public class DocumentGraphFactory {
         Page page = context.getPage(originalTextBlock.getPage());
 
         GenericSemanticNode node;
-        if (pageBlockIsHeadline(originalTextBlock)) {
+        if (originalTextBlock.isHeadline()) {
             node = Headline.builder().documentTree(context.getDocumentTree()).build();
         } else {
             node = Paragraph.builder().documentTree(context.getDocumentTree()).build();
@@ -94,17 +87,9 @@ public class DocumentGraphFactory {
         List<ClassificationTextBlock> textBlocks = new LinkedList<>(textBlocksToMerge);
         textBlocks.add(originalTextBlock);
         AtomicTextBlock textBlock = context.textBlockFactory.buildAtomicTextBlock(TextPositionOperations.mergeAndSortTextPositionSequenceByYThenX(textBlocks), node, context, page);
-
+        List<Integer> treeId = context.documentTree.createNewChildEntryAndReturnId(parentNode, node.getType(), node);
-        if (node instanceof Headline headline) {
+        node.setLeafTextBlock(textBlock);
-            List<Integer> tocId = context.documentTree.createNewChildEntryAndReturnId(parentNode, NodeType.HEADLINE, node);
+        node.setTreeId(treeId);
-            headline.setLeafTextBlock(textBlock);
-            headline.setTreeId(tocId);
-        }
-        if (node instanceof Paragraph paragraph) {
-            List<Integer> tocId = context.documentTree.createNewChildEntryAndReturnId(parentNode, NodeType.PARAGRAPH, node);
-            paragraph.setLeafTextBlock(textBlock);
-            paragraph.setTreeId(tocId);
-        }
     }
 
 
@@ -178,11 +163,7 @@ public class DocumentGraphFactory {
 
         Page page = context.getPage(textBlocks.get(0).getPage());
         Header header = Header.builder().documentTree(context.getDocumentTree()).build();
-        AtomicTextBlock textBlock = context.textBlockFactory.buildAtomicTextBlock(TextPositionOperations.mergeAndSortTextPositionSequenceByYThenX(textBlocks),
+        AtomicTextBlock textBlock = context.textBlockFactory.buildAtomicTextBlock(TextPositionOperations.mergeAndSortTextPositionSequenceByYThenX(textBlocks), header, 0, page);
-                header,
-                context,
-                0,
-                page);
         List<Integer> tocId = context.getDocumentTree().createNewMainEntryAndReturnId(NodeType.HEADER, header);
         header.setTreeId(tocId);
         header.setLeafTextBlock(textBlock);
@@ -214,30 +195,46 @@ public class DocumentGraphFactory {
     }
 
 
-    private Page buildPage(ClassificationPage p) {
-
-        return Page.builder()
-                .height((int) p.getPageHeight())
-                .width((int) p.getPageWidth())
-                .number(p.getPageNumber())
-                .rotation(p.getRotation())
-                .mainBody(new LinkedList<>())
-                .build();
-    }
-
-
     @Getter
     @Builder
+    @AllArgsConstructor
     @FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
-    static final class Context {
+    public static final class Context {
 
         DocumentTree documentTree;
-        Map<Page, AtomicInteger> pages;
+        Map<Page, Integer> pages;
         List<Section> sections;
         List<ClassifiedImage> images;
         TextBlockFactory textBlockFactory;
 
 
+        public Context(Document document) {
+
+            documentTree = new DocumentTree(document);
+            pages = new HashMap<>();
+            sections = new LinkedList<>();
+            images = new LinkedList<>();
+            textBlockFactory = new TextBlockFactory();
+        }
+
+
+        public void buildAndAddPageWithCounter(ClassificationPage classificationPage) {
+
+            Page page = Page.fromClassificationPage(classificationPage);
+            //this counter counts the TextBlocks per page
+            //initial value is set to 1, because 0 is reserved for Header
+            pages.put(page, 1);
+        }
+
+
+        public int getAndIncrementTextBlockNumberOnPage(Page page) {
+
+            Integer textBlockNumberOnPage = pages.get(page);
+            pages.merge(page, 1, Integer::sum);
+            return textBlockNumberOnPage;
+        }
+
+
         public Page getPage(int pageIndex) {
 
             return pages.keySet()

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/factory/SearchTextWithTextPositionDto.java

@@ -1,6 +1,7 @@
 package com.iqser.red.service.redaction.v1.server.layoutparsing.document.factory;
 
 import java.awt.geom.Rectangle2D;
+import java.util.Collections;
 import java.util.List;
 
 import lombok.AccessLevel;
@@ -18,4 +19,15 @@ public class SearchTextWithTextPositionDto {
     List<Integer> stringCoordsToPositionCoords;
     List<Rectangle2D> positions;
 
+
+    public static SearchTextWithTextPositionDto empty() {
+
+        return SearchTextWithTextPositionDto.builder()
+                .searchText("")
+                .lineBreaks(Collections.emptyList())
+                .positions(Collections.emptyList())
+                .stringCoordsToPositionCoords(Collections.emptyList())
+                .build();
+    }
+
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/factory/SearchTextWithTextPositionFactory.java

@@ -2,7 +2,6 @@ package com.iqser.red.service.redaction.v1.server.layoutparsing.document.factory
 
 import java.awt.geom.AffineTransform;
 import java.awt.geom.Rectangle2D;
-import java.util.Collections;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Objects;
@@ -19,21 +18,10 @@ public class SearchTextWithTextPositionFactory {
     public static SearchTextWithTextPositionDto buildSearchTextToTextPositionModel(List<TextPositionSequence> sequences) {
 
         if (sequences.isEmpty() || sequences.stream().allMatch(sequence -> sequence.getTextPositions().isEmpty())) {
-            return SearchTextWithTextPositionDto.builder()
+            return SearchTextWithTextPositionDto.empty();
-                    .searchText("")
-                    .lineBreaks(Collections.emptyList())
-                    .positions(Collections.emptyList())
-                    .stringCoordsToPositionCoords(Collections.emptyList())
-                    .build();
         }
 
-        List<Integer> stringIdxToPositionIdx = new LinkedList<>();
+        Context context = new Context();
-        List<Integer> lineBreaksStringIdx = new LinkedList<>();
-        StringBuilder sb = new StringBuilder();
-
-        int stringIdx = 0;
-        int positionIdx = 0;
-        int lastHyphenIdx = -3;
 
         RedTextPosition currentTextPosition = sequences.get(0).getTextPositions().get(0);
         RedTextPosition previousTextPosition = RedTextPosition.builder().unicode(" ").position(currentTextPosition.getPosition()).build();
@@ -43,55 +31,72 @@ public class SearchTextWithTextPositionFactory {
 
                 currentTextPosition = word.getTextPositions().get(i);
 
-                if (isLineBreak(currentTextPosition, previousTextPosition)) {
+                removeHyphenLinebreaks(currentTextPosition, previousTextPosition, context);
-
-                    if (stringIdx - lastHyphenIdx < 3) {
-                        sb.delete(lastHyphenIdx, sb.length());
-                        stringIdxToPositionIdx = stringIdxToPositionIdx.subList(0, lastHyphenIdx);
-                        stringIdx = lastHyphenIdx;
-                        lastHyphenIdx = -3;
-                    }
-                    lineBreaksStringIdx.add(stringIdx);
-                }
                 if (!isRepeatedWhitespace(currentTextPosition.getUnicode(), previousTextPosition.getUnicode())) {
-
                     if (isHyphen(currentTextPosition.getUnicode())) {
-                        lastHyphenIdx = stringIdx;
+                        context.lastHyphenIdx = context.stringIdx;
                     }
-                    sb.append(currentTextPosition.getUnicode());
+                    appendCurrentTextPosition(context, currentTextPosition);
-
-                    // unicode characters with more than 16 bit encoding have a length > 1 in java strings
-                    for (int j = 0; j < currentTextPosition.getUnicode().length(); j++) {
-                        stringIdxToPositionIdx.add(positionIdx);
-                    }
-                    stringIdx += currentTextPosition.getUnicode().length();
                 }
 
                 previousTextPosition = currentTextPosition;
-                ++positionIdx;
+                ++context.positionIdx;
             }
 
             previousTextPosition = RedTextPosition.builder().unicode(" ").position(previousTextPosition.getPosition()).build();
-            sb.append(" ");
+            context.sb.append(" ");
-            stringIdxToPositionIdx.add(positionIdx);
+            context.stringIdxToPositionIdx.add(context.positionIdx);
-            ++stringIdx;
+            ++context.stringIdx;
         }
 
-        assert sb.length() == stringIdxToPositionIdx.size();
+        assert context.sb.length() == context.stringIdxToPositionIdx.size();
 
         List<Rectangle2D> positions = sequences.stream()
                 .flatMap(sequence -> sequence.getTextPositions().stream().map(textPosition -> mapRedTextPositionToInitialUserSpace(textPosition, sequence)))
                 .toList();
 
         return SearchTextWithTextPositionDto.builder()
-                .searchText(sb.toString())
+                .searchText(context.sb.toString())
-                .lineBreaks(lineBreaksStringIdx)
+                .lineBreaks(context.lineBreaksStringIdx)
-                .stringCoordsToPositionCoords(stringIdxToPositionIdx)
+                .stringCoordsToPositionCoords(context.stringIdxToPositionIdx)
                 .positions(positions)
                 .build();
     }
 
 
+    private static void appendCurrentTextPosition(Context context, RedTextPosition currentTextPosition) {
+
+        context.sb.append(currentTextPosition.getUnicode());
+
+        // unicode characters with more than 16-bit encoding have a length > 1 in java strings
+        for (int j = 0; j < currentTextPosition.getUnicode().length(); j++) {
+            context.stringIdxToPositionIdx.add(context.positionIdx);
+        }
+        context.stringIdx += currentTextPosition.getUnicode().length();
+    }
+
+
+    private static void removeHyphenLinebreaks(RedTextPosition currentTextPosition, RedTextPosition previousTextPosition, Context context) {
+
+        if (isLineBreak(currentTextPosition, previousTextPosition)) {
+
+            if (lastHyphenDirectlyBeforeLineBreak(context)) {
+                context.sb.delete(context.lastHyphenIdx, context.sb.length());
+                context.stringIdxToPositionIdx = context.stringIdxToPositionIdx.subList(0, context.lastHyphenIdx);
+                context.stringIdx = context.lastHyphenIdx;
+                context.lastHyphenIdx = -3;
+            }
+            context.lineBreaksStringIdx.add(context.stringIdx);
+        }
+    }
+
+
+    private static boolean lastHyphenDirectlyBeforeLineBreak(Context context) {
+
+        return context.stringIdx - context.lastHyphenIdx < 3;
+    }
+
+
     private static boolean isLineBreak(RedTextPosition currentTextPosition, RedTextPosition previousTextPosition) {
 
         return Objects.equals(currentTextPosition.getUnicode(), "\n") || isDeltaYLargerThanTextHeight(currentTextPosition, previousTextPosition);
@@ -156,4 +161,22 @@ public class SearchTextWithTextPositionFactory {
         return transform.createTransformedShape(rectangle2D).getBounds2D();
     }
 
+
+    private static class Context {
+
+        List<Integer> stringIdxToPositionIdx = new LinkedList<>();
+        List<Integer> lineBreaksStringIdx = new LinkedList<>();
+        StringBuilder sb = new StringBuilder();
+
+        int stringIdx;
+        int positionIdx;
+        // when checking for a hyphen linebreak, we need to check after a linebreak if the last hyphen was less than three symbols away.
+        // We detect a linebreak as either a "\n" character or if two adjacent symbol's position differ in y-coordinates by at least one character height.
+        // If there is a hyphen linebreak, the hyphen will be 1 position in front of a "\n" or 2 positions in front of the character which has a lower y-coordinate
+        // This is why, we need to initialize this to <= -3, otherwise, if the very first symbol is a \n we would detect a hyphen linebreak that isn't there.
+        // Also, Integer.MIN_VALUE is a bad idea due to potential overflow during arithmetic operations. This is why the default should be -3.
+        int lastHyphenIdx = -3;
+
+    }
+
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/factory/SectionNodeFactory.java

@@ -1,6 +1,5 @@
 package com.iqser.red.service.redaction.v1.server.layoutparsing.document.factory;
 
-import static com.iqser.red.service.redaction.v1.server.layoutparsing.document.factory.DocumentGraphFactory.pageBlockIsHeadline;
 import static java.util.Collections.emptyList;
 import static java.util.stream.Collectors.groupingBy;
 
@@ -36,19 +35,13 @@ public class SectionNodeFactory {
         context.getSections().add(section);
         blocksPerPage.keySet().forEach(pageNumber -> addSectionNodeToPageNode(context, section, pageNumber));
 
-        List<Integer> tocId;
+        section.setTreeId(getTreeId(parentNode, context, section));
-        if (parentNode == null) {
-            tocId = context.getDocumentTree().createNewMainEntryAndReturnId(NodeType.SECTION, section);
-        } else {
-            tocId = context.getDocumentTree().createNewChildEntryAndReturnId(parentNode, NodeType.SECTION, section);
-        }
-        section.setTreeId(tocId);
 
         addFirstHeadlineDirectlyToSection(pageBlocks, context, section);
         if (containsTablesAndTextBlocks(pageBlocks)) {
             splitPageBlocksIntoSubSections(pageBlocks).forEach(subSectionPageBlocks -> addSection(section, subSectionPageBlocks, emptyList(), context));
         } else {
-            addTablesOrParagraphsOrHeadlinesToSection(pageBlocks, context, section);
+            addTablesAndParagraphsAndHeadlinesToSection(pageBlocks, context, section);
             if (listIsTextBlockOnly(pageBlocks)) {
                 section.setExcludesTables(true);
             }
@@ -58,7 +51,17 @@ public class SectionNodeFactory {
     }
 
 
-    private static boolean listIsTextBlockOnly(List<AbstractPageBlock> pageBlocks) {
+    private static List<Integer> getTreeId(GenericSemanticNode parentNode, DocumentGraphFactory.Context context, Section section) {
+
+        if (parentNode == null) {
+            return context.getDocumentTree().createNewMainEntryAndReturnId(NodeType.SECTION, section);
+        } else {
+            return context.getDocumentTree().createNewChildEntryAndReturnId(parentNode, NodeType.SECTION, section);
+        }
+    }
+
+
+    private boolean listIsTextBlockOnly(List<AbstractPageBlock> pageBlocks) {
 
         return pageBlocks.stream().allMatch(abstractPageBlock -> abstractPageBlock instanceof ClassificationTextBlock);
     }
@@ -66,14 +69,14 @@ public class SectionNodeFactory {
 
     private void addFirstHeadlineDirectlyToSection(List<AbstractPageBlock> pageBlocks, DocumentGraphFactory.Context context, Section section) {
 
-        if (pageBlockIsHeadline(pageBlocks.get(0))) {
+        if (pageBlocks.get(0).isHeadline()) {
-            addTablesOrParagraphsOrHeadlinesToSection(List.of(pageBlocks.get(0)), context, section);
+            addTablesAndParagraphsAndHeadlinesToSection(List.of(pageBlocks.get(0)), context, section);
             pageBlocks.remove(0);
         }
     }
 
 
-    private void addTablesOrParagraphsOrHeadlinesToSection(List<AbstractPageBlock> pageBlocks, DocumentGraphFactory.Context context, Section section) {
+    private void addTablesAndParagraphsAndHeadlinesToSection(List<AbstractPageBlock> pageBlocks, DocumentGraphFactory.Context context, Section section) {
 
         Set<AbstractPageBlock> alreadyMerged = new HashSet<>();
         List<AbstractPageBlock> remainingBlocks = new LinkedList<>(pageBlocks);
@@ -100,7 +103,7 @@ public class SectionNodeFactory {
     }
 
 
-    private static boolean containsTablesAndTextBlocks(List<AbstractPageBlock> pageBlocks) {
+    private boolean containsTablesAndTextBlocks(List<AbstractPageBlock> pageBlocks) {
 
         return pageBlocks.stream().anyMatch(pageBlock -> pageBlock instanceof TablePageBlock) && pageBlocks.stream()
                 .anyMatch(pageBlock -> pageBlock instanceof ClassificationTextBlock);
@@ -109,8 +112,8 @@ public class SectionNodeFactory {
 
     /**
      * This function splits the list of PageBlocks around TablePageBlocks, such that SubSections can be created, that don't include tables.
-     * First, the function splits the list into coherent Lists, which include only either type.
+     * This is needed so we can execute rules on sections, that do not contain tables.
-     * If a Table is directly preceded by a Headline, it is moved to the Table list.
+     * See: <a href="https://knecon.atlassian.net/wiki/spaces/RED/pages/14765218/Document+Structure">document structure wiki</a>
      *
      * @param pageBlocks a List of AbstractPageBlocks, which have at least one TablePageBlock and one ClassificationTextBlock
      * @return List of Lists of AbstractPageBlocks, which include either a single Headline ClassificationTextBlock and a TablePageBlock or only ClassificationTextBlocks.
@@ -123,13 +126,13 @@ public class SectionNodeFactory {
     }
 
 
-    private static void movePrecedingHeadlineToTableList(List<List<AbstractPageBlock>> splitList) {
+    private void movePrecedingHeadlineToTableList(List<List<AbstractPageBlock>> splitList) {
 
         for (int i = 0; i < splitList.size(); i++) {
-            if (listIsTableOnly(splitList.get(i)) && i > 0) {
+            if (listIsTablesOnly(splitList.get(i)) && i > 0) {
                 List<AbstractPageBlock> previousList = splitList.get(i - 1);
                 AbstractPageBlock lastPageBlockInPreviousList = previousList.get(previousList.size() - 1);
-                if (pageBlockIsHeadline(lastPageBlockInPreviousList)) {
+                if (lastPageBlockInPreviousList.isHeadline()) {
                     previousList.remove(i - 1);
                     splitList.get(i).add(0, lastPageBlockInPreviousList);
                 }
@@ -138,7 +141,7 @@ public class SectionNodeFactory {
     }
 
 
-    private static boolean listIsTableOnly(List<AbstractPageBlock> abstractPageBlocks) {
+    private boolean listIsTablesOnly(List<AbstractPageBlock> abstractPageBlocks) {
 
         return abstractPageBlocks.stream().allMatch(abstractPageBlock -> abstractPageBlock instanceof TablePageBlock);
     }
@@ -148,7 +151,7 @@ public class SectionNodeFactory {
      * @param pageBlocks a List of AbstractPageBlocks, which have at least one TablePageBlock and one ClassificationTextBlock
      * @return List of Lists of AbstractPageBlocks, which are exclusively of type ClassificationTextBlock or TablePageBlock
      */
-    private static List<List<AbstractPageBlock>> splitIntoCoherentList(List<AbstractPageBlock> pageBlocks) {
+    private List<List<AbstractPageBlock>> splitIntoCoherentList(List<AbstractPageBlock> pageBlocks) {
 
         List<List<AbstractPageBlock>> splitList = new LinkedList<>();
         List<AbstractPageBlock> currentList = new LinkedList<>();
@@ -169,7 +172,7 @@ public class SectionNodeFactory {
     }
 
 
-    private static List<ClassificationTextBlock> findTextBlocksWithSameClassificationAndAlignsY(AbstractPageBlock atc, List<AbstractPageBlock> pageBlocks) {
+    private List<ClassificationTextBlock> findTextBlocksWithSameClassificationAndAlignsY(AbstractPageBlock atc, List<AbstractPageBlock> pageBlocks) {
 
         return pageBlocks.stream()
                 .filter(abstractTextContainer -> !abstractTextContainer.equals(atc))

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/factory/TableNodeFactory.java

@@ -30,7 +30,7 @@ public class TableNodeFactory {
 
     public void addTable(GenericSemanticNode parentNode, List<TablePageBlock> tablesToMerge, DocumentGraphFactory.Context context) {
 
-        setPageNumberInCellTextBlocks(tablesToMerge);
+        setPageNumberInCells(tablesToMerge);
         Set<Page> pages = tablesToMerge.stream().map(AbstractPageBlock::getPage).map(context::getPage).collect(Collectors.toSet());
         List<List<Cell>> mergedRows = tablesToMerge.stream().map(TablePageBlock::getRows).flatMap(Collection::stream).toList();
         Table table = Table.builder().documentTree(context.getDocumentTree()).numberOfCols(mergedRows.get(0).size()).numberOfRows(mergedRows.size()).build();
@@ -41,22 +41,32 @@ public class TableNodeFactory {
         table.setTreeId(tocId);
         addTableCells(mergedRows, table, context);
 
-        IfTableHasNoHeadersAssumeFirstRowAreHeaders(table);
+        ifTableHasNoHeadersAssumeFirstRowAreHeaders(table);
     }
 
 
-    private static void setPageNumberInCellTextBlocks(List<TablePageBlock> tablesToMerge) {
+    private void setPageNumberInCells(List<TablePageBlock> tablesToMerge) {
 
+        // For some reason I can't figure out, in some table cells, the ClassificationTextBlocks have 0 as page number
+        // So I am fixing this here, but this should actually be fixed upstream.
         tablesToMerge.forEach(table -> table.getRows()
                 .stream()
                 .flatMap(Collection::stream)
                 .peek(cell -> cell.setPageNumber(table.getPage()))
-                .forEach(cell -> cell.getTextBlocks().stream().filter(tb -> tb.getPage() == 0).forEach(tb -> tb.setPage(table.getPage()))));
+                .forEach(cell -> setPageNumberInTextBlocksWithPageNumberSetTo0(table, cell)));
+    }
+
+
+    private static void setPageNumberInTextBlocksWithPageNumberSetTo0(TablePageBlock table, Cell cell) {
+
+        cell.getTextBlocks().stream()//
+                .filter(tb -> tb.getPage() == 0)//
+                .forEach(tb -> tb.setPage(table.getPage()));
     }
 
 
     @SuppressWarnings("PMD.UnusedPrivateMethod") // PMD actually flags this wrong
-    private static void addTableToPage(Page page, SemanticNode parentNode, Table table) {
+    private void addTableToPage(Page page, SemanticNode parentNode, Table table) {
 
         if (!page.getMainBody().contains(parentNode)) {
             parentNode.getPages().add(page);
@@ -66,7 +76,7 @@ public class TableNodeFactory {
     }
 
 
-    private static void IfTableHasNoHeadersAssumeFirstRowAreHeaders(Table table) {
+    private void ifTableHasNoHeadersAssumeFirstRowAreHeaders(Table table) {
 
         if (table.streamHeaders().findAny().isEmpty()) {
             table.streamRow(0).forEach(tableCellNode -> tableCellNode.setHeader(true));
@@ -124,13 +134,13 @@ public class TableNodeFactory {
     }
 
 
-    private static boolean cellAreaIsSmallerThanPageAreaTimesThreshold(Cell cell, Page page) {
+    private boolean cellAreaIsSmallerThanPageAreaTimesThreshold(Cell cell, Page page) {
 
         return cell.getArea() < TABLE_CELL_MERGE_CONTENTS_SIZE_THRESHOLD * page.getHeight() * page.getWidth();
     }
 
 
-    private static boolean firstTextBlockIsHeadline(Cell cell) {
+    private boolean firstTextBlockIsHeadline(Cell cell) {
 
         String classification = cell.getTextBlocks().get(0).getClassification();
         return classification != null && classification.startsWith("H");

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/factory/TextBlockFactory.java

@@ -1,74 +1,53 @@
 package com.iqser.red.service.redaction.v1.server.layoutparsing.document.factory;
 
-import java.util.Collections;
 import java.util.List;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.atomic.AtomicLong;
 
 import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.TextPositionSequence;
-import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.Boundary;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.Page;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.SemanticNode;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.AtomicTextBlock;
 
+import lombok.AccessLevel;
+import lombok.experimental.FieldDefaults;
+
+@FieldDefaults(level = AccessLevel.PRIVATE)
 public class TextBlockFactory {
 
-    AtomicInteger stringOffset;
+    int stringOffset;
-    AtomicLong textBlockIdx;
+    long textBlockIdx;
-
-
-    public TextBlockFactory() {
-
-        stringOffset = new AtomicInteger();
-        textBlockIdx = new AtomicLong();
-    }
 
 
     public AtomicTextBlock buildAtomicTextBlock(List<TextPositionSequence> sequences, SemanticNode parent, DocumentGraphFactory.Context context, Page page) {
 
-        Integer numberOnPage = context.getPages().get(page).getAndIncrement();
+        Integer numberOnPage = context.getAndIncrementTextBlockNumberOnPage(page);
-        return buildAtomicTextBlock(sequences, parent, context, numberOnPage, page);
+        return buildAtomicTextBlock(sequences, parent, numberOnPage, page);
     }
 
 
-    public AtomicTextBlock buildAtomicTextBlock(List<TextPositionSequence> sequences, SemanticNode parent, DocumentGraphFactory.Context context, Integer numberOnPage, Page page) {
+    public AtomicTextBlock buildAtomicTextBlock(List<TextPositionSequence> sequences, SemanticNode parent, Integer numberOnPage, Page page) {
 
         SearchTextWithTextPositionDto searchTextWithTextPositionDto = SearchTextWithTextPositionFactory.buildSearchTextToTextPositionModel(sequences);
-        int offset = stringOffset.getAndAdd(searchTextWithTextPositionDto.getSearchText().length());
+        int offset = stringOffset;
-
+        stringOffset += searchTextWithTextPositionDto.getSearchText().length();
-        return AtomicTextBlock.builder()
+        long idx = textBlockIdx;
-                .id(textBlockIdx.getAndIncrement())
+        textBlockIdx++;
-                .parent(parent)
+        return AtomicTextBlock.fromSearchTextWithTextPositionDto(searchTextWithTextPositionDto, parent, offset, idx, numberOnPage, page);
-                .searchText(searchTextWithTextPositionDto.getSearchText())
-                .numberOnPage(numberOnPage)
-                .page(page)
-                .lineBreaks(searchTextWithTextPositionDto.getLineBreaks())
-                .positions(searchTextWithTextPositionDto.getPositions())
-                .stringIdxToPositionIdx(searchTextWithTextPositionDto.getStringCoordsToPositionCoords())
-                .boundary(new Boundary(offset, offset + searchTextWithTextPositionDto.getSearchText().length()))
-                .build();
     }
 
 
     public AtomicTextBlock emptyTextBlock(SemanticNode parent, DocumentGraphFactory.Context context, Page page) {
 
-        return emptyTextBlock(parent, context.getPages().get(page).getAndIncrement(), page);
+        long idx = textBlockIdx;
+        textBlockIdx++;
+        return AtomicTextBlock.empty(idx, stringOffset, page, context.getAndIncrementTextBlockNumberOnPage(page), parent);
     }
 
 
     public AtomicTextBlock emptyTextBlock(SemanticNode parent, Integer numberOnPage, Page page) {
 
-        return AtomicTextBlock.builder()
+        long idx = textBlockIdx;
-                .id(textBlockIdx.getAndIncrement())
+        textBlockIdx++;
-                .boundary(new Boundary(stringOffset.get(), stringOffset.get()))
+        return AtomicTextBlock.empty(idx, stringOffset, page, numberOnPage, parent);
-                .searchText("")
-                .lineBreaks(Collections.emptyList())
-                .page(page)
-                .numberOnPage(numberOnPage)
-                .stringIdxToPositionIdx(Collections.emptyList())
-                .positions(Collections.emptyList())
-                .parent(parent)
-                .build();
     }
 
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/Document.java

@@ -38,6 +38,13 @@ public class Document implements GenericSemanticNode {
     Set<RedactionEntity> entities = new HashSet<>();
 
 
+    @Override
+    public NodeType getType() {
+
+        return NodeType.DOCUMENT;
+    }
+
+
     public TextBlock buildTextBlock() {
 
         return streamTerminalTextBlocksInOrder().collect(new TextBlockCollector());

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/Footer.java

@@ -37,6 +37,13 @@ public class Footer implements GenericSemanticNode {
     Set<RedactionEntity> entities = new HashSet<>();
 
 
+    @Override
+    public NodeType getType() {
+
+        return NodeType.FOOTER;
+    }
+
+
     @Override
     public TextBlock buildTextBlock() {
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/Header.java

@@ -37,6 +37,13 @@ public class Header implements GenericSemanticNode {
     Set<RedactionEntity> entities = new HashSet<>();
 
 
+    @Override
+    public NodeType getType() {
+
+        return NodeType.HEADER;
+    }
+
+
     @Override
     public TextBlock buildTextBlock() {
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/Headline.java

@@ -37,6 +37,13 @@ public class Headline implements GenericSemanticNode {
     Set<RedactionEntity> entities = new HashSet<>();
 
 
+    @Override
+    public NodeType getType() {
+
+        return NodeType.HEADLINE;
+    }
+
+
     @Override
     public TextBlock buildTextBlock() {
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/Image.java

@@ -55,6 +55,13 @@ public class Image implements GenericSemanticNode {
     Set<RedactionEntity> entities = new HashSet<>();
 
 
+    @Override
+    public NodeType getType() {
+
+        return NodeType.IMAGE;
+    }
+
+
     @Override
     public TextBlock buildTextBlock() {
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/Page.java

@@ -1,9 +1,11 @@
 package com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes;
 
 import java.util.HashSet;
+import java.util.LinkedList;
 import java.util.List;
 import java.util.Set;
 
+import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.ClassificationPage;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionEntity;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.TextBlock;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.TextBlockCollector;
@@ -44,6 +46,18 @@ public class Page {
     Set<Image> images = new HashSet<>();
 
 
+    public static Page fromClassificationPage(ClassificationPage classificationPage) {
+
+        return Page.builder()
+                .height((int) classificationPage.getPageHeight())
+                .width((int) classificationPage.getPageWidth())
+                .number(classificationPage.getPageNumber())
+                .rotation(classificationPage.getRotation())
+                .mainBody(new LinkedList<>())
+                .build();
+    }
+
+
     public TextBlock getMainBodyTextBlock() {
 
         return mainBody.stream().filter(SemanticNode::isLeaf).map(SemanticNode::getLeafTextBlock).collect(new TextBlockCollector());

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/Paragraph.java

@@ -35,6 +35,13 @@ public class Paragraph implements GenericSemanticNode {
     Set<RedactionEntity> entities = new HashSet<>();
 
 
+    @Override
+    public NodeType getType() {
+
+        return NodeType.PARAGRAPH;
+    }
+
+
     @Override
     public TextBlock buildTextBlock() {
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/Section.java

@@ -36,6 +36,13 @@ public class Section implements GenericSemanticNode {
     Set<RedactionEntity> entities = new HashSet<>();
 
 
+    @Override
+    public NodeType getType() {
+
+        return NodeType.SECTION;
+    }
+
+
     @Override
     public TextBlock buildTextBlock() {
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/SemanticNode.java

@@ -19,6 +19,14 @@ import com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.Re
 
 public interface SemanticNode {
 
+    /**
+     * Returns the type of this node, such as Section, Paragraph, etc.
+     *
+     * @return NodeType of this node
+     */
+    NodeType getType();
+
+
     /**
      * Searches all Nodes located underneath this Node in the DocumentTree and concatenates their AtomicTextBlocks into a single TextBlockEntity.
      * So, for a Section all TextBlocks of Subsections, Paragraphs, and Tables are concatenated into a single TextBlockEntity

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/Table.java

@@ -271,6 +271,13 @@ public class Table implements SemanticNode {
     }
 
 
+    @Override
+    public NodeType getType() {
+
+        return NodeType.TABLE;
+    }
+
+
     @Override
     public TextBlock buildTextBlock() {
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/TableCell.java

@@ -55,6 +55,13 @@ public class TableCell implements GenericSemanticNode {
     }
 
 
+    @Override
+    public NodeType getType() {
+
+        return NodeType.TABLE_CELL;
+    }
+
+
     @Override
     public TextBlock buildTextBlock() {
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/textblock/AtomicTextBlock.java

@@ -10,9 +10,9 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
-import com.google.common.primitives.Ints;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.data.AtomicPositionBlockData;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.data.AtomicTextBlockData;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.document.factory.SearchTextWithTextPositionDto;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.Boundary;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.Page;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.SemanticNode;
@@ -55,6 +55,43 @@ public class AtomicTextBlock implements TextBlock {
     }
 
 
+    public static AtomicTextBlock fromSearchTextWithTextPositionDto(SearchTextWithTextPositionDto searchTextWithTextPositionDto,
+                                                                    SemanticNode parent,
+                                                                    int stringOffset,
+                                                                    Long textBlockIdx,
+                                                                    Integer numberOnPage,
+                                                                    Page page) {
+
+        return AtomicTextBlock.builder()
+                .id(textBlockIdx)
+                .parent(parent)
+                .searchText(searchTextWithTextPositionDto.getSearchText())
+                .numberOnPage(numberOnPage)
+                .page(page)
+                .lineBreaks(searchTextWithTextPositionDto.getLineBreaks())
+                .positions(searchTextWithTextPositionDto.getPositions())
+                .stringIdxToPositionIdx(searchTextWithTextPositionDto.getStringCoordsToPositionCoords())
+                .boundary(new Boundary(stringOffset, stringOffset + searchTextWithTextPositionDto.getSearchText().length()))
+                .build();
+    }
+
+
+    public static AtomicTextBlock empty(Long textBlockIdx, int stringOffset, Page page, int numberOnPage, SemanticNode parent) {
+
+        return AtomicTextBlock.builder()
+                .id(textBlockIdx)
+                .boundary(new Boundary(stringOffset, stringOffset))
+                .searchText("")
+                .lineBreaks(Collections.emptyList())
+                .page(page)
+                .numberOnPage(numberOnPage)
+                .stringIdxToPositionIdx(Collections.emptyList())
+                .positions(Collections.emptyList())
+                .parent(parent)
+                .build();
+    }
+
+
     public static AtomicTextBlock fromAtomicTextBlockData(AtomicTextBlockData atomicTextBlockData,
                                                           AtomicPositionBlockData atomicPositionBlockData,
                                                           SemanticNode parent,
@@ -66,8 +103,8 @@ public class AtomicTextBlock implements TextBlock {
                 .page(page)
                 .boundary(new Boundary(atomicTextBlockData.getStart(), atomicTextBlockData.getEnd()))
                 .searchText(atomicTextBlockData.getSearchText())
-                .lineBreaks(Ints.asList(atomicTextBlockData.getLineBreaks()))
+                .lineBreaks(Arrays.stream(atomicTextBlockData.getLineBreaks()).boxed().toList())
-                .stringIdxToPositionIdx(Ints.asList(atomicPositionBlockData.getStringIdxToPositionIdx()))
+                .stringIdxToPositionIdx(Arrays.stream(atomicPositionBlockData.getStringIdxToPositionIdx()).boxed().toList())
                 .positions(toRectangle2DList(atomicPositionBlockData.getPositions()))
                 .parent(parent)
                 .build();

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/textblock/ConcatenatedTextBlock.java

@@ -3,6 +3,7 @@ package com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.t
 import static java.lang.String.format;
 
 import java.awt.geom.Rectangle2D;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.LinkedList;
 import java.util.List;
@@ -25,6 +26,12 @@ public class ConcatenatedTextBlock implements TextBlock {
     Boundary boundary;
 
 
+    public static ConcatenatedTextBlock empty() {
+
+        return new ConcatenatedTextBlock(Collections.emptyList());
+    }
+
+
     public ConcatenatedTextBlock(List<AtomicTextBlock> atomicTextBlocks) {
 
         this.atomicTextBlocks = new LinkedList<>();

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/textblock/TextBlockCollector.java

@@ -1,6 +1,5 @@
 package com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock;
 
-import java.util.Collections;
 import java.util.Set;
 import java.util.function.BiConsumer;
 import java.util.function.BinaryOperator;
@@ -16,7 +15,7 @@ public class TextBlockCollector implements Collector<TextBlock, ConcatenatedText
     @Override
     public Supplier<ConcatenatedTextBlock> supplier() {
 
-        return () -> new ConcatenatedTextBlock(Collections.emptyList());
+        return ConcatenatedTextBlock::empty;
     }
 
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/AnalyzeService.java

@@ -33,7 +33,7 @@ import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.mo
 import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.SimplifiedSectionText;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.SimplifiedText;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.service.PdfSegmentationService;
-import com.iqser.red.service.redaction.v1.server.layoutparsing.document.data.mapper.DocumentDataMapper;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.document.data.DocumentData;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.data.mapper.DocumentGraphMapper;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.factory.DocumentGraphFactory;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.Document;
@@ -109,7 +109,7 @@ public class AnalyzeService {
         sectionGridCreatorService.createSectionGrid(classifiedDoc, pageCount);
 
         log.info("Store document graph, text, simplified text, and section grid for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
-        redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.TEXT, DocumentDataMapper.toDocumentData(document));
+        redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.TEXT, DocumentData.fromDocument(document));
         redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.SIMPLIFIED_TEXT, toSimplifiedText(document));
         redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.SECTION_GRID, classifiedDoc.getSectionGrid());
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/TextNormalizationUtilities.java

@@ -16,4 +16,16 @@ public final class TextNormalizationUtilities {
         return text.replaceAll("([^\\s\\d\\-]{2,500})[\\-\\u00AD]\\R", "$1");
     }
 
+
+    public static String removeLineBreaks(String text) {
+
+        return text.replaceAll("\n", " ");
+    }
+
+
+    public static String removeRepeatingWhitespaces(String text) {
+
+        return text.replaceAll(" {2}", " ");
+    }
+
 }

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentMappingIntegrationTest.java

@@ -10,7 +10,6 @@ import com.iqser.red.service.redaction.v1.server.layoutparsing.document.data.Ato
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.data.DocumentData;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.data.DocumentTreeData;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.data.PageData;
-import com.iqser.red.service.redaction.v1.server.layoutparsing.document.data.mapper.DocumentDataMapper;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.data.mapper.DocumentGraphMapper;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.data.mapper.PropertiesMapper;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.Document;
@@ -29,7 +28,7 @@ public class DocumentMappingIntegrationTest extends BuildDocumentIntegrationTest
         String filename = "files/new/crafted document";
 
         Document document = buildGraph(filename);
-        DocumentData documentData = DocumentDataMapper.toDocumentData(document);
+        DocumentData documentData = DocumentData.fromDocument(document);
 
         storageService.storeJSONObject(TenantContext.getTenantId(), filename + "_PAGES" + ".json", documentData.getPages());
         storageService.storeJSONObject(TenantContext.getTenantId(), filename + "_TEXT" + ".json", documentData.getAtomicTextBlocks());

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl

@@ -135,7 +135,6 @@ rule "4: Redact Names and Addresses if no_redaction_indicator and redaction_indi
     end
 
 rule "5: Do not redact Names and Addresses if published information found"
-
     when
         $section: Section(hasEntitiesOfType("vertebrate"),
                               hasEntitiesOfType("published_information"),
@@ -663,8 +662,8 @@ rule "remove Entity of type RECOMMENDATION when contained by ENTITY"
 rule "remove Entity of lower rank, when equal boundaries and entityType"
     salience 32
     when
-        $higherRank: RedactionEntity($type: type, $entityType: entityType, $boundary: boundary)
+        $higherRank: RedactionEntity($type: type, $entityType: entityType)
-        $lowerRank: RedactionEntity($boundary == boundary, type != $type, entityType == $entityType, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !redaction)
+        $lowerRank: RedactionEntity(intersects($higherRank), type != $type, entityType == $entityType, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !resized, !skipRemoveEntitiesContainedInLarger)
     then
         $lowerRank.removeFromGraph();
         retract($lowerRank);