RED-6009 - Document Tree Structure

* implemented table merging when no headers present

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/model/table/Cell.java

@@ -27,6 +27,8 @@ public class Cell extends Rectangle {
 
     private static final int MIN_SIZE = 1;
 
+    private int pageNumber;
+
 
     public Cell(Point2D topLeft, Point2D bottomRight) {
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/factory/SectionNodeFactory.java

@@ -18,6 +18,7 @@ import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.no
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.PageNode;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.SectionNode;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.SemanticNode;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.TableMergingUtility;
 
 import lombok.experimental.UtilityClass;
 
@@ -75,19 +76,25 @@ public class SectionNodeFactory {
     private void addTablesOrParagraphsOrHeadlinesToSection(List<AbstractPageBlock> pageBlocks, DocumentGraphFactory.Context context, SectionNode sectionNode) {
 
         Set<AbstractPageBlock> alreadyMerged = new HashSet<>();
+        List<AbstractPageBlock> remainingBlocks = new LinkedList<>(pageBlocks);
         for (AbstractPageBlock abstractPageBlock : pageBlocks) {
 
             if (alreadyMerged.contains(abstractPageBlock)) {
                 continue;
             }
 
+            remainingBlocks.removeAll(alreadyMerged);
+
             if (abstractPageBlock instanceof ClassificationTextBlock) {
-                List<ClassificationTextBlock> textBlocks = findTextBlocksWithSameClassificationAndAlignsY(abstractPageBlock, pageBlocks);
+                List<ClassificationTextBlock> textBlocks = findTextBlocksWithSameClassificationAndAlignsY(abstractPageBlock, remainingBlocks);
                 alreadyMerged.addAll(textBlocks);
                 DocumentGraphFactory.addParagraphOrHeadline(sectionNode, (ClassificationTextBlock) abstractPageBlock, context, textBlocks);
             }
-            if (abstractPageBlock instanceof TablePageBlock) {
+            if (abstractPageBlock instanceof TablePageBlock tablePageBlock) {
-                TableNodeFactory.addTable(sectionNode, (TablePageBlock) abstractPageBlock, context);
+
+                List<TablePageBlock> tablesToMerge = TableMergingUtility.findConsecutiveTablesWithSameColCountAndSameHeaders(tablePageBlock, remainingBlocks);
+                alreadyMerged.addAll(tablesToMerge);
+                TableNodeFactory.addTable(sectionNode, tablesToMerge, context);
             }
         }
     }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/factory/TableNodeFactory.java

@@ -2,7 +2,10 @@ package com.iqser.red.service.redaction.v1.server.layoutparsing.document.factory
 
 import static java.util.Collections.emptyList;
 
+import java.util.Collection;
 import java.util.List;
+import java.util.Set;
+import java.util.stream.Collectors;
 
 import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.AbstractPageBlock;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.table.Cell;
@@ -23,23 +26,40 @@ public class TableNodeFactory {
     public static final double TABLE_CELL_MERGE_CONTENTS_SIZE_THRESHOLD = 0.05;
 
 
-    public void addTable(SemanticNode parentNode, TablePageBlock table, DocumentGraphFactory.Context context) {
+    public void addTable(SemanticNode parentNode, List<TablePageBlock> tablesToMerge, DocumentGraphFactory.Context context) {
 
-        PageNode page = context.getPage(table.getPage());
+        setPageNumberInCellTextBlocks(tablesToMerge);
-        TableNode tableNode = TableNode.builder().tableOfContents(context.getTableOfContents()).numberOfCols(table.getColCount()).numberOfRows(table.getRowCount()).build();
+        Set<PageNode> pages = tablesToMerge.stream().map(AbstractPageBlock::getPage).map(context::getPage).collect(Collectors.toSet());
+        List<List<Cell>> mergedRows = tablesToMerge.stream().map(TablePageBlock::getRows).flatMap(Collection::stream).toList();
+        TableNode tableNode = TableNode.builder().tableOfContents(context.getTableOfContents()).numberOfCols(mergedRows.get(0).size()).numberOfRows(mergedRows.size()).build();
+
+        pages.forEach(page -> addTableToPage(page, parentNode, tableNode));
+
+        List<Integer> tocId = context.getTableOfContents().createNewChildEntryAndReturnId(parentNode.getTocId(), NodeType.TABLE, tableNode);
+        tableNode.setTocId(tocId);
+        addTableCells(mergedRows, tableNode, context);
+
+        IfTableHasNoHeadersAssumeFirstRowAreHeaders(tableNode);
+    }
+
+
+    private static void setPageNumberInCellTextBlocks(List<TablePageBlock> tablesToMerge) {
+
+        tablesToMerge.forEach(table -> table.getRows()
+                .stream()
+                .flatMap(Collection::stream)
+                .peek(cell -> cell.setPageNumber(table.getPage()))
+                .forEach(cell -> cell.getTextBlocks().stream().filter(tb -> tb.getPage() == 0).forEach(tb -> tb.setPage(table.getPage()))));
+    }
+
+
+    private static void addTableToPage(PageNode page, SemanticNode parentNode, TableNode tableNode) {
 
         if (!page.getMainBody().contains(parentNode)) {
             parentNode.getPages().add(page);
         }
 
         page.getMainBody().add(tableNode);
-
-        List<Integer> tocId = context.getTableOfContents().createNewChildEntryAndReturnId(parentNode.getTocId(), NodeType.TABLE, tableNode);
-        tableNode.setTocId(tocId);
-
-        addTableCells(table.getRows(), tableNode, context, table.getPage());
-
-        IfTableHasNoHeadersAssumeFirstRowAreHeaders(tableNode);
     }
 
 
@@ -51,21 +71,20 @@ public class TableNodeFactory {
     }
 
 
-    private void addTableCells(List<List<Cell>> rows, TableNode tableNode, DocumentGraphFactory.Context context, int pageNumber) {
+    private void addTableCells(List<List<Cell>> rows, TableNode tableNode, DocumentGraphFactory.Context context) {
 
         for (int rowIndex = 0; rowIndex < rows.size(); rowIndex++) {
             for (int colIndex = 0; colIndex < rows.get(rowIndex).size(); colIndex++) {
-                addTableCell(rows.get(rowIndex).get(colIndex), rowIndex, colIndex, tableNode, pageNumber, context);
+                addTableCell(rows.get(rowIndex).get(colIndex), rowIndex, colIndex, tableNode, context);
             }
         }
     }
 
 
     @SuppressWarnings("PMD.UnusedPrivateMethod") // PMD actually flags this wrong
-    private void addTableCell(Cell cell, int rowIndex, int colIndex, SemanticNode parentNode, int pageNumber, DocumentGraphFactory.Context context) {
+    private void addTableCell(Cell cell, int rowIndex, int colIndex, SemanticNode parentNode, DocumentGraphFactory.Context context) {
 
-        PageNode page = context.getPage(pageNumber);
+        PageNode page = context.getPage(cell.getPageNumber());
-        cell.getTextBlocks().stream().filter(tb -> tb.getPage() == 0).forEach(tb -> tb.setPage(pageNumber));
 
         TableCellNode tableCellNode = TableCellNode.builder()
                 .tableOfContents(context.getTableOfContents())

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/services/EntityCreationService.java

@@ -1,7 +1,7 @@
 package com.iqser.red.service.redaction.v1.server.layoutparsing.document.services;
 
-import static com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RedactionSearchUtils.getExpandedEndByRegex;
+import static com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RedactionSearchUtility.getExpandedEndByRegex;
-import static com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RedactionSearchUtils.getExpandedStartByRegex;
+import static com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RedactionSearchUtility.getExpandedStartByRegex;
 import static com.iqser.red.service.redaction.v1.server.redaction.utils.SeparatorUtils.boundaryIsSurroundedBySeparators;
 import static com.iqser.red.service.redaction.v1.server.redaction.utils.SeparatorUtils.isWhiteSpacesOrSeparatorsOnly;
 
@@ -27,7 +27,7 @@ import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.no
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.SemanticNode;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.TableCellNode;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.TextBlock;
-import com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RedactionSearchUtils;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RedactionSearchUtility;
 import com.iqser.red.service.redaction.v1.server.redaction.adapter.NerEntities;
 import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType;
 import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.SearchImplementation;
@@ -47,8 +47,8 @@ public class EntityCreationService {
     public Stream<RedactionEntity> betweenStrings(String start, String stop, String type, EntityType entityType, SemanticNode node) {
 
         TextBlock textBlock = node.buildTextBlock();
-        List<Boundary> startBoundaries = RedactionSearchUtils.findBoundariesByString(start, textBlock);
+        List<Boundary> startBoundaries = RedactionSearchUtility.findBoundariesByString(start, textBlock);
-        List<Boundary> stopBoundaries = RedactionSearchUtils.findBoundariesByString(stop, textBlock);
+        List<Boundary> stopBoundaries = RedactionSearchUtility.findBoundariesByString(stop, textBlock);
 
         List<Boundary> entityBoundaries = new LinkedList<>();
         if (startBoundaries.isEmpty() || stopBoundaries.isEmpty()) {
@@ -90,7 +90,7 @@ public class EntityCreationService {
     public Stream<RedactionEntity> lineAfterString(String string, String type, EntityType entityType, SemanticNode node) {
 
         TextBlock textBlock = node.buildTextBlock();
-        return RedactionSearchUtils.findBoundariesByString(string, textBlock)
+        return RedactionSearchUtility.findBoundariesByString(string, textBlock)
                 .stream()
                 .map(boundary -> toLineAfterBoundary(textBlock, boundary))
                 .filter(boundary -> isValidEntityBoundary(textBlock, boundary))
@@ -100,13 +100,13 @@ public class EntityCreationService {
 
     public Stream<RedactionEntity> byRegex(String regexPattern, String type, EntityType entityType, SemanticNode node) {
 
-        return RedactionSearchUtils.findBoundariesByRegex(regexPattern, node.buildTextBlock()).stream().map(boundary -> byBoundary(boundary, type, entityType, node));
+        return RedactionSearchUtility.findBoundariesByRegex(regexPattern, node.buildTextBlock()).stream().map(boundary -> byBoundary(boundary, type, entityType, node));
     }
 
 
     public Stream<RedactionEntity> byString(String keyword, String type, EntityType entityType, SemanticNode node) {
 
-        return RedactionSearchUtils.findBoundariesByString(keyword, node.buildTextBlock()).stream().map(boundary -> byBoundary(boundary, type, entityType, node));
+        return RedactionSearchUtility.findBoundariesByString(keyword, node.buildTextBlock()).stream().map(boundary -> byBoundary(boundary, type, entityType, node));
     }
 
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/utils/RedactionSearchUtility.java

@@ -12,7 +12,10 @@ import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.en
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.TextBlock;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.Patterns;
 
-public class RedactionSearchUtils {
+import lombok.experimental.UtilityClass;
+
+@UtilityClass
+public class RedactionSearchUtility {
 
     public static boolean anyMatch(CharSequence charSequence, String regexPattern) {
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/utils/TableMergingUtility.java

@@ -0,0 +1,42 @@
+package com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils;
+
+import java.util.Collection;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.stream.Stream;
+
+import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.AbstractPageBlock;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.table.Cell;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.table.TablePageBlock;
+
+import lombok.experimental.UtilityClass;
+
+@UtilityClass
+public class TableMergingUtility {
+
+    public List<TablePageBlock> findConsecutiveTablesWithSameColCountAndSameHeaders(TablePageBlock originalTablePageBlock, List<AbstractPageBlock> pageBlocks) {
+
+        List<TablePageBlock> consecutiveTables = pageBlocks.stream()
+                .map(abstractPageBlock -> (TablePageBlock) abstractPageBlock)
+                .filter(tablePageBlock -> !tablePageBlock.equals(originalTablePageBlock))
+                .toList();
+        assert consecutiveTables.size() == pageBlocks.size() - 1;
+
+        List<TablePageBlock> consecutiveTablesWithSameColCountAndHeaders = new LinkedList<>();
+        for (TablePageBlock consecutiveTable : consecutiveTables) {
+            if (consecutiveTable.getColCount() == originalTablePageBlock.getColCount() && !hasTableHeader(consecutiveTable)) {
+                consecutiveTablesWithSameColCountAndHeaders.add(consecutiveTable);
+            } else {
+                break;
+            }
+        }
+        return Stream.concat(Stream.of(originalTablePageBlock), consecutiveTablesWithSameColCountAndHeaders.stream()).toList();
+    }
+
+
+    private boolean hasTableHeader(TablePageBlock table) {
+
+        return table.getRows().stream().flatMap(Collection::stream).anyMatch(Cell::isHeaderCell);
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/manual_redaction_rules.drl

@@ -1,8 +1,8 @@
 package drools
 
 import static java.lang.String.format;
-import static com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RedactionSearchUtils.anyMatch;
+import static com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RedactionSearchUtility.anyMatch;
-import static com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RedactionSearchUtils.exactMatch;
+import static com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RedactionSearchUtility.exactMatch;
 import static com.iqser.red.service.redaction.v1.server.layoutparsing.document.data.mapper.PropertiesMapper.parseImageType;
 
 import java.util.List;
@@ -37,7 +37,7 @@ import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.Bo
 import java.util.stream.Collectors
 import java.util.Collection
 import java.util.stream.Stream
-import com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RedactionSearchUtils;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RedactionSearchUtility;
 
 global DocumentGraph document
 global ManualRedactionApplicationService manualRedactionApplicationService

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl

@@ -1,8 +1,8 @@
 package drools
 
 import static java.lang.String.format;
-import static com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RedactionSearchUtils.anyMatch;
+import static com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RedactionSearchUtility.anyMatch;
-import static com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RedactionSearchUtils.exactMatch;
+import static com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RedactionSearchUtility.exactMatch;
 import static com.iqser.red.service.redaction.v1.server.layoutparsing.document.data.mapper.PropertiesMapper.parseImageType;
 
 import java.util.List;
@@ -37,7 +37,7 @@ import com.iqser.red.service.redaction.v1.server.redaction.adapter.NerEntities;
 import java.util.stream.Collectors;
 import java.util.Collection;
 import java.util.stream.Stream;
-import com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RedactionSearchUtils;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RedactionSearchUtility;
 
 global DocumentGraph document
 global EntityCreationService entityCreationService
@@ -509,7 +509,7 @@ rule "102: Guidelines FileAttributes"
     when
         $section: SectionNode(excludesTables, (containsString("DATA REQUIREMENT(S):") || containsString("TEST GUIDELINE(S):")) && (containsString("OECD") || containsString("EPA") || containsString("OPPTS")))
     then
-        RedactionSearchUtils.findBoundariesByRegex("OECD (No\\.? )?\\d{3}( \\(\\d{4}\\))?", $section.buildTextBlock()).stream()
+        RedactionSearchUtility.findBoundariesByRegex("OECD (No\\.? )?\\d{3}( \\(\\d{4}\\))?", $section.buildTextBlock()).stream()
             .map(boundary -> $section.buildTextBlock().subSequence(boundary).toString())
             .map(value -> FileAttribute.builder().label("OECD Number").value(value).build())
             .forEach(fileAttribute -> insert(fileAttribute));
@@ -640,17 +640,27 @@ rule "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATIO
         retract($recommendation);
     end
 
-rule "remove Entity of type RECOMMENDATION when contained by ENTITY"
+rule "remove Entity of type RECOMMENDATION when intersected by ENTITY with same type"
     salience 256
     when
         $entity: RedactionEntity($type: type, entityType == EntityType.ENTITY)
-        $recommendation: RedactionEntity(containedBy($entity), type == $type, entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger)
+        $recommendation: RedactionEntity(intersects($entity), type == $type, entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger)
     then
         $entity.addEngines($recommendation.getEngines());
         $recommendation.removeFromGraph();
         retract($recommendation);
     end
 
+rule "remove Entity of type RECOMMENDATION when contained by ENTITY"
+    salience 256
+    when
+        $entity: RedactionEntity(entityType == EntityType.ENTITY)
+        $recommendation: RedactionEntity(containedBy($entity), entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger)
+    then
+        $recommendation.removeFromGraph();
+        retract($recommendation);
+    end
+
 rule "remove Entity of lower rank, when equal boundaries and entityType"
     salience 32
     when

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_v2.drl

@@ -1,8 +1,8 @@
 package drools
 
 import static java.lang.String.format;
-import static com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RedactionSearchUtils.anyMatch;
+import static com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RedactionSearchUtility.anyMatch;
-import static com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RedactionSearchUtils.exactMatch;
+import static com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RedactionSearchUtility.exactMatch;
 import static com.iqser.red.service.redaction.v1.server.layoutparsing.document.data.mapper.PropertiesMapper.parseImageType;
 
 import java.util.List;
@@ -37,7 +37,7 @@ import com.iqser.red.service.redaction.v1.server.redaction.adapter.NerEntities;
 import java.util.stream.Collectors;
 import java.util.Collection;
 import java.util.stream.Stream;
-import com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RedactionSearchUtils;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RedactionSearchUtility;
 
 global DocumentGraph document
 global EntityCreationService entityCreationService