TAAS-104: merge visually intersecting Paragraphs

* fix static import

.gitlab-ci.yml

@@ -16,8 +16,6 @@ deploy:
     reports:
       dotenv: version.env
   rules:
-    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
+    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH 
     - if: $CI_COMMIT_BRANCH =~ /^release/
     - if: $CI_COMMIT_TAG
-pmd:
-  allow_failure: true

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/queue/LayoutParsingRequest.java

@@ -36,6 +36,9 @@ public record LayoutParsingRequest(
         @Schema(description = "Path where the Simplified Text File will be stored.")//
         @NonNull String simplifiedTextStorageId,//
         @Schema(description = "Path where the Viewer Document PDF will be stored.")//
-        @NonNull String viewerDocumentStorageId) {
+        @NonNull String viewerDocumentStorageId,//
+        @Deprecated//
+        @Schema(description = "Path where the Section Grid will be stored.")//
+        @NonNull String sectionGridStorageId) {
 
 }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/LayoutParsingPipeline.java

@@ -35,6 +35,7 @@ import com.knecon.fforesight.service.layoutparser.processor.python_api.model.tab
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableServiceResponse;
 import com.knecon.fforesight.service.layoutparser.processor.services.BodyTextFrameService;
 import com.knecon.fforesight.service.layoutparser.processor.services.RulingCleaningService;
+import com.knecon.fforesight.service.layoutparser.processor.services.SectionGridCreatorService;
 import com.knecon.fforesight.service.layoutparser.processor.services.SectionsBuilderService;
 import com.knecon.fforesight.service.layoutparser.processor.services.SimplifiedSectionTextService;
 import com.knecon.fforesight.service.layoutparser.processor.services.TableExtractionService;
@@ -64,6 +65,7 @@ public class LayoutParsingPipeline {
     private final CvTableParsingAdapter cvTableParsingAdapter;
     private final LayoutParsingStorageService layoutParsingStorageService;
     private final SectionsBuilderService sectionsBuilderService;
+    private final SectionGridCreatorService sectionGridCreatorService;
     private final TaasClassificationService taasClassificationService;
     private final RedactManagerClassificationService redactManagerClassificationService;
     private final DocuMineClassificationService docuMineClassificationService;
@@ -97,6 +99,7 @@ public class LayoutParsingPipeline {
 
             int numberOfPages = originDocument.getNumberOfPages();
 
+            layoutParsingStorageService.storeSectionGrid(layoutParsingRequest, sectionGridCreatorService.createSectionGrid(documentGraph));
             layoutParsingStorageService.storeDocumentData(layoutParsingRequest, DocumentDataMapper.toDocumentData(documentGraph));
             layoutParsingStorageService.storeSimplifiedText(layoutParsingRequest, simplifiedSectionTextService.toSimplifiedText(documentGraph));
 
@@ -143,13 +146,13 @@ public class LayoutParsingPipeline {
 
         return String.format("%d pages with %d sections, %d headlines, %d paragraphs, %d tables with %d cells, %d headers, and %d footers parsed",
                 numberOfPages,
-                semanticNodeCounts.get(NodeType.SECTION) == null ? 0 : semanticNodeCounts.get(NodeType.SECTION),
-                semanticNodeCounts.get(NodeType.HEADLINE) == null ? 0 : semanticNodeCounts.get(NodeType.HEADLINE),
-                semanticNodeCounts.get(NodeType.PARAGRAPH) == null ? 0 : semanticNodeCounts.get(NodeType.PARAGRAPH),
-                semanticNodeCounts.get(NodeType.TABLE) == null ? 0 : semanticNodeCounts.get(NodeType.TABLE),
-                semanticNodeCounts.get(NodeType.TABLE_CELL) == null ? 0 : semanticNodeCounts.get(NodeType.TABLE_CELL),
-                semanticNodeCounts.get(NodeType.HEADER) == null ? 0 : semanticNodeCounts.get(NodeType.HEADER),
-                semanticNodeCounts.get(NodeType.FOOTER) == null ? 0 : semanticNodeCounts.get(NodeType.FOOTER));
+                semanticNodeCounts.get(NodeType.SECTION),
+                semanticNodeCounts.get(NodeType.HEADLINE),
+                semanticNodeCounts.get(NodeType.PARAGRAPH),
+                semanticNodeCounts.get(NodeType.TABLE),
+                semanticNodeCounts.get(NodeType.TABLE_CELL),
+                semanticNodeCounts.get(NodeType.HEADER),
+                semanticNodeCounts.get(NodeType.FOOTER));
     }
 
 
@@ -187,7 +190,10 @@ public class LayoutParsingPipeline {
             boolean isLandscape = pdr.getWidth() > pdr.getHeight() && (rotation == 0 || rotation == 180) || pdr.getHeight() > pdr.getWidth() && (rotation == 90 || rotation == 270);
 
             PDRectangle cropbox = pdPage.getCropBox();
-            CleanRulings cleanRulings = rulingCleaningService.getCleanRulings(pdfTableCells.get(pageNumber), stripper.getRulings());
+            CleanRulings cleanRulings = rulingCleaningService.getCleanRulings(pdfTableCells.get(pageNumber),
+                    stripper.getRulings(),
+                    stripper.getMinCharWidth(),
+                    stripper.getMaxCharHeight());
 
             ClassificationPage classificationPage = switch (layoutParsingType) {
                 case REDACT_MANAGER -> redactManagerBlockificationService.blockify(stripper.getTextPositionSequences(), cleanRulings.getHorizontal(), cleanRulings.getVertical());
@@ -210,8 +216,7 @@ public class LayoutParsingPipeline {
                 imageServiceResponseAdapter.findOcr(classificationPage);
             }
 
-            tableExtractionService.extractTables(cleanRulings, classificationPage);
-
+            tableExtractionService.extractTables(cleanRulings, classificationPage, layoutParsingType);
             buildPageStatistics(classificationPage);
             increaseDocumentStatistics(classificationPage, classificationDocument);
 
@@ -244,9 +249,9 @@ public class LayoutParsingPipeline {
 
     private void increaseDocumentStatistics(ClassificationPage classificationPage, ClassificationDocument document) {
 
-       if (!classificationPage.isLandscape()) {
-        document.getFontSizeCounter().addAll(classificationPage.getFontSizeCounter().getCountPerValue());
-      }
+        if (!classificationPage.isLandscape()) {
+            document.getFontSizeCounter().addAll(classificationPage.getFontSizeCounter().getCountPerValue());
+        }
         document.getFontCounter().addAll(classificationPage.getFontCounter().getCountPerValue());
         document.getTextHeightCounter().addAll(classificationPage.getTextHeightCounter().getCountPerValue());
         document.getFontStyleCounter().addAll(classificationPage.getFontStyleCounter().getCountPerValue());

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/LayoutParsingStorageService.java

@@ -17,6 +17,7 @@ import org.apache.pdfbox.pdmodel.PDDocument;
 import org.springframework.stereotype.Service;
 
 import com.fasterxml.jackson.databind.ObjectMapper;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.SectionGrid;
 import com.iqser.red.storage.commons.service.StorageService;
 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentData;
 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.SimplifiedText;
@@ -83,6 +84,11 @@ public class LayoutParsingStorageService {
     }
 
 
+    public void storeSectionGrid(LayoutParsingRequest layoutParsingRequest, SectionGrid sectionGrid) {
+
+        storageService.storeJSONObject(TenantContext.getTenantId(), layoutParsingRequest.sectionGridStorageId(), sectionGrid);
+    }
+
 
     public void storeResearchDocumentData(LayoutParsingRequest layoutParsingRequest, ResearchDocumentData researchDocumentData) {
 

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/ClassificationDocument.java

@@ -3,6 +3,7 @@ package com.knecon.fforesight.service.layoutparser.processor.model;
 import java.util.ArrayList;
 import java.util.List;
 
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.SectionGrid;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.StringFrequencyCounter;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.UnclassifiedText;
 
@@ -24,6 +25,7 @@ public class ClassificationDocument {
     private StringFrequencyCounter fontStyleCounter = new StringFrequencyCounter();
     private boolean headlines;
 
+    private SectionGrid sectionGrid = new SectionGrid();
     private long rulesVersion;
 
 }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/Table.java

@@ -34,6 +34,7 @@ public class Table implements SemanticNode {
 
     int numberOfRows;
     int numberOfCols;
+
     TextBlock textBlock;
 
     @Builder.Default
@@ -207,6 +208,7 @@ public class Table implements SemanticNode {
         return IntStream.range(0, numberOfCols).boxed().map(col -> getCell(row, col));
     }
 
+
     /**
      * Streams all TableCells row-wise and filters them with header == true.
      *

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/table/TablePageBlock.java

@@ -1,14 +1,12 @@
 package com.knecon.fforesight.service.layoutparser.processor.model.table;
 
 import java.awt.geom.Point2D;
-import java.awt.geom.Rectangle2D;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
 import java.util.TreeMap;
-import java.util.stream.Collectors;
 
 import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBlock;
 import com.knecon.fforesight.service.layoutparser.processor.model.PageBlockType;
@@ -254,8 +252,7 @@ public class TablePageBlock extends AbstractPageBlock {
                 if (prevY != null && prevX != null) {
                     var cell = new Cell(new Point2D.Float(prevX, prevY), new Point2D.Float(x, y));
 
-                    var intersectionCell = cells.stream().filter(c -> intersects(cell, c)).findFirst();
-
+                    var intersectionCell = cells.stream().filter(c -> cell.intersects(c) && cell.overlapRatio(c) > 0.1f).findFirst();
                     intersectionCell.ifPresent(value -> cell.getTextBlocks().addAll(value.getTextBlocks()));
                     if (cell.hasMinimumSize()) {
                         row.add(cell);
@@ -276,21 +273,6 @@ public class TablePageBlock extends AbstractPageBlock {
     }
 
 
-
-    public boolean intersects(Cell cell1, Cell cell2) {
-        if (cell1.getHeight() <= 0 || cell2.getHeight() <= 0) {
-            return false;
-        }
-        double x0 = cell1.getX() + 2;
-        double y0 = cell1.getY() + 2;
-        return (cell2.x + cell2.width > x0 &&
-                cell2.y + cell2.height > y0 &&
-                cell2.x < x0 + cell1.getWidth() -2 &&
-                cell2.y < y0 + cell1.getHeight() -2);
-    }
-
-
-
     @Override
     public String getText() {
 

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/text/RedTextPosition.java

@@ -17,6 +17,7 @@ import lombok.SneakyThrows;
 @AllArgsConstructor
 public class RedTextPosition {
 
+    private String textMatrix;
     private float[] position;
 
     @JsonIgnore
@@ -55,6 +56,8 @@ public class RedTextPosition {
 
         pos.setFontSizeInPt(textPosition.getFontSizeInPt());
 
+        pos.setTextMatrix(textPosition.getTextMatrix().toString());
+
         var position = new float[4];
 
         position[0] = textPosition.getXDirAdj();

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/text/TextPositionSequence.java

@@ -196,12 +196,6 @@ public class TextPositionSequence implements CharSequence {
     }
 
 
-    public float getTextHeightNoPadding() {
-
-        return textPositions.get(0).getHeightDir();
-    }
-
-
     @JsonIgnore
     @JsonAttribute(ignore = true)
     public float getTextHeight() {
@@ -230,9 +224,6 @@ public class TextPositionSequence implements CharSequence {
     @JsonAttribute(ignore = true)
     public String getFont() {
 
-        if (textPositions.get(0).getFontName() == null) {
-            return "none";
-        }
         return textPositions.get(0).getFontName().toLowerCase(Locale.ROOT).replaceAll(",bold", "").replaceAll(",italic", "");
     }
 
@@ -241,9 +232,6 @@ public class TextPositionSequence implements CharSequence {
     @JsonAttribute(ignore = true)
     public String getFontStyle() {
 
-        if (textPositions.get(0).getFontName() == null) {
-            return "standard";
-        }
         String lowercaseFontName = textPositions.get(0).getFontName().toLowerCase(Locale.ROOT);
 
         if (lowercaseFontName.contains("bold") && lowercaseFontName.contains("italic")) {
@@ -255,6 +243,7 @@ public class TextPositionSequence implements CharSequence {
         } else {
             return "standard";
         }
+
     }
 
 

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/python_api/adapter/CvTableParsingAdapter.java

@@ -8,7 +8,6 @@ import java.util.Map;
 
 import org.springframework.stereotype.Service;
 
-import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.PageInfo;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableCells;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableServiceResponse;
 
@@ -25,26 +24,24 @@ public class CvTableParsingAdapter {
         Map<Integer, List<TableCells>> tableCells = new HashMap<>();
         tableServiceResponse.getData()
                 .forEach(tableData -> tableCells.computeIfAbsent(tableData.getPageInfo().getNumber(), tableCell -> new ArrayList<>())
-                        .addAll(convertTableCells(tableData.getTableCells(), tableData.getPageInfo())));
+                        .addAll(convertTableCells(tableData.getTableCells())));
 
         return tableCells;
     }
 
 
-    private Collection<TableCells> convertTableCells(List<TableCells> tableCells, PageInfo pageInfo) {
+    private Collection<TableCells> convertTableCells(List<TableCells> tableCells) {
 
         List<TableCells> cvParsedTableCells = new ArrayList<>();
 
-        tableCells.stream()
-                .filter(cell -> cell.getWidth() < pageInfo.getWidth() * 0.98 && cell.getHeight() < pageInfo.getHeight() * 0.98)
-                .forEach(t -> cvParsedTableCells.add(TableCells.builder()
-                        .y0(t.getY0())
-                        .x1(t.getX1())
-                        .y1(t.getY1())
-                        .x0(t.getX0())
-                        .width(t.getWidth())
-                        .height(t.getHeight())
-                        .build()));
+        tableCells.forEach(t -> cvParsedTableCells.add(TableCells.builder()
+                .y0(t.getY0())
+                .x1(t.getX1())
+                .y1(t.getY1())
+                .x0(t.getX0())
+                .width(t.getWidth())
+                .height(t.getHeight())
+                .build()));
 
         return cvParsedTableCells;
     }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/RulingCleaningService.java

@@ -12,9 +12,9 @@ import java.util.Map;
 
 import org.springframework.stereotype.Service;
 
+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableCells;
 import com.knecon.fforesight.service.layoutparser.processor.model.table.CleanRulings;
 import com.knecon.fforesight.service.layoutparser.processor.model.table.Ruling;
-import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableCells;
 import com.knecon.fforesight.service.layoutparser.processor.utils.DoubleComparisons;
 
 import lombok.RequiredArgsConstructor;
@@ -25,13 +25,10 @@ import lombok.extern.slf4j.Slf4j;
 @RequiredArgsConstructor
 public class RulingCleaningService {
 
-    private static final float THRESHOLD = 6;
-
-
-    public CleanRulings getCleanRulings(List<TableCells> tableCells, List<Ruling> rulings) {
+    public CleanRulings getCleanRulings(List<TableCells> tableCells, List<Ruling> rulings, float minCharWidth, float maxCharHeight) {
 
         if (!rulings.isEmpty()) {
-            snapPoints(rulings);
+            snapPoints(rulings, minCharWidth, maxCharHeight);
         }
 
         List<Ruling> vrs = new ArrayList<>();
@@ -56,11 +53,14 @@ public class RulingCleaningService {
         }
         List<Ruling> horizontalRulingLines = collapseOrientedRulings(hrs);
 
-        return CleanRulings.builder().vertical(verticalRulingLines).horizontal(horizontalRulingLines).build();
+        return CleanRulings.builder()
+                .vertical(verticalRulingLines)
+                .horizontal(horizontalRulingLines)
+                .build();
     }
 
 
-    public void snapPoints(List<? extends Line2D.Float> rulings) {
+    public void snapPoints(List<? extends Line2D.Float> rulings, float xThreshold, float yThreshold) {
 
         // collect points and keep a Line -> p1,p2 map
         Map<Line2D.Float, Point2D[]> linesToPoints = new HashMap<>();
@@ -81,7 +81,7 @@ public class RulingCleaningService {
 
         for (Point2D p : points.subList(1, points.size() - 1)) {
             List<Point2D> last = groupedPoints.get(groupedPoints.size() - 1);
-            if (Math.abs(p.getX() - last.get(0).getX()) < THRESHOLD) {
+            if (Math.abs(p.getX() - last.get(0).getX()) < xThreshold) {
                 groupedPoints.get(groupedPoints.size() - 1).add(p);
             } else {
                 groupedPoints.add(new ArrayList<>(Collections.singletonList(p)));
@@ -108,7 +108,7 @@ public class RulingCleaningService {
 
         for (Point2D p : points.subList(1, points.size() - 1)) {
             List<Point2D> last = groupedPoints.get(groupedPoints.size() - 1);
-            if (Math.abs(p.getY() - last.get(0).getY()) < THRESHOLD) {
+            if (Math.abs(p.getY() - last.get(0).getY()) < yThreshold) {
                 groupedPoints.get(groupedPoints.size() - 1).add(p);
             } else {
                 groupedPoints.add(new ArrayList<>(Collections.singletonList(p)));

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/SectionGridCreatorService.java

@@ -0,0 +1,146 @@
+package com.knecon.fforesight.service.layoutparser.processor.services;
+
+import java.awt.geom.Rectangle2D;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import java.util.function.BiConsumer;
+import java.util.function.BinaryOperator;
+import java.util.function.Function;
+import java.util.function.Supplier;
+import java.util.stream.Collector;
+import java.util.stream.Stream;
+
+import org.springframework.stereotype.Service;
+
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Point;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.CellRectangle;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.SectionGrid;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.SectionRectangle;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Page;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.SemanticNode;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Table;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.TableCell;
+
+import lombok.RequiredArgsConstructor;
+
+@Service
+@RequiredArgsConstructor
+public class SectionGridCreatorService {
+
+    public SectionGrid createSectionGrid(Document document) {
+
+        Map<Integer, List<SectionRectangle>> sectionBBox = document.streamAllSubNodesOfType(NodeType.SECTION).map(SemanticNode::getBBox).collect(new SectionGridCollector());
+        Map<Integer, List<SectionRectangle>> paragraphBBox = document.streamAllSubNodesOfType(NodeType.PARAGRAPH).map(SemanticNode::getBBox).collect(new SectionGridCollector());
+        Map<Integer, List<SectionRectangle>> headlineBBox = document.streamAllSubNodesOfType(NodeType.HEADLINE).map(SemanticNode::getBBox).collect(new SectionGridCollector());
+        Map<Integer, List<SectionRectangle>> tableBBox = document.streamAllSubNodesOfType(NodeType.TABLE).map(node -> (Table) node).collect(new TableGridCollector());
+        var sectionGrid = new SectionGrid();
+
+        sectionGrid.setRectanglesPerPage(mergeMapsByConcatenatingLists(//
+                mergeMapsByConcatenatingLists(paragraphBBox, headlineBBox), //
+                mergeMapsByConcatenatingLists(sectionBBox, tableBBox)));
+
+        return sectionGrid;
+    }
+
+
+    private static abstract class GridCollector<T> implements Collector<T, Map<Integer, List<SectionRectangle>>, Map<Integer, List<SectionRectangle>>> {
+
+        @Override
+        public Supplier<Map<Integer, List<SectionRectangle>>> supplier() {
+
+            return HashMap::new;
+        }
+
+
+        @Override
+        public Function<Map<Integer, List<SectionRectangle>>, Map<Integer, List<SectionRectangle>>> finisher() {
+
+            return Function.identity();
+        }
+
+
+        @Override
+        public BinaryOperator<Map<Integer, List<SectionRectangle>>> combiner() {
+
+            return SectionGridCreatorService::mergeMapsByConcatenatingLists;
+        }
+
+
+        @Override
+        public Set<Characteristics> characteristics() {
+
+            return Set.of(Characteristics.IDENTITY_FINISH, Characteristics.CONCURRENT, Characteristics.UNORDERED);
+        }
+
+    }
+
+    private static class TableGridCollector extends GridCollector<Table> {
+
+        @Override
+        public BiConsumer<Map<Integer, List<SectionRectangle>>, Table> accumulator() {
+
+            return (map, table) -> table.getPages()
+                    .forEach(page -> map.merge(page.getNumber(), List.of(toSectionRectangle(table, page, table.getPages().size())), SectionGridCreatorService::concatLists));
+        }
+
+
+        private static SectionRectangle toSectionRectangle(Table table, Page page, int numberOfParts) {
+
+            Rectangle2D rect = table.getBBox().get(page);
+            List<CellRectangle> tableCellRectangles = table.streamTableCells()
+                    .map(TableCell::getBBox)
+                    .map(map -> map.get(page))
+                    .filter(Objects::nonNull)
+                    .map(rectangle2D -> new CellRectangle(new Point((float) rectangle2D.getX(), (float) rectangle2D.getY()),
+                            (float) rectangle2D.getWidth(),
+                            (float) rectangle2D.getHeight()))
+                    .toList();
+            return new SectionRectangle(new Point((float) rect.getX(), (float) rect.getY()),
+                    (float) rect.getWidth(),
+                    (float) rect.getHeight(),
+                    1,
+                    numberOfParts,
+                    tableCellRectangles);
+        }
+
+    }
+
+    private static class SectionGridCollector extends GridCollector<Map<Page, Rectangle2D>> {
+
+        @Override
+        public BiConsumer<Map<Integer, List<SectionRectangle>>, Map<Page, Rectangle2D>> accumulator() {
+
+            return (mapToKeep, mapToMerge) -> mapToMerge.forEach((page, rectangle) -> mapToKeep.merge(page.getNumber(),
+                    List.of(toSectionRectangle(rectangle, mapToMerge.values().size())),
+                    SectionGridCreatorService::concatLists));
+
+        }
+
+
+        private static SectionRectangle toSectionRectangle(Rectangle2D rect, int numberOfParts) {
+
+            return new SectionRectangle(new Point((float) rect.getX(), (float) rect.getY()), (float) rect.getWidth(), (float) rect.getHeight(), 1, numberOfParts, null);
+        }
+
+    }
+
+
+    private static Map<Integer, List<SectionRectangle>> mergeMapsByConcatenatingLists(Map<Integer, List<SectionRectangle>> mapToKeep,
+                                                                                      Map<Integer, List<SectionRectangle>> mapToMerge) {
+
+        mapToMerge.forEach((page, rectangle) -> mapToKeep.merge(page, rectangle, SectionGridCreatorService::concatLists));
+        return mapToKeep;
+    }
+
+
+    private static List<SectionRectangle> concatLists(List<SectionRectangle> l1, List<SectionRectangle> l2) {
+
+        return Stream.concat(l1.stream(), l2.stream()).toList();
+    }
+
+}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/TableExtractionService.java

@@ -12,6 +12,7 @@ import java.util.Set;
 
 import org.springframework.stereotype.Service;
 
+import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
 import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBlock;
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationPage;
 import com.knecon.fforesight.service.layoutparser.processor.model.table.Cell;
@@ -65,17 +66,6 @@ public class TableExtractionService {
     };
 
 
-    public boolean contains(Cell cell, double x, double y, double w, double h) {
-
-        if (cell.isEmpty() || w <= 0 || h <= 0) {
-            return false;
-        }
-        double x0 = cell.getX();
-        double y0 = cell.getY();
-        return (x >= x0 - 2 && y >= y0 - 2 && (x + w) <= x0 + cell.getWidth() + 2 && (y + h) <= y0 + cell.getHeight() + 2);
-    }
-
-
     /**
      * Finds tables on a page and moves textblocks into cells of the found tables.
      * Note: This algorithm uses Pdf Coordinate System where {0,0} rotated with the page rotation.
@@ -89,17 +79,16 @@ public class TableExtractionService {
      * @param cleanRulings The lines used to build the table.
      * @param page         Page object that contains textblocks and statistics.
      */
-    public void extractTables(CleanRulings cleanRulings, ClassificationPage page) {
+    public void extractTables(CleanRulings cleanRulings, ClassificationPage page, LayoutParsingType layoutParsingType) {
 
-        List<Cell> cells = findCells(cleanRulings.getHorizontal(), cleanRulings.getVertical());
+        List<Cell> cells = findCells(cleanRulings.getHorizontal(), cleanRulings.getVertical(), layoutParsingType);
 
         List<TextPageBlock> toBeRemoved = new ArrayList<>();
 
         for (AbstractPageBlock abstractPageBlock : page.getTextBlocks()) {
             TextPageBlock textBlock = (TextPageBlock) abstractPageBlock;
             for (Cell cell : cells) {
-                if (cell.hasMinimumSize() && contains(cell,
-                        textBlock.getPdfMinX(),
+                if (cell.hasMinimumSize() && cell.intersects(textBlock.getPdfMinX(),
                         textBlock.getPdfMinY(),
                         textBlock.getPdfMaxX() - textBlock.getPdfMinX(),
                         textBlock.getPdfMaxY() - textBlock.getPdfMinY())) {
@@ -113,7 +102,7 @@ public class TableExtractionService {
         cells = new ArrayList<>(new HashSet<>(cells));
         DoubleComparisons.sort(cells, Rectangle.ILL_DEFINED_ORDER);
 
-        List<Rectangle> spreadsheetAreas = findSpreadsheetsFromCells(cells);
+        List<Rectangle> spreadsheetAreas = findSpreadsheetsFromCells(cells).stream().filter(r -> r.getWidth() > 0f && r.getHeight() > 0f).toList();
 
         List<TablePageBlock> tables = new ArrayList<>();
         for (Rectangle area : spreadsheetAreas) {
@@ -146,14 +135,16 @@ public class TableExtractionService {
     }
 
 
-    public List<Cell> findCells(List<Ruling> horizontalRulingLines, List<Ruling> verticalRulingLines) {
+    public List<Cell> findCells(List<Ruling> horizontalRulingLines, List<Ruling> verticalRulingLines, LayoutParsingType layoutParsingType) {
 
-        // Fix for 211.pdf
-        for (Ruling r : horizontalRulingLines) {
-            if (r.getX2() < r.getX1()) {
-                double a = r.getX2();
-                r.x2 = (float) r.getX1();
-                r.x1 = (float) a;
+        if (layoutParsingType.equals(LayoutParsingType.TAAS)) {
+            // TODO: breaks some tables, for example "1 Abamectin Prr.pdf" try to fix this upstream in RulingCleaningService
+            for (Ruling r : horizontalRulingLines) {
+                if (r.getX2() < r.getX1()) {
+                    double a = r.getX2();
+                    r.x2 = (float) r.getX1();
+                    r.x1 = (float) a;
+                }
             }
         }
 

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/blockification/TaasBlockificationService.java

@@ -1,8 +1,18 @@
 package com.knecon.fforesight.service.layoutparser.processor.services.blockification;
 
+import static com.knecon.fforesight.service.layoutparser.processor.services.factory.SearchTextWithTextPositionFactory.HEIGHT_PADDING;
 
-// TODO: figure out, why this fails the build
-// import static com.knecon.fforesight.service.layoutparser.processor.services.factory.SearchTextWithTextPositionFactory.HEIGHT_PADDING;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Set;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import java.util.stream.Stream;
+
+import org.springframework.stereotype.Service;
 
 import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBlock;
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationPage;
@@ -11,12 +21,6 @@ import com.knecon.fforesight.service.layoutparser.processor.model.table.Ruling;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
 import com.knecon.fforesight.service.layoutparser.processor.utils.RulingTextDirAdjustUtil;
-import org.springframework.stereotype.Service;
-
-import java.util.*;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-import java.util.stream.Stream;
 
 @Service
 @SuppressWarnings("all")
@@ -24,13 +28,12 @@ public class TaasBlockificationService {
 
     private static final float THRESHOLD = 1f;
     private static final float Y_GAP_SPLIT_HEIGHT_MODIFIER = 1.25f; // multiplied with text height
-    private static final float INTERSECTS_Y_THRESHOLD = 4;// 2 * HEIGHT_PADDING // This is exactly 2 times our position height padding. This is required to find boxes that are visually intersecting.
+    private static final float INTERSECTS_Y_THRESHOLD =  2 * HEIGHT_PADDING; // This is exactly 2 times our position height padding. This is required to find boxes that are visually intersecting.
     private static final int X_GAP_SPLIT_CONSTANT = 50;
     public static final int X_ALIGNMENT_THRESHOLD = 1;
+    public static final int SMALL_Y_GAP_THRESHOLD = 5;
     public static final int NEGATIVE_X_GAP_THRESHOLD = -5;
 
-    private Pattern listIdentifier = Pattern.compile("^(?:(?:[1-9]|1\\d|20|[ivxlc]|[a-z])\\s*(?:[.)]))|\\uF0B7", Pattern.CASE_INSENSITIVE);
-
 
     /**
      * This method is building blocks by expanding the minX/maxX and minY/maxY value on each word that is not split by the conditions.
@@ -75,29 +78,16 @@ public class TaasBlockificationService {
         List<TextPageBlock> currentTextBlocksToMerge = new LinkedList<>();
         textBlocksToMerge.add(currentTextBlocksToMerge);
         TextPageBlock previousTextBlock = null;
-        Float lastLineGap = null;
         for (TextPageBlock currentTextBlock : classificationTextBlocks) {
             if (previousTextBlock == null) {
                 currentTextBlocksToMerge.add(currentTextBlock);
                 previousTextBlock = currentTextBlock;
                 continue;
             }
-
-
-            Matcher listIdentifierPattern = listIdentifier.matcher(currentTextBlock.getText());
-            boolean isListIdentifier = listIdentifierPattern.find();
-
-            boolean yGap = Math.abs(currentTextBlock.getPdfMaxY() - previousTextBlock.getPdfMinY()) < previousTextBlock.getMostPopularWordHeight() * Y_GAP_SPLIT_HEIGHT_MODIFIER;
-
-            boolean sameFont = previousTextBlock.getMostPopularWordFont().equals(currentTextBlock.getMostPopularWordFont()) && previousTextBlock.getMostPopularWordFontSize() == currentTextBlock.getMostPopularWordFontSize();
-//            boolean yGap = previousTextBlock != null && currentTextBlock.getMinYDirAdj() - maxY > Math.min(word.getHeight(), prev.getHeight()) * Y_GAP_SPLIT_HEIGHT_MODIFIER;
-
             boolean alignsXRight = Math.abs(currentTextBlock.getPdfMaxX() - previousTextBlock.getPdfMaxX()) < X_ALIGNMENT_THRESHOLD;
-            boolean alignsXLeft = Math.abs(currentTextBlock.getPdfMinX() - previousTextBlock.getPdfMinX()) < X_ALIGNMENT_THRESHOLD;
-//            boolean smallYGap = Math.abs(currentTextBlock.getPdfMaxY() - previousTextBlock.getPdfMinY()) < yGap;
-            if (yGap && sameFont && !isListIdentifier) {
+            boolean smallYGap = Math.abs(currentTextBlock.getPdfMaxY() - previousTextBlock.getPdfMinY()) < SMALL_Y_GAP_THRESHOLD;
+            if (alignsXRight && smallYGap) {
                 currentTextBlocksToMerge.add(currentTextBlock);
-
             } else {
                 currentTextBlocksToMerge = new LinkedList<>();
                 currentTextBlocksToMerge.add(currentTextBlock);
@@ -178,8 +168,8 @@ public class TaasBlockificationService {
 
 
     private List<TextPageBlock> constructFineGranularTextPageBlocks(List<TextPositionSequence> textPositions,
-                                                                    List<Ruling> horizontalRulingLines,
-                                                                    List<Ruling> verticalRulingLines) {
+                                                                                        List<Ruling> horizontalRulingLines,
+                                                                                        List<Ruling> verticalRulingLines) {
 
         int indexOnPage = 0;
         List<TextPositionSequence> wordClusterToCombine = new ArrayList<>();
@@ -188,13 +178,13 @@ public class TaasBlockificationService {
         float minX = 1000, maxX = 0, minY = 1000, maxY = 0;
         TextPositionSequence prev = null;
         // TODO: make static final constant
-
+        var listIdentitifier = Pattern.compile("\\b(?:[1-9]|1\\d|20|[ivxlc]|[a-z])\\s*(?:[.)])", Pattern.CASE_INSENSITIVE);
 
         boolean wasSplitted = false;
         Float splitX1 = null;
         for (TextPositionSequence word : textPositions) {
 
-            Matcher listIdentifierPattern = listIdentifier.matcher(word.toString());
+            Matcher listIdentifierPattern = listIdentitifier.matcher(word.toString());
 
             boolean yGap = prev != null && word.getMinYDirAdj() - maxY > Math.min(word.getHeight(), prev.getHeight()) * Y_GAP_SPLIT_HEIGHT_MODIFIER;
             boolean sameLine = prev != null && equalsWithThreshold(prev.getMinYDirAdj(), word.getMinYDirAdj());

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/factory/SearchTextWithTextPositionFactory.java

@@ -18,7 +18,7 @@ import lombok.experimental.UtilityClass;
 @UtilityClass
 public class SearchTextWithTextPositionFactory {
 
-    public final int HEIGHT_PADDING = 2;
+    public static final int HEIGHT_PADDING = 2;
     // when checking for a hyphen linebreak, we need to check after a linebreak if the last hyphen was less than three symbols away.
     // We detect a linebreak as either a "\n" character or if two adjacent symbol's position differ in y-coordinates by at least one character height.
     // If there is a hyphen linebreak, the hyphen will be 1 position in front of a "\n" or 2 positions in front of the character which has a lower y-coordinate
@@ -133,7 +133,7 @@ public class SearchTextWithTextPositionFactory {
 
     private static void addTextPositionWithFontType(RedTextPosition currentTextPosition, String fontType, List<Integer> fontTypePositions, int stringIdx) {
 
-        if (currentTextPosition.getFontName() != null && currentTextPosition.getFontName().toLowerCase(Locale.ROOT).contains(fontType)) {
+        if (currentTextPosition.getFontName().toLowerCase(Locale.ROOT).contains(fontType)) {
             fontTypePositions.add(stringIdx);
         }
     }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/mapper/PropertiesMapper.java

@@ -1,7 +1,6 @@
 package com.knecon.fforesight.service.layoutparser.processor.services.mapper;
 
 import java.awt.geom.Rectangle2D;
-import java.util.Collections;
 import java.util.HashMap;
 import java.util.Locale;
 import java.util.Map;
@@ -9,7 +8,6 @@ import java.util.Map;
 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentStructure;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Image;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.ImageType;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Page;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Table;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.TableCell;
 

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/parsing/PDFLinesTextStripper.java

@@ -1,34 +1,18 @@
 package com.knecon.fforesight.service.layoutparser.processor.services.parsing;
 
-import java.awt.color.CMMException;
-import java.awt.geom.Point2D;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-
+import com.knecon.fforesight.service.layoutparser.processor.model.table.Ruling;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.RedTextPosition;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
+import lombok.Getter;
+import lombok.Setter;
+import lombok.SneakyThrows;
+import lombok.extern.slf4j.Slf4j;
 import org.apache.pdfbox.contentstream.operator.Operator;
 import org.apache.pdfbox.contentstream.operator.OperatorName;
-import org.apache.pdfbox.contentstream.operator.color.SetNonStrokingColor;
-import org.apache.pdfbox.contentstream.operator.color.SetNonStrokingColorN;
-import org.apache.pdfbox.contentstream.operator.color.SetNonStrokingColorSpace;
-import org.apache.pdfbox.contentstream.operator.color.SetNonStrokingDeviceCMYKColor;
-import org.apache.pdfbox.contentstream.operator.color.SetNonStrokingDeviceGrayColor;
-import org.apache.pdfbox.contentstream.operator.color.SetNonStrokingDeviceRGBColor;
-import org.apache.pdfbox.contentstream.operator.color.SetStrokingColor;
-import org.apache.pdfbox.contentstream.operator.color.SetStrokingColorN;
-import org.apache.pdfbox.contentstream.operator.color.SetStrokingColorSpace;
-import org.apache.pdfbox.contentstream.operator.color.SetStrokingDeviceCMYKColor;
-import org.apache.pdfbox.contentstream.operator.color.SetStrokingDeviceGrayColor;
-import org.apache.pdfbox.contentstream.operator.color.SetStrokingDeviceRGBColor;
+import org.apache.pdfbox.contentstream.operator.color.*;
 import org.apache.pdfbox.contentstream.operator.markedcontent.BeginMarkedContentSequenceWithProperties;
 import org.apache.pdfbox.contentstream.operator.markedcontent.EndMarkedContentSequence;
-import org.apache.pdfbox.contentstream.operator.state.SetFlatness;
-import org.apache.pdfbox.contentstream.operator.state.SetLineCapStyle;
-import org.apache.pdfbox.contentstream.operator.state.SetLineDashPattern;
-import org.apache.pdfbox.contentstream.operator.state.SetLineJoinStyle;
-import org.apache.pdfbox.contentstream.operator.state.SetLineMiterLimit;
-import org.apache.pdfbox.contentstream.operator.state.SetLineWidth;
-import org.apache.pdfbox.contentstream.operator.state.SetRenderingIntent;
+import org.apache.pdfbox.contentstream.operator.state.*;
 import org.apache.pdfbox.contentstream.operator.text.SetFontAndSize;
 import org.apache.pdfbox.cos.COSBase;
 import org.apache.pdfbox.cos.COSNumber;
@@ -37,14 +21,11 @@ import org.apache.pdfbox.pdmodel.PDPage;
 import org.apache.pdfbox.pdmodel.graphics.color.PDColor;
 import org.apache.pdfbox.text.TextPosition;
 
-import com.knecon.fforesight.service.layoutparser.processor.model.table.Ruling;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.RedTextPosition;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
-
-import lombok.Getter;
-import lombok.Setter;
-import lombok.SneakyThrows;
-import lombok.extern.slf4j.Slf4j;
+import java.awt.color.CMMException;
+import java.awt.geom.Point2D;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
 
 @Getter
 @Slf4j
@@ -55,6 +36,11 @@ public class PDFLinesTextStripper extends PDFTextStripper {
     private final List<Ruling> graphicsPath = new ArrayList<>();
     @Setter
     protected PDPage pdpage;
+    private int minCharWidth;
+    private int maxCharWidth;
+    private int minCharHeight;
+    private int maxCharHeight;
+
 
     private float path_x;
     private float path_y;
@@ -87,6 +73,7 @@ public class PDFLinesTextStripper extends PDFTextStripper {
         this.addOperator(new SetFontAndSize(this));
         this.addOperator(new SetLineWidth(this));
 
+
         addOperator(new BeginMarkedContentSequenceWithProperties(this));
 //        addOperator(new BeginMarkedContentSequence(this));
         addOperator(new EndMarkedContentSequence(this));
@@ -245,15 +232,33 @@ public class PDFLinesTextStripper extends PDFTextStripper {
                         .get(textPositionSequences.get(textPositionSequences.size() - 1).getTextPositions().size() - 1);
             }
 
+            int charWidth = (int) textPositions.get(i).getWidthDirAdj();
+            if (charWidth < minCharWidth) {
+                minCharWidth = charWidth;
+            }
+            if (charWidth > maxCharWidth) {
+                maxCharWidth = charWidth;
+            }
+
+            int charHeight = (int) textPositions.get(i).getHeightDir();
+            if (charHeight < minCharHeight) {
+                minCharHeight = charHeight;
+            }
+            if (charWidth > maxCharHeight) {
+                maxCharHeight = charHeight;
+            }
+
             if (i == 0 && (textPositions.get(i).getUnicode().equals(" ") || textPositions.get(i).getUnicode().equals("\u00A0") || textPositions.get(i).getUnicode().equals("\t"))) {
                 startIndex++;
                 continue;
             }
 
             // Strange but sometimes this is happening, for example: Metolachlor2.pdf
-            if (checkIfCurrentPositionIsToTheRightOfPreviousPosition(i, textPositions)) {
+            if (i > 0 && textPositions.get(i).getXDirAdj() < textPositions.get(i - 1).getXDirAdj()) {
                 List<TextPosition> sublist = textPositions.subList(startIndex, i);
-                if (checkIfSequenceContainsOnlyWhitespaces(sublist)) {
+                if (!(sublist.isEmpty() || sublist.size() == 1 && (sublist.get(0).getUnicode().equals(" ") || sublist.get(0).getUnicode().equals("\u00A0") || sublist.get(0)
+                        .getUnicode()
+                        .equals("\t")))) {
                     textPositionSequences.add(new TextPositionSequence(sublist, pageNumber, i == textPositions.size() - 1 && isParagraphStart));
                 }
                 startIndex = i;
@@ -261,7 +266,9 @@ public class PDFLinesTextStripper extends PDFTextStripper {
 
             if (textPositions.get(i).getRotation() == 0 && i > 0 && textPositions.get(i).getX() > textPositions.get(i - 1).getEndX() + 1) {
                 List<TextPosition> sublist = textPositions.subList(startIndex, i);
-                if (checkIfSequenceContainsOnlyWhitespaces(sublist)) {
+                if (!(sublist.isEmpty() || sublist.size() == 1 && (sublist.get(0).getUnicode().equals(" ") || sublist.get(0).getUnicode().equals("\u00A0") || sublist.get(0)
+                        .getUnicode()
+                        .equals("\t")))) {
                     textPositionSequences.add(new TextPositionSequence(sublist, pageNumber, i == textPositions.size() - 1 && isParagraphStart));
                 }
                 startIndex = i;
@@ -271,10 +278,13 @@ public class PDFLinesTextStripper extends PDFTextStripper {
                     .getUnicode()
                     .equals("\t")) && i <= textPositions.size() - 2) {
                 List<TextPosition> sublist = textPositions.subList(startIndex, i);
-                if (checkIfSequenceContainsOnlyWhitespaces(sublist)) {
+                if (!(sublist.isEmpty() || sublist.size() == 1 && (sublist.get(0).getUnicode().equals(" ") || sublist.get(0).getUnicode().equals("\u00A0") || sublist.get(0)
+                        .getUnicode()
+                        .equals("\t")))) {
 
                     // Remove false sequence ends (whitespaces)
-                    if (checkIfGapSizeBetweenCharactersSmallerThanMaximum(previous, sublist, 0.01f)) {
+                    if (previous != null && sublist.get(0).getYDirAdj() == previous.getYDirAdj() && sublist.get(0)
+                            .getXDirAdj() - (previous.getXDirAdj() + previous.getWidthDirAdj()) < 0.01) {
                         for (TextPosition t : sublist) {
                             textPositionSequences.get(textPositionSequences.size() - 1).add(t);
                         }
@@ -309,29 +319,13 @@ public class PDFLinesTextStripper extends PDFTextStripper {
     }
 
 
-    public boolean checkIfCurrentPositionIsToTheRightOfPreviousPosition(int i, List<TextPosition> textPositions) {
-
-        return i > 0 && textPositions.get(i).getXDirAdj() < textPositions.get(i - 1).getXDirAdj();
-    }
-
-
-    public boolean checkIfSequenceContainsOnlyWhitespaces(List<TextPosition> sublist) {
-
-        return !(sublist.isEmpty() || sublist.size() == 1 && (sublist.get(0).getUnicode().equals(" ") || sublist.get(0).getUnicode().equals("\u00A0") || sublist.get(0)
-                .getUnicode()
-                .equals("\t")));
-    }
-
-
-    public boolean checkIfGapSizeBetweenCharactersSmallerThanMaximum(RedTextPosition previous, List<TextPosition> sublist, float maximumGapSize) {
-
-        return previous != null && sublist.get(0).getYDirAdj() == previous.getYDirAdj() && sublist.get(0)
-                .getXDirAdj() - (previous.getXDirAdj() + previous.getWidthDirAdj()) < maximumGapSize;
-    }
-
     @Override
     public String getText(PDDocument doc) throws IOException {
 
+        minCharWidth = Integer.MAX_VALUE;
+        maxCharWidth = 0;
+        minCharHeight = Integer.MAX_VALUE;
+        maxCharHeight = 0;
         textPositionSequences.clear();
         rulings.clear();
         graphicsPath.clear();

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/parsing/PDFTextStripper.java

@@ -1711,7 +1711,7 @@ public class PDFTextStripper extends LegacyPDFStreamEngine {
         int numberOfStrings = line.size();
         for (int i = 0; i < numberOfStrings; i++) {
             WordWithTextPositions word = line.get(i);
-            word.getTextPositions().sort(Comparator.comparing(TextPosition::getXDirAdj));
+            word.getTextPositions().sort(Comparator.comparing(TextPosition::getX));
             writeString(word.getText(), word.getTextPositions(), isParagraphEnd && i == numberOfStrings - 1);
             if (i < numberOfStrings - 1) {
                 writeWordSeparator();

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/utils/MarkedContentUtils.java

@@ -9,7 +9,6 @@ import org.apache.pdfbox.text.TextPosition;
 
 import java.awt.geom.Rectangle2D;
 import java.util.Collection;
-import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 import java.util.stream.Collectors;
@@ -23,7 +22,7 @@ public class MarkedContentUtils {
     public List<Rectangle2D> getMarkedContentBboxPerLine(List<PDMarkedContent> markedContents, String subtype) {
 
         if (markedContents == null) {
-            return Collections.emptyList();
+            return null;
         }
 
         var markedContentByYPosition = markedContents.stream()
@@ -38,7 +37,7 @@ public class MarkedContentUtils {
                 .collect(Collectors.groupingBy(TextPosition::getY));
 
         if (markedContentByYPosition.isEmpty()) {
-            return Collections.emptyList();
+            return null;
         }
 
         return markedContentByYPosition.values().stream()

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/utils/PositionUtils.java

@@ -19,10 +19,9 @@ public final class PositionUtils {
 
         double threshold = textBlock.getMostPopularWordHeight() * 3;
 
-        if (textBlock.getPdfMinX() + threshold > btf.getTopLeft().getX()
-                && textBlock.getPdfMaxX() - threshold < btf.getTopLeft().getX() + btf.getWidth()
-                && textBlock.getPdfMinY() + threshold > btf.getTopLeft().getY()
-                && textBlock.getPdfMaxY() - threshold < btf.getTopLeft().getY() + btf.getHeight()) {
+        if (textBlock.getPdfMinX() + threshold > btf.getTopLeft().getX() && textBlock.getPdfMaxX() - threshold < btf.getTopLeft()
+                .getX() + btf.getWidth() && textBlock.getPdfMinY() + threshold > btf.getTopLeft().getY() && textBlock.getPdfMaxY() - threshold < btf.getTopLeft()
+                .getY() + btf.getHeight()) {
             return true;
         } else {
             return false;

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/utils/TextPositionSequenceComparator.java

@@ -28,13 +28,15 @@ import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPosit
  *
  * @author Ben Litchfield
  */
-public class TextPositionSequenceComparator implements Comparator<TextPositionSequence> {
-
+public class TextPositionSequenceComparator implements Comparator<TextPositionSequence>
+{
     @Override
-    public int compare(TextPositionSequence pos1, TextPositionSequence pos2) {
+    public int compare(TextPositionSequence pos1, TextPositionSequence pos2)
+    {
         // only compare text that is in the same direction
         int cmp1 = Float.compare(pos1.getDir().getDegrees(), pos2.getDir().getDegrees());
-        if (cmp1 != 0) {
+        if (cmp1 != 0)
+        {
             return cmp1;
         }
 
@@ -46,19 +48,25 @@ public class TextPositionSequenceComparator implements Comparator<TextPositionSe
         float pos2YBottom = pos2.getMaxYDirAdj();
 
         // note that the coordinates have been adjusted so 0,0 is in upper left
-        float pos1YTop = pos1YBottom - pos1.getTextHeightNoPadding();
-        float pos2YTop = pos2YBottom - pos2.getTextHeightNoPadding();
+        float pos1YTop = pos1YBottom - pos1.getTextHeight();
+        float pos2YTop = pos2YBottom - pos2.getTextHeight();
 
         float yDifference = Math.abs(pos1YBottom - pos2YBottom);
 
         // we will do a simple tolerance comparison
-        if (yDifference < .1 || pos2YBottom >= pos1YTop && pos2YBottom <= pos1YBottom || pos1YBottom >= pos2YTop && pos1YBottom <= pos2YBottom) {
+        if (yDifference < .1 ||
+                pos2YBottom >= pos1YTop && pos2YBottom <= pos1YBottom ||
+                pos1YBottom >= pos2YTop && pos1YBottom <= pos2YBottom)
+        {
             return Float.compare(x1, x2);
-        } else if (pos1YBottom < pos2YBottom) {
+        }
+        else if (pos1YBottom < pos2YBottom)
+        {
             return -1;
-        } else {
+        }
+        else
+        {
             return 1;
         }
     }
-
 }

layoutparser-service/layoutparser-service-server/build.gradle.kts

@@ -19,8 +19,8 @@ dependencies {
     implementation(project(":layoutparser-service-processor"))
     implementation(project(":layoutparser-service-internal-api"))
 
-    implementation("com.iqser.red.commons:storage-commons:2.45.0")
-    implementation("com.knecon.fforesight:tenant-commons:0.14.0")
+    implementation("com.iqser.red.commons:storage-commons:2.40.0")
+    implementation("com.knecon.fforesight:tenant-commons:0.10.0")
     implementation("org.springframework.boot:spring-boot-starter-actuator:3.1.3")
     implementation("com.amazonaws:aws-java-sdk-s3:1.12.536")
     implementation("org.springframework.boot:spring-boot-starter-amqp:3.1.3")

layoutparser-service/layoutparser-service-server/src/main/java/com/knecon/fforesight/service/layoutparser/server/queue/MessagingConfiguration.java

@@ -24,7 +24,8 @@ public class MessagingConfiguration {
     @Bean
     public Queue layoutparsingResponseQueue() {
 
-        return QueueBuilder.durable(LayoutParsingQueueNames.LAYOUT_PARSING_FINISHED_EVENT_QUEUE).build();
+        return QueueBuilder.durable(LayoutParsingQueueNames.LAYOUT_PARSING_FINISHED_EVENT_QUEUE)//
+                .withArgument("x-dead-letter-exchange", "").withArgument("x-dead-letter-routing-key", LayoutParsingQueueNames.LAYOUT_PARSING_DLQ).build();
     }
 
 

layoutparser-service/layoutparser-service-server/src/test/java/com/knecon/fforesight/service/layoutparser/server/graph/DocumentGraphVisualizationTest.java

@@ -47,7 +47,7 @@ public class DocumentGraphVisualizationTest extends BuildDocumentTest {
     @Disabled
     public void visualizeCraftedDocument() {
 
-        String filename = "files/1 Abamectin_prr.pdf";
+        String filename = "files/crafted document.pdf";
         visualizePdf(filename);
     }
 

layoutparser-service/layoutparser-service-server/src/test/java/com/knecon/fforesight/service/layoutparser/server/graph/ExtractMarkedContentTest.java

@@ -53,7 +53,7 @@ public class ExtractMarkedContentTest {
     @SneakyThrows
     public void testExtractTestWPhromma() throws IOException {
         System.out.printf("\n\n===\n%s\n===\n", "testWPhromma.pdf");
-        try (PDDocument document = Loader.loadPDF(new ClassPathResource("files/bdr/Drucksache_19_9865.pdf").getFile())) {
+            PDDocument document = Loader.loadPDF(new ClassPathResource("files/bdr/Drucksache_19_9865.pdf").getFile());
 
             Map<PDPage, Map<Integer, PDMarkedContent>> markedContents = new HashMap<>();
 
@@ -70,8 +70,6 @@ public class ExtractMarkedContentTest {
 
             PDStructureNode root = document.getDocumentCatalog().getStructureTreeRoot();
             showStructure(root, markedContents);
-            document.close();
-        }
     }
 
     /**
@@ -89,8 +87,7 @@ public class ExtractMarkedContentTest {
     @Test
     public void testExtractResMultipage() throws IOException {
         System.out.printf("\n\n===\n%s\n===\n", "res_multipage.pdf");
-
-        try(PDDocument document = Loader.loadPDF(new ClassPathResource("files/bdr/Drucksache_19_9865.pdf").getFile())) {
+            PDDocument document = Loader.loadPDF(new ClassPathResource("files/bdr/Drucksache_19_9865.pdf").getFile());
 
             Map<PDPage, Map<Integer, PDMarkedContent>> markedContents = new HashMap<>();
 
@@ -107,8 +104,6 @@ public class ExtractMarkedContentTest {
 
             PDStructureNode root = document.getDocumentCatalog().getStructureTreeRoot();
             showStructure(root, markedContents);
-            document.close();
-        }
     }
 
     /**
@@ -126,7 +121,7 @@ public class ExtractMarkedContentTest {
     @Test
     public void testExtractDailyReport() throws IOException {
         System.out.printf("\n\n===\n%s\n===\n", "Daily Report.pdf");
-        try (PDDocument document = Loader.loadPDF(new ClassPathResource("files/bdr/Drucksache_19_9865.pdf").getFile())) {
+            PDDocument document = Loader.loadPDF(new ClassPathResource("files/bdr/Drucksache_19_9865.pdf").getFile());
 
             Map<PDPage, Map<Integer, PDMarkedContent>> markedContents = new HashMap<>();
 
@@ -143,9 +138,7 @@ public class ExtractMarkedContentTest {
 
             PDStructureNode root = document.getDocumentCatalog().getStructureTreeRoot();
             showStructure(root, markedContents);
-            document.close();
         }
-    }
 
     /**
      * @see #testExtractTestWPhromma()

layoutparser-service/layoutparser-service-server/src/test/java/com/knecon/fforesight/service/layoutparser/server/graph/ViewerDocumentTest.java

@@ -2,31 +2,14 @@ package com.knecon.fforesight.service.layoutparser.server.graph;
 
 import java.io.FileOutputStream;
 import java.nio.file.Path;
-import java.util.List;
 
 import org.apache.pdfbox.Loader;
-import org.apache.pdfbox.pdmodel.PDDocument;
 import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.Test;
-import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.core.io.ClassPathResource;
 
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentData;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentStructure;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
-import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationDocument;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Table;
-import com.knecon.fforesight.service.layoutparser.processor.model.table.Cell;
-import com.knecon.fforesight.service.layoutparser.processor.model.table.TablePageBlock;
-import com.knecon.fforesight.service.layoutparser.processor.python_api.model.image.ImageServiceResponse;
-import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableServiceResponse;
-import com.knecon.fforesight.service.layoutparser.processor.services.SectionsBuilderService;
-import com.knecon.fforesight.service.layoutparser.processor.services.classification.RedactManagerClassificationService;
-import com.knecon.fforesight.service.layoutparser.processor.services.factory.DocumentGraphFactory;
-import com.knecon.fforesight.service.layoutparser.processor.services.mapper.DocumentDataMapper;
-import com.knecon.fforesight.service.layoutparser.processor.services.mapper.PropertiesMapper;
 import com.knecon.fforesight.service.layoutparser.processor.services.visualization.LayoutGridService;
 import com.knecon.fforesight.service.layoutparser.processor.services.visualization.ViewerDocumentService;
 import com.knecon.fforesight.service.layoutparser.server.utils.BuildDocumentTest;
@@ -35,38 +18,19 @@ import lombok.SneakyThrows;
 
 public class ViewerDocumentTest extends BuildDocumentTest {
 
-    @Autowired
-    private SectionsBuilderService sectionsBuilderService;
-
-    @Autowired
-    private RedactManagerClassificationService redactManagerClassificationService;
-
     @Test
+    @Disabled
     @SneakyThrows
     public void testViewerDocument() {
 
-        String fileName = "files/bdr/notMergedParagraphs.pdf";
-        String tmpFileName = "/tmp/" + Path.of(fileName).getFileName() + "_VIEWER.pdf";
         LayoutGridService layoutGridService = new LayoutGridService();
         ViewerDocumentService viewerDocumentService = new ViewerDocumentService(layoutGridService);
+        String fileName = "files/bdr/notMergedParagraphs.pdf";
         Document document = buildGraph(fileName, LayoutParsingType.TAAS);
+        String tmpFileName = "/tmp/" + Path.of(fileName).getFileName() + "_VIEWER.pdf";
         try (var pdDocument = Loader.loadPDF(new ClassPathResource(fileName).getFile()); var out = new FileOutputStream(tmpFileName)) {
             viewerDocumentService.createViewerDocument(pdDocument, document, out, true);
         }
     }
 
-    public ClassificationDocument buildClassificationDocument(PDDocument originDocument) {
-
-        ClassificationDocument classificationDocument = layoutParsingPipeline.parseLayout(LayoutParsingType.REDACT_MANAGER,
-                originDocument,
-                new ImageServiceResponse(),
-                new TableServiceResponse());
-
-        redactManagerClassificationService.classifyDocument(classificationDocument);
-
-        sectionsBuilderService.buildSections(classificationDocument);
-
-        return classificationDocument;
-    }
-
 }

layoutparser-service/layoutparser-service-server/src/test/java/com/knecon/fforesight/service/layoutparser/server/model/TextPositionSequenceTest.java

@@ -27,14 +27,14 @@ public class TextPositionSequenceTest {
             + "  \"pageWidth\": 600\n"  //
             + "}";
 
-    private final ObjectSerializer objectSerializer = new ObjectSerializer(new ObjectMapper());
+    private final ObjectSerializer objectSerializer = new ObjectSerializer(new ObjectMapper(), new StorageProperties());
 
 
     @Test
     @SneakyThrows
     public void testDeserializationWithJackson() {
 
-        TextPositionSequence textPositionSequence = objectSerializer.deserialize(new ByteArrayInputStream(TEXT_POSITION_SEQUENCE_AS_JSON.getBytes(StandardCharsets.UTF_8)),
+        TextPositionSequence textPositionSequence = objectSerializer.deserializeWithJackson(new ByteArrayInputStream(TEXT_POSITION_SEQUENCE_AS_JSON.getBytes(StandardCharsets.UTF_8)),
                 TextPositionSequence.class);
 
         assertPropertiesAfterJsonDeserialization(textPositionSequence);

layoutparser-service/layoutparser-service-server/src/test/java/com/knecon/fforesight/service/layoutparser/server/segmentation/PdfSegmentationServiceTest.java

@@ -1,27 +1,5 @@
 package com.knecon.fforesight.service.layoutparser.server.segmentation;
 
-import static org.assertj.core.api.Assertions.assertThat;
-
-import java.awt.geom.Rectangle2D;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.nio.file.Path;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Locale;
-import java.util.Map;
-import java.util.stream.Collectors;
-
-import org.apache.pdfbox.Loader;
-import org.apache.pdfbox.pdmodel.PDDocument;
-import org.junit.jupiter.api.Disabled;
-import org.junit.jupiter.api.Test;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.core.io.ClassPathResource;
-
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
 import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingPipeline;
@@ -37,8 +15,19 @@ import com.knecon.fforesight.service.layoutparser.processor.python_api.model.tab
 import com.knecon.fforesight.service.layoutparser.processor.services.SectionsBuilderService;
 import com.knecon.fforesight.service.layoutparser.processor.services.classification.RedactManagerClassificationService;
 import com.knecon.fforesight.service.layoutparser.server.utils.AbstractTest;
-
 import lombok.SneakyThrows;
+import org.apache.pdfbox.Loader;
+import org.apache.pdfbox.pdmodel.PDDocument;
+import org.junit.jupiter.api.Test;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.core.io.ClassPathResource;
+
+import java.awt.geom.Rectangle2D;
+import java.io.IOException;
+import java.util.*;
+import java.util.stream.Collectors;
+
+import static org.assertj.core.api.Assertions.assertThat;
 
 public class PdfSegmentationServiceTest extends AbstractTest {
 
@@ -61,13 +50,12 @@ public class PdfSegmentationServiceTest extends AbstractTest {
     private SectionsBuilderService sectionsBuilderService;
 
 
-    @SneakyThrows
-    public ClassificationDocument buildClassificationDocument(PDDocument originDocument, TableServiceResponse tableServiceResponse) {
+    public ClassificationDocument buildClassificationDocument(PDDocument originDocument) {
 
-        ClassificationDocument classificationDocument = layoutParsingPipeline.parseLayout(LayoutParsingType.REDACT_MANAGER,
+        ClassificationDocument classificationDocument = layoutParsingPipeline.parseLayout(LayoutParsingType.DOCUMINE,
                 originDocument,
                 new ImageServiceResponse(),
-                tableServiceResponse);
+                new TableServiceResponse());
 
         redactManagerClassificationService.classifyDocument(classificationDocument);
 
@@ -77,58 +65,6 @@ public class PdfSegmentationServiceTest extends AbstractTest {
     }
 
 
-    @SneakyThrows
-    public ClassificationDocument buildClassificationDocument(PDDocument originDocument) {
-
-        return buildClassificationDocument(originDocument, new TableServiceResponse());
-    }
-
-
-    @Test
-    public void tablesToHtmlDebugger() throws IOException {
-
-        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/A20622A izRMS (CZ) fRR Part B9_Page185.pdf");
-
-        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getFile()));
-
-        toHtml(document, "/tmp/A20622A izRMS (CZ) fRR Part B9_Page185.html");
-
-    }
-
-
-    @Test
-    public void tablesToHtmlDebuggerWithCVResponse() throws IOException {
-
-        ClassPathResource pdfFileResource = new ClassPathResource("files/cv_tables/ScanRotationBorder.pdf");
-        ClassPathResource cvTablesResource = new ClassPathResource("files/cv_tables/ScanRotationBorder.TABLES.json");
-        var tableServiceResponse = objectMapper.readValue(cvTablesResource.getInputStream(), TableServiceResponse.class);
-
-        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getFile()), tableServiceResponse);
-
-        toHtml(document, "/tmp/ScanRotationBorder.html");
-    }
-
-
-    @Test
-    public void testScanRotationBorderIsIgnored() throws IOException {
-
-        ClassPathResource pdfFileResource = new ClassPathResource("files/cv_tables/ScanRotationBorder.pdf");
-        ClassPathResource cvTablesResource = new ClassPathResource("files/cv_tables/ScanRotationBorder.TABLES.json");
-        var tableServiceResponse = objectMapper.readValue(cvTablesResource.getInputStream(), TableServiceResponse.class);
-
-        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getFile()), tableServiceResponse);
-        assertThat(document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList())).isNotEmpty();
-        var tables = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).toList();
-
-        // Quality of the table parsing is not good, because the file is rotated at scanning.
-        // We only asset that the table border is not the page border.
-        tables.forEach(table -> {
-            assertThat(table.getMinX()).isGreaterThan(75);
-            assertThat(table.getMaxX()).isLessThan(512);
-        });
-    }
-
-
     @Test
     @SneakyThrows
     public void testMapping() {
@@ -219,7 +155,7 @@ public class PdfSegmentationServiceTest extends AbstractTest {
     }
 
 
-    @Test // Non-sense test
+    @Test
     public void testDoc56Page170() throws IOException {
 
         ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/56 Fludioxonil_RAR_12_Volume_3CA_B-7_2018-02-21_Page170.pdf");
@@ -230,25 +166,8 @@ public class PdfSegmentationServiceTest extends AbstractTest {
 
         validateTable(document, 0, 1, 1, 0, 0);
         validateTable(document, 1, 2, 2, 0, 0);
-        validateTable(document, 2, 6, 20, 0, 0);
-        validateTable(document, 3, 7, 31, 0, 0);
-
-    }
-
-
-    @Test
-    public void testDoc211() throws IOException {
-
-        ClassPathResource pdfFileResource = new ClassPathResource("files/211.pdf");
-
-        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getFile()));
-
-        validateTableSize(document, 4);
-
-        validateTable(document, 0, 5, 4, 0, 0);
-        validateTable(document, 1, 5, 15, 14, 0);
-        validateTable(document, 2, 5, 14, 11, 0);
-        validateTable(document, 3, 5, 3, 0, 0);
+        validateTable(document, 2, 7, 20, 0, 140);
+        validateTable(document, 3, 8, 31, 0, 170);
 
     }
 
@@ -262,7 +181,7 @@ public class PdfSegmentationServiceTest extends AbstractTest {
 
         validateTableSize(document, 1);
 
-        validateTable(document, 0, 8, 8, 0, 0);
+        validateTable(document, 0, 8, 8, 0, 2);
 
         List<List<String>> values = Arrays.asList(Arrays.asList("Annex point Reference within DAR/RAR",
                         "Author, date",
@@ -272,18 +191,18 @@ public class PdfSegmentationServiceTest extends AbstractTest {
                         "Method meets analytical validation criteria",
                         "Remarks (in case validation criteria are not met)",
                         "Acceptability of the method"),
-                Arrays.asList("Part (a) Methods in soil, water, sediment, air and any additional matrices used in support of environmental fate studies",
-                        "Part (a) Methods in soil, water, sediment, air and any additional matrices used in support of environmental fate studies",
+                Arrays.asList("",
                         "Part (a) Methods in soil, water, sediment, air and any additional matrices used in support of environmental fate studies",
                         "Part (a) Methods in soil, water, sediment, air and any additional matrices used in support of environmental fate studies",
                         "Part (a) Methods in soil, water, sediment, air and any additional matrices used in support of environmental fate studies",
                         "Part (a) Methods in soil, water, sediment, air and any additional matrices used in support of environmental fate studies",
+                        "",
                         "Part (a) Methods in soil, water, sediment, air and any additional matrices used in support of environmental fate studies",
                         "Part (a) Methods in soil, water, sediment, air and any additional matrices used in support of environmental fate studies"),
                 Arrays.asList("CA 7.1.2.1.1 DAR (2009)",
                         "Evans P.G. 2001 TMJ4569B, VV-323245",
                         "Azoxystrobin Laboratory Degradation Study in Three Soil Types, Sampled from Holland and the United Kingdom",
-                        "Method: RAM 269 Johnson R.I., Tummon O.J., Earl M. 1995 RJ1864B, VV-377731 Johnson R.I., Tummon O.J., Earl M. 1998 RAM 269/02, VV-124072 Johnson R.I., Tummon O.J., Earl M. 2000 RAM 269/03, VV-123986 Validation: Robinson N.J. 2001 TMJ4617B, VV-895845",
+                        "Method: RAM 269 Johnson R.I., Tummon O.J., Earl M. 1995 RJ1864B, VV-377731 Johnson R.I., Tummon O.J., Earl M. 1998 RAM 269/02, VV-124072 Johnson R.I., Tummon O.J., Earl M. 2000 RAM 269/03, VV-123986 Validation: Robinson N.J. 2001 TMJ4617B, VV-895845 in a Trial Carried",
                         "LC-MS/MS LOQ: 0.01 mg/kg (R401553 (SYN50165 7), R402173 (SYN501114 )) or 0.02 mg/kg (azoxystrobin, R230310, R234886) Working range: 0.02-1.0 or 0.01-0.5 mg/kg (depending on analyte) Other supporting quantificati on methods: HPLC-UV GC-MSD",
                         "Y",
                         "N/A",
@@ -301,8 +220,6 @@ public class PdfSegmentationServiceTest extends AbstractTest {
 
         ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getFile()));
 
-        toHtml(document, "/tmp/html.html");
-
         validateTableSize(document, 4);
 
         validateTable(document, 0, 3, 2, 0, 0);
@@ -314,29 +231,17 @@ public class PdfSegmentationServiceTest extends AbstractTest {
 
 
     @Test
-    @Disabled // FIXME Fake Redactions leads to more cells, no solution for this currently
     public void testDocA20622APartB9Page185() throws IOException {
 
         ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/A20622A izRMS (CZ) fRR Part B9_Page185.pdf");
 
         ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getFile()));
 
-        validateTableSize(document, 1);
+        validateTableSize(document, 2);
 
-        validateTable(document, 0, 7, 4, 0, 0);
-    }
+        validateTable(document, 0, 5, 5, 0, 23);
+        validateTable(document, 1, 11, 9, 0, 36);
 
-
-    @Test
-    public void testDocA20622APartB9Page185FixedDoc() throws IOException {
-
-        ClassPathResource pdfFileResource = new ClassPathResource("files/SinglePages/A20622A izRMS (CZ) fRR Part B9_Page185_fixed.pdf");
-
-        ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getFile()));
-
-        validateTableSize(document, 1);
-
-        validateTable(document, 0, 7, 4, 0, 0);
     }
 
 
@@ -423,7 +328,7 @@ public class PdfSegmentationServiceTest extends AbstractTest {
         ClassificationDocument document = buildClassificationDocument(Loader.loadPDF(pdfFileResource.getFile()));
 
         validateTableSize(document, 1);
-        validateTable(document, 0, 10, 6, 0, 0);
+        validateTable(document, 0, 10, 6, 0, 1);
 
     }
 
@@ -545,8 +450,8 @@ public class PdfSegmentationServiceTest extends AbstractTest {
 
         validateTableSize(document, 2);
 
-        validateTable(document, 0, 6, 8, 0, 0);
-        validateTable(document, 1, 6, 8, 0, 0);
+        validateTable(document, 0, 6, 8, 0, 2);
+        validateTable(document, 1, 6, 8, 0, 1);
 
     }
 
@@ -579,37 +484,12 @@ public class PdfSegmentationServiceTest extends AbstractTest {
     }
 
 
-    @SneakyThrows
-    private void toHtml(ClassificationDocument document, String filename) {
-
-        var tables = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).toList();
-        StringBuilder sb = new StringBuilder();
-
-        int currentPage = 1;
-        for (var table : tables) {
-            if (currentPage != table.getPage()) {
-                currentPage = table.getPage();
-                sb.append("---------------------- Page ").append(currentPage).append("--------------\n");
-            }
-            sb.append("\n\n");
-            sb.append(table.getTextAsHtml());
-        }
-
-        try (FileOutputStream fileOutputStream = new FileOutputStream(Path.of(filename).toFile())) {
-            fileOutputStream.write(sb.toString().getBytes());
-        }
-    }
-
-
     private void validateTable(ClassificationDocument document, int tableIndex, int colCount, int rowCount, int emptyCellsCountCorrect, int emptyCellsCountIncorrect) {
 
         TablePageBlock table = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).toList().get(tableIndex);
         List<List<Cell>> rows = table.getRows();
         int emptyCellsFoundFound = rows.stream().flatMap(List::stream).toList().stream().filter(f -> f.toString().equals("")).toList().size();
 
-        for (List<Cell> row : table.getRows()) {
-            row.forEach(r -> System.out.println(r.toString()));
-        }
         assertThat(emptyCellsFoundFound).isEqualTo(emptyCellsCountCorrect + emptyCellsCountIncorrect);
 
         assertThat(table.getColCount()).isEqualTo(colCount);

layoutparser-service/layoutparser-service-server/src/test/java/com/knecon/fforesight/service/layoutparser/server/services/RulingCleaningServiceTest.java

@@ -1,39 +1,21 @@
 package com.knecon.fforesight.service.layoutparser.server.services;
 
-import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.Collections;
 import java.util.LinkedList;
 import java.util.List;
 
-import org.apache.pdfbox.Loader;
-import org.apache.pdfbox.pdmodel.PDDocument;
 import org.junit.jupiter.api.Test;
-import org.springframework.core.io.ClassPathResource;
 
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentData;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentStructure;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
-import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
 import com.knecon.fforesight.service.layoutparser.processor.model.PageContents;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Table;
 import com.knecon.fforesight.service.layoutparser.processor.model.table.CleanRulings;
-import com.knecon.fforesight.service.layoutparser.processor.python_api.model.image.ImageServiceResponse;
-import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableServiceResponse;
 import com.knecon.fforesight.service.layoutparser.processor.services.PageContentExtractor;
 import com.knecon.fforesight.service.layoutparser.processor.services.RulingCleaningService;
-import com.knecon.fforesight.service.layoutparser.processor.services.factory.DocumentGraphFactory;
-import com.knecon.fforesight.service.layoutparser.processor.services.mapper.DocumentDataMapper;
-import com.knecon.fforesight.service.layoutparser.processor.services.mapper.PropertiesMapper;
-import com.knecon.fforesight.service.layoutparser.processor.services.visualization.LayoutGridService;
-import com.knecon.fforesight.service.layoutparser.processor.services.visualization.ViewerDocumentService;
-import com.knecon.fforesight.service.layoutparser.server.utils.BuildDocumentTest;
 import com.knecon.fforesight.service.layoutparser.server.utils.visualizations.PdfDraw;
 
 import lombok.SneakyThrows;
 
-public class RulingCleaningServiceTest extends BuildDocumentTest {
+public class RulingCleaningServiceTest {
 
     @Test
 //    @Disabled
@@ -43,96 +25,13 @@ public class RulingCleaningServiceTest extends BuildDocumentTest {
         String fileName = "files/211.pdf";
         String lineFileName = "/tmp/" + Path.of(fileName).getFileName().toString() + "_LINES.pdf";
         List<PageContents> pageContents = PageContentExtractor.getSortedPageContents(fileName);
-        RulingCleaningService rulingCleaningService = new RulingCleaningService();
         PdfDraw.drawLinesPerPage(fileName, pageContents.stream().map(PageContents::getRulings).toList(), lineFileName);
+
+        RulingCleaningService rulingCleaningService = new RulingCleaningService();
         List<CleanRulings> cleanRulingsPerPage = new LinkedList<>();
         for (PageContents pageContent : pageContents) {
-            cleanRulingsPerPage.add(rulingCleaningService.getCleanRulings(Collections.emptyList(), pageContent.getRulings()));
+            cleanRulingsPerPage.add(rulingCleaningService.getCleanRulings(Collections.emptyList(), pageContent.getRulings(), 8, 20));
         }
-
-    }
-
-
-    @Test
-    @SneakyThrows
-    public void testTableExtraction() {
-
-        LayoutGridService layoutGridService = new LayoutGridService();
-        ViewerDocumentService viewerDocumentService = new ViewerDocumentService(layoutGridService);
-
-        ClassPathResource resource = new ClassPathResource("files");
-        List<String> pdfFileNames = Files.walk(resource.getFile().toPath())
-                .filter(path -> path.getFileName().toString().endsWith(".pdf"))
-                .map(Path::toAbsolutePath)
-                .map(Path::toString)
-                .toList();
-
-        for (int i = 0; i < pdfFileNames.size(); i++) {
-            writeJsons(Path.of(pdfFileNames.get(i)));
-        }
-    }
-
-
-    @SneakyThrows
-    private void writeJsons(Path filename) {
-
-        Document documentGraphBefore = DocumentGraphFactory.buildDocumentGraph(layoutParsingPipeline.parseLayout(LayoutParsingType.REDACT_MANAGER,
-                Loader.loadPDF(filename.toFile()),
-                new ImageServiceResponse(),
-                new TableServiceResponse()));
-        Document documentGraphAfter = DocumentGraphFactory.buildDocumentGraph(layoutParsingPipeline.parseLayout(LayoutParsingType.REDACT_MANAGER,
-                Loader.loadPDF(filename.toFile()),
-                new ImageServiceResponse(),
-                new TableServiceResponse()));
-        DocumentData documentDataBefore = DocumentDataMapper.toDocumentData(documentGraphBefore);
-        DocumentData documentDataAfter = DocumentDataMapper.toDocumentData(documentGraphAfter);
-        if (!compareStructures(documentDataBefore.getDocumentStructure(), documentDataAfter.getDocumentStructure(), filename.getFileName().toString())) {
-            String tmpFileNameBefore = "C:/Users/YANNIK~1/AppData/Local/Temp/before." + filename.getFileName().toString();
-            try (PDDocument pdDocument = Loader.loadPDF(filename.toFile())) {
-                PdfDraw.drawDocumentGraph(pdDocument, documentGraphBefore);
-                pdDocument.save(tmpFileNameBefore);
-            }
-            String tmpFileNameAfter = "C:/Users/YANNIK~1/AppData/Local/Temp/after." + filename.getFileName().toString();
-            try (PDDocument pdDocument = Loader.loadPDF(filename.toFile())) {
-                PdfDraw.drawDocumentGraph(pdDocument, documentGraphAfter);
-                pdDocument.save(tmpFileNameAfter);
-
-            }
-        }
-    }
-
-
-    @SneakyThrows
-    private boolean compareStructures(DocumentStructure structure1, DocumentStructure structure2, String pdfName) {
-
-        List listStructure1 = structure1.streamAllEntries()
-                .filter(entryData -> entryData.getType().equals(NodeType.TABLE))
-                .map(DocumentStructure.EntryData::getProperties)
-                .map(properties -> {
-                    var builder = Table.builder();
-                    PropertiesMapper.parseTableProperties(properties, builder);
-                    return builder.build();
-                })
-                .toList();
-
-        List listStructure2 = structure2.streamAllEntries()
-                .filter(entryData -> entryData.getType().equals(NodeType.TABLE))
-                .map(DocumentStructure.EntryData::getProperties)
-                .map(properties -> {
-                    var builder = Table.builder();
-                    PropertiesMapper.parseTableProperties(properties, builder);
-                    return builder.build();
-                })
-                .toList();
-
-        for (int i = 0; i < listStructure1.size(); i++) {
-            Table tableNode1 = (Table) listStructure1.get(i);
-            Table tableNode2 = (Table) listStructure2.get(i);
-            if (tableNode1.getNumberOfRows() != tableNode2.getNumberOfRows() || tableNode1.getNumberOfCols() != tableNode2.getNumberOfCols()) {
-                return false;
-            }
-        }
-        return true;
     }
 
 }

layoutparser-service/layoutparser-service-server/src/test/java/com/knecon/fforesight/service/layoutparser/server/utils/AbstractTest.java

@@ -1,15 +1,8 @@
 package com.knecon.fforesight.service.layoutparser.server.utils;
 
-import com.iqser.red.commons.jackson.ObjectMapperFactory;
-import com.iqser.red.storage.commons.service.StorageService;
-import com.iqser.red.storage.commons.utils.FileSystemBackedStorageService;
-import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingRequest;
-import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
-import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingStorageService;
-import com.knecon.fforesight.service.layoutparser.server.Application;
-import com.knecon.fforesight.tenantcommons.TenantContext;
-import com.knecon.fforesight.tenantcommons.TenantsClient;
-import lombok.SneakyThrows;
+import java.io.InputStream;
+import java.util.Optional;
+
 import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.extension.ExtendWith;
@@ -19,12 +12,21 @@ import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
 import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration;
 import org.springframework.boot.test.context.SpringBootTest;
 import org.springframework.boot.test.mock.mockito.MockBean;
-import org.springframework.context.annotation.*;
+import org.springframework.context.annotation.ComponentScan;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.context.annotation.Import;
 import org.springframework.core.io.ClassPathResource;
 import org.springframework.test.context.junit.jupiter.SpringExtension;
 
-import java.io.InputStream;
-import java.util.Optional;
+import com.iqser.red.storage.commons.service.StorageService;
+import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingRequest;
+import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
+import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingStorageService;
+import com.knecon.fforesight.service.layoutparser.server.Application;
+import com.knecon.fforesight.tenantcommons.TenantContext;
+import com.knecon.fforesight.tenantcommons.TenantsClient;
+
+import lombok.SneakyThrows;
 
 @ExtendWith(SpringExtension.class)
 @SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
@@ -51,6 +53,7 @@ public abstract class AbstractTest {
     protected final static String POSITION_FILE_ID = "positions";
     protected final static String PAGES_FILE_ID = "pages";
     protected final static String TENANT_ID = "tenant";
+    protected final static String SECTION_GRID_ID = "section";
     protected final static String VIEWER_DOCUMENT_ID = "viewer";
     protected final static String SIMPLIFIED_ID = "simplified";
 
@@ -67,6 +70,7 @@ public abstract class AbstractTest {
                 .positionBlockFileStorageId(POSITION_FILE_ID)
                 .pageFileStorageId(PAGES_FILE_ID)
                 .simplifiedTextStorageId(SIMPLIFIED_ID)
+                .sectionGridStorageId(SECTION_GRID_ID)
                 .viewerDocumentStorageId(VIEWER_DOCUMENT_ID)
                 .build();
     }
@@ -112,6 +116,7 @@ public abstract class AbstractTest {
                 .positionBlockFileStorageId(POSITION_FILE_ID)
                 .pageFileStorageId(PAGES_FILE_ID)
                 .simplifiedTextStorageId(SIMPLIFIED_ID)
+                .sectionGridStorageId(SECTION_GRID_ID)
                 .viewerDocumentStorageId(VIEWER_DOCUMENT_ID)
                 .build();
     }
@@ -152,14 +157,6 @@ public abstract class AbstractTest {
     @ComponentScan("com.knecon.fforesight.service.layoutparser")
     public static class TestConfiguration {
 
-
-        @Bean
-        @Primary
-        public StorageService inmemoryStorage() {
-
-            return new FileSystemBackedStorageService(ObjectMapperFactory.create());
-        }
-
     }
 
 }

layoutparser-service/layoutparser-service-server/src/test/java/com/knecon/fforesight/service/layoutparser/server/utils/FileSystemBackedStorageService.java

@@ -0,0 +1,124 @@
+package com.knecon.fforesight.service.layoutparser.server.utils;
+
+import static java.io.File.createTempFile;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.InputStream;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+import org.apache.commons.io.IOUtils;
+import org.springframework.stereotype.Service;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.iqser.red.commons.jackson.ObjectMapperFactory;
+import com.iqser.red.storage.commons.exception.StorageObjectDoesNotExist;
+import com.iqser.red.storage.commons.service.StorageService;
+
+import lombok.SneakyThrows;
+
+@Service
+public class FileSystemBackedStorageService implements StorageService {
+
+    private final Map<String, File> dataMap = new HashMap<>();
+
+
+    public FileSystemBackedStorageService() {
+
+    }
+
+
+    @SneakyThrows
+    @Override
+    public void downloadTo(String tenantId, String objectId, File destinationFile) {
+
+        var res = dataMap.get(objectId);
+        if (res == null) {
+            throw new StorageObjectDoesNotExist(new RuntimeException());
+        }
+
+        IOUtils.copy(new FileInputStream(res), new FileOutputStream(destinationFile));
+    }
+
+
+    @Override
+    public void deleteObject(String tenantId, String objectId) {
+
+        dataMap.remove(objectId);
+    }
+
+
+    @Override
+    public boolean objectExists(String tenantId, String objectId) {
+
+        return dataMap.containsKey(objectId);
+    }
+
+
+    @Override
+    @SneakyThrows
+    public <T> void storeJSONObject(String tenantId, String objectId, T any) {
+
+        File tempFile = createTempFile("test", ".tmp");
+        getMapper().writeValue(new FileOutputStream(tempFile), any);
+        dataMap.put(objectId, tempFile);
+    }
+
+
+    private ObjectMapper getMapper() {
+
+        return ObjectMapperFactory.create();
+    }
+
+
+    @Override
+    @SneakyThrows
+    public <T> T readJSONObject(String tenantId, String objectId, Class<T> clazz) {
+
+        if (dataMap.get(objectId) == null || !dataMap.get(objectId).exists()) {
+            throw new StorageObjectDoesNotExist("Stored object not found");
+        }
+        return getMapper().readValue(new FileInputStream(dataMap.get(objectId)), clazz);
+    }
+
+
+    public List<String> listPaths() {
+
+        return new ArrayList<>(dataMap.keySet());
+    }
+
+
+    public List<String> listFilePaths() {
+
+        return dataMap.values().stream().map(File::getAbsolutePath).collect(Collectors.toList());
+    }
+
+
+    @Override
+    @SneakyThrows
+    public void storeObject(String tenantId, String objectId, InputStream stream) {
+
+        File tempFile = createTempFile("test", ".tmp");
+
+        try (var fileOutputStream = new FileOutputStream(tempFile)) {
+            IOUtils.copy(stream, fileOutputStream);
+        }
+
+        dataMap.put(objectId, tempFile);
+    }
+
+
+    public void clearStorage() {
+
+        this.dataMap.forEach((k, v) -> {
+            v.delete();
+        });
+        this.dataMap.clear();
+    }
+
+}