Merge branch 'RED-9374' into 'main'

hotfix for table/paragraph section creation on document start before first headline

See merge request fforesight/layout-parser!170

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/data/redaction/LayoutEngine.java

@@ -2,5 +2,6 @@ package com.knecon.fforesight.service.layoutparser.internal.api.data.redaction;
 
 public enum LayoutEngine {
     ALGORITHM,
-    AI
+    AI,
+    OUTLINE
 }

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/data/redaction/SimplifiedSectionText.java

@@ -14,7 +14,7 @@ import lombok.NoArgsConstructor;
 public class SimplifiedSectionText {
 
     @Schema(description = "The number of this Section. This is used to map the simplified section text back to the original Section.")
-    private int sectionNumber;
+    private String sectionNumber;
     @Schema(description = "The text in this Section.")
     private String text;
 

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/queue/LayoutParsingType.java

@@ -4,6 +4,7 @@ public enum LayoutParsingType {
     REDACT_MANAGER,
     REDACT_MANAGER_OLD,
     REDACT_MANAGER_PARAGRAPH_DEBUG,
+    REDACT_MANAGER_WITHOUT_DUPLICATE_PARAGRAPH,
     DOCUMINE,
     DOCUMINE_OLD,
     CLARIFYND,

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/LayoutParsingPipeline.java

@@ -51,8 +51,8 @@ import com.knecon.fforesight.service.layoutparser.processor.services.RulingClean
 import com.knecon.fforesight.service.layoutparser.processor.services.SectionsBuilderService;
 import com.knecon.fforesight.service.layoutparser.processor.services.SimplifiedSectionTextService;
 import com.knecon.fforesight.service.layoutparser.processor.services.TableExtractionService;
-import com.knecon.fforesight.service.layoutparser.processor.services.blockification.BlockificationPostprocessingService;
 import com.knecon.fforesight.service.layoutparser.processor.services.TextRulingsClassifier;
+import com.knecon.fforesight.service.layoutparser.processor.services.blockification.BlockificationPostprocessingService;
 import com.knecon.fforesight.service.layoutparser.processor.services.blockification.DocstrumBlockificationService;
 import com.knecon.fforesight.service.layoutparser.processor.services.blockification.DocuMineBlockificationService;
 import com.knecon.fforesight.service.layoutparser.processor.services.blockification.RedactManagerBlockificationService;
@@ -319,10 +319,16 @@ public class LayoutParsingPipeline {
                 case REDACT_MANAGER_OLD ->
                         redactManagerBlockificationService.blockify(stripper.getTextPositionSequences(), cleanRulings, classificationDocument.getVisualizations());
                 case DOCUMINE_OLD -> docuMineBlockificationService.blockify(words, cleanRulings);
-                case DOCUMINE, REDACT_MANAGER, REDACT_MANAGER_PARAGRAPH_DEBUG ->
-                        docstrumBlockificationService.blockify(words, cleanRulings, true, classificationDocument.getVisualizations(), layoutParsingType);
-                case CLARIFYND, CLARIFYND_PARAGRAPH_DEBUG ->
-                        docstrumBlockificationService.blockify(words, cleanRulings, false, classificationDocument.getVisualizations(), layoutParsingType);
+                case DOCUMINE, REDACT_MANAGER, REDACT_MANAGER_PARAGRAPH_DEBUG, REDACT_MANAGER_WITHOUT_DUPLICATE_PARAGRAPH -> docstrumBlockificationService.blockify(words,
+                                                                                                                                                                    cleanRulings,
+                                                                                                                                                                    true,
+                                                                                                                                                                    classificationDocument.getVisualizations(),
+                                                                                                                                                                    layoutParsingType);
+                case CLARIFYND, CLARIFYND_PARAGRAPH_DEBUG -> docstrumBlockificationService.blockify(words,
+                                                                                                    cleanRulings,
+                                                                                                    false,
+                                                                                                    classificationDocument.getVisualizations(),
+                                                                                                    layoutParsingType);
             };
 
             classificationPage.setCleanRulings(cleanRulings);
@@ -381,8 +387,8 @@ public class LayoutParsingPipeline {
         }
         log.info("Classify TextBlocks for {}", identifier);
         switch (layoutParsingType) {
-            case REDACT_MANAGER, REDACT_MANAGER_PARAGRAPH_DEBUG, REDACT_MANAGER_OLD, CLARIFYND_PARAGRAPH_DEBUG ->
-                    redactManagerClassificationService.classifyDocument(classificationDocument);
+            case REDACT_MANAGER, REDACT_MANAGER_PARAGRAPH_DEBUG, REDACT_MANAGER_OLD, CLARIFYND_PARAGRAPH_DEBUG, REDACT_MANAGER_WITHOUT_DUPLICATE_PARAGRAPH -> redactManagerClassificationService.classifyDocument(
+                    classificationDocument);
             case DOCUMINE_OLD, DOCUMINE -> docuMineClassificationService.classifyDocument(classificationDocument);
             case CLARIFYND -> clarifyndClassificationService.classifyDocument(classificationDocument);
         }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/AbstractPageBlock.java

@@ -1,9 +1,10 @@
 package com.knecon.fforesight.service.layoutparser.processor.model;
 
-import java.awt.geom.Rectangle2D;
+import java.util.HashSet;
+import java.util.Set;
 
 import com.fasterxml.jackson.annotation.JsonIgnore;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
 import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.BoundingBox;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
 
@@ -20,6 +21,9 @@ public abstract class AbstractPageBlock extends BoundingBox {
 
     @JsonIgnore
     protected PageBlockType classification;
+
+    Set<LayoutEngine> engines = new HashSet<>();
+
     @JsonIgnore
     protected int page;
 

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/Document.java

@@ -43,6 +43,11 @@ public class Document extends AbstractSemanticNode {
     }
 
 
+    /**
+     * Gets the sections of the document as a list.
+     *
+     * @return A list of all sections within the document.
+     */
     public List<Section> getAllSections() {
 
         return streamAllSubNodesOfType(NodeType.SECTION).map(node -> (Section) node)
@@ -50,6 +55,34 @@ public class Document extends AbstractSemanticNode {
     }
 
 
+    /**
+     * Gets the main sections of the document as a list.
+     *
+     * @return A list of main sections within the document
+     * @deprecated This method is marked for removal.
+     *  Use {@link #streamChildrenOfType(NodeType)} instead,
+     *  or {@link #getChildrenOfTypeSectionOrSuperSection()} which returns children of type SECTION as well as SUPER_SECTION.
+     */
+    @Deprecated(forRemoval = true)
+    public List<Section> getMainSections() {
+
+        return streamChildrenOfType(NodeType.SECTION).map(node -> (Section) node)
+                .collect(Collectors.toList());
+    }
+
+
+    /**
+     * Gets the direct children of type SECTION or SUPER_SECTION of the document as a list of SemanticNode objects.
+     *
+     * @return A list of all children of type SECTION or SUPER_SECTION.
+     */
+    public List<SemanticNode> getChildrenOfTypeSectionOrSuperSection() {
+
+        return streamChildren().filter(semanticNode -> semanticNode.getType().equals(NodeType.SECTION) || semanticNode.getType().equals(NodeType.SUPER_SECTION))
+                .toList();
+    }
+
+
     public List<Header> getHeaders() {
 
         return streamChildrenOfType(NodeType.HEADER).map(node -> (Header) node)

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/outline/TOCEnrichmentService.java

@@ -29,7 +29,7 @@ public class TOCEnrichmentService {
         TableOfContents toc = document.getTableOfContents();
         Iterator<TableOfContentItem> iterator = toc.iterator();
         TableOfContentItem currentTOCItem = null;
-        if(iterator.hasNext()) {
+        if (iterator.hasNext()) {
             currentTOCItem = iterator.next();
         }
         List<AbstractPageBlock> startBlocks = new ArrayList<>();
@@ -79,7 +79,7 @@ public class TOCEnrichmentService {
                     currentTOCItem.getSectionBlocks().add(current);
                     currentPageTOCItems.add(currentTOCItem);
 
-                    if(iterator.hasNext()) {
+                    if (iterator.hasNext()) {
                         currentTOCItem = iterator.next();
                     }
                 } else if (!foundFirstHeadline) {
@@ -149,9 +149,9 @@ public class TOCEnrichmentService {
                     }
                 }
                 if (!image.isAppendedToSection()) {
-                    log.debug("Image uses first paragraph");
+                    log.debug("Image uses last found section");
                     if (!lastFoundTOCItems.isEmpty()) {
-                        lastFoundTOCItems.get(0).getImages().add(image);
+                        lastFoundTOCItems.get(lastFoundTOCItems.size() - 1).getImages().add(image);
                     } else {
                         startImages.add(image);
                     }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/SimplifiedSectionTextService.java

@@ -42,7 +42,9 @@ public class SimplifiedSectionTextService {
 
         return SimplifiedSectionText.builder()
                 .sectionNumber(section.getTreeId()
-                                       .get(0))
+                                       .stream()
+                                       .map(String::valueOf)
+                                       .collect(Collectors.joining(".")))
                 .text(section.getTextBlock().getSearchText())
                 .build();
     }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/blockification/BlockificationPostprocessingService.java

@@ -12,6 +12,7 @@ import java.util.function.Function;
 import org.apache.commons.lang3.StringUtils;
 import org.springframework.stereotype.Service;
 
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationPage;
 import com.knecon.fforesight.service.layoutparser.processor.model.PageBlockType;
 import com.knecon.fforesight.service.layoutparser.processor.model.SectionIdentifier;
@@ -204,21 +205,28 @@ public class BlockificationPostprocessingService {
             return false;
         }
         if (minDistance == distanceToDirectMatch) {
-            directMatch.setClassification(headlineType);
+            setClassificationAndAddOutlineEngine(directMatch, headlineType);
         } else if (minDistance == distanceToSplitCandidate) {
             SplitBlockResult splitBlockResult = splitBlock(classificationPage, splitCandidate, context.sectionIdentifier, outlineObject.getTitle());
             if (splitBlockResult.modifiedBlockToSplit) {
-                splitCandidate.setClassification(headlineType);
+                setClassificationAndAddOutlineEngine(splitCandidate, headlineType);
             }
             splitBlockResult.otherBlocks.forEach(other -> other.setClassification(null));
         } else {
             var merged = mergeBlocks(classificationPage, bestMergeCandidateCombination);
-            merged.setClassification(headlineType);
+            setClassificationAndAddOutlineEngine(merged, headlineType);
         }
         return true;
     }
 
 
+    private static void setClassificationAndAddOutlineEngine(TextPageBlock block, PageBlockType headlineType) {
+
+        block.setClassification(headlineType);
+        block.getEngines().add(LayoutEngine.OUTLINE);
+    }
+
+
     private SplitBlockResult splitBlock(ClassificationPage classificationPage, TextPageBlock blockToSplit, SectionIdentifier sectionIdentifier, String title) {
 
         List<TextPageBlock> otherBlocks = new ArrayList<>();

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/blockification/DocstrumBlockificationService.java

@@ -26,6 +26,7 @@ import lombok.RequiredArgsConstructor;
 @RequiredArgsConstructor
 public class DocstrumBlockificationService {
 
+    public static final float Y_THRESHOLD = 5f;
     private final DocstrumSegmentationService docstrumSegmentationService;
 
     static final float THRESHOLD = 1f;
@@ -58,8 +59,10 @@ public class DocstrumBlockificationService {
 
         mergeIntersectingBlocks(classificationPage, usedRulings, 0, 0);
 
-        if (layoutParsingType == LayoutParsingType.DOCUMINE || layoutParsingType == LayoutParsingType.REDACT_MANAGER) {
-            combineBlocks(classificationPage);
+        if (layoutParsingType == LayoutParsingType.DOCUMINE
+            || layoutParsingType == LayoutParsingType.REDACT_MANAGER
+            || layoutParsingType == LayoutParsingType.REDACT_MANAGER_WITHOUT_DUPLICATE_PARAGRAPH) {
+            combineBlocks(classificationPage, layoutParsingType);
         }
 
         if (layoutParsingType == LayoutParsingType.CLARIFYND) {
@@ -105,7 +108,7 @@ public class DocstrumBlockificationService {
     }
 
 
-    public void combineBlocks(ClassificationPage page) {
+    public void combineBlocks(ClassificationPage page, LayoutParsingType layoutParsingType) {
 
         TextPageBlock previous = new TextPageBlock();
         ListIterator<AbstractPageBlock> itty = page.getTextBlocks().listIterator();
@@ -137,7 +140,8 @@ public class DocstrumBlockificationService {
                 }
 
                 if (areTheOnlyTwoBlocksOnHeightsWithBothMoreThanTwoLines(previous, current, page)) {
-                    previous = combineBlocksAndResetIterator(previous, current, itty, true);
+//                    previous = combineBlocksAndResetIterator(previous, current, itty, true);
+                    previous = combineBlocksAndResetIterator(previous, current, itty, layoutParsingType != LayoutParsingType.REDACT_MANAGER_WITHOUT_DUPLICATE_PARAGRAPH);
                     continue;
                 }
 
@@ -160,7 +164,7 @@ public class DocstrumBlockificationService {
             previous = current;
         }
 
-        mergeIntersectingBlocks(page, usedRulings, 0, 6.5f);
+        mergeIntersectingBlocks(page, usedRulings, 0, Y_THRESHOLD);
     }
 
 

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/classification/ClarifyndClassificationService.java

@@ -21,33 +21,31 @@ import lombok.extern.slf4j.Slf4j;
 @RequiredArgsConstructor
 public class ClarifyndClassificationService {
 
-    private final HeadlineClassificationService headlineClassificationService;
-
     public void classifyDocument(ClassificationDocument document) {
 
         List<Float> headlineFontSizes = document.getFontSizeCounter().getHighterThanMostPopular();
 
         log.debug("Document FontSize counters are: {}", document.getFontSizeCounter().getCountPerValue());
 
-        headlineClassificationService.resetContext();
+        HeadlineClassificationService headlineClassificationService = new HeadlineClassificationService();
 
         for (ClassificationPage page : document.getPages()) {
-            classifyPage(page, document, headlineFontSizes);
+            classifyPage(headlineClassificationService, page, document, headlineFontSizes);
         }
     }
 
 
-    private void classifyPage(ClassificationPage page, ClassificationDocument document, List<Float> headlineFontSizes) {
+    private void classifyPage(HeadlineClassificationService headlineClassificationService, ClassificationPage page, ClassificationDocument document, List<Float> headlineFontSizes) {
 
         for (AbstractPageBlock textBlock : page.getTextBlocks()) {
             if (textBlock instanceof TextPageBlock) {
-                classifyBlock((TextPageBlock) textBlock, page, document, headlineFontSizes);
+                classifyBlock(headlineClassificationService, (TextPageBlock) textBlock, page, document, headlineFontSizes);
             }
         }
     }
 
 
-    private void classifyBlock(TextPageBlock textBlock, ClassificationPage page, ClassificationDocument document, List<Float> headlineFontSizes) {
+    private void classifyBlock(HeadlineClassificationService headlineClassificationService, TextPageBlock textBlock, ClassificationPage page, ClassificationDocument document, List<Float> headlineFontSizes) {
 
         var bodyTextFrame = page.getBodyTextFrame();
 

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/classification/DocuMineClassificationService.java

@@ -24,10 +24,9 @@ import lombok.extern.slf4j.Slf4j;
 @RequiredArgsConstructor
 public class DocuMineClassificationService {
 
-    private final HeadlineClassificationService headlineClassificationService;
-    private static final Pattern pattern = Pattern.compile("^([1-9]\\d?\\.){1,3}\\d{1,2}\\.?\\s[0-9A-Za-z \\[\\]]{2,50}", Pattern.CASE_INSENSITIVE);
-    private static final Pattern pattern2 = Pattern.compile("\\p{L}{3,}", Pattern.CASE_INSENSITIVE);
-    private static final Pattern pattern3 = Pattern.compile("^(\\d{1,1}\\.){1,3}\\d{1,2}\\.?\\s[a-z]{1,2}\\/[a-z]{1,2}.*");
+    private static final Pattern HEADLINE_WITH_IDENTIFER_PATTERN = Pattern.compile("^([1-9]\\d?\\.){0,3}\\d{1,2}\\.?\\s[0-9A-Za-z \\[\\]]{2,50}", Pattern.CASE_INSENSITIVE);
+    private static final Pattern AT_LEAST_3_PATTERN = Pattern.compile("\\p{L}{3,}", Pattern.CASE_INSENSITIVE);
+    private static final Pattern HEADLINE_PATTTERN_WITH_SLASHES = Pattern.compile("^(\\d{1,1}\\.){0,3}\\d{1,2}\\.?\\s[a-z]{1,2}\\/[a-z]{1,2}.*");
 
 
     public void classifyDocument(ClassificationDocument document) {
@@ -36,32 +35,32 @@ public class DocuMineClassificationService {
 
         log.debug("Document FontSize counters are: {}", document.getFontSizeCounter().getCountPerValue());
 
-        headlineClassificationService.resetContext();
+        HeadlineClassificationService headlineClassificationService = new HeadlineClassificationService();
 
         for (ClassificationPage page : document.getPages()) {
-            classifyPage(page, document, headlineFontSizes);
+            classifyPage(headlineClassificationService, page, document, headlineFontSizes);
         }
     }
 
 
-    private void classifyPage(ClassificationPage page, ClassificationDocument document, List<Float> headlineFontSizes) {
+    private void classifyPage(HeadlineClassificationService headlineClassificationService, ClassificationPage page, ClassificationDocument document, List<Float> headlineFontSizes) {
 
         for (AbstractPageBlock textBlock : page.getTextBlocks()) {
             if (textBlock instanceof TextPageBlock) {
-                classifyBlock((TextPageBlock) textBlock, page, document, headlineFontSizes);
+                classifyBlock(headlineClassificationService, (TextPageBlock) textBlock, page, document, headlineFontSizes);
             }
         }
     }
 
 
-    private void classifyBlock(TextPageBlock textBlock, ClassificationPage page, ClassificationDocument document, List<Float> headlineFontSizes) {
+    private void classifyBlock(HeadlineClassificationService headlineClassificationService, TextPageBlock textBlock, ClassificationPage page, ClassificationDocument document, List<Float> headlineFontSizes) {
 
         log.debug("headlineFontSizes: {}", headlineFontSizes);
         var bodyTextFrame = page.getBodyTextFrame();
 
-        Matcher matcher = pattern.matcher(textBlock.toString());
-        Matcher matcher2 = pattern2.matcher(textBlock.toString());
-        Matcher matcher3 = pattern3.matcher(textBlock.toString());
+        Matcher headlineWithIdentifierMatcher = HEADLINE_WITH_IDENTIFER_PATTERN.matcher(textBlock.toString());
+        Matcher atLeast3Matcher = AT_LEAST_3_PATTERN.matcher(textBlock.toString());
+        Matcher headlineWithSlashesMatcher = HEADLINE_PATTTERN_WITH_SLASHES.matcher(textBlock.toString());
 
         if (textBlock.getClassification() != null && textBlock.getClassification().isHeadline()) {
             headlineClassificationService.setLastHeadlineFromOutline(textBlock);
@@ -102,19 +101,20 @@ public class DocuMineClassificationService {
                        || textBlock.getMostPopularWordFontSize() > document.getFontSizeCounter().getMostPopular())
                    && PositionUtils.getApproxLineCount(textBlock) < 5.9
 
-                   && (textBlock.getMostPopularWordStyle().contains("bold") && Character.isDigit(textBlock.toString().charAt(0)) && matcher2.reset().find() && !textBlock.toString()
+                   && (textBlock.getMostPopularWordStyle().contains("bold") && Character.isDigit(textBlock.toString().charAt(0)) && atLeast3Matcher.reset().find() && !textBlock.toString()
                 .contains(":")
-                       || textBlock.toString().equals(textBlock.toString().toUpperCase(Locale.ROOT)) && matcher2.reset().find() && !textBlock.toString().contains(":")
+                       || textBlock.toString().equals(textBlock.toString().toUpperCase(Locale.ROOT)) && atLeast3Matcher.reset().find() && !textBlock.toString().contains(":")
                        || textBlock.toString().startsWith("APPENDIX")
                        || textBlock.toString().startsWith("FIGURE")
+                       || textBlock.toString().startsWith("Continued TABLE")
                        || textBlock.toString().startsWith("TABLE"))
                    && !textBlock.toString().endsWith(":")
-                   && matcher2.reset().find()) {
+                   && atLeast3Matcher.reset().find()) {
             PageBlockType headlineType = PageBlockType.getHeadlineType(1);
             headlineClassificationService.classifyHeadline(textBlock, headlineType);
             document.setHeadlines(true);
 
-        } else if (matcher.reset().find() && PositionUtils.getApproxLineCount(textBlock) < 2.9 && matcher2.reset().find() && !matcher3.reset().matches()) {
+        } else if (headlineWithIdentifierMatcher.reset().find() && PositionUtils.getApproxLineCount(textBlock) < 2.9 && atLeast3Matcher.reset().find() && !headlineWithSlashesMatcher.reset().matches()) {
             PageBlockType headlineType = PageBlockType.getHeadlineType(2);
             headlineClassificationService.classifyHeadline(textBlock, headlineType);
             document.setHeadlines(true);

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/classification/HeadlineClassificationService.java

@@ -2,15 +2,12 @@ package com.knecon.fforesight.service.layoutparser.processor.services.classifica
 
 import static com.knecon.fforesight.service.layoutparser.processor.model.PageBlockType.getHeadlineNumber;
 
-import org.springframework.stereotype.Service;
-
 import com.knecon.fforesight.service.layoutparser.processor.model.PageBlockType;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
 
 import lombok.Getter;
 import lombok.Setter;
 
-@Service
 @Getter
 @Setter
 public class HeadlineClassificationService {
@@ -19,13 +16,6 @@ public class HeadlineClassificationService {
     PageBlockType originalClassifiedBlockType;
     TextPageBlock lastHeadlineFromOutline;
 
-    public void resetContext() {
-        setLastHeadline(null);
-        setOriginalClassifiedBlockType(null);
-        setLastHeadlineFromOutline(null);
-    }
-
-
     public void setLastHeadlineFromOutline(TextPageBlock lastHeadlineFromOutline) {
 
         this.lastHeadlineFromOutline = lastHeadlineFromOutline;

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/classification/RedactManagerClassificationService.java

@@ -22,8 +22,6 @@ import lombok.extern.slf4j.Slf4j;
 @RequiredArgsConstructor
 public class RedactManagerClassificationService {
 
-    private final HeadlineClassificationService headlineClassificationService;
-
 
     public void classifyDocument(ClassificationDocument document) {
 
@@ -31,25 +29,25 @@ public class RedactManagerClassificationService {
 
         log.debug("Document FontSize counters are: {}", document.getFontSizeCounter().getCountPerValue());
 
-        headlineClassificationService.resetContext();
+        HeadlineClassificationService headlineClassificationService = new HeadlineClassificationService();
 
         for (ClassificationPage page : document.getPages()) {
-            classifyPage(page, document, headlineFontSizes);
+            classifyPage(headlineClassificationService, page, document, headlineFontSizes);
         }
     }
 
 
-    private void classifyPage(ClassificationPage page, ClassificationDocument document, List<Float> headlineFontSizes) {
+    private void classifyPage(HeadlineClassificationService headlineClassificationService, ClassificationPage page, ClassificationDocument document, List<Float> headlineFontSizes) {
 
         for (AbstractPageBlock textBlock : page.getTextBlocks()) {
             if (textBlock instanceof TextPageBlock) {
-                classifyBlock((TextPageBlock) textBlock, page, document, headlineFontSizes);
+                classifyBlock(headlineClassificationService, (TextPageBlock) textBlock, page, document, headlineFontSizes);
             }
         }
     }
 
 
-    private void classifyBlock(TextPageBlock textBlock, ClassificationPage page, ClassificationDocument document, List<Float> headlineFontSizes) {
+    private void classifyBlock(HeadlineClassificationService headlineClassificationService, TextPageBlock textBlock, ClassificationPage page, ClassificationDocument document, List<Float> headlineFontSizes) {
 
         var bodyTextFrame = page.getBodyTextFrame();
 

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/factory/DocumentGraphFactory.java

@@ -122,6 +122,7 @@ public class DocumentGraphFactory {
         List<Integer> treeId = context.documentTree.createNewChildEntryAndReturnId(parentNode, node);
         node.setLeafTextBlock(textBlock);
         node.setTreeId(treeId);
+        node.getEngines().addAll(originalTextBlock.getEngines());
     }
 
 

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/factory/SectionNodeFactory.java

@@ -14,11 +14,11 @@ import java.util.Set;
 
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
 import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBlock;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.AbstractSemanticNode;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.GenericSemanticNode;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Page;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Section;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.AbstractSemanticNode;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.SuperSection;
 import com.knecon.fforesight.service.layoutparser.processor.model.image.ClassifiedImage;
 import com.knecon.fforesight.service.layoutparser.processor.model.table.TablePageBlock;
@@ -55,7 +55,8 @@ public class SectionNodeFactory {
                 .collect(groupingBy(AbstractPageBlock::getPage));
 
         AbstractSemanticNode section;
-        if (isLeaf) {
+        boolean containsTablesAndTextBlocks = containsTablesAndTextBlocks(pageBlocks);
+        if (isLeaf && !containsTablesAndTextBlocks) {
             section = Section.builder().documentTree(context.getDocumentTree()).build();
         } else {
             section = SuperSection.builder().documentTree(context.getDocumentTree()).build();
@@ -68,7 +69,7 @@ public class SectionNodeFactory {
         section.setTreeId(getTreeId(parentNode, context, section));
 
         addFirstHeadlineDirectlyToSection(layoutParsingType, pageBlocks, context, section, document);
-        if (containsTablesAndTextBlocks(pageBlocks)) {
+        if (containsTablesAndTextBlocks) {
             splitPageBlocksIntoSubSections(pageBlocks).forEach(subSectionPageBlocks -> addSection(layoutParsingType,
                                                                                                   section,
                                                                                                   true,
@@ -132,7 +133,7 @@ public class SectionNodeFactory {
             if (abstractPageBlock instanceof TextPageBlock) {
 
                 switch (layoutParsingType) {
-                    case REDACT_MANAGER, DOCUMINE, CLARIFYND, CLARIFYND_PARAGRAPH_DEBUG -> {
+                    case REDACT_MANAGER, DOCUMINE, CLARIFYND, CLARIFYND_PARAGRAPH_DEBUG, REDACT_MANAGER_WITHOUT_DUPLICATE_PARAGRAPH -> {
                         alreadyMerged.add(abstractPageBlock);
                         remainingBlocks.remove(abstractPageBlock);
                         DocumentGraphFactory.addParagraphOrHeadline(section, (TextPageBlock) abstractPageBlock, context, new ArrayList<>(), layoutParsingType);

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/visualization/LayoutGridService.java

@@ -9,6 +9,7 @@ import java.io.File;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.Comparator;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
@@ -23,6 +24,7 @@ import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Do
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.DuplicatedParagraph;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Page;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.SemanticNode;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.SuperSection;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Table;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.TableCell;
 import com.knecon.fforesight.service.layoutparser.processor.utils.RectangleTransformations;
@@ -62,6 +64,12 @@ public class LayoutGridService {
     static Color HEADER_COLOR = new Color(171, 131, 6);
     static Color IMAGE_COLOR = new Color(253, 63, 146);
 
+    private record RectangleIdentifier(List<Integer> treeId, Integer pageNumber) {
+
+    }
+
+    HashMap<RectangleIdentifier, Rectangle2D> rectangleMap = new HashMap<>();
+
 
     @SneakyThrows
     @Observed(name = "ViewerDocumentService", contextualName = "create-viewer-document")
@@ -165,8 +173,10 @@ public class LayoutGridService {
             List<Double> ys = yStream.collect(Collectors.toList());
             ys.remove(0);
 
-            Rectangle2D tableBBox = table.getBBox().get(page);
-            List<ColoredLine> coloredLines = layoutGrid.getVisualizationsPerPages().get(page.getNumber() - 1).getColoredLines();
+            Rectangle2D tableBBox = table.getBBox()
+                    .get(page);
+            List<ColoredLine> coloredLines = layoutGrid.getVisualizationsPerPages()
+                    .get(page.getNumber() - 1).getColoredLines();
             xs.forEach(x -> {
                 Line2D line = new Line2D.Double(new Point2D.Double(x, tableBBox.getMaxY()), new Point2D.Double(x, tableBBox.getMinY()));
                 coloredLines.add(new ColoredLine(line, INNER_LINES_COLOR, LINE_WIDTH));
@@ -190,25 +200,20 @@ public class LayoutGridService {
     private void addSection(SemanticNode semanticNode, LayoutGrid layoutGrid, Color color) {
 
         Map<Page, Rectangle2D> bBoxMap = semanticNode.getBBox();
+
         List<SemanticNode> subSections = semanticNode.streamAllSubNodesOfType(NodeType.SECTION)
                 .toList();
+        Integer maxChildDepth = subSections.stream()
+                .map(node -> node.getTreeId().size())
+                .max(Integer::compareTo)
+                .orElse(semanticNode.getTreeId().size());
+        int ownDepth = semanticNode.getTreeId().size();
+
         Page firstPage = semanticNode.getFirstPage();
         String treeIdString = buildTreeIdString(semanticNode);
-        if (!subSections.isEmpty()) {
-            addPlacedText(firstPage, bBoxMap.get(firstPage), treeIdString, layoutGrid);
-        } else {
-            bBoxMap.forEach(((page, textBBox) -> addPlacedText(page, textBBox, treeIdString, layoutGrid)));
-        }
+
         if (bBoxMap.values().size() == 1) {
-            Rectangle2D r = RectangleTransformations.pad(bBoxMap.get(firstPage), LINE_WIDTH, LINE_WIDTH);
-            List<ColoredLine> coloredLines = layoutGrid.getVisualizationsPerPages().get(firstPage.getNumber() - 1).getColoredLines();
-            List<Line2D> lines = createLinesFromRectangle(r, firstPage.getRotation());
-            // add string to top line
-            var firstLine = lines.remove(0);
-            coloredLines.add(new ColoredLine(firstLine, color, LINE_WIDTH));
-            for (Line2D line : lines) {
-                coloredLines.add(new ColoredLine(line, color, LINE_WIDTH));
-            }
+            handleSinglePage(semanticNode, layoutGrid, color, firstPage, bBoxMap.get(firstPage), treeIdString, maxChildDepth, ownDepth);
             return;
         }
         List<Page> pagesInOrder = bBoxMap.keySet()
@@ -216,99 +221,203 @@ public class LayoutGridService {
                 .sorted(Comparator.comparingInt(Page::getNumber))
                 .collect(Collectors.toList());
         pagesInOrder.remove(0);
-        addLinesForFirstPageOfSection(semanticNode, color, firstPage, layoutGrid);
-        var lastPage = pagesInOrder.remove(pagesInOrder.size() - 1);
-        addLinesForLastPageOfSection(semanticNode, color, lastPage, layoutGrid);
-        for (Page middlePage : pagesInOrder) {
-            addLinesForMiddlePageOfSection(semanticNode, color, middlePage, layoutGrid);
+        handleFirstPageOfSection(semanticNode, color, firstPage, bBoxMap.get(firstPage), treeIdString, layoutGrid, maxChildDepth, ownDepth);
+        if (semanticNode instanceof SuperSection) {
+            return;
         }
+        for (Page middlePage : pagesInOrder.subList(0, pagesInOrder.size() - 1)) {
+            handleForMiddlePageOfSection(semanticNode, color, middlePage, bBoxMap.get(middlePage), treeIdString, layoutGrid, maxChildDepth, ownDepth);
+        }
+        var lastPage = pagesInOrder.remove(pagesInOrder.size() - 1);
+        handleLastPageOfSection(semanticNode, color, lastPage, bBoxMap.get(lastPage), treeIdString, layoutGrid, maxChildDepth, ownDepth);
     }
 
 
     @SneakyThrows
-    private void addPlacedText(Page page, Rectangle2D textBBox, String s, LayoutGrid layoutGrid) {
+    private void addPlacedText(Page page, Rectangle2D textBBox, Rectangle2D highestParentRect, String s, LayoutGrid layoutGrid, Integer maxChildDepth) {
 
         // translates text, such that its right edge is a bit to the left of the drawn box
-        float translationAmount = ((FONT.getStringWidth(s) / 1000) * FONT_SIZE + (2 * LINE_WIDTH) + 4);
+        float translationAmount = ((FONT.getStringWidth(s) / 1000) * FONT_SIZE + LINE_WIDTH + 2 * maxChildDepth);
 
         Point2D upperLeftCorner;
         Point2D translationVector;
         switch (page.getRotation()) {
             case 90 -> {
-                upperLeftCorner = new Point2D.Double(textBBox.getMinX(), textBBox.getMinY());
+                if (highestParentRect != null) {
+                    upperLeftCorner = new Point2D.Double(highestParentRect.getMinX(), textBBox.getMinY());
+                } else {
+                    upperLeftCorner = new Point2D.Double(textBBox.getMinX(), textBBox.getMinY());
+                }
                 translationVector = new Point2D.Double(FONT_SIZE, -translationAmount);
             }
             case 180 -> {
-                upperLeftCorner = new Point2D.Double(textBBox.getMaxX(), textBBox.getMinY());
+                if (highestParentRect != null) {
+                    upperLeftCorner = new Point2D.Double(highestParentRect.getMaxX(), textBBox.getMinY());
+                } else {
+                    upperLeftCorner = new Point2D.Double(textBBox.getMaxX(), textBBox.getMinY());
+                }
                 translationVector = new Point2D.Double(translationAmount, FONT_SIZE);
             }
             case 270 -> {
-                upperLeftCorner = new Point2D.Double(textBBox.getMaxX(), textBBox.getMaxY());
+
+                if (highestParentRect != null) {
+                    upperLeftCorner = new Point2D.Double(highestParentRect.getMaxX(), textBBox.getMaxY());
+                } else {
+                    upperLeftCorner = new Point2D.Double(textBBox.getMaxX(), textBBox.getMaxY());
+                }
                 translationVector = new Point2D.Double(-FONT_SIZE, translationAmount);
             }
             default -> {
-                upperLeftCorner = new Point2D.Double(textBBox.getMinX(), textBBox.getMaxY());
+
+                if (highestParentRect != null) {
+                    upperLeftCorner = new Point2D.Double(highestParentRect.getMinX(), textBBox.getMaxY());
+                } else {
+                    upperLeftCorner = new Point2D.Double(textBBox.getMinX(), textBBox.getMaxY());
+                }
                 translationVector = new Point2D.Double(-translationAmount, -FONT_SIZE);
             }
         }
 
         upperLeftCorner = add(upperLeftCorner, translationVector);
 
-        var placedTexts = layoutGrid.getVisualizationsPerPages().get(page.getNumber() - 1).getPlacedTexts();
-        placedTexts.add(PlacedText.textFacingUp(s, upperLeftCorner, FONT_SIZE, Color.BLACK, FONT));
-    }
+        List<PlacedText> placedTexts = layoutGrid.getVisualizationsPerPages()
+                .get(page.getNumber() - 1).getPlacedTexts();
 
+        PlacedText newText = PlacedText.textFacingUp(s, upperLeftCorner, FONT_SIZE, Color.BLACK, FONT);
 
-    private void addLinesForMiddlePageOfSection(SemanticNode semanticNode, Color color, Page middlePage, LayoutGrid layoutGrid) {
+        Optional<PlacedText> conflictingText = placedTexts.stream()
+                .filter(pt -> Math.abs(pt.lineStart().getY() - newText.lineStart().getY()) <= FONT_SIZE)
+                .findFirst();
 
-        List<ColoredLine> coloredLines = layoutGrid.getVisualizationsPerPages().get(middlePage.getNumber() - 1).getColoredLines();
-        Rectangle2D r = RectangleTransformations.pad(semanticNode.getBBox().get(middlePage), LINE_WIDTH, LINE_WIDTH);
-        var midPageLines = createLinesFromRectangle(r, middlePage.getRotation());
-        // remove top line
-        midPageLines.remove(0);
-        // remove top line
-        midPageLines.remove(1);
-        // add string to left line
-        var leftLine = midPageLines.remove(1);
-        coloredLines.add(new ColoredLine(leftLine, color, LINE_WIDTH));
-        for (Line2D line : midPageLines) {
-            coloredLines.add(new ColoredLine(line, color, LINE_WIDTH));
+        if (conflictingText.isPresent()) {
+            PlacedText existingText = conflictingText.get();
+            if (newText.text().length() > existingText.text().length()) {
+                placedTexts.remove(existingText);
+                placedTexts.add(newText);
+            }
+        } else {
+            placedTexts.add(newText);
         }
+
     }
 
 
-    private void addLinesForLastPageOfSection(SemanticNode semanticNode, Color color, Page lastPage, LayoutGrid layoutGrid) {
+    private void handleSinglePage(SemanticNode semanticNode,
+                                  LayoutGrid layoutGrid,
+                                  Color color,
+                                  Page page,
+                                  Rectangle2D rectangle2D,
+                                  String treeIdString,
+                                  Integer maxChildDepth,
+                                  Integer ownDepth) {
 
-        List<ColoredLine> coloredLines = layoutGrid.getVisualizationsPerPages().get(lastPage.getNumber() - 1).getColoredLines();
-        Rectangle2D r = RectangleTransformations.pad(semanticNode.getBBox().get(lastPage), LINE_WIDTH, LINE_WIDTH);
-        var lastPageLines = createLinesFromRectangle(r, lastPage.getRotation());
-        // remove top line
-        lastPageLines.remove(0);
-        // add string to left line
-        var leftLine = lastPageLines.remove(2);
-        coloredLines.add(new ColoredLine(leftLine, color, LINE_WIDTH));
-        for (Line2D line : lastPageLines) {
-            coloredLines.add(new ColoredLine(line, color, LINE_WIDTH));
-        }
-    }
-
-
-    private void addLinesForFirstPageOfSection(SemanticNode semanticNode, Color color, Page firstPage, LayoutGrid layoutGrid) {
-
-        List<ColoredLine> coloredLines = layoutGrid.getVisualizationsPerPages().get(firstPage.getNumber() - 1).getColoredLines();
-        Rectangle2D r = RectangleTransformations.pad(semanticNode.getBBox().get(firstPage), LINE_WIDTH, LINE_WIDTH);
-        var firstPageLines = createLinesFromRectangle(r, firstPage.getRotation());
-        // remove bottom line
-        firstPageLines.remove(2);
+        RectangleAndLinesResult result = createLinesAndPlaceText(semanticNode, page, rectangle2D, treeIdString, layoutGrid, maxChildDepth, ownDepth);
         // add string to top line
-        var firstLine = firstPageLines.remove(0);
-        coloredLines.add(new ColoredLine(firstLine, color, LINE_WIDTH));
-        for (Line2D line : firstPageLines) {
-            coloredLines.add(new ColoredLine(line, color, LINE_WIDTH));
+        var firstLine = result.pageLines().remove(0);
+        result.coloredLines().add(new ColoredLine(firstLine, color, LINE_WIDTH));
+        for (Line2D line : result.pageLines()) {
+            result.coloredLines().add(new ColoredLine(line, color, LINE_WIDTH));
         }
     }
 
 
+    private void handleFirstPageOfSection(SemanticNode semanticNode,
+                                          Color color,
+                                          Page firstPage,
+                                          Rectangle2D rectangle2D,
+                                          String treeIdString,
+                                          LayoutGrid layoutGrid,
+                                          Integer maxChildDepth,
+                                          Integer ownDepth) {
+
+        RectangleAndLinesResult result = createLinesAndPlaceText(semanticNode, firstPage, rectangle2D, treeIdString, layoutGrid, maxChildDepth, ownDepth);
+        // remove bottom line
+        result.pageLines().remove(2);
+        // add string to top line
+        var firstLine = result.pageLines().remove(0);
+        result.coloredLines().add(new ColoredLine(firstLine, color, LINE_WIDTH));
+        for (Line2D line : result.pageLines()) {
+            result.coloredLines().add(new ColoredLine(line, color, LINE_WIDTH));
+        }
+    }
+
+
+    private void handleForMiddlePageOfSection(SemanticNode semanticNode,
+                                              Color color,
+                                              Page middlePage,
+                                              Rectangle2D rectangle2D,
+                                              String treeIdString,
+                                              LayoutGrid layoutGrid,
+                                              Integer maxChildDepth,
+                                              Integer ownDepth) {
+
+        RectangleAndLinesResult result = createLinesAndPlaceText(semanticNode, middlePage, rectangle2D, treeIdString, layoutGrid, maxChildDepth, ownDepth);
+        // remove top line
+        result.pageLines().remove(0);
+        // remove bottom line
+        result.pageLines().remove(1);
+        // add string to left line
+        var leftLine = result.pageLines().remove(1);
+        result.coloredLines().add(new ColoredLine(leftLine, color, LINE_WIDTH));
+        for (Line2D line : result.pageLines()) {
+            result.coloredLines().add(new ColoredLine(line, color, LINE_WIDTH));
+        }
+    }
+
+
+    private void handleLastPageOfSection(SemanticNode semanticNode,
+                                         Color color,
+                                         Page lastPage,
+                                         Rectangle2D rectangle2D,
+                                         String treeIdString,
+                                         LayoutGrid layoutGrid,
+                                         Integer maxChildDepth,
+                                         Integer ownDepth) {
+
+        RectangleAndLinesResult result = createLinesAndPlaceText(semanticNode, lastPage, rectangle2D, treeIdString, layoutGrid, maxChildDepth, ownDepth);
+        // remove top line
+        result.pageLines().remove(0);
+        // add string to left line
+        var leftLine = result.pageLines().remove(2);
+        result.coloredLines().add(new ColoredLine(leftLine, color, LINE_WIDTH));
+        for (Line2D line : result.pageLines()) {
+            result.coloredLines().add(new ColoredLine(line, color, LINE_WIDTH));
+        }
+    }
+
+
+    private RectangleAndLinesResult createLinesAndPlaceText(SemanticNode semanticNode,
+                                                            Page page,
+                                                            Rectangle2D rectangle2D,
+                                                            String treeIdString,
+                                                            LayoutGrid layoutGrid,
+                                                            Integer maxChildDepth,
+                                                            Integer ownDepth) {
+
+        List<ColoredLine> coloredLines = layoutGrid.getVisualizationsPerPages()
+                .get(page.getNumber() - 1).getColoredLines();
+        int lineWidthModifier = maxChildDepth - ownDepth;
+        Rectangle2D r = RectangleTransformations.pad(semanticNode.getBBox()
+                                                             .get(page), LINE_WIDTH * (1 + lineWidthModifier), LINE_WIDTH * (1 + lineWidthModifier));
+        var lastPageLines = createLinesFromRectangle(r, page.getRotation());
+
+        SemanticNode highestParent = semanticNode.getHighestParent();
+        Rectangle2D highestParentRect = rectangleMap.get(new RectangleIdentifier(highestParent.getTreeId(), page.getNumber()));
+        addPlacedText(page, rectangle2D, highestParentRect, treeIdString, layoutGrid, maxChildDepth);
+
+        if (semanticNode instanceof SuperSection) {
+            rectangleMap.put(new RectangleIdentifier(semanticNode.getTreeId(), page.getNumber()), r);
+        }
+
+        return new RectangleAndLinesResult(coloredLines, r, lastPageLines);
+    }
+
+
+    private record RectangleAndLinesResult(List<ColoredLine> coloredLines, Rectangle2D rectangle, List<Line2D> pageLines) {
+
+    }
+
+
     private String buildTreeIdString(SemanticNode semanticNode) {
 
         return semanticNode.getTreeId()
@@ -366,7 +475,10 @@ public class LayoutGridService {
 
     private static boolean isNotSectionOrTableCellOrDocument(SemanticNode semanticNode) {
 
-        return !(semanticNode.getType().equals(NodeType.DOCUMENT) || semanticNode.getType().equals(NodeType.SECTION) || semanticNode.getType().equals(NodeType.TABLE_CELL));
+        return !(semanticNode.getType().equals(NodeType.DOCUMENT)
+                 || semanticNode.getType().equals(NodeType.SECTION)
+                 || semanticNode.getType().equals(NodeType.SUPER_SECTION)
+                 || semanticNode.getType().equals(NodeType.TABLE_CELL));
     }
 
 
@@ -374,9 +486,7 @@ public class LayoutGridService {
 
         semanticNode.getBBox()
                 .forEach((page, textBBox) -> layoutGrid.getVisualizationsPerPages()
-                        .get(page.getNumber() - 1)
-                        .getColoredRectangles()
-                        .add(new ColoredRectangle(textBBox, color, LINE_WIDTH)));
+                        .get(page.getNumber() - 1).getColoredRectangles().add(new ColoredRectangle(textBBox, color, LINE_WIDTH)));
     }
 
 

layoutparser-service/layoutparser-service-server/src/test/java/com/knecon/fforesight/service/layoutparser/server/LayoutparserEnd2EndTest.java

@@ -70,7 +70,7 @@ public class LayoutparserEnd2EndTest extends AbstractTest {
             file = new File(filePath);
         }
 
-        LayoutParsingRequest layoutParsingRequest = buildDefaultLayoutParsingRequest(fileName, LayoutParsingType.REDACT_MANAGER, true);
+        LayoutParsingRequest layoutParsingRequest = buildDefaultLayoutParsingRequest(fileName, LayoutParsingType.REDACT_MANAGER_WITHOUT_DUPLICATE_PARAGRAPH, true);
         prepareStorage(layoutParsingRequest, file);
 
         LayoutParsingFinishedEvent finishedEvent = layoutParsingPipeline.parseLayoutAndSaveFilesToStorage(layoutParsingRequest);

layoutparser-service/layoutparser-service-server/src/test/java/com/knecon/fforesight/service/layoutparser/server/OutlineDetectionTest.java

@@ -0,0 +1,231 @@
+package com.knecon.fforesight.service.layoutparser.server;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.io.File;
+import java.nio.file.Path;
+import java.util.Collection;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.function.Predicate;
+
+import org.apache.commons.lang3.StringUtils;
+import org.junit.jupiter.api.Test;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.core.io.ClassPathResource;
+
+import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingRequest;
+import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
+import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingPipeline;
+import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationDocument;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Section;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.SemanticNode;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.SuperSection;
+import com.knecon.fforesight.service.layoutparser.processor.model.outline.OutlineObject;
+import com.knecon.fforesight.service.layoutparser.processor.model.outline.OutlineObjectTree;
+import com.knecon.fforesight.service.layoutparser.processor.model.outline.TableOfContents;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableServiceResponse;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.VisualLayoutParsingResponse;
+import com.knecon.fforesight.service.layoutparser.processor.services.factory.DocumentGraphFactory;
+import com.knecon.fforesight.service.layoutparser.processor.services.visualization.LayoutGridService;
+import com.knecon.fforesight.service.layoutparser.server.utils.AbstractTest;
+import com.knecon.fforesight.service.viewerdoc.service.ViewerDocumentService;
+
+import lombok.SneakyThrows;
+
+public class OutlineDetectionTest extends AbstractTest {
+
+    ViewerDocumentService viewerDocumentService = new ViewerDocumentService(null);
+    LayoutGridService layoutGridService = new LayoutGridService(viewerDocumentService);
+
+    @Autowired
+    protected LayoutParsingPipeline layoutParsingPipeline;
+
+
+    @Test
+    @SneakyThrows
+    public void testOutlinesToSections() {
+
+        String fileName = "files/new/crafted_outline_test_doc.pdf";
+
+        String tmpFileName = "/tmp/" + Path.of(fileName).getFileName() + "_VIEWER.pdf";
+
+        var documentFile = new ClassPathResource(fileName).getFile();
+
+        long start = System.currentTimeMillis();
+        ClassificationDocument classificationDocument = parseLayout(fileName, LayoutParsingType.REDACT_MANAGER_WITHOUT_DUPLICATE_PARAGRAPH);
+
+        OutlineObjectTree outlineObjectTree = classificationDocument.getOutlineObjectTree();
+        assertEquals(outlineObjectTree.getRootNodes().size(), 8);
+        assertEquals(outlineObjectTree.getOutlineObjectsPerPage()
+                             .get(1).size(), 1);
+        assertEquals(outlineObjectTree.getOutlineObjectsPerPage()
+                             .get(3).size(), 1);
+        assertEquals(outlineObjectTree.getOutlineObjectsPerPage()
+                             .get(4).size(), 1);
+        assertEquals(outlineObjectTree.getOutlineObjectsPerPage()
+                             .get(5).size(), 1);
+        assertEquals(outlineObjectTree.getOutlineObjectsPerPage()
+                             .get(6).size(), 2);
+        assertEquals(outlineObjectTree.getOutlineObjectsPerPage()
+                             .get(7).size(), 3);
+        assertEquals(outlineObjectTree.getOutlineObjectsPerPage()
+                             .get(8).size(), 2);
+        assertEquals(outlineObjectTree.getOutlineObjectsPerPage()
+                             .get(10).size(), 1);
+        assertEquals(outlineObjectTree.getOutlineObjectsPerPage()
+                             .get(11).size(), 4);
+        assertEquals(outlineObjectTree.getOutlineObjectsPerPage()
+                             .get(12).size(), 1);
+        assertEquals(outlineObjectTree.getOutlineObjectsPerPage()
+                             .get(13).size(), 2);
+        assertTrue(outlineObjectTree.getOutlineObjectsPerPage().values()
+                           .stream()
+                           .flatMap(Collection::stream)
+                           .allMatch(OutlineObject::isFound));
+
+        TableOfContents tableOfContents = classificationDocument.getTableOfContents();
+
+        assertEquals(tableOfContents.getMainSections().size(), 9);
+        assertEquals(tableOfContents.getMainSections().subList(1, 9)
+                             .stream()
+                             .map(tableOfContentItem -> sanitizeString(tableOfContentItem.getHeadline().toString()))
+                             .toList(),
+                     outlineObjectTree.getRootNodes()
+                             .stream()
+                             .map(outlineObjectTreeNode -> sanitizeString(outlineObjectTreeNode.getOutlineObject().getTitle()))
+                             .toList());
+        assertEquals(tableOfContents.getMainSections()
+                             .get(5).getChildren().size(), 6);
+        assertEquals(tableOfContents.getMainSections()
+                             .get(7).getChildren().size(), 3);
+        assertEquals(tableOfContents.getMainSections()
+                             .get(8).getChildren().size(), 3);
+        assertEquals(tableOfContents.getMainSections()
+                             .get(8).getChildren()
+                             .get(2).getChildren().size(), 1);
+        assertEquals(tableOfContents.getMainSections()
+                             .get(8).getChildren()
+                             .get(2).getChildren()
+                             .get(0).getChildren().size(), 3);
+
+        assertEquals(tableOfContents.getMainSections()
+                             .get(0).getImages().size(), 1);
+        assertEquals(tableOfContents.getMainSections()
+                             .get(6).getImages().size(), 1);
+        assertEquals(tableOfContents.getMainSections()
+                             .get(8).getChildren()
+                             .get(2).getChildren()
+                             .get(0).getChildren()
+                             .get(2).getImages().size(), 1);
+
+        Document document = buildGraph(fileName, classificationDocument);
+
+        assertTrue(tableOfContents.getAllTableOfContentItems()
+                           .stream()
+                           .allMatch(tableOfContentItem -> tableOfContentItem.getSection() != null));
+        assertTrue(tableOfContents.getAllTableOfContentItems()
+                           .stream()
+                           .filter(tableOfContentItem -> tableOfContentItem.getChildren().isEmpty())
+                           .allMatch(tableOfContentItem -> tableOfContentItem.getSection() instanceof Section));
+        assertTrue(tableOfContents.getAllTableOfContentItems()
+                           .stream()
+                           .filter(tableOfContentItem -> !tableOfContentItem.getChildren().isEmpty())
+                           .allMatch(tableOfContentItem -> tableOfContentItem.getSection() instanceof SuperSection));
+
+        List<SemanticNode> childrenOfTypeSectionOrSuperSection = document.getChildrenOfTypeSectionOrSuperSection();
+
+        assertEquals(childrenOfTypeSectionOrSuperSection.size(), 9);
+        assertEquals(childrenOfTypeSectionOrSuperSection.subList(1, 9)
+                             .stream()
+                             .map(section -> sanitizeString(section.getHeadline().getLeafTextBlock().toString()))
+                             .toList(),
+                     outlineObjectTree.getRootNodes()
+                             .stream()
+                             .map(outlineObjectTreeNode -> sanitizeString(outlineObjectTreeNode.getOutlineObject().getTitle()))
+                             .toList());
+        Predicate<SemanticNode> isSectionOrSuperSection = semanticNode -> semanticNode instanceof Section || semanticNode instanceof SuperSection;
+        assertEquals(childrenOfTypeSectionOrSuperSection.get(5).streamChildren()
+                             .filter(isSectionOrSuperSection)
+                             .count(), 6 + 1); // 1 additional for main text of parent section
+        assertEquals(childrenOfTypeSectionOrSuperSection.get(7).streamChildren()
+                             .filter(isSectionOrSuperSection)
+                             .count(), 3 + 1);
+        assertEquals(childrenOfTypeSectionOrSuperSection.get(8).streamChildren()
+                             .filter(isSectionOrSuperSection)
+                             .count(), 3 + 1);
+        assertEquals(childrenOfTypeSectionOrSuperSection.get(8).streamChildren()
+                             .filter(isSectionOrSuperSection)
+                             .toList()
+                             .get(3).streamChildren()
+                             .filter(isSectionOrSuperSection)
+                             .count(), 1 + 1);
+        assertEquals(childrenOfTypeSectionOrSuperSection.get(8).streamChildren()
+                             .filter(isSectionOrSuperSection)
+                             .toList()
+                             .get(3).streamChildren()
+                             .filter(isSectionOrSuperSection)
+                             .toList()
+                             .get(1).streamChildren()
+                             .filter(isSectionOrSuperSection)
+                             .count(), 3 + 1);
+
+        List<List<Integer>> imageTreeIdList = document.streamAllImages()
+                .map(image -> image.getParent().getTreeId())
+                .toList();
+
+        assertEquals(imageTreeIdList.get(0), List.of(0));
+        assertEquals(imageTreeIdList.get(1), List.of(6));
+        assertEquals(imageTreeIdList.get(2), List.of(8, 4, 2, 4));
+
+        layoutGridService.addLayoutGrid(documentFile, document, new File(tmpFileName), true, false);
+        System.out.printf("Total time: %.2fs%n", ((float) (System.currentTimeMillis() - start)) / 1000);
+    }
+
+
+    private static String sanitizeString(String text) {
+
+        return StringUtils.deleteWhitespace(text).toLowerCase(Locale.ROOT);
+    }
+
+
+    @SneakyThrows
+    protected ClassificationDocument parseLayout(String filename, LayoutParsingType layoutParsingType) {
+
+        File fileResource = new ClassPathResource(filename).getFile();
+        prepareStorage(filename, "cv_table_parsing_response/empty.json", "image_service_response/crafted_outline_test_doc.IMAGE_INFO.json");
+        return layoutParsingPipeline.parseLayout(layoutParsingType,
+                                                 fileResource,
+                                                 layoutParsingStorageService.getImagesFile(IMAGE_FILE_ID),
+                                                 new TableServiceResponse(),
+                                                 new VisualLayoutParsingResponse(),
+                                                 Map.of("file", filename, "debug", "true"));
+    }
+
+
+    @SneakyThrows
+    protected Document buildGraph(String filename, ClassificationDocument classificationDocument) {
+
+        if (!filename.startsWith("files") && filename.startsWith("/")) {
+            LayoutParsingRequest layoutParsingRequest = buildDefaultLayoutParsingRequest(Path.of(filename).getFileName().toString(), LayoutParsingType.REDACT_MANAGER, true);
+            prepareStorage(layoutParsingRequest, new File(filename));
+            return DocumentGraphFactory.buildDocumentGraph(LayoutParsingType.REDACT_MANAGER,
+                                                           layoutParsingPipeline.parseLayout(LayoutParsingType.REDACT_MANAGER,
+                                                                                             new File(filename),
+                                                                                             layoutParsingStorageService.getImagesFile(layoutParsingRequest.imagesFileStorageId()
+                                                                                                                                               .get()),
+                                                                                             new TableServiceResponse(),
+                                                                                             new VisualLayoutParsingResponse(),
+                                                                                             layoutParsingRequest.identifier()));
+        } else {
+            prepareStorage(filename);
+            return DocumentGraphFactory.buildDocumentGraph(LayoutParsingType.REDACT_MANAGER, classificationDocument);
+        }
+
+    }
+
+}
+

layoutparser-service/layoutparser-service-server/src/test/java/com/knecon/fforesight/service/layoutparser/server/SimplifiedTextServiceTest.java

@@ -64,8 +64,8 @@ public class SimplifiedTextServiceTest
     @SneakyThrows
     protected Document buildGraph(File file) {
 
-        return DocumentGraphFactory.buildDocumentGraph(LayoutParsingType.REDACT_MANAGER,
-                                                       layoutParsingPipeline.parseLayout(LayoutParsingType.REDACT_MANAGER,
+        return DocumentGraphFactory.buildDocumentGraph(LayoutParsingType.REDACT_MANAGER_WITHOUT_DUPLICATE_PARAGRAPH,
+                                                       layoutParsingPipeline.parseLayout(LayoutParsingType.REDACT_MANAGER_WITHOUT_DUPLICATE_PARAGRAPH,
                                                                                          file,
                                                                                          new ImageServiceResponse(),
                                                                                          new TableServiceResponse(),

layoutparser-service/layoutparser-service-server/src/test/java/com/knecon/fforesight/service/layoutparser/server/graph/DocumentGraphJsonWritingTest.java

@@ -56,8 +56,8 @@ public class DocumentGraphJsonWritingTest extends BuildDocumentTest {
     @SneakyThrows
     private void writeJsons(Path filename) {
 
-        Document documentGraph = DocumentGraphFactory.buildDocumentGraph(LayoutParsingType.REDACT_MANAGER_OLD,
-                layoutParsingPipeline.parseLayout(LayoutParsingType.REDACT_MANAGER_OLD,
+        Document documentGraph = DocumentGraphFactory.buildDocumentGraph(LayoutParsingType.REDACT_MANAGER_WITHOUT_DUPLICATE_PARAGRAPH,
+                layoutParsingPipeline.parseLayout(LayoutParsingType.REDACT_MANAGER_WITHOUT_DUPLICATE_PARAGRAPH,
                         filename.toFile(),
                         new ImageServiceResponse(),
                         new TableServiceResponse(),

layoutparser-service/layoutparser-service-server/src/test/java/com/knecon/fforesight/service/layoutparser/server/graph/ViewerDocumentTest.java

@@ -31,14 +31,14 @@ public class ViewerDocumentTest extends BuildDocumentTest {
     @SneakyThrows
     public void testViewerDocument() {
 
-        String fileName = "files/syngenta/CustomerFiles/SinglePages/S4_S-Metolachlor_RAR_01_Volume_1_2018-09-06.pdf";
+        String fileName = "files/syngenta/CustomerFiles/S-Metolachlor_RAR_01_Volume_1_2018-09-06.pdf";
 
         String tmpFileName = "/tmp/" + Path.of(fileName).getFileName() + "_VIEWER.pdf";
 
         var documentFile = new ClassPathResource(fileName).getFile();
 
         long start = System.currentTimeMillis();
-        Document document = buildGraph(fileName, LayoutParsingType.REDACT_MANAGER);
+        Document document = buildGraph(fileName, LayoutParsingType.REDACT_MANAGER_WITHOUT_DUPLICATE_PARAGRAPH);
         layoutGridService.addLayoutGrid(documentFile, document, new File(tmpFileName), true, false);
         System.out.printf("Total time: %.2fs%n", ((float) (System.currentTimeMillis() - start)) / 1000);
     }

layoutparser-service/layoutparser-service-server/src/test/java/com/knecon/fforesight/service/layoutparser/server/segmentation/PdfSegmentationServiceTest.java

@@ -54,7 +54,7 @@ public class PdfSegmentationServiceTest extends AbstractTest {
     @SneakyThrows
     public ClassificationDocument buildClassificationDocument(File originDocument, TableServiceResponse tableServiceResponse) {
 
-        return layoutParsingPipeline.parseLayout(LayoutParsingType.REDACT_MANAGER,
+        return layoutParsingPipeline.parseLayout(LayoutParsingType.REDACT_MANAGER_WITHOUT_DUPLICATE_PARAGRAPH,
                                                  originDocument,
                                                  new ImageServiceResponse(),
                                                  tableServiceResponse,
@@ -114,7 +114,7 @@ public class PdfSegmentationServiceTest extends AbstractTest {
         ClassificationDocument classificationDocument = buildClassificationDocument(pdfFileResource.getFile());
 
         assertThat(classificationDocument.getHeaders()
-                           .get(0).getTextBlocks().size()).isEqualTo(2);
+                           .get(0).getTextBlocks().size()).isEqualTo(3);
         assertThat(classificationDocument.getHeaders()
                            .get(0).getTextBlocks()
                            .get(0).getSequences().size()).isEqualTo(8);
@@ -122,7 +122,7 @@ public class PdfSegmentationServiceTest extends AbstractTest {
                            .get(0).getTextBlocks()
                            .get(0).toString()).contains(textToSearch);
 
-        Document document = DocumentGraphFactory.buildDocumentGraph(LayoutParsingType.REDACT_MANAGER, classificationDocument);
+        Document document = DocumentGraphFactory.buildDocumentGraph(LayoutParsingType.REDACT_MANAGER_WITHOUT_DUPLICATE_PARAGRAPH, classificationDocument);
 
         TextBlock leafTextBlock = document.getFirstPage().getHeader().getLeafTextBlock();
         assertThat(leafTextBlock.getSearchText().contains(textToSearch)).isTrue();

layoutparser-service/layoutparser-service-server/src/test/java/com/knecon/fforesight/service/layoutparser/server/services/RulingCleaningServiceTest.java

@@ -103,15 +103,15 @@ public class RulingCleaningServiceTest extends BuildDocumentTest {
     @SneakyThrows
     private void writeJsons(Path filename) {
 
-        Document documentGraphBefore = DocumentGraphFactory.buildDocumentGraph(LayoutParsingType.REDACT_MANAGER,
-                layoutParsingPipeline.parseLayout(LayoutParsingType.REDACT_MANAGER,
+        Document documentGraphBefore = DocumentGraphFactory.buildDocumentGraph(LayoutParsingType.REDACT_MANAGER_WITHOUT_DUPLICATE_PARAGRAPH,
+                layoutParsingPipeline.parseLayout(LayoutParsingType.REDACT_MANAGER_WITHOUT_DUPLICATE_PARAGRAPH,
                         filename.toFile(),
                         new ImageServiceResponse(),
                         new TableServiceResponse(),
                         new VisualLayoutParsingResponse(),
                         Map.of("file",filename.toFile().toString())));
-        Document documentGraphAfter = DocumentGraphFactory.buildDocumentGraph(LayoutParsingType.REDACT_MANAGER,
-                layoutParsingPipeline.parseLayout(LayoutParsingType.REDACT_MANAGER,
+        Document documentGraphAfter = DocumentGraphFactory.buildDocumentGraph(LayoutParsingType.REDACT_MANAGER_WITHOUT_DUPLICATE_PARAGRAPH,
+                layoutParsingPipeline.parseLayout(LayoutParsingType.REDACT_MANAGER_WITHOUT_DUPLICATE_PARAGRAPH,
                         filename.toFile(),
                         new ImageServiceResponse(),
                         new TableServiceResponse(),

layoutparser-service/layoutparser-service-server/src/test/java/com/knecon/fforesight/service/layoutparser/server/utils/BuildDocumentTest.java

@@ -48,8 +48,9 @@ public abstract class BuildDocumentTest extends AbstractTest {
     @SneakyThrows
     protected Document buildGraph(String filename, LayoutParsingType layoutParsingType) {
 
+
         if (!filename.startsWith("files") && filename.startsWith("/")) {
-            LayoutParsingRequest layoutParsingRequest = buildDefaultLayoutParsingRequest(Path.of(filename).getFileName().toString(), LayoutParsingType.REDACT_MANAGER, true);
+            LayoutParsingRequest layoutParsingRequest = buildDefaultLayoutParsingRequest(Path.of(filename).getFileName().toString(), LayoutParsingType.REDACT_MANAGER_WITHOUT_DUPLICATE_PARAGRAPH, true);
             prepareStorage(layoutParsingRequest, new File(filename));
             return DocumentGraphFactory.buildDocumentGraph(layoutParsingType,
                                                            layoutParsingPipeline.parseLayout(layoutParsingType,

layoutparser-service/layoutparser-service-server/src/test/resources/files/new/crafted_outline_test_doc.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8afb731a307e1a3f827c59e902164b10bdabef96e14193b949fe081cd3aa859f
+size 168878

layoutparser-service/layoutparser-service-server/src/test/resources/image_service_response/crafted_outline_test_doc.IMAGE_INFO.json

@@ -0,0 +1,135 @@
+{
+  "dossierId": "a91f19ff-11ba-4735-9f60-c650243f64a9",
+  "fileId": "6e8c5f114e2b71e103a32a20c5273188",
+  "targetFileExtension": "ORIGIN.pdf.gz",
+  "responseFileExtension": "IMAGE_INFO.json.gz",
+  "X-TENANT-ID": "redaction",
+  "data": [
+    {
+      "classification": {
+        "label": "other",
+        "probabilities": {
+          "other": 0.9126,
+          "formula": 0.0588,
+          "signature": 0.0261,
+          "logo": 0.0024
+        }
+      },
+      "representation": "70E1070C1030E081B7EF7FFFF",
+      "position": {
+        "x1": 61,
+        "x2": 394,
+        "y1": 155,
+        "y2": 470,
+        "pageNumber": 1
+      },
+      "geometry": {
+        "width": 333,
+        "height": 315
+      },
+      "alpha": false,
+      "filters": {
+        "geometry": {
+          "imageSize": {
+            "quotient": 0.5976,
+            "tooLarge": false,
+            "tooSmall": false
+          },
+          "imageFormat": {
+            "quotient": 1.0571,
+            "tooTall": false,
+            "tooWide": false
+          }
+        },
+        "probability": {
+          "unconfident": false
+        },
+        "allPassed": true
+      }
+    },
+    {
+      "classification": {
+        "label": "other",
+        "probabilities": {
+          "other": 0.9126,
+          "formula": 0.0588,
+          "signature": 0.0261,
+          "logo": 0.0024
+        }
+      },
+      "representation": "70E1070C1030E081B7EF7FFFF",
+      "position": {
+        "x1": 61,
+        "x2": 394,
+        "y1": 202,
+        "y2": 517,
+        "pageNumber": 11
+      },
+      "geometry": {
+        "width": 333,
+        "height": 315
+      },
+      "alpha": false,
+      "filters": {
+        "geometry": {
+          "imageSize": {
+            "quotient": 0.5976,
+            "tooLarge": false,
+            "tooSmall": false
+          },
+          "imageFormat": {
+            "quotient": 1.0571,
+            "tooTall": false,
+            "tooWide": false
+          }
+        },
+        "probability": {
+          "unconfident": false
+        },
+        "allPassed": true
+      }
+    },
+    {
+      "classification": {
+        "label": "other",
+        "probabilities": {
+          "other": 0.9126,
+          "formula": 0.0588,
+          "signature": 0.0261,
+          "logo": 0.0024
+        }
+      },
+      "representation": "70E1070C1030E081B7EF7FFFF",
+      "position": {
+        "x1": 47,
+        "x2": 379,
+        "y1": 289,
+        "y2": 604,
+        "pageNumber": 16
+      },
+      "geometry": {
+        "width": 332,
+        "height": 315
+      },
+      "alpha": false,
+      "filters": {
+        "geometry": {
+          "imageSize": {
+            "quotient": 0.5967,
+            "tooLarge": false,
+            "tooSmall": false
+          },
+          "imageFormat": {
+            "quotient": 1.054,
+            "tooTall": false,
+            "tooWide": false
+          }
+        },
+        "probability": {
+          "unconfident": false
+        },
+        "allPassed": true
+      }
+    }
+  ]
+}