Merge branch 'RED-9760-4.1' into 'release/0.142.x'

RED-9760: Fixed missing newLines See merge request fforesight/layout-parser!185
RED-9760: Fixed missing newLines
2024-07-30 16:06:30 +02:00 · 2024-07-30 15:46:40 +02:00 · 2024-07-25 14:11:03 +02:00 · 2024-07-24 16:32:54 +03:00 · 2024-07-15 17:58:59 +02:00 · 2024-07-15 17:58:59 +02:00
52 changed files with 942 additions and 528 deletions
@@ -2,5 +2,6 @@ package com.knecon.fforesight.service.layoutparser.internal.api.data.redaction;

 public enum LayoutEngine {
    ALGORITHM,
-    AI
+    AI,
+    OUTLINE
 }
@@ -14,7 +14,7 @@ import lombok.NoArgsConstructor;
 public class SimplifiedSectionText {

    @Schema(description = "The number of this Section. This is used to map the simplified section text back to the original Section.")
-    private int sectionNumber;
+    private String sectionNumber;
    @Schema(description = "The text in this Section.")
    private String text;

@@ -4,6 +4,7 @@ public enum LayoutParsingType {
    REDACT_MANAGER,
    REDACT_MANAGER_OLD,
    REDACT_MANAGER_PARAGRAPH_DEBUG,
+    REDACT_MANAGER_WITHOUT_DUPLICATE_PARAGRAPH,
    DOCUMINE,
    DOCUMINE_OLD,
    CLARIFYND,
@@ -51,8 +51,8 @@ import com.knecon.fforesight.service.layoutparser.processor.services.RulingClean
 import com.knecon.fforesight.service.layoutparser.processor.services.SectionsBuilderService;
 import com.knecon.fforesight.service.layoutparser.processor.services.SimplifiedSectionTextService;
 import com.knecon.fforesight.service.layoutparser.processor.services.TableExtractionService;
-import com.knecon.fforesight.service.layoutparser.processor.services.blockification.BlockificationPostprocessingService;
 import com.knecon.fforesight.service.layoutparser.processor.services.TextRulingsClassifier;
+import com.knecon.fforesight.service.layoutparser.processor.services.blockification.BlockificationPostprocessingService;
 import com.knecon.fforesight.service.layoutparser.processor.services.blockification.DocstrumBlockificationService;
 import com.knecon.fforesight.service.layoutparser.processor.services.blockification.DocuMineBlockificationService;
 import com.knecon.fforesight.service.layoutparser.processor.services.blockification.RedactManagerBlockificationService;
@@ -116,8 +116,8 @@ public class LayoutParsingPipeline {
        log.info("Starting layout parsing for {}", layoutParsingRequest.identifier());

        File originFile = layoutParsingStorageService.getOriginFile(layoutParsingRequest.originFileStorageId());
-        File viewerDocumentFile = layoutParsingStorageService.getViewerDocFile(layoutParsingRequest.viewerDocumentStorageId())
-                .orElse(originFile);
+//        File viewerDocumentFile = layoutParsingStorageService.getViewerDocFile(layoutParsingRequest.viewerDocumentStorageId()).orElse(originFile);
+        File viewerDocumentFile = originFile;

        VisualLayoutParsingResponse visualLayoutParsingResponse = new VisualLayoutParsingResponse();
        if (layoutParsingRequest.visualLayoutParsingFileId()
@@ -319,10 +319,16 @@ public class LayoutParsingPipeline {
                case REDACT_MANAGER_OLD ->
                        redactManagerBlockificationService.blockify(stripper.getTextPositionSequences(), cleanRulings, classificationDocument.getVisualizations());
                case DOCUMINE_OLD -> docuMineBlockificationService.blockify(words, cleanRulings);
-                case DOCUMINE, REDACT_MANAGER, REDACT_MANAGER_PARAGRAPH_DEBUG ->
-                        docstrumBlockificationService.blockify(words, cleanRulings, true, classificationDocument.getVisualizations(), layoutParsingType);
-                case CLARIFYND, CLARIFYND_PARAGRAPH_DEBUG ->
-                        docstrumBlockificationService.blockify(words, cleanRulings, false, classificationDocument.getVisualizations(), layoutParsingType);
+                case DOCUMINE, REDACT_MANAGER, REDACT_MANAGER_PARAGRAPH_DEBUG, REDACT_MANAGER_WITHOUT_DUPLICATE_PARAGRAPH -> docstrumBlockificationService.blockify(words,
+                                                                                                                                                                    cleanRulings,
+                                                                                                                                                                    true,
+                                                                                                                                                                    classificationDocument.getVisualizations(),
+                                                                                                                                                                    layoutParsingType);
+                case CLARIFYND, CLARIFYND_PARAGRAPH_DEBUG -> docstrumBlockificationService.blockify(words,
+                                                                                                    cleanRulings,
+                                                                                                    false,
+                                                                                                    classificationDocument.getVisualizations(),
+                                                                                                    layoutParsingType);
            };

            classificationPage.setCleanRulings(cleanRulings);
@@ -381,8 +387,8 @@ public class LayoutParsingPipeline {
        }
        log.info("Classify TextBlocks for {}", identifier);
        switch (layoutParsingType) {
-            case REDACT_MANAGER, REDACT_MANAGER_PARAGRAPH_DEBUG, REDACT_MANAGER_OLD, CLARIFYND_PARAGRAPH_DEBUG ->
-                    redactManagerClassificationService.classifyDocument(classificationDocument);
+            case REDACT_MANAGER, REDACT_MANAGER_PARAGRAPH_DEBUG, REDACT_MANAGER_OLD, CLARIFYND_PARAGRAPH_DEBUG, REDACT_MANAGER_WITHOUT_DUPLICATE_PARAGRAPH -> redactManagerClassificationService.classifyDocument(
+                    classificationDocument);
            case DOCUMINE_OLD, DOCUMINE -> docuMineClassificationService.classifyDocument(classificationDocument);
            case CLARIFYND -> clarifyndClassificationService.classifyDocument(classificationDocument);
        }
@@ -401,9 +407,7 @@ public class LayoutParsingPipeline {

        switch (layoutParsingType) {
            case CLARIFYND_PARAGRAPH_DEBUG, REDACT_MANAGER_PARAGRAPH_DEBUG -> sectionsBuilderService.buildParagraphDebugSections(classificationDocument);
-            default -> {
-                tocEnrichmentService.assignSectionBlocksAndImages(classificationDocument);
-            }
+            default -> tocEnrichmentService.assignSectionBlocksAndImages(classificationDocument);
        }

        return classificationDocument;
@@ -1,9 +1,10 @@
 package com.knecon.fforesight.service.layoutparser.processor.model;

-import java.awt.geom.Rectangle2D;
+import java.util.HashSet;
+import java.util.Set;

 import com.fasterxml.jackson.annotation.JsonIgnore;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
 import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.BoundingBox;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;

@@ -20,6 +21,9 @@ public abstract class AbstractPageBlock extends BoundingBox {

    @JsonIgnore
    protected PageBlockType classification;
+
+    Set<LayoutEngine> engines = new HashSet<>();
+
    @JsonIgnore
    protected int page;

@@ -7,7 +7,6 @@ import java.util.Map;
 import java.util.Set;

 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.DocumentTree;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.entity.RedactionEntity;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlock;
@@ -28,7 +27,7 @@ import lombok.extern.slf4j.Slf4j;
@AllArgsConstructor
@NoArgsConstructor
@FieldDefaults(level = AccessLevel.PRIVATE)
-public abstract class SectionNode implements GenericSemanticNode {
+public abstract class AbstractSemanticNode implements GenericSemanticNode {

    @Builder.Default
    Set<LayoutEngine> engines = new HashSet<>(Set.of(LayoutEngine.ALGORITHM));
@@ -46,25 +45,6 @@ public abstract class SectionNode implements GenericSemanticNode {
    Map<Page, Rectangle2D> bBoxCache;


-    @Override
-    public NodeType getType() {
-
-        return NodeType.SECTION;
-    }
-
-
-    public boolean hasTables() {
-
-        return streamAllSubNodesOfType(NodeType.TABLE).findAny()
-                .isPresent();
-    }
-
-    public boolean isLeafSection() {
-
-        return streamAllSubNodesOfType(NodeType.SECTION).findAny()
-                .isEmpty();
-    }
-
    @Override
    public TextBlock getTextBlock() {

@@ -78,16 +58,7 @@ public abstract class SectionNode implements GenericSemanticNode {
    @Override
    public String toString() {

-        return treeId.toString() + ": " + NodeType.SECTION + ": " + this.getTextBlock().buildSummary();
-    }
-
-
-    public Headline getHeadline() {
-
-        return streamChildrenOfType(NodeType.HEADLINE)//
-                .map(node -> (Headline) node)//
-                .findFirst()//
-                .orElseGet(() -> getParent().getHeadline());
+        return treeId.toString() + ": " + getType() + ": " + this.getTextBlock().buildSummary();
    }


@@ -3,43 +3,35 @@ package com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes;
 import java.awt.geom.Rectangle2D;
 import java.util.Collections;
 import java.util.HashMap;
-import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;

-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
-import com.knecon.fforesight.service.layoutparser.processor.visualization.LayoutparsingVisualizations;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.DocumentTree;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.entity.RedactionEntity;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlock;
+import com.knecon.fforesight.service.layoutparser.processor.visualization.LayoutparsingVisualizations;

 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
-import lombok.Builder;
 import lombok.Data;
+import lombok.EqualsAndHashCode;
 import lombok.NoArgsConstructor;
 import lombok.experimental.FieldDefaults;
+import lombok.experimental.SuperBuilder;

@Data
-@Builder
+@SuperBuilder
@AllArgsConstructor
@NoArgsConstructor
+@EqualsAndHashCode(callSuper = true)
@FieldDefaults(level = AccessLevel.PRIVATE)
-public class Document implements GenericSemanticNode {
-
-    @Builder.Default
-    Set<LayoutEngine> engines = new HashSet<>(Set.of(LayoutEngine.ALGORITHM));
+public class Document extends AbstractSemanticNode {

    Set<Page> pages;
-    DocumentTree documentTree;
    Integer numberOfPages;
-    TextBlock textBlock;
-    @Builder.Default
-    Set<RedactionEntity> entities = new HashSet<>();

    LayoutparsingVisualizations visualizations;

@@ -51,15 +43,27 @@ public class Document implements GenericSemanticNode {
    }


-    public TextBlock getTextBlock() {
+    /**
+     * Gets the sections of the document as a list.
+     *
+     * @return A list of all sections within the document.
+     */
+    public List<Section> getAllSections() {

-        if (textBlock == null) {
-            textBlock = GenericSemanticNode.super.getTextBlock();
-        }
-        return textBlock;
+        return streamAllSubNodesOfType(NodeType.SECTION).map(node -> (Section) node)
+                .collect(Collectors.toList());
    }


+    /**
+     * Gets the main sections of the document as a list.
+     *
+     * @return A list of main sections within the document
+     * @deprecated This method is marked for removal.
+     *  Use {@link #streamChildrenOfType(NodeType)} instead,
+     *  or {@link #getChildrenOfTypeSectionOrSuperSection()} which returns children of type SECTION as well as SUPER_SECTION.
+     */
+    @Deprecated(forRemoval = true)
    public List<Section> getMainSections() {

        return streamChildrenOfType(NodeType.SECTION).map(node -> (Section) node)
@@ -67,6 +71,18 @@ public class Document implements GenericSemanticNode {
    }


+    /**
+     * Gets the direct children of type SECTION or SUPER_SECTION of the document as a list of SemanticNode objects.
+     *
+     * @return A list of all children of type SECTION or SUPER_SECTION.
+     */
+    public List<SemanticNode> getChildrenOfTypeSectionOrSuperSection() {
+
+        return streamChildren().filter(semanticNode -> semanticNode.getType().equals(NodeType.SECTION) || semanticNode.getType().equals(NodeType.SUPER_SECTION))
+                .toList();
+    }
+
+
    public List<Header> getHeaders() {

        return streamChildrenOfType(NodeType.HEADER).map(node -> (Header) node)
@@ -81,6 +97,15 @@ public class Document implements GenericSemanticNode {
    }


+    @Override
+    public Headline getHeadline() {
+
+        return streamAllSubNodesOfType(NodeType.HEADLINE).map(node -> (Headline) node)
+                .findFirst()
+                .orElse(Headline.builder().build());
+    }
+
+
    public Stream<TextBlock> streamTerminalTextBlocksInOrder() {

        return streamAllNodes().filter(SemanticNode::isLeaf)
@@ -102,18 +127,9 @@ public class Document implements GenericSemanticNode {
    }


-    @Override
-    public Headline getHeadline() {
-
-        return streamAllSubNodesOfType(NodeType.HEADLINE).map(node -> (Headline) node)
-                .findFirst()
-                .orElse(Headline.builder().build());
-    }
-
-
    private Stream<SemanticNode> streamAllNodes() {

-        return documentTree.allEntriesInOrder()
+        return getDocumentTree().allEntriesInOrder()
                .map(DocumentTree.Entry::getNode);
    }

@@ -20,7 +20,8 @@ public class DuplicatedParagraph extends Paragraph {
    @Override
    public TextBlock getTextBlock() {

-        return Stream.of(leafTextBlock, unsortedLeafTextBlock).collect(new TextBlockCollector());
+        return Stream.of(super.getLeafTextBlock(), unsortedLeafTextBlock)
+                .collect(new TextBlockCollector());

    }

@@ -1,48 +1,24 @@
 package com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes;

-import java.awt.geom.Rectangle2D;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.DocumentTree;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.entity.RedactionEntity;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlock;

 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
-import lombok.Builder;
 import lombok.Data;
 import lombok.EqualsAndHashCode;
-import lombok.NoArgsConstructor;
 import lombok.experimental.FieldDefaults;
+import lombok.experimental.SuperBuilder;

@Data
-@Builder
+@SuperBuilder
@AllArgsConstructor
-@NoArgsConstructor
+@EqualsAndHashCode(callSuper = true)
@FieldDefaults(level = AccessLevel.PRIVATE)
-public class Footer implements GenericSemanticNode {
+public class Footer extends AbstractSemanticNode {

-    @Builder.Default
-    Set<LayoutEngine> engines = new HashSet<>(Set.of(LayoutEngine.ALGORITHM));
-
-    List<Integer> treeId;
    TextBlock leafTextBlock;

-    @EqualsAndHashCode.Exclude
-    DocumentTree documentTree;
-
-    @Builder.Default
-    @EqualsAndHashCode.Exclude
-    Set<RedactionEntity> entities = new HashSet<>();
-
-    @EqualsAndHashCode.Exclude
-    Map<Page, Rectangle2D> bBoxCache;
-

    @Override
    public NodeType getType() {
@@ -68,17 +44,7 @@ public class Footer implements GenericSemanticNode {
    @Override
    public String toString() {

-        return treeId + ": " + NodeType.FOOTER + ": " + leafTextBlock.buildSummary();
-    }
-
-
-    @Override
-    public Map<Page, Rectangle2D> getBBox() {
-
-        if (bBoxCache == null) {
-            bBoxCache = GenericSemanticNode.super.getBBox();
-        }
-        return bBoxCache;
+        return getTreeId() + ": " + NodeType.FOOTER + ": " + leafTextBlock.buildSummary();
    }

 }
@@ -1,47 +1,24 @@
 package com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes;

-import java.awt.geom.Rectangle2D;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.DocumentTree;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.entity.RedactionEntity;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlock;

 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
-import lombok.Builder;
 import lombok.Data;
 import lombok.EqualsAndHashCode;
-import lombok.NoArgsConstructor;
 import lombok.experimental.FieldDefaults;
+import lombok.experimental.SuperBuilder;

@Data
-@Builder
+@SuperBuilder
@AllArgsConstructor
-@NoArgsConstructor
+@EqualsAndHashCode(callSuper = true)
@FieldDefaults(level = AccessLevel.PRIVATE)
-public class Header implements GenericSemanticNode {
+public class Header extends AbstractSemanticNode {

-    @Builder.Default
-    Set<LayoutEngine> engines = new HashSet<>(Set.of(LayoutEngine.ALGORITHM));
-    List<Integer> treeId;
    TextBlock leafTextBlock;

-    @EqualsAndHashCode.Exclude
-    DocumentTree documentTree;
-
-    @Builder.Default
-    @EqualsAndHashCode.Exclude
-    Set<RedactionEntity> entities = new HashSet<>();
-
-    @EqualsAndHashCode.Exclude
-    Map<Page, Rectangle2D> bBoxCache;
-

    @Override
    public boolean isLeaf() {
@@ -67,17 +44,7 @@ public class Header implements GenericSemanticNode {
    @Override
    public String toString() {

-        return treeId + ": " + NodeType.HEADER + ": " + leafTextBlock.buildSummary();
-    }
-
-
-    @Override
-    public Map<Page, Rectangle2D> getBBox() {
-
-        if (bBoxCache == null) {
-            bBoxCache = GenericSemanticNode.super.getBBox();
-        }
-        return bBoxCache;
+        return getTreeId() + ": " + NodeType.HEADER + ": " + leafTextBlock.buildSummary();
    }

 }
@@ -1,47 +1,24 @@
 package com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes;

-import java.awt.geom.Rectangle2D;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.DocumentTree;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.entity.RedactionEntity;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlock;

 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
-import lombok.Builder;
 import lombok.Data;
 import lombok.EqualsAndHashCode;
-import lombok.NoArgsConstructor;
 import lombok.experimental.FieldDefaults;
+import lombok.experimental.SuperBuilder;

@Data
-@Builder
+@SuperBuilder
@AllArgsConstructor
-@NoArgsConstructor
+@EqualsAndHashCode(callSuper = true)
@FieldDefaults(level = AccessLevel.PRIVATE)
-public class Headline implements GenericSemanticNode {
+public class Headline extends AbstractSemanticNode {

-    @Builder.Default
-    Set<LayoutEngine> engines = new HashSet<>(Set.of(LayoutEngine.ALGORITHM));
-    List<Integer> treeId;
    TextBlock leafTextBlock;

-    @EqualsAndHashCode.Exclude
-    DocumentTree documentTree;
-
-    @Builder.Default
-    @EqualsAndHashCode.Exclude
-    Set<RedactionEntity> entities = new HashSet<>();
-
-    @EqualsAndHashCode.Exclude
-    Map<Page, Rectangle2D> bBoxCache;
-

    @Override
    public NodeType getType() {
@@ -67,7 +44,7 @@ public class Headline implements GenericSemanticNode {
    @Override
    public String toString() {

-        return treeId + ": " + NodeType.HEADLINE + ": " + leafTextBlock.buildSummary();
+        return getTreeId() + ": " + NodeType.HEADLINE + ": " + leafTextBlock.buildSummary();
    }


@@ -77,14 +54,4 @@ public class Headline implements GenericSemanticNode {
        return this;
    }

-
-    @Override
-    public Map<Page, Rectangle2D> getBBox() {
-
-        if (bBoxCache == null) {
-            bBoxCache = GenericSemanticNode.super.getBBox();
-        }
-        return bBoxCache;
-    }
-
 }
@@ -3,15 +3,10 @@ package com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes;
 import java.awt.geom.Rectangle2D;
 import java.util.Collections;
 import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
 import java.util.Map;
 import java.util.Set;

-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.DocumentTree;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.entity.RedactionEntity;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlock;

 import lombok.AccessLevel;
@@ -21,18 +16,16 @@ import lombok.Data;
 import lombok.EqualsAndHashCode;
 import lombok.NoArgsConstructor;
 import lombok.experimental.FieldDefaults;
+import lombok.experimental.SuperBuilder;

@Data
-@Builder
+@SuperBuilder
@AllArgsConstructor
@NoArgsConstructor
+@EqualsAndHashCode(callSuper = true)
@FieldDefaults(level = AccessLevel.PRIVATE)
-public class Image implements GenericSemanticNode {
+public class Image extends AbstractSemanticNode {

-    @Builder.Default
-    Set<LayoutEngine> engines = new HashSet<>(Set.of(LayoutEngine.ALGORITHM));
-
-    List<Integer> treeId;
    String id;

    ImageType imageType;
@@ -53,13 +46,6 @@ public class Image implements GenericSemanticNode {
    @EqualsAndHashCode.Exclude
    Page page;

-    @EqualsAndHashCode.Exclude
-    DocumentTree documentTree;
-
-    @Builder.Default
-    @EqualsAndHashCode.Exclude
-    Set<RedactionEntity> entities = new HashSet<>();
-

    @Override
    public NodeType getType() {
@@ -85,7 +71,7 @@ public class Image implements GenericSemanticNode {
    @Override
    public String toString() {

-        return treeId + ": " + NodeType.IMAGE + ": " + imageType.toString() + " " + position;
+        return getTreeId() + ": " + NodeType.IMAGE + ": " + imageType.toString() + " " + position;
    }


@@ -1,20 +1,10 @@
 package com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes;

-import java.awt.geom.Rectangle2D;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.DocumentTree;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.entity.RedactionEntity;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlock;

 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
-import lombok.Builder;
 import lombok.Data;
 import lombok.EqualsAndHashCode;
 import lombok.experimental.FieldDefaults;
@@ -23,25 +13,12 @@ import lombok.experimental.SuperBuilder;
@Data
@SuperBuilder
@AllArgsConstructor
+@EqualsAndHashCode(callSuper = true)
@FieldDefaults(level = AccessLevel.PROTECTED)
-public class Paragraph implements GenericSemanticNode {
+public class Paragraph extends AbstractSemanticNode {

-    @Builder.Default
-    Set<LayoutEngine> engines = new HashSet<>(Set.of(LayoutEngine.ALGORITHM));
-
-    List<Integer> treeId;
    TextBlock leafTextBlock;

-    @EqualsAndHashCode.Exclude
-    DocumentTree documentTree;
-
-    @Builder.Default
-    @EqualsAndHashCode.Exclude
-    Set<RedactionEntity> entities = new HashSet<>();
-
-    @EqualsAndHashCode.Exclude
-    Map<Page, Rectangle2D> bBoxCache;
-

    @Override
    public NodeType getType() {
@@ -63,21 +40,4 @@ public class Paragraph implements GenericSemanticNode {
        return leafTextBlock;
    }

-
-    @Override
-    public String toString() {
-
-        return treeId + ": " + NodeType.PARAGRAPH + ": " + leafTextBlock.buildSummary();
-    }
-
-
-    @Override
-    public Map<Page, Rectangle2D> getBBox() {
-
-        if (bBoxCache == null) {
-            bBoxCache = GenericSemanticNode.super.getBBox();
-        }
-        return bBoxCache;
-    }
-
 }
@@ -1,37 +1,47 @@
 package com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes;

-import java.awt.geom.Rectangle2D;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.DocumentTree;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.entity.RedactionEntity;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlock;

 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
-import lombok.Builder;
 import lombok.Data;
 import lombok.EqualsAndHashCode;
 import lombok.experimental.FieldDefaults;
 import lombok.experimental.SuperBuilder;
-import lombok.extern.slf4j.Slf4j;

@Data
@SuperBuilder
@AllArgsConstructor
@FieldDefaults(level = AccessLevel.PRIVATE)
@EqualsAndHashCode(callSuper = true)
-public class Section extends SectionNode {
+public class Section extends AbstractSemanticNode {
+
+    @Override
+    public NodeType getType() {
+
+        return NodeType.SECTION;
+    }
+
+
+    public Headline getHeadline() {
+
+        return streamChildrenOfType(NodeType.HEADLINE).map(node -> (Headline) node)
+                .findFirst()
+                .orElseGet(() -> getParent().getHeadline());
+    }
+
+
+    public boolean hasTables() {
+
+        return streamAllSubNodesOfType(NodeType.TABLE).findAny()
+                .isPresent();
+    }
+

    @Override
    public String toString() {

-        return super.toString();
+        return getTreeId() + ": " + NodeType.SECTION + ": " + this.getTextBlock().buildSummary();
    }

 }
@@ -1,19 +1,11 @@
 package com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes;

-import java.util.List;
-import java.util.Set;
-
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.DocumentTree;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.entity.RedactionEntity;

 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
-import lombok.Builder;
 import lombok.Data;
 import lombok.EqualsAndHashCode;
-import lombok.ToString;
 import lombok.experimental.FieldDefaults;
 import lombok.experimental.SuperBuilder;

@@ -22,7 +14,7 @@ import lombok.experimental.SuperBuilder;
@AllArgsConstructor
@FieldDefaults(level = AccessLevel.PRIVATE)
@EqualsAndHashCode(callSuper = true)
-public class SuperSection extends SectionNode {
+public class SuperSection extends AbstractSemanticNode {

    @Override
    public NodeType getType() {
@@ -31,10 +23,18 @@ public class SuperSection extends SectionNode {
    }


+    public Headline getHeadline() {
+
+        return streamChildrenOfType(NodeType.HEADLINE).map(node -> (Headline) node)
+                .findFirst()
+                .orElseGet(() -> getParent().getHeadline());
+    }
+
+
    @Override
    public String toString() {

-        return super.toString();
+        return getTreeId() + ": " + NodeType.SUPER_SECTION + ": " + this.getTextBlock().buildSummary();
    }

 }
@@ -2,34 +2,26 @@ package com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes;

 import java.awt.geom.Rectangle2D;
 import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
 import java.util.Map;
-import java.util.Set;

-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.DocumentTree;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.entity.RedactionEntity;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlock;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlockCollector;

 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
-import lombok.Builder;
 import lombok.Data;
 import lombok.EqualsAndHashCode;
 import lombok.experimental.FieldDefaults;
+import lombok.experimental.SuperBuilder;

@Data
-@Builder
+@SuperBuilder
@AllArgsConstructor
+@EqualsAndHashCode(callSuper = true)
@FieldDefaults(level = AccessLevel.PRIVATE)
-public class TableCell implements GenericSemanticNode {
+public class TableCell extends AbstractSemanticNode {

-    @Builder.Default
-    Set<LayoutEngine> engines = new HashSet<>(Set.of(LayoutEngine.ALGORITHM));
-    List<Integer> treeId;
    int row;
    int col;
    boolean header;
@@ -40,13 +32,6 @@ public class TableCell implements GenericSemanticNode {

    TextBlock textBlock;

-    @EqualsAndHashCode.Exclude
-    DocumentTree documentTree;
-
-    @Builder.Default
-    @EqualsAndHashCode.Exclude
-    Set<RedactionEntity> entities = new HashSet<>();
-

    @Override
    public Map<Page, Rectangle2D> getBBox() {
@@ -96,7 +81,7 @@ public class TableCell implements GenericSemanticNode {
    @Override
    public String toString() {

-        return treeId + ": " + NodeType.TABLE_CELL + ": " + this.getTextBlock().buildSummary();
+        return getTreeId() + ": " + NodeType.TABLE_CELL + ": " + this.getTextBlock().buildSummary();
    }

 }
@@ -33,6 +33,8 @@ public class OutlineValidationService {

            if (parentDepth == null) {
                mainSections.add(tocItem);
+                lastItemsPerDepth = new HashMap<>();
+                depths = new TreeSet<>();

            } else {
                assert last != null;
@@ -29,7 +29,7 @@ public class TOCEnrichmentService {
        TableOfContents toc = document.getTableOfContents();
        Iterator<TableOfContentItem> iterator = toc.iterator();
        TableOfContentItem currentTOCItem = null;
-        if(iterator.hasNext()) {
+        if (iterator.hasNext()) {
            currentTOCItem = iterator.next();
        }
        List<AbstractPageBlock> startBlocks = new ArrayList<>();
@@ -79,12 +79,10 @@ public class TOCEnrichmentService {
                    currentTOCItem.getSectionBlocks().add(current);
                    currentPageTOCItems.add(currentTOCItem);

-                    if(iterator.hasNext()) {
+                    if (iterator.hasNext()) {
                        currentTOCItem = iterator.next();
                    }
-                }
-
-                if (!foundFirstHeadline) {
+                } else if (!foundFirstHeadline) {
                    startBlocks.add(current);
                } else {
                    currentSection.getSectionBlocks().add(current);
@@ -151,9 +149,9 @@ public class TOCEnrichmentService {
                    }
                }
                if (!image.isAppendedToSection()) {
-                    log.debug("Image uses first paragraph");
+                    log.debug("Image uses last found section");
                    if (!lastFoundTOCItems.isEmpty()) {
-                        lastFoundTOCItems.get(0).getImages().add(image);
+                        lastFoundTOCItems.get(lastFoundTOCItems.size() - 1).getImages().add(image);
                    } else {
                        startImages.add(image);
                    }
@@ -5,8 +5,7 @@ import java.util.List;
 import java.util.stream.Collectors;

 import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBlock;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Section;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.SectionNode;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.AbstractSemanticNode;
 import com.knecon.fforesight.service.layoutparser.processor.model.image.ClassifiedImage;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;

@@ -25,7 +24,7 @@ public class TableOfContentItem {
    private List<AbstractPageBlock> sectionBlocks = new ArrayList<>();
    private List<ClassifiedImage> images = new ArrayList<>();

-    private SectionNode section;
+    private AbstractSemanticNode section;


    public TableOfContentItem(TextPageBlock headline) {
@@ -68,7 +68,7 @@ public class RedTextPosition extends BoundingBox {
        // I guess if we start with the initial user space positions and transform them the same way we do the rulings it would work.
        pos.setBBox(new Rectangle2D.Float(textPosition.getX(), textPosition.getY(), textPosition.getWidthDirAdj(), textPosition.getHeight()));

-        float textHeight = textPosition.getHeight() + HEIGHT_PADDING;
+        float textHeight = textPosition.getHeight() + 2 * HEIGHT_PADDING;
        Rectangle2D.Float dirAdjPosition = new Rectangle2D.Float(textPosition.getXDirAdj(),
                                                                 textPosition.getYDirAdj() - textHeight,
                                                                 textPosition.getWidthDirAdj(),
@@ -1,5 +1,7 @@
 package com.knecon.fforesight.service.layoutparser.processor.model.text;

+import static com.knecon.fforesight.service.layoutparser.processor.utils.ParsingConstants.NEW_LINE_TEXT_HEIGHT_PERCENTAGE;
+
 import java.awt.geom.Rectangle2D;
 import java.util.ArrayList;
 import java.util.List;
@@ -208,7 +210,7 @@ public class TextPageBlock extends AbstractPageBlock {
        TextPositionSequence previous = null;
        for (TextPositionSequence word : sequences) {
            if (previous != null) {
-                if (Math.abs(previous.getMaxYDirAdj() - word.getMaxYDirAdj()) > word.getTextHeight()) {
+                if (Math.abs(previous.getMaxYDirAdj() - word.getMaxYDirAdj()) > word.getTextHeight() * NEW_LINE_TEXT_HEIGHT_PERCENTAGE) {
                    sb.append('\n');
                } else {
                    sb.append(' ');
@@ -228,7 +230,7 @@ public class TextPageBlock extends AbstractPageBlock {
        TextPositionSequence previous = null;
        for (TextPositionSequence word : sequences) {
            if (previous != null) {
-                if (word.getMaxYDirAdj() - previous.getMaxYDirAdj() > word.getTextHeight()) {
+                if (word.getMaxYDirAdj() - previous.getMaxYDirAdj() > word.getTextHeight() * NEW_LINE_TEXT_HEIGHT_PERCENTAGE) {
                    numberOfLines++;
                }
            }
@@ -9,7 +9,6 @@ import org.springframework.stereotype.Service;
 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.SimplifiedSectionText;
 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.SimplifiedText;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.GenericSemanticNode;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.SemanticNode;

 import lombok.extern.slf4j.Slf4j;
@@ -20,7 +19,7 @@ public class SimplifiedSectionTextService {

    public SimplifiedText toSimplifiedText(Document document) {

-        List<SimplifiedSectionText> simplifiedMainSectionsList = document.getMainSections()
+        List<SimplifiedSectionText> simplifiedMainSectionsList = document.getAllSections()
                .stream()
                .map(this::toSimplifiedSectionText)
                .toList();
@@ -43,7 +42,9 @@ public class SimplifiedSectionTextService {

        return SimplifiedSectionText.builder()
                .sectionNumber(section.getTreeId()
-                                       .get(0))
+                                       .stream()
+                                       .map(String::valueOf)
+                                       .collect(Collectors.joining(".")))
                .text(section.getTextBlock().getSearchText())
                .build();
    }
@@ -12,6 +12,7 @@ import java.util.function.Function;
 import org.apache.commons.lang3.StringUtils;
 import org.springframework.stereotype.Service;

+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationPage;
 import com.knecon.fforesight.service.layoutparser.processor.model.PageBlockType;
 import com.knecon.fforesight.service.layoutparser.processor.model.SectionIdentifier;
@@ -204,21 +205,28 @@ public class BlockificationPostprocessingService {
            return false;
        }
        if (minDistance == distanceToDirectMatch) {
-            directMatch.setClassification(headlineType);
+            setClassificationAndAddOutlineEngine(directMatch, headlineType);
        } else if (minDistance == distanceToSplitCandidate) {
            SplitBlockResult splitBlockResult = splitBlock(classificationPage, splitCandidate, context.sectionIdentifier, outlineObject.getTitle());
            if (splitBlockResult.modifiedBlockToSplit) {
-                splitCandidate.setClassification(headlineType);
+                setClassificationAndAddOutlineEngine(splitCandidate, headlineType);
            }
            splitBlockResult.otherBlocks.forEach(other -> other.setClassification(null));
        } else {
            var merged = mergeBlocks(classificationPage, bestMergeCandidateCombination);
-            merged.setClassification(headlineType);
+            setClassificationAndAddOutlineEngine(merged, headlineType);
        }
        return true;
    }


+    private static void setClassificationAndAddOutlineEngine(TextPageBlock block, PageBlockType headlineType) {
+
+        block.setClassification(headlineType);
+        block.getEngines().add(LayoutEngine.OUTLINE);
+    }
+
+
    private SplitBlockResult splitBlock(ClassificationPage classificationPage, TextPageBlock blockToSplit, SectionIdentifier sectionIdentifier, String title) {

        List<TextPageBlock> otherBlocks = new ArrayList<>();
@@ -26,6 +26,7 @@ import lombok.RequiredArgsConstructor;
@RequiredArgsConstructor
 public class DocstrumBlockificationService {

+    public static final float Y_THRESHOLD = 5f;
    private final DocstrumSegmentationService docstrumSegmentationService;

    static final float THRESHOLD = 1f;
@@ -58,12 +59,14 @@ public class DocstrumBlockificationService {

        mergeIntersectingBlocks(classificationPage, usedRulings, 0, 0);

-        if (layoutParsingType == LayoutParsingType.DOCUMINE || layoutParsingType == LayoutParsingType.REDACT_MANAGER) {
-            combineBlocks(classificationPage);
+        if (layoutParsingType == LayoutParsingType.DOCUMINE
+            || layoutParsingType == LayoutParsingType.REDACT_MANAGER
+            || layoutParsingType == LayoutParsingType.REDACT_MANAGER_WITHOUT_DUPLICATE_PARAGRAPH) {
+            combineBlocks(classificationPage, layoutParsingType);
        }

        if (layoutParsingType == LayoutParsingType.CLARIFYND) {
-            mergeIntersectingBlocks(classificationPage, usedRulings, 0, 6.5f);
+            mergeIntersectingBlocks(classificationPage, usedRulings, 0, 0);
        }

        return classificationPage;
@@ -105,7 +108,7 @@ public class DocstrumBlockificationService {
    }


-    public void combineBlocks(ClassificationPage page) {
+    public void combineBlocks(ClassificationPage page, LayoutParsingType layoutParsingType) {

        TextPageBlock previous = new TextPageBlock();
        ListIterator<AbstractPageBlock> itty = page.getTextBlocks().listIterator();
@@ -137,7 +140,8 @@ public class DocstrumBlockificationService {
                }

                if (areTheOnlyTwoBlocksOnHeightsWithBothMoreThanTwoLines(previous, current, page)) {
-                    previous = combineBlocksAndResetIterator(previous, current, itty, true);
+//                    previous = combineBlocksAndResetIterator(previous, current, itty, true);
+                    previous = combineBlocksAndResetIterator(previous, current, itty, layoutParsingType != LayoutParsingType.REDACT_MANAGER_WITHOUT_DUPLICATE_PARAGRAPH);
                    continue;
                }

@@ -160,7 +164,7 @@ public class DocstrumBlockificationService {
            previous = current;
        }

-        mergeIntersectingBlocks(page, usedRulings, 0, 6.5f);
+        mergeIntersectingBlocks(page, usedRulings, 0, Y_THRESHOLD);
    }


@@ -21,33 +21,31 @@ import lombok.extern.slf4j.Slf4j;
@RequiredArgsConstructor
 public class ClarifyndClassificationService {

-    private final HeadlineClassificationService headlineClassificationService;
-
    public void classifyDocument(ClassificationDocument document) {

        List<Float> headlineFontSizes = document.getFontSizeCounter().getHighterThanMostPopular();

        log.debug("Document FontSize counters are: {}", document.getFontSizeCounter().getCountPerValue());

-        headlineClassificationService.resetContext();
+        HeadlineClassificationService headlineClassificationService = new HeadlineClassificationService();

        for (ClassificationPage page : document.getPages()) {
-            classifyPage(page, document, headlineFontSizes);
+            classifyPage(headlineClassificationService, page, document, headlineFontSizes);
        }
    }


-    private void classifyPage(ClassificationPage page, ClassificationDocument document, List<Float> headlineFontSizes) {
+    private void classifyPage(HeadlineClassificationService headlineClassificationService, ClassificationPage page, ClassificationDocument document, List<Float> headlineFontSizes) {

        for (AbstractPageBlock textBlock : page.getTextBlocks()) {
            if (textBlock instanceof TextPageBlock) {
-                classifyBlock((TextPageBlock) textBlock, page, document, headlineFontSizes);
+                classifyBlock(headlineClassificationService, (TextPageBlock) textBlock, page, document, headlineFontSizes);
            }
        }
    }


-    private void classifyBlock(TextPageBlock textBlock, ClassificationPage page, ClassificationDocument document, List<Float> headlineFontSizes) {
+    private void classifyBlock(HeadlineClassificationService headlineClassificationService, TextPageBlock textBlock, ClassificationPage page, ClassificationDocument document, List<Float> headlineFontSizes) {

        var bodyTextFrame = page.getBodyTextFrame();

@@ -24,10 +24,9 @@ import lombok.extern.slf4j.Slf4j;
@RequiredArgsConstructor
 public class DocuMineClassificationService {

-    private final HeadlineClassificationService headlineClassificationService;
-    private static final Pattern pattern = Pattern.compile("^([1-9]\\d?\\.){1,3}\\d{1,2}\\.?\\s[0-9A-Za-z \\[\\]]{2,50}", Pattern.CASE_INSENSITIVE);
-    private static final Pattern pattern2 = Pattern.compile("\\p{L}{3,}", Pattern.CASE_INSENSITIVE);
-    private static final Pattern pattern3 = Pattern.compile("^(\\d{1,1}\\.){1,3}\\d{1,2}\\.?\\s[a-z]{1,2}\\/[a-z]{1,2}.*");
+    private static final Pattern HEADLINE_WITH_IDENTIFER_PATTERN = Pattern.compile("^([1-9]\\d?\\.){1,3}\\d{1,2}\\.?\\s[0-9A-Za-z \\[\\]]{2,50}", Pattern.CASE_INSENSITIVE);
+    private static final Pattern AT_LEAST_3_PATTERN = Pattern.compile("\\p{L}{3,}", Pattern.CASE_INSENSITIVE);
+    private static final Pattern HEADLINE_PATTTERN_WITH_SLASHES = Pattern.compile("^(\\d{1,1}\\.){1,3}\\d{1,2}\\.?\\s[a-z]{1,2}\\/[a-z]{1,2}.*");


    public void classifyDocument(ClassificationDocument document) {
@@ -36,32 +35,39 @@ public class DocuMineClassificationService {

        log.debug("Document FontSize counters are: {}", document.getFontSizeCounter().getCountPerValue());

-        headlineClassificationService.resetContext();
+        HeadlineClassificationService headlineClassificationService = new HeadlineClassificationService();

        for (ClassificationPage page : document.getPages()) {
-            classifyPage(page, document, headlineFontSizes);
+            classifyPage(headlineClassificationService, page, document, headlineFontSizes);
        }
    }


-    private void classifyPage(ClassificationPage page, ClassificationDocument document, List<Float> headlineFontSizes) {
+    private void classifyPage(HeadlineClassificationService headlineClassificationService,
+                              ClassificationPage page,
+                              ClassificationDocument document,
+                              List<Float> headlineFontSizes) {

        for (AbstractPageBlock textBlock : page.getTextBlocks()) {
            if (textBlock instanceof TextPageBlock) {
-                classifyBlock((TextPageBlock) textBlock, page, document, headlineFontSizes);
+                classifyBlock(headlineClassificationService, (TextPageBlock) textBlock, page, document, headlineFontSizes);
            }
        }
    }


-    private void classifyBlock(TextPageBlock textBlock, ClassificationPage page, ClassificationDocument document, List<Float> headlineFontSizes) {
+    private void classifyBlock(HeadlineClassificationService headlineClassificationService,
+                               TextPageBlock textBlock,
+                               ClassificationPage page,
+                               ClassificationDocument document,
+                               List<Float> headlineFontSizes) {

        log.debug("headlineFontSizes: {}", headlineFontSizes);
        var bodyTextFrame = page.getBodyTextFrame();

-        Matcher matcher = pattern.matcher(textBlock.toString());
-        Matcher matcher2 = pattern2.matcher(textBlock.toString());
-        Matcher matcher3 = pattern3.matcher(textBlock.toString());
+        Matcher headlineWithIdentifierMatcher = HEADLINE_WITH_IDENTIFER_PATTERN.matcher(textBlock.toString());
+        Matcher atLeast3Matcher = AT_LEAST_3_PATTERN.matcher(textBlock.toString());
+        Matcher headlineWithSlashesMatcher = HEADLINE_PATTTERN_WITH_SLASHES.matcher(textBlock.toString());

        if (textBlock.getClassification() != null && textBlock.getClassification().isHeadline()) {
            headlineClassificationService.setLastHeadlineFromOutline(textBlock);
@@ -71,15 +77,10 @@ public class DocuMineClassificationService {
            textBlock.setClassification(PageBlockType.OTHER);
            return;
        }
-        if (MarkedContentUtils.intersects(textBlock, page.getMarkedContentBboxPerType(), MarkedContentUtils.HEADER)
-            || (PositionUtils.isOverBodyTextFrame(bodyTextFrame,
-                                                  textBlock,
-                                                  page.getRotation()) && (document.getFontSizeCounter().getMostPopular()
-                                                                          == null
-                                                                          || textBlock.getHighestFontSize()
-                                                                             <= document.getFontSizeCounter()
-                                                                                     .getMostPopular()))
-            || HeaderFooterDetection.isLikelyHeader(textBlock, document, page)) {
+        if (MarkedContentUtils.intersects(textBlock, page.getMarkedContentBboxPerType(), MarkedContentUtils.HEADER) //
+            || (PositionUtils.isOverBodyTextFrame(bodyTextFrame, textBlock, page.getRotation()) //
+                && (document.getFontSizeCounter().getMostPopular() == null //
+                    || textBlock.getHighestFontSize() <= document.getFontSizeCounter().getMostPopular()))) {
            textBlock.setClassification(PageBlockType.HEADER);

        } else if (MarkedContentUtils.intersects(textBlock, page.getMarkedContentBboxPerType(), MarkedContentUtils.FOOTER)
@@ -103,19 +104,25 @@ public class DocuMineClassificationService {
                       || textBlock.getMostPopularWordFontSize() > document.getFontSizeCounter().getMostPopular())
                   && PositionUtils.getApproxLineCount(textBlock) < 5.9

-                   && (textBlock.getMostPopularWordStyle().contains("bold") && Character.isDigit(textBlock.toString().charAt(0)) && matcher2.reset().find() && !textBlock.toString()
-                .contains(":")
-                       || textBlock.toString().equals(textBlock.toString().toUpperCase(Locale.ROOT)) && matcher2.reset().find() && !textBlock.toString().contains(":")
-                       || textBlock.toString().startsWith("APPENDIX")
-                       || textBlock.toString().startsWith("FIGURE")
+                   && (textBlock.getMostPopularWordStyle().contains("bold")
+                       && Character.isDigit(textBlock.toString().charAt(0))
+                       && atLeast3Matcher.reset().find()
+                       && !textBlock.toString().contains(":") //
+                       || textBlock.toString().equals(textBlock.toString().toUpperCase(Locale.ROOT)) && atLeast3Matcher.reset().find() && !textBlock.toString().contains(":") //
+                       || textBlock.toString().startsWith("APPENDIX") //
+                       || textBlock.toString().startsWith("FIGURE") //
+                       || textBlock.toString().startsWith("Continued TABLE") //
                       || textBlock.toString().startsWith("TABLE"))
                   && !textBlock.toString().endsWith(":")
-                   && matcher2.reset().find()) {
+                   && atLeast3Matcher.reset().find()) {
            PageBlockType headlineType = PageBlockType.getHeadlineType(1);
            headlineClassificationService.classifyHeadline(textBlock, headlineType);
            document.setHeadlines(true);

-        } else if (matcher.reset().find() && PositionUtils.getApproxLineCount(textBlock) < 2.9 && matcher2.reset().find() && !matcher3.reset().matches()) {
+        } else if (headlineWithIdentifierMatcher.reset().find()
+                   && PositionUtils.getApproxLineCount(textBlock) < 2.9
+                   && atLeast3Matcher.reset().find()
+                   && !headlineWithSlashesMatcher.reset().matches()) {
            PageBlockType headlineType = PageBlockType.getHeadlineType(2);
            headlineClassificationService.classifyHeadline(textBlock, headlineType);
            document.setHeadlines(true);
@@ -2,15 +2,12 @@ package com.knecon.fforesight.service.layoutparser.processor.services.classifica

 import static com.knecon.fforesight.service.layoutparser.processor.model.PageBlockType.getHeadlineNumber;

-import org.springframework.stereotype.Service;
-
 import com.knecon.fforesight.service.layoutparser.processor.model.PageBlockType;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;

 import lombok.Getter;
 import lombok.Setter;

-@Service
@Getter
@Setter
 public class HeadlineClassificationService {
@@ -19,13 +16,6 @@ public class HeadlineClassificationService {
    PageBlockType originalClassifiedBlockType;
    TextPageBlock lastHeadlineFromOutline;

-    public void resetContext() {
-        setLastHeadline(null);
-        setOriginalClassifiedBlockType(null);
-        setLastHeadlineFromOutline(null);
-    }
-
-
    public void setLastHeadlineFromOutline(TextPageBlock lastHeadlineFromOutline) {

        this.lastHeadlineFromOutline = lastHeadlineFromOutline;
@@ -22,8 +22,6 @@ import lombok.extern.slf4j.Slf4j;
@RequiredArgsConstructor
 public class RedactManagerClassificationService {

-    private final HeadlineClassificationService headlineClassificationService;
-

    public void classifyDocument(ClassificationDocument document) {

@@ -31,25 +29,25 @@ public class RedactManagerClassificationService {

        log.debug("Document FontSize counters are: {}", document.getFontSizeCounter().getCountPerValue());

-        headlineClassificationService.resetContext();
+        HeadlineClassificationService headlineClassificationService = new HeadlineClassificationService();

        for (ClassificationPage page : document.getPages()) {
-            classifyPage(page, document, headlineFontSizes);
+            classifyPage(headlineClassificationService, page, document, headlineFontSizes);
        }
    }


-    private void classifyPage(ClassificationPage page, ClassificationDocument document, List<Float> headlineFontSizes) {
+    private void classifyPage(HeadlineClassificationService headlineClassificationService, ClassificationPage page, ClassificationDocument document, List<Float> headlineFontSizes) {

        for (AbstractPageBlock textBlock : page.getTextBlocks()) {
            if (textBlock instanceof TextPageBlock) {
-                classifyBlock((TextPageBlock) textBlock, page, document, headlineFontSizes);
+                classifyBlock(headlineClassificationService, (TextPageBlock) textBlock, page, document, headlineFontSizes);
            }
        }
    }


-    private void classifyBlock(TextPageBlock textBlock, ClassificationPage page, ClassificationDocument document, List<Float> headlineFontSizes) {
+    private void classifyBlock(HeadlineClassificationService headlineClassificationService, TextPageBlock textBlock, ClassificationPage page, ClassificationDocument document, List<Float> headlineFontSizes) {

        var bodyTextFrame = page.getBodyTextFrame();

@@ -31,8 +31,7 @@ import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.He
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Image;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Page;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Paragraph;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Section;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.SectionNode;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.AbstractSemanticNode;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.AtomicTextBlock;
 import com.knecon.fforesight.service.layoutparser.processor.model.image.ClassifiedImage;
 import com.knecon.fforesight.service.layoutparser.processor.model.outline.TableOfContentItem;
@@ -75,13 +74,13 @@ public class DocumentGraphFactory {

        for (TableOfContentItem tocItem : classificationDocument.getTableOfContents()) {
            var parent = tocItem.getParent() == null ? null : tocItem.getParent().getSection();
-            Optional<SectionNode> section = SectionNodeFactory.addSection(layoutParsingType,
-                                                                      parent,
-                                                                      tocItem.getChildren().isEmpty(),
-                                                                      tocItem.getNonEmptySectionBlocks(),
-                                                                      tocItem.getImages(),
-                                                                      context,
-                                                                      document);
+            Optional<AbstractSemanticNode> section = SectionNodeFactory.addSection(layoutParsingType,
+                                                                                   parent,
+                                                                                   tocItem.getChildren().isEmpty(),
+                                                                                   tocItem.getNonEmptySectionBlocks(),
+                                                                                   tocItem.getImages(),
+                                                                                   context,
+                                                                                   document);
            tocItem.setSection(section.orElse(null));
        }
    }
@@ -123,6 +122,7 @@ public class DocumentGraphFactory {
        List<Integer> treeId = context.documentTree.createNewChildEntryAndReturnId(parentNode, node);
        node.setLeafTextBlock(textBlock);
        node.setTreeId(treeId);
+        node.getEngines().addAll(originalTextBlock.getEngines());
    }


@@ -240,7 +240,7 @@ public class DocumentGraphFactory {

        DocumentTree documentTree;
        Map<Page, Integer> pages;
-        List<SectionNode> sections;
+        List<AbstractSemanticNode> sections;
        List<ClassifiedImage> images;
        TextBlockFactory textBlockFactory;

@@ -1,5 +1,7 @@
 package com.knecon.fforesight.service.layoutparser.processor.services.factory;

+import static com.knecon.fforesight.service.layoutparser.processor.utils.ParsingConstants.NEW_LINE_TEXT_HEIGHT_PERCENTAGE;
+
 import java.awt.geom.AffineTransform;
 import java.awt.geom.Rectangle2D;
 import java.util.Collection;
@@ -67,7 +69,6 @@ public class SearchTextWithTextPositionFactory {
            ++context.stringIdx;
        }

-
        List<Rectangle2D> positions = sequences.stream()
                .map(TextPositionSequence::getTextPositions)
                .flatMap(Collection::stream)
@@ -161,7 +162,7 @@ public class SearchTextWithTextPositionFactory {
        }

        double deltaY = Math.abs(currentPosition.getYDirAdj() - previousPosition.getYDirAdj());
-        return deltaY >= currentPosition.getHeightDir();
+        return deltaY >= currentPosition.getHeightDir() * NEW_LINE_TEXT_HEIGHT_PERCENTAGE;
    }


@@ -191,9 +192,9 @@ public class SearchTextWithTextPositionFactory {

        float textHeight = sequence.getTextHeight() + HEIGHT_PADDING;
        Rectangle2D rectangle2D = new Rectangle2D.Double(textPosition.getXDirAdj(),
-                textPosition.getYDirAdj() - textHeight,
-                textPosition.getWidthDirAdj(),
-                textHeight + HEIGHT_PADDING);
+                                                         textPosition.getYDirAdj() - textHeight,
+                                                         textPosition.getWidthDirAdj(),
+                                                         textHeight + HEIGHT_PADDING);

        AffineTransform transform = new AffineTransform();

@@ -14,11 +14,11 @@ import java.util.Set;

 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
 import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBlock;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.AbstractSemanticNode;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.GenericSemanticNode;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Page;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Section;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.SectionNode;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.SuperSection;
 import com.knecon.fforesight.service.layoutparser.processor.model.image.ClassifiedImage;
 import com.knecon.fforesight.service.layoutparser.processor.model.table.TablePageBlock;
@@ -30,13 +30,13 @@ import lombok.experimental.UtilityClass;
@UtilityClass
 public class SectionNodeFactory {

-    public Optional<SectionNode> addSection(LayoutParsingType layoutParsingType,
-                                        GenericSemanticNode parentNode,
-                                        boolean isLeaf,
-                                        List<AbstractPageBlock> pageBlocks,
-                                        List<ClassifiedImage> images,
-                                        DocumentGraphFactory.Context context,
-                                        Document document) {
+    public Optional<AbstractSemanticNode> addSection(LayoutParsingType layoutParsingType,
+                                                     GenericSemanticNode parentNode,
+                                                     boolean isLeaf,
+                                                     List<AbstractPageBlock> pageBlocks,
+                                                     List<ClassifiedImage> images,
+                                                     DocumentGraphFactory.Context context,
+                                                     Document document) {

        // This is for the case where we have images on a page without any text/footer/header.
        // The pageBlocks list is empty, but we still need to add those images to the document.
@@ -54,8 +54,9 @@ public class SectionNodeFactory {
        Map<Integer, List<AbstractPageBlock>> blocksPerPage = pageBlocks.stream()
                .collect(groupingBy(AbstractPageBlock::getPage));

-        SectionNode section;
-        if (isLeaf) {
+        AbstractSemanticNode section;
+        boolean containsTablesAndTextBlocks = containsTablesAndTextBlocks(pageBlocks);
+        if (isLeaf && !containsTablesAndTextBlocks) {
            section = Section.builder().documentTree(context.getDocumentTree()).build();
        } else {
            section = SuperSection.builder().documentTree(context.getDocumentTree()).build();
@@ -68,12 +69,7 @@ public class SectionNodeFactory {
        section.setTreeId(getTreeId(parentNode, context, section));

        addFirstHeadlineDirectlyToSection(layoutParsingType, pageBlocks, context, section, document);
-        if (containsTablesAndTextBlocks(pageBlocks)) {
-
-            if (pageBlocks.get(0).isHeadline()) {
-                pageBlocks.remove(0);
-            }
-
+        if (containsTablesAndTextBlocks) {
            splitPageBlocksIntoSubSections(pageBlocks).forEach(subSectionPageBlocks -> addSection(layoutParsingType,
                                                                                                  section,
                                                                                                  true,
@@ -82,10 +78,6 @@ public class SectionNodeFactory {
                                                                                                  context,
                                                                                                  document));
        } else if (!isLeaf) {
-
-            if (pageBlocks.get(0).isHeadline()) {
-                pageBlocks.remove(0);
-            }
            addSection(layoutParsingType, section, true, pageBlocks, emptyList(), context, document);
        } else {
            addTablesAndParagraphsAndHeadlinesToSection(layoutParsingType, pageBlocks, context, section, document);
@@ -99,7 +91,7 @@ public class SectionNodeFactory {
    }


-    private List<Integer> getTreeId(GenericSemanticNode parentNode, DocumentGraphFactory.Context context, SectionNode section) {
+    private List<Integer> getTreeId(GenericSemanticNode parentNode, DocumentGraphFactory.Context context, AbstractSemanticNode section) {

        if (parentNode == null) {
            return context.getDocumentTree().createNewMainEntryAndReturnId(section);
@@ -112,7 +104,7 @@ public class SectionNodeFactory {
    private void addFirstHeadlineDirectlyToSection(LayoutParsingType layoutParsingType,
                                                   List<AbstractPageBlock> pageBlocks,
                                                   DocumentGraphFactory.Context context,
-                                                   SectionNode section,
+                                                   AbstractSemanticNode section,
                                                   Document document) {

        if (pageBlocks.get(0).isHeadline()) {
@@ -125,7 +117,7 @@ public class SectionNodeFactory {
    private void addTablesAndParagraphsAndHeadlinesToSection(LayoutParsingType layoutParsingType,
                                                             List<AbstractPageBlock> pageBlocks,
                                                             DocumentGraphFactory.Context context,
-                                                             SectionNode section,
+                                                             AbstractSemanticNode section,
                                                             Document document) {

        Set<AbstractPageBlock> alreadyMerged = new HashSet<>();
@@ -141,7 +133,7 @@ public class SectionNodeFactory {
            if (abstractPageBlock instanceof TextPageBlock) {

                switch (layoutParsingType) {
-                    case REDACT_MANAGER, DOCUMINE, CLARIFYND, CLARIFYND_PARAGRAPH_DEBUG -> {
+                    case REDACT_MANAGER, DOCUMINE, CLARIFYND, CLARIFYND_PARAGRAPH_DEBUG, REDACT_MANAGER_WITHOUT_DUPLICATE_PARAGRAPH -> {
                        alreadyMerged.add(abstractPageBlock);
                        remainingBlocks.remove(abstractPageBlock);
                        DocumentGraphFactory.addParagraphOrHeadline(section, (TextPageBlock) abstractPageBlock, context, new ArrayList<>(), layoutParsingType);
@@ -250,7 +242,7 @@ public class SectionNodeFactory {
    }


-    private void addSectionNodeToPageNode(DocumentGraphFactory.Context context, SectionNode section, Integer pageNumber) {
+    private void addSectionNodeToPageNode(DocumentGraphFactory.Context context, AbstractSemanticNode section, Integer pageNumber) {

        Page page = context.getPage(pageNumber);
        page.getMainBody().add(section);
@@ -12,7 +12,6 @@ import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBl
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.GenericSemanticNode;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Page;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Section;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.SemanticNode;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Table;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.TableCell;
@@ -23,6 +23,7 @@ import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Pa
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Paragraph;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Section;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.SemanticNode;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.SuperSection;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Table;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.TableCell;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.AtomicTextBlock;
@@ -61,7 +62,8 @@ public class DocumentGraphMapper {
            List<Page> pages = Arrays.stream(entryData.getPageNumbers()).map(pageNumber -> getPage(pageNumber, context)).toList();

            SemanticNode node = switch (entryData.getType()) {
-                case SECTION, SUPER_SECTION -> buildSection(context);
+                case SECTION -> buildSection(context);
+                case SUPER_SECTION -> buildSuperSection(context);
                case PARAGRAPH -> buildParagraph(context, entryData.getProperties());
                case HEADLINE -> buildHeadline(context);
                case HEADER -> buildHeader(context);
@@ -109,7 +111,7 @@ public class DocumentGraphMapper {

    private TableCell buildTableCell(Context context, Map<String, String> properties) {

-        TableCell.TableCellBuilder builder = TableCell.builder();
+        TableCell.TableCellBuilder<?, ?> builder = TableCell.builder();
        PropertiesMapper.parseTableCellProperties(properties, builder);
        return builder.documentTree(context.documentTree).build();
    }
@@ -140,6 +142,11 @@ public class DocumentGraphMapper {
        return Section.builder().documentTree(context.documentTree).build();
    }

+    private SuperSection buildSuperSection(Context context) {
+
+        return SuperSection.builder().documentTree(context.documentTree).build();
+    }
+

    private Paragraph buildParagraph(Context context, Map<String, String> properties) {

@@ -82,6 +82,7 @@ public class LegacyPDFStreamEngine extends PDFStreamEngine {

    private int pageRotation;
    private PDRectangle pageSize;
+    private Matrix translateMatrix;
    private final GlyphList glyphList;
    private final Map<COSDictionary, Float> fontHeightMap = new WeakHashMap<COSDictionary, Float>();

@@ -133,6 +134,12 @@ public class LegacyPDFStreamEngine extends PDFStreamEngine {
        this.pageRotation = page.getRotation();
        this.pageSize = page.getCropBox();

+        if (pageSize.getLowerLeftX() == 0 && pageSize.getLowerLeftY() == 0) {
+            translateMatrix = null;
+        } else {
+            // translation matrix for cropbox
+            translateMatrix = Matrix.getTranslateInstance(-pageSize.getLowerLeftX(), -pageSize.getLowerLeftY());
+        }
        super.processPage(page);
    }

@@ -257,13 +264,22 @@ public class LegacyPDFStreamEngine extends PDFStreamEngine {
                return;
            }
        }
+        // adjust for cropbox if needed
+        Matrix translatedTextRenderingMatrix;
+        if (translateMatrix == null) {
+            translatedTextRenderingMatrix = textRenderingMatrix;
+        } else {
+            translatedTextRenderingMatrix = Matrix.concatenate(translateMatrix, textRenderingMatrix);
+            nextX -= pageSize.getLowerLeftX();
+            nextY -= pageSize.getLowerLeftY();
+        }

        // This is a hack for unicode letter with 2 chars e.g. RA see unicodeProblem.pdf
        if (unicodeMapping.length() == 2) {
            processTextPosition(new TextPosition(pageRotation,
                                                 pageSize.getWidth(),
                                                 pageSize.getHeight(),
-                                                 textRenderingMatrix,
+                                                 translatedTextRenderingMatrix,
                                                 nextX,
                                                 nextY,
                                                 Math.abs(dyDisplay),
@@ -277,7 +293,7 @@ public class LegacyPDFStreamEngine extends PDFStreamEngine {
            processTextPosition(new TextPosition(pageRotation,
                                                 pageSize.getWidth(),
                                                 pageSize.getHeight(),
-                                                 textRenderingMatrix,
+                                                 translatedTextRenderingMatrix,
                                                 nextX,
                                                 nextY,
                                                 Math.abs(dyDisplay),
@@ -293,7 +309,7 @@ public class LegacyPDFStreamEngine extends PDFStreamEngine {
            processTextPosition(new TextPosition(pageRotation,
                                                 pageSize.getWidth(),
                                                 pageSize.getHeight(),
-                                                 textRenderingMatrix,
+                                                 translatedTextRenderingMatrix,
                                                 nextX,
                                                 nextY,
                                                 Math.abs(dyDisplay),
@@ -9,6 +9,7 @@ import java.io.File;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.Comparator;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
@@ -23,6 +24,7 @@ import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Do
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.DuplicatedParagraph;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Page;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.SemanticNode;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.SuperSection;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Table;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.TableCell;
 import com.knecon.fforesight.service.layoutparser.processor.utils.RectangleTransformations;
@@ -62,6 +64,12 @@ public class LayoutGridService {
    static Color HEADER_COLOR = new Color(171, 131, 6);
    static Color IMAGE_COLOR = new Color(253, 63, 146);

+    private record RectangleIdentifier(List<Integer> treeId, Integer pageNumber) {
+
+    }
+
+    HashMap<RectangleIdentifier, Rectangle2D> rectangleMap = new HashMap<>();
+

    @SneakyThrows
    @Observed(name = "ViewerDocumentService", contextualName = "create-viewer-document")
@@ -105,7 +113,7 @@ public class LayoutGridService {
                    Color color = switch (semanticNode.getType()) {
                        case PARAGRAPH -> PARAGRAPH_COLOR;
                        case TABLE -> TABLE_COLOR;
-                        case SECTION -> SECTION_COLOR;
+                        case SECTION, SUPER_SECTION -> SECTION_COLOR;
                        case HEADLINE -> HEADLINE_COLOR;
                        case HEADER, FOOTER -> HEADER_COLOR;
                        case IMAGE -> IMAGE_COLOR;
@@ -119,7 +127,7 @@ public class LayoutGridService {
                    if (isNotSectionOrTableCellOrDocument(semanticNode)) {
                        addAsRectangle(semanticNode, layoutGrid, color);
                    }
-                    if (semanticNode.getType().equals(NodeType.SECTION)) {
+                    if (semanticNode.getType().equals(NodeType.SECTION) || semanticNode.getType().equals(NodeType.SUPER_SECTION)) {
                        addSection(semanticNode, layoutGrid, color);
                    }
                    if (semanticNode.getType().equals(NodeType.TABLE)) {
@@ -165,8 +173,10 @@ public class LayoutGridService {
            List<Double> ys = yStream.collect(Collectors.toList());
            ys.remove(0);

-            Rectangle2D tableBBox = table.getBBox().get(page);
-            List<ColoredLine> coloredLines = layoutGrid.getVisualizationsPerPages().get(page.getNumber() - 1).getColoredLines();
+            Rectangle2D tableBBox = table.getBBox()
+                    .get(page);
+            List<ColoredLine> coloredLines = layoutGrid.getVisualizationsPerPages()
+                    .get(page.getNumber() - 1).getColoredLines();
            xs.forEach(x -> {
                Line2D line = new Line2D.Double(new Point2D.Double(x, tableBBox.getMaxY()), new Point2D.Double(x, tableBBox.getMinY()));
                coloredLines.add(new ColoredLine(line, INNER_LINES_COLOR, LINE_WIDTH));
@@ -190,25 +200,20 @@ public class LayoutGridService {
    private void addSection(SemanticNode semanticNode, LayoutGrid layoutGrid, Color color) {

        Map<Page, Rectangle2D> bBoxMap = semanticNode.getBBox();
+
        List<SemanticNode> subSections = semanticNode.streamAllSubNodesOfType(NodeType.SECTION)
                .toList();
+        Integer maxChildDepth = subSections.stream()
+                .map(node -> node.getTreeId().size())
+                .max(Integer::compareTo)
+                .orElse(semanticNode.getTreeId().size());
+        int ownDepth = semanticNode.getTreeId().size();
+
        Page firstPage = semanticNode.getFirstPage();
        String treeIdString = buildTreeIdString(semanticNode);
-        if (!subSections.isEmpty()) {
-            addPlacedText(firstPage, bBoxMap.get(firstPage), treeIdString, layoutGrid);
-        } else {
-            bBoxMap.forEach(((page, textBBox) -> addPlacedText(page, textBBox, treeIdString, layoutGrid)));
-        }
+
        if (bBoxMap.values().size() == 1) {
-            Rectangle2D r = RectangleTransformations.pad(bBoxMap.get(firstPage), LINE_WIDTH, LINE_WIDTH);
-            List<ColoredLine> coloredLines = layoutGrid.getVisualizationsPerPages().get(firstPage.getNumber() - 1).getColoredLines();
-            List<Line2D> lines = createLinesFromRectangle(r, firstPage.getRotation());
-            // add string to top line
-            var firstLine = lines.remove(0);
-            coloredLines.add(new ColoredLine(firstLine, color, LINE_WIDTH));
-            for (Line2D line : lines) {
-                coloredLines.add(new ColoredLine(line, color, LINE_WIDTH));
-            }
+            handleSinglePage(semanticNode, layoutGrid, color, firstPage, bBoxMap.get(firstPage), treeIdString, maxChildDepth, ownDepth);
            return;
        }
        List<Page> pagesInOrder = bBoxMap.keySet()
@@ -216,99 +221,203 @@ public class LayoutGridService {
                .sorted(Comparator.comparingInt(Page::getNumber))
                .collect(Collectors.toList());
        pagesInOrder.remove(0);
-        addLinesForFirstPageOfSection(semanticNode, color, firstPage, layoutGrid);
-        var lastPage = pagesInOrder.remove(pagesInOrder.size() - 1);
-        addLinesForLastPageOfSection(semanticNode, color, lastPage, layoutGrid);
-        for (Page middlePage : pagesInOrder) {
-            addLinesForMiddlePageOfSection(semanticNode, color, middlePage, layoutGrid);
+        handleFirstPageOfSection(semanticNode, color, firstPage, bBoxMap.get(firstPage), treeIdString, layoutGrid, maxChildDepth, ownDepth);
+        if (semanticNode instanceof SuperSection) {
+            return;
        }
+        for (Page middlePage : pagesInOrder.subList(0, pagesInOrder.size() - 1)) {
+            handleForMiddlePageOfSection(semanticNode, color, middlePage, bBoxMap.get(middlePage), treeIdString, layoutGrid, maxChildDepth, ownDepth);
+        }
+        var lastPage = pagesInOrder.remove(pagesInOrder.size() - 1);
+        handleLastPageOfSection(semanticNode, color, lastPage, bBoxMap.get(lastPage), treeIdString, layoutGrid, maxChildDepth, ownDepth);
    }


    @SneakyThrows
-    private void addPlacedText(Page page, Rectangle2D textBBox, String s, LayoutGrid layoutGrid) {
+    private void addPlacedText(Page page, Rectangle2D textBBox, Rectangle2D highestParentRect, String s, LayoutGrid layoutGrid, Integer maxChildDepth) {

        // translates text, such that its right edge is a bit to the left of the drawn box
-        float translationAmount = ((FONT.getStringWidth(s) / 1000) * FONT_SIZE + (2 * LINE_WIDTH) + 4);
+        float translationAmount = ((FONT.getStringWidth(s) / 1000) * FONT_SIZE + LINE_WIDTH + 2 * maxChildDepth);

        Point2D upperLeftCorner;
        Point2D translationVector;
        switch (page.getRotation()) {
            case 90 -> {
-                upperLeftCorner = new Point2D.Double(textBBox.getMinX(), textBBox.getMinY());
+                if (highestParentRect != null) {
+                    upperLeftCorner = new Point2D.Double(highestParentRect.getMinX(), textBBox.getMinY());
+                } else {
+                    upperLeftCorner = new Point2D.Double(textBBox.getMinX(), textBBox.getMinY());
+                }
                translationVector = new Point2D.Double(FONT_SIZE, -translationAmount);
            }
            case 180 -> {
-                upperLeftCorner = new Point2D.Double(textBBox.getMaxX(), textBBox.getMinY());
+                if (highestParentRect != null) {
+                    upperLeftCorner = new Point2D.Double(highestParentRect.getMaxX(), textBBox.getMinY());
+                } else {
+                    upperLeftCorner = new Point2D.Double(textBBox.getMaxX(), textBBox.getMinY());
+                }
                translationVector = new Point2D.Double(translationAmount, FONT_SIZE);
            }
            case 270 -> {
-                upperLeftCorner = new Point2D.Double(textBBox.getMaxX(), textBBox.getMaxY());
+
+                if (highestParentRect != null) {
+                    upperLeftCorner = new Point2D.Double(highestParentRect.getMaxX(), textBBox.getMaxY());
+                } else {
+                    upperLeftCorner = new Point2D.Double(textBBox.getMaxX(), textBBox.getMaxY());
+                }
                translationVector = new Point2D.Double(-FONT_SIZE, translationAmount);
            }
            default -> {
-                upperLeftCorner = new Point2D.Double(textBBox.getMinX(), textBBox.getMaxY());
+
+                if (highestParentRect != null) {
+                    upperLeftCorner = new Point2D.Double(highestParentRect.getMinX(), textBBox.getMaxY());
+                } else {
+                    upperLeftCorner = new Point2D.Double(textBBox.getMinX(), textBBox.getMaxY());
+                }
                translationVector = new Point2D.Double(-translationAmount, -FONT_SIZE);
            }
        }

        upperLeftCorner = add(upperLeftCorner, translationVector);

-        var placedTexts = layoutGrid.getVisualizationsPerPages().get(page.getNumber() - 1).getPlacedTexts();
-        placedTexts.add(PlacedText.textFacingUp(s, upperLeftCorner, FONT_SIZE, Color.BLACK, FONT));
-    }
+        List<PlacedText> placedTexts = layoutGrid.getVisualizationsPerPages()
+                .get(page.getNumber() - 1).getPlacedTexts();

+        PlacedText newText = PlacedText.textFacingUp(s, upperLeftCorner, FONT_SIZE, Color.BLACK, FONT);

-    private void addLinesForMiddlePageOfSection(SemanticNode semanticNode, Color color, Page middlePage, LayoutGrid layoutGrid) {
+        Optional<PlacedText> conflictingText = placedTexts.stream()
+                .filter(pt -> Math.abs(pt.lineStart().getY() - newText.lineStart().getY()) <= FONT_SIZE)
+                .findFirst();

-        List<ColoredLine> coloredLines = layoutGrid.getVisualizationsPerPages().get(middlePage.getNumber() - 1).getColoredLines();
-        Rectangle2D r = RectangleTransformations.pad(semanticNode.getBBox().get(middlePage), LINE_WIDTH, LINE_WIDTH);
-        var midPageLines = createLinesFromRectangle(r, middlePage.getRotation());
-        // remove top line
-        midPageLines.remove(0);
-        // remove top line
-        midPageLines.remove(1);
-        // add string to left line
-        var leftLine = midPageLines.remove(1);
-        coloredLines.add(new ColoredLine(leftLine, color, LINE_WIDTH));
-        for (Line2D line : midPageLines) {
-            coloredLines.add(new ColoredLine(line, color, LINE_WIDTH));
+        if (conflictingText.isPresent()) {
+            PlacedText existingText = conflictingText.get();
+            if (newText.text().length() > existingText.text().length()) {
+                placedTexts.remove(existingText);
+                placedTexts.add(newText);
+            }
+        } else {
+            placedTexts.add(newText);
        }
+
    }


-    private void addLinesForLastPageOfSection(SemanticNode semanticNode, Color color, Page lastPage, LayoutGrid layoutGrid) {
+    private void handleSinglePage(SemanticNode semanticNode,
+                                  LayoutGrid layoutGrid,
+                                  Color color,
+                                  Page page,
+                                  Rectangle2D rectangle2D,
+                                  String treeIdString,
+                                  Integer maxChildDepth,
+                                  Integer ownDepth) {

-        List<ColoredLine> coloredLines = layoutGrid.getVisualizationsPerPages().get(lastPage.getNumber() - 1).getColoredLines();
-        Rectangle2D r = RectangleTransformations.pad(semanticNode.getBBox().get(lastPage), LINE_WIDTH, LINE_WIDTH);
-        var lastPageLines = createLinesFromRectangle(r, lastPage.getRotation());
-        // remove top line
-        lastPageLines.remove(0);
-        // add string to left line
-        var leftLine = lastPageLines.remove(2);
-        coloredLines.add(new ColoredLine(leftLine, color, LINE_WIDTH));
-        for (Line2D line : lastPageLines) {
-            coloredLines.add(new ColoredLine(line, color, LINE_WIDTH));
-        }
-    }
-
-
-    private void addLinesForFirstPageOfSection(SemanticNode semanticNode, Color color, Page firstPage, LayoutGrid layoutGrid) {
-
-        List<ColoredLine> coloredLines = layoutGrid.getVisualizationsPerPages().get(firstPage.getNumber() - 1).getColoredLines();
-        Rectangle2D r = RectangleTransformations.pad(semanticNode.getBBox().get(firstPage), LINE_WIDTH, LINE_WIDTH);
-        var firstPageLines = createLinesFromRectangle(r, firstPage.getRotation());
-        // remove bottom line
-        firstPageLines.remove(2);
+        RectangleAndLinesResult result = createLinesAndPlaceText(semanticNode, page, rectangle2D, treeIdString, layoutGrid, maxChildDepth, ownDepth);
        // add string to top line
-        var firstLine = firstPageLines.remove(0);
-        coloredLines.add(new ColoredLine(firstLine, color, LINE_WIDTH));
-        for (Line2D line : firstPageLines) {
-            coloredLines.add(new ColoredLine(line, color, LINE_WIDTH));
+        var firstLine = result.pageLines().remove(0);
+        result.coloredLines().add(new ColoredLine(firstLine, color, LINE_WIDTH));
+        for (Line2D line : result.pageLines()) {
+            result.coloredLines().add(new ColoredLine(line, color, LINE_WIDTH));
        }
    }


+    private void handleFirstPageOfSection(SemanticNode semanticNode,
+                                          Color color,
+                                          Page firstPage,
+                                          Rectangle2D rectangle2D,
+                                          String treeIdString,
+                                          LayoutGrid layoutGrid,
+                                          Integer maxChildDepth,
+                                          Integer ownDepth) {
+
+        RectangleAndLinesResult result = createLinesAndPlaceText(semanticNode, firstPage, rectangle2D, treeIdString, layoutGrid, maxChildDepth, ownDepth);
+        // remove bottom line
+        result.pageLines().remove(2);
+        // add string to top line
+        var firstLine = result.pageLines().remove(0);
+        result.coloredLines().add(new ColoredLine(firstLine, color, LINE_WIDTH));
+        for (Line2D line : result.pageLines()) {
+            result.coloredLines().add(new ColoredLine(line, color, LINE_WIDTH));
+        }
+    }
+
+
+    private void handleForMiddlePageOfSection(SemanticNode semanticNode,
+                                              Color color,
+                                              Page middlePage,
+                                              Rectangle2D rectangle2D,
+                                              String treeIdString,
+                                              LayoutGrid layoutGrid,
+                                              Integer maxChildDepth,
+                                              Integer ownDepth) {
+
+        RectangleAndLinesResult result = createLinesAndPlaceText(semanticNode, middlePage, rectangle2D, treeIdString, layoutGrid, maxChildDepth, ownDepth);
+        // remove top line
+        result.pageLines().remove(0);
+        // remove bottom line
+        result.pageLines().remove(1);
+        // add string to left line
+        var leftLine = result.pageLines().remove(1);
+        result.coloredLines().add(new ColoredLine(leftLine, color, LINE_WIDTH));
+        for (Line2D line : result.pageLines()) {
+            result.coloredLines().add(new ColoredLine(line, color, LINE_WIDTH));
+        }
+    }
+
+
+    private void handleLastPageOfSection(SemanticNode semanticNode,
+                                         Color color,
+                                         Page lastPage,
+                                         Rectangle2D rectangle2D,
+                                         String treeIdString,
+                                         LayoutGrid layoutGrid,
+                                         Integer maxChildDepth,
+                                         Integer ownDepth) {
+
+        RectangleAndLinesResult result = createLinesAndPlaceText(semanticNode, lastPage, rectangle2D, treeIdString, layoutGrid, maxChildDepth, ownDepth);
+        // remove top line
+        result.pageLines().remove(0);
+        // add string to left line
+        var leftLine = result.pageLines().remove(2);
+        result.coloredLines().add(new ColoredLine(leftLine, color, LINE_WIDTH));
+        for (Line2D line : result.pageLines()) {
+            result.coloredLines().add(new ColoredLine(line, color, LINE_WIDTH));
+        }
+    }
+
+
+    private RectangleAndLinesResult createLinesAndPlaceText(SemanticNode semanticNode,
+                                                            Page page,
+                                                            Rectangle2D rectangle2D,
+                                                            String treeIdString,
+                                                            LayoutGrid layoutGrid,
+                                                            Integer maxChildDepth,
+                                                            Integer ownDepth) {
+
+        List<ColoredLine> coloredLines = layoutGrid.getVisualizationsPerPages()
+                .get(page.getNumber() - 1).getColoredLines();
+        int lineWidthModifier = maxChildDepth - ownDepth;
+        Rectangle2D r = RectangleTransformations.pad(semanticNode.getBBox()
+                                                             .get(page), LINE_WIDTH * (1 + lineWidthModifier), LINE_WIDTH * (1 + lineWidthModifier));
+        var lastPageLines = createLinesFromRectangle(r, page.getRotation());
+
+        SemanticNode highestParent = semanticNode.getHighestParent();
+        Rectangle2D highestParentRect = rectangleMap.get(new RectangleIdentifier(highestParent.getTreeId(), page.getNumber()));
+        addPlacedText(page, rectangle2D, highestParentRect, treeIdString, layoutGrid, maxChildDepth);
+
+        if (semanticNode instanceof SuperSection) {
+            rectangleMap.put(new RectangleIdentifier(semanticNode.getTreeId(), page.getNumber()), r);
+        }
+
+        return new RectangleAndLinesResult(coloredLines, r, lastPageLines);
+    }
+
+
+    private record RectangleAndLinesResult(List<ColoredLine> coloredLines, Rectangle2D rectangle, List<Line2D> pageLines) {
+
+    }
+
+
    private String buildTreeIdString(SemanticNode semanticNode) {

        return semanticNode.getTreeId()
@@ -366,7 +475,10 @@ public class LayoutGridService {

    private static boolean isNotSectionOrTableCellOrDocument(SemanticNode semanticNode) {

-        return !(semanticNode.getType().equals(NodeType.DOCUMENT) || semanticNode.getType().equals(NodeType.SECTION) || semanticNode.getType().equals(NodeType.TABLE_CELL));
+        return !(semanticNode.getType().equals(NodeType.DOCUMENT)
+                 || semanticNode.getType().equals(NodeType.SECTION)
+                 || semanticNode.getType().equals(NodeType.SUPER_SECTION)
+                 || semanticNode.getType().equals(NodeType.TABLE_CELL));
    }


@@ -374,9 +486,7 @@ public class LayoutGridService {

        semanticNode.getBBox()
                .forEach((page, textBBox) -> layoutGrid.getVisualizationsPerPages()
-                        .get(page.getNumber() - 1)
-                        .getColoredRectangles()
-                        .add(new ColoredRectangle(textBBox, color, LINE_WIDTH)));
+                        .get(page.getNumber() - 1).getColoredRectangles().add(new ColoredRectangle(textBBox, color, LINE_WIDTH)));
    }


@@ -0,0 +1,7 @@
+package com.knecon.fforesight.service.layoutparser.processor.utils;
+
+public class ParsingConstants {
+
+    public final static float NEW_LINE_TEXT_HEIGHT_PERCENTAGE = 0.6f;
+
+}
@@ -51,13 +51,47 @@ public class TextPositionSequenceComparator implements Comparator<TextPositionSe

        double yDifference = Math.abs(pos1YBottom - pos2YBottom);

-        // we will do a simple tolerance comparison
-        if (yDifference < .1 || pos2YBottom >= pos1YTop && pos2YBottom <= pos1YBottom || pos1YBottom >= pos2YTop && pos1YBottom <= pos2YBottom) {
-            return Double.compare(x1, x2);
-        } else if (pos1YBottom < pos2YBottom) {
-            return -1;
-        } else {
-            return 1;
+        // Adjust for text rotation
+        switch (pos1.getRotation()) {
+            case 0:
+                // 0 degrees (horizontal, top to bottom and left to right): Sort primarily by y-coordinates from top to bottom (pos1YBottom < pos2YBottom).
+                if (yDifference < .1 || (pos2YBottom >= pos1YTop && pos2YBottom <= pos1YBottom) || (pos1YBottom >= pos2YTop && pos1YBottom <= pos2YBottom)) {
+                    return Double.compare(x1, x2);
+                } else if (pos1YBottom < pos2YBottom) {
+                    return -1;
+                } else {
+                    return 1;
+                }
+            case 90:
+                // 90 degrees (vertical, right to left): Sort by x-coordinates first (x1 > x2), then by y-coordinates from top to bottom (pos1YBottom < pos2YBottom).
+                if (x1 > x2) {
+                    return -1;
+                } else if (x1 < x2) {
+                    return 1;
+                } else {
+                    return Double.compare(pos1YBottom, pos2YBottom);
+                }
+            case 180:
+                // 180 degrees (horizontal, bottom to top and right to left): Sort primarily by y-coordinates from bottom to top (pos1YBottom > pos2YBottom).
+                if (yDifference < .1 || (pos2YBottom >= pos1YTop && pos2YBottom <= pos1YBottom) || (pos1YBottom >= pos2YTop && pos1YBottom <= pos2YBottom)) {
+                    return Double.compare(x2, x1);
+
+                } else if (pos1YBottom > pos2YBottom) {
+                    return -1;
+                } else {
+                    return 1;
+                }
+            case 270:
+                // 270 degrees (vertical, left to right): Sort by x-coordinates in reverse (x2 > x1), then by y-coordinates from bottom to top (pos2YBottom > pos1YBottom).
+                if (x2 > x1) {
+                    return -1;
+                } else if (x2 < x1) {
+                    return 1;
+                } else {
+                    return Double.compare(pos2YBottom, pos1YBottom);
+                }
+            default:
+                throw new RuntimeException("Rotation not supported. Only 0/90/180/270 degree rotation is supported.");
        }
    }

@@ -32,6 +32,7 @@ dependencies {
    implementation("com.iqser.red.commons:storage-commons:2.45.0")
    implementation("com.knecon.fforesight:tenant-commons:0.21.0")
    implementation("com.knecon.fforesight:tracing-commons:0.5.0")
+    implementation("com.knecon.fforesight:lifecycle-commons:0.6.0")
    implementation("org.springframework.boot:spring-boot-starter-actuator:${springBootStarterVersion}")
    implementation("org.springframework.boot:spring-boot-starter-amqp:${springBootStarterVersion}")
    implementation("com.amazonaws:aws-java-sdk-s3:1.12.536")
@@ -6,19 +6,22 @@ import org.springframework.boot.autoconfigure.ImportAutoConfiguration;
 import org.springframework.boot.autoconfigure.SpringBootApplication;
 import org.springframework.boot.autoconfigure.security.servlet.SecurityAutoConfiguration;
 import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.EnableAspectJAutoProxy;
 import org.springframework.context.annotation.Import;

 import com.amazonaws.services.s3.model.metrics.MetricsConfiguration;
 import com.iqser.red.storage.commons.StorageAutoConfiguration;
+import com.knecon.fforesight.lifecyclecommons.LifecycleAutoconfiguration;
 import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingServiceProcessorConfiguration;
 import com.knecon.fforesight.tenantcommons.MultiTenancyAutoConfiguration;

 import io.micrometer.observation.ObservationRegistry;
 import io.micrometer.observation.aop.ObservedAspect;

-@ImportAutoConfiguration({MultiTenancyAutoConfiguration.class})
+@ImportAutoConfiguration({MultiTenancyAutoConfiguration.class, LifecycleAutoconfiguration.class})
@Import({MetricsConfiguration.class, StorageAutoConfiguration.class, LayoutParsingServiceProcessorConfiguration.class})
@SpringBootApplication(exclude = {SecurityAutoConfiguration.class, ManagementWebSecurityAutoConfiguration.class})
+@EnableAspectJAutoProxy
 public class Application {

    public static void main(String[] args) {
@@ -15,6 +15,9 @@ project.version: 1.0-SNAPSHOT
 server:
  port: 8080

+lifecycle:
+  base-package: com.knecon.fforesight.service.layoutparser
+
 spring:
  application:
    name: layoutparser-service
@@ -70,7 +70,7 @@ public class LayoutparserEnd2EndTest extends AbstractTest {
            file = new File(filePath);
        }

-        LayoutParsingRequest layoutParsingRequest = buildDefaultLayoutParsingRequest(fileName, LayoutParsingType.REDACT_MANAGER, true);
+        LayoutParsingRequest layoutParsingRequest = buildDefaultLayoutParsingRequest(fileName, LayoutParsingType.REDACT_MANAGER_WITHOUT_DUPLICATE_PARAGRAPH, true);
        prepareStorage(layoutParsingRequest, file);

        LayoutParsingFinishedEvent finishedEvent = layoutParsingPipeline.parseLayoutAndSaveFilesToStorage(layoutParsingRequest);
@@ -0,0 +1,231 @@
+package com.knecon.fforesight.service.layoutparser.server;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.io.File;
+import java.nio.file.Path;
+import java.util.Collection;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.function.Predicate;
+
+import org.apache.commons.lang3.StringUtils;
+import org.junit.jupiter.api.Test;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.core.io.ClassPathResource;
+
+import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingRequest;
+import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
+import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingPipeline;
+import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationDocument;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Section;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.SemanticNode;
+import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.SuperSection;
+import com.knecon.fforesight.service.layoutparser.processor.model.outline.OutlineObject;
+import com.knecon.fforesight.service.layoutparser.processor.model.outline.OutlineObjectTree;
+import com.knecon.fforesight.service.layoutparser.processor.model.outline.TableOfContents;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableServiceResponse;
+import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.VisualLayoutParsingResponse;
+import com.knecon.fforesight.service.layoutparser.processor.services.factory.DocumentGraphFactory;
+import com.knecon.fforesight.service.layoutparser.processor.services.visualization.LayoutGridService;
+import com.knecon.fforesight.service.layoutparser.server.utils.AbstractTest;
+import com.knecon.fforesight.service.viewerdoc.service.ViewerDocumentService;
+
+import lombok.SneakyThrows;
+
+public class OutlineDetectionTest extends AbstractTest {
+
+    ViewerDocumentService viewerDocumentService = new ViewerDocumentService(null);
+    LayoutGridService layoutGridService = new LayoutGridService(viewerDocumentService);
+
+    @Autowired
+    protected LayoutParsingPipeline layoutParsingPipeline;
+
+
+    @Test
+    @SneakyThrows
+    public void testOutlinesToSections() {
+
+        String fileName = "files/new/crafted_outline_test_doc.pdf";
+
+        String tmpFileName = "/tmp/" + Path.of(fileName).getFileName() + "_VIEWER.pdf";
+
+        var documentFile = new ClassPathResource(fileName).getFile();
+
+        long start = System.currentTimeMillis();
+        ClassificationDocument classificationDocument = parseLayout(fileName, LayoutParsingType.REDACT_MANAGER_WITHOUT_DUPLICATE_PARAGRAPH);
+
+        OutlineObjectTree outlineObjectTree = classificationDocument.getOutlineObjectTree();
+        assertEquals(outlineObjectTree.getRootNodes().size(), 8);
+        assertEquals(outlineObjectTree.getOutlineObjectsPerPage()
+                             .get(1).size(), 1);
+        assertEquals(outlineObjectTree.getOutlineObjectsPerPage()
+                             .get(3).size(), 1);
+        assertEquals(outlineObjectTree.getOutlineObjectsPerPage()
+                             .get(4).size(), 1);
+        assertEquals(outlineObjectTree.getOutlineObjectsPerPage()
+                             .get(5).size(), 1);
+        assertEquals(outlineObjectTree.getOutlineObjectsPerPage()
+                             .get(6).size(), 2);
+        assertEquals(outlineObjectTree.getOutlineObjectsPerPage()
+                             .get(7).size(), 3);
+        assertEquals(outlineObjectTree.getOutlineObjectsPerPage()
+                             .get(8).size(), 2);
+        assertEquals(outlineObjectTree.getOutlineObjectsPerPage()
+                             .get(10).size(), 1);
+        assertEquals(outlineObjectTree.getOutlineObjectsPerPage()
+                             .get(11).size(), 4);
+        assertEquals(outlineObjectTree.getOutlineObjectsPerPage()
+                             .get(12).size(), 1);
+        assertEquals(outlineObjectTree.getOutlineObjectsPerPage()
+                             .get(13).size(), 2);
+        assertTrue(outlineObjectTree.getOutlineObjectsPerPage().values()
+                           .stream()
+                           .flatMap(Collection::stream)
+                           .allMatch(OutlineObject::isFound));
+
+        TableOfContents tableOfContents = classificationDocument.getTableOfContents();
+
+        assertEquals(tableOfContents.getMainSections().size(), 9);
+        assertEquals(tableOfContents.getMainSections().subList(1, 9)
+                             .stream()
+                             .map(tableOfContentItem -> sanitizeString(tableOfContentItem.getHeadline().toString()))
+                             .toList(),
+                     outlineObjectTree.getRootNodes()
+                             .stream()
+                             .map(outlineObjectTreeNode -> sanitizeString(outlineObjectTreeNode.getOutlineObject().getTitle()))
+                             .toList());
+        assertEquals(tableOfContents.getMainSections()
+                             .get(5).getChildren().size(), 6);
+        assertEquals(tableOfContents.getMainSections()
+                             .get(7).getChildren().size(), 3);
+        assertEquals(tableOfContents.getMainSections()
+                             .get(8).getChildren().size(), 3);
+        assertEquals(tableOfContents.getMainSections()
+                             .get(8).getChildren()
+                             .get(2).getChildren().size(), 1);
+        assertEquals(tableOfContents.getMainSections()
+                             .get(8).getChildren()
+                             .get(2).getChildren()
+                             .get(0).getChildren().size(), 3);
+
+        assertEquals(tableOfContents.getMainSections()
+                             .get(0).getImages().size(), 1);
+        assertEquals(tableOfContents.getMainSections()
+                             .get(6).getImages().size(), 1);
+        assertEquals(tableOfContents.getMainSections()
+                             .get(8).getChildren()
+                             .get(2).getChildren()
+                             .get(0).getChildren()
+                             .get(2).getImages().size(), 1);
+
+        Document document = buildGraph(fileName, classificationDocument);
+
+        assertTrue(tableOfContents.getAllTableOfContentItems()
+                           .stream()
+                           .allMatch(tableOfContentItem -> tableOfContentItem.getSection() != null));
+        assertTrue(tableOfContents.getAllTableOfContentItems()
+                           .stream()
+                           .filter(tableOfContentItem -> tableOfContentItem.getChildren().isEmpty())
+                           .allMatch(tableOfContentItem -> tableOfContentItem.getSection() instanceof Section));
+        assertTrue(tableOfContents.getAllTableOfContentItems()
+                           .stream()
+                           .filter(tableOfContentItem -> !tableOfContentItem.getChildren().isEmpty())
+                           .allMatch(tableOfContentItem -> tableOfContentItem.getSection() instanceof SuperSection));
+
+        List<SemanticNode> childrenOfTypeSectionOrSuperSection = document.getChildrenOfTypeSectionOrSuperSection();
+
+        assertEquals(childrenOfTypeSectionOrSuperSection.size(), 9);
+        assertEquals(childrenOfTypeSectionOrSuperSection.subList(1, 9)
+                             .stream()
+                             .map(section -> sanitizeString(section.getHeadline().getLeafTextBlock().toString()))
+                             .toList(),
+                     outlineObjectTree.getRootNodes()
+                             .stream()
+                             .map(outlineObjectTreeNode -> sanitizeString(outlineObjectTreeNode.getOutlineObject().getTitle()))
+                             .toList());
+        Predicate<SemanticNode> isSectionOrSuperSection = semanticNode -> semanticNode instanceof Section || semanticNode instanceof SuperSection;
+        assertEquals(childrenOfTypeSectionOrSuperSection.get(5).streamChildren()
+                             .filter(isSectionOrSuperSection)
+                             .count(), 6 + 1); // 1 additional for main text of parent section
+        assertEquals(childrenOfTypeSectionOrSuperSection.get(7).streamChildren()
+                             .filter(isSectionOrSuperSection)
+                             .count(), 3 + 1);
+        assertEquals(childrenOfTypeSectionOrSuperSection.get(8).streamChildren()
+                             .filter(isSectionOrSuperSection)
+                             .count(), 3 + 1);
+        assertEquals(childrenOfTypeSectionOrSuperSection.get(8).streamChildren()
+                             .filter(isSectionOrSuperSection)
+                             .toList()
+                             .get(3).streamChildren()
+                             .filter(isSectionOrSuperSection)
+                             .count(), 1 + 1);
+        assertEquals(childrenOfTypeSectionOrSuperSection.get(8).streamChildren()
+                             .filter(isSectionOrSuperSection)
+                             .toList()
+                             .get(3).streamChildren()
+                             .filter(isSectionOrSuperSection)
+                             .toList()
+                             .get(1).streamChildren()
+                             .filter(isSectionOrSuperSection)
+                             .count(), 3 + 1);
+
+        List<List<Integer>> imageTreeIdList = document.streamAllImages()
+                .map(image -> image.getParent().getTreeId())
+                .toList();
+
+        assertEquals(imageTreeIdList.get(0), List.of(0));
+        assertEquals(imageTreeIdList.get(1), List.of(6));
+        assertEquals(imageTreeIdList.get(2), List.of(8, 4, 2, 4));
+
+        layoutGridService.addLayoutGrid(documentFile, document, new File(tmpFileName), true, false);
+        System.out.printf("Total time: %.2fs%n", ((float) (System.currentTimeMillis() - start)) / 1000);
+    }
+
+
+    private static String sanitizeString(String text) {
+
+        return StringUtils.deleteWhitespace(text).toLowerCase(Locale.ROOT);
+    }
+
+
+    @SneakyThrows
+    protected ClassificationDocument parseLayout(String filename, LayoutParsingType layoutParsingType) {
+
+        File fileResource = new ClassPathResource(filename).getFile();
+        prepareStorage(filename, "cv_table_parsing_response/empty.json", "image_service_response/crafted_outline_test_doc.IMAGE_INFO.json");
+        return layoutParsingPipeline.parseLayout(layoutParsingType,
+                                                 fileResource,
+                                                 layoutParsingStorageService.getImagesFile(IMAGE_FILE_ID),
+                                                 new TableServiceResponse(),
+                                                 new VisualLayoutParsingResponse(),
+                                                 Map.of("file", filename, "debug", "true"));
+    }
+
+
+    @SneakyThrows
+    protected Document buildGraph(String filename, ClassificationDocument classificationDocument) {
+
+        if (!filename.startsWith("files") && filename.startsWith("/")) {
+            LayoutParsingRequest layoutParsingRequest = buildDefaultLayoutParsingRequest(Path.of(filename).getFileName().toString(), LayoutParsingType.REDACT_MANAGER, true);
+            prepareStorage(layoutParsingRequest, new File(filename));
+            return DocumentGraphFactory.buildDocumentGraph(LayoutParsingType.REDACT_MANAGER,
+                                                           layoutParsingPipeline.parseLayout(LayoutParsingType.REDACT_MANAGER,
+                                                                                             new File(filename),
+                                                                                             layoutParsingStorageService.getImagesFile(layoutParsingRequest.imagesFileStorageId()
+                                                                                                                                               .get()),
+                                                                                             new TableServiceResponse(),
+                                                                                             new VisualLayoutParsingResponse(),
+                                                                                             layoutParsingRequest.identifier()));
+        } else {
+            prepareStorage(filename);
+            return DocumentGraphFactory.buildDocumentGraph(LayoutParsingType.REDACT_MANAGER, classificationDocument);
+        }
+
+    }
+
+}
+
@@ -64,8 +64,8 @@ public class SimplifiedTextServiceTest
    @SneakyThrows
    protected Document buildGraph(File file) {

-        return DocumentGraphFactory.buildDocumentGraph(LayoutParsingType.REDACT_MANAGER,
-                                                       layoutParsingPipeline.parseLayout(LayoutParsingType.REDACT_MANAGER,
+        return DocumentGraphFactory.buildDocumentGraph(LayoutParsingType.REDACT_MANAGER_WITHOUT_DUPLICATE_PARAGRAPH,
+                                                       layoutParsingPipeline.parseLayout(LayoutParsingType.REDACT_MANAGER_WITHOUT_DUPLICATE_PARAGRAPH,
                                                                                         file,
                                                                                         new ImageServiceResponse(),
                                                                                         new TableServiceResponse(),
@@ -56,8 +56,8 @@ public class DocumentGraphJsonWritingTest extends BuildDocumentTest {
    @SneakyThrows
    private void writeJsons(Path filename) {

-        Document documentGraph = DocumentGraphFactory.buildDocumentGraph(LayoutParsingType.REDACT_MANAGER_OLD,
-                layoutParsingPipeline.parseLayout(LayoutParsingType.REDACT_MANAGER_OLD,
+        Document documentGraph = DocumentGraphFactory.buildDocumentGraph(LayoutParsingType.REDACT_MANAGER_WITHOUT_DUPLICATE_PARAGRAPH,
+                layoutParsingPipeline.parseLayout(LayoutParsingType.REDACT_MANAGER_WITHOUT_DUPLICATE_PARAGRAPH,
                        filename.toFile(),
                        new ImageServiceResponse(),
                        new TableServiceResponse(),
@@ -38,7 +38,7 @@ public class ViewerDocumentTest extends BuildDocumentTest {
        var documentFile = new ClassPathResource(fileName).getFile();

        long start = System.currentTimeMillis();
-        Document document = buildGraph(fileName, LayoutParsingType.REDACT_MANAGER);
+        Document document = buildGraph(fileName, LayoutParsingType.REDACT_MANAGER_WITHOUT_DUPLICATE_PARAGRAPH);
        layoutGridService.addLayoutGrid(documentFile, document, new File(tmpFileName), true, false);
        System.out.printf("Total time: %.2fs%n", ((float) (System.currentTimeMillis() - start)) / 1000);
    }
@@ -54,7 +54,7 @@ public class PdfSegmentationServiceTest extends AbstractTest {
    @SneakyThrows
    public ClassificationDocument buildClassificationDocument(File originDocument, TableServiceResponse tableServiceResponse) {

-        return layoutParsingPipeline.parseLayout(LayoutParsingType.REDACT_MANAGER,
+        return layoutParsingPipeline.parseLayout(LayoutParsingType.REDACT_MANAGER_WITHOUT_DUPLICATE_PARAGRAPH,
                                                 originDocument,
                                                 new ImageServiceResponse(),
                                                 tableServiceResponse,
@@ -122,7 +122,7 @@ public class PdfSegmentationServiceTest extends AbstractTest {
                           .get(0).getTextBlocks()
                           .get(0).toString()).contains(textToSearch);

-        Document document = DocumentGraphFactory.buildDocumentGraph(LayoutParsingType.REDACT_MANAGER, classificationDocument);
+        Document document = DocumentGraphFactory.buildDocumentGraph(LayoutParsingType.REDACT_MANAGER_WITHOUT_DUPLICATE_PARAGRAPH, classificationDocument);

        TextBlock leafTextBlock = document.getFirstPage().getHeader().getLeafTextBlock();
        assertThat(leafTextBlock.getSearchText().contains(textToSearch)).isTrue();
@@ -103,15 +103,15 @@ public class RulingCleaningServiceTest extends BuildDocumentTest {
    @SneakyThrows
    private void writeJsons(Path filename) {

-        Document documentGraphBefore = DocumentGraphFactory.buildDocumentGraph(LayoutParsingType.REDACT_MANAGER,
-                layoutParsingPipeline.parseLayout(LayoutParsingType.REDACT_MANAGER,
+        Document documentGraphBefore = DocumentGraphFactory.buildDocumentGraph(LayoutParsingType.REDACT_MANAGER_WITHOUT_DUPLICATE_PARAGRAPH,
+                layoutParsingPipeline.parseLayout(LayoutParsingType.REDACT_MANAGER_WITHOUT_DUPLICATE_PARAGRAPH,
                        filename.toFile(),
                        new ImageServiceResponse(),
                        new TableServiceResponse(),
                        new VisualLayoutParsingResponse(),
                        Map.of("file",filename.toFile().toString())));
-        Document documentGraphAfter = DocumentGraphFactory.buildDocumentGraph(LayoutParsingType.REDACT_MANAGER,
-                layoutParsingPipeline.parseLayout(LayoutParsingType.REDACT_MANAGER,
+        Document documentGraphAfter = DocumentGraphFactory.buildDocumentGraph(LayoutParsingType.REDACT_MANAGER_WITHOUT_DUPLICATE_PARAGRAPH,
+                layoutParsingPipeline.parseLayout(LayoutParsingType.REDACT_MANAGER_WITHOUT_DUPLICATE_PARAGRAPH,
                        filename.toFile(),
                        new ImageServiceResponse(),
                        new TableServiceResponse(),
@@ -48,8 +48,9 @@ public abstract class BuildDocumentTest extends AbstractTest {
    @SneakyThrows
    protected Document buildGraph(String filename, LayoutParsingType layoutParsingType) {

+
        if (!filename.startsWith("files") && filename.startsWith("/")) {
-            LayoutParsingRequest layoutParsingRequest = buildDefaultLayoutParsingRequest(Path.of(filename).getFileName().toString(), LayoutParsingType.REDACT_MANAGER, true);
+            LayoutParsingRequest layoutParsingRequest = buildDefaultLayoutParsingRequest(Path.of(filename).getFileName().toString(), LayoutParsingType.REDACT_MANAGER_WITHOUT_DUPLICATE_PARAGRAPH, true);
            prepareStorage(layoutParsingRequest, new File(filename));
            return DocumentGraphFactory.buildDocumentGraph(layoutParsingType,
                                                           layoutParsingPipeline.parseLayout(layoutParsingType,
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8afb731a307e1a3f827c59e902164b10bdabef96e14193b949fe081cd3aa859f
+size 168878
@@ -0,0 +1,135 @@
+{
+  "dossierId": "a91f19ff-11ba-4735-9f60-c650243f64a9",
+  "fileId": "6e8c5f114e2b71e103a32a20c5273188",
+  "targetFileExtension": "ORIGIN.pdf.gz",
+  "responseFileExtension": "IMAGE_INFO.json.gz",
+  "X-TENANT-ID": "redaction",
+  "data": [
+    {
+      "classification": {
+        "label": "other",
+        "probabilities": {
+          "other": 0.9126,
+          "formula": 0.0588,
+          "signature": 0.0261,
+          "logo": 0.0024
+        }
+      },
+      "representation": "70E1070C1030E081B7EF7FFFF",
+      "position": {
+        "x1": 61,
+        "x2": 394,
+        "y1": 155,
+        "y2": 470,
+        "pageNumber": 1
+      },
+      "geometry": {
+        "width": 333,
+        "height": 315
+      },
+      "alpha": false,
+      "filters": {
+        "geometry": {
+          "imageSize": {
+            "quotient": 0.5976,
+            "tooLarge": false,
+            "tooSmall": false
+          },
+          "imageFormat": {
+            "quotient": 1.0571,
+            "tooTall": false,
+            "tooWide": false
+          }
+        },
+        "probability": {
+          "unconfident": false
+        },
+        "allPassed": true
+      }
+    },
+    {
+      "classification": {
+        "label": "other",
+        "probabilities": {
+          "other": 0.9126,
+          "formula": 0.0588,
+          "signature": 0.0261,
+          "logo": 0.0024
+        }
+      },
+      "representation": "70E1070C1030E081B7EF7FFFF",
+      "position": {
+        "x1": 61,
+        "x2": 394,
+        "y1": 202,
+        "y2": 517,
+        "pageNumber": 11
+      },
+      "geometry": {
+        "width": 333,
+        "height": 315
+      },
+      "alpha": false,
+      "filters": {
+        "geometry": {
+          "imageSize": {
+            "quotient": 0.5976,
+            "tooLarge": false,
+            "tooSmall": false
+          },
+          "imageFormat": {
+            "quotient": 1.0571,
+            "tooTall": false,
+            "tooWide": false
+          }
+        },
+        "probability": {
+          "unconfident": false
+        },
+        "allPassed": true
+      }
+    },
+    {
+      "classification": {
+        "label": "other",
+        "probabilities": {
+          "other": 0.9126,
+          "formula": 0.0588,
+          "signature": 0.0261,
+          "logo": 0.0024
+        }
+      },
+      "representation": "70E1070C1030E081B7EF7FFFF",
+      "position": {
+        "x1": 47,
+        "x2": 379,
+        "y1": 289,
+        "y2": 604,
+        "pageNumber": 16
+      },
+      "geometry": {
+        "width": 332,
+        "height": 315
+      },
+      "alpha": false,
+      "filters": {
+        "geometry": {
+          "imageSize": {
+            "quotient": 0.5967,
+            "tooLarge": false,
+            "tooSmall": false
+          },
+          "imageFormat": {
+            "quotient": 1.054,
+            "tooTall": false,
+            "tooWide": false
+          }
+        },
+        "probability": {
+          "unconfident": false
+        },
+        "allPassed": true
+      }
+    }
+  ]
+}
@@ -52,17 +52,16 @@ public class ContentStreamUtility {

    public static Optional<COSName> findLayoutGridOCGName(PDPage page) {

-        Optional<COSName> layoutGridOCGName = Optional.empty();
        var resourceIterator = page.getResources().getPropertiesNames();
        for (COSName cosName : resourceIterator) {
            COSBase cosBase = page.getResources().getProperties(cosName).getCOSObject().getDictionaryObject(COSName.NAME);
            if (cosBase instanceof COSString string) {
                if (ContentStreams.KNECON_LAYOUT.name().equals(string.getString())) {
-                    layoutGridOCGName = Optional.of(cosName);
+                    return Optional.of(cosName);
                }
            }
        }
-        return layoutGridOCGName;
+        return Optional.empty();
    }


@@ -1,6 +1,7 @@
 package com.knecon.fforesight.service.viewerdoc.service;

 import java.awt.geom.AffineTransform;
+import java.awt.geom.Line2D;
 import java.awt.geom.Rectangle2D;
 import java.io.File;
 import java.io.FileOutputStream;
@@ -84,6 +85,7 @@ public class ViewerDocumentService {
            pdPage.setContents(ContentStreamUtility.removeLayerFromContentStreams(allLayers, classifiers));

            AffineTransform textDeRotationMatrix = getTextDeRotationTransform(pdPage);
+            AffineTransform pageTransformationMatrix = getPageTransformationMatrix(pdPage);

            if (!ContentStreamClassifier.areAllContentStreamsEscaped(classifiers)) {
                // We need to save the graphics state before, such that our appended content cannot be affected by previous content streams with side effects,
@@ -106,7 +108,11 @@ public class ViewerDocumentService {

                    contentStream.saveGraphicsState();

-                    drawVisualizationsToContentStream(pdDocument, visualization.getVisualizationsOnPages().get(pageNumber), contentStream, textDeRotationMatrix);
+                    drawVisualizationsToContentStream(pdDocument,
+                                                      visualization.getVisualizationsOnPages().get(pageNumber),
+                                                      contentStream,
+                                                      textDeRotationMatrix,
+                                                      pageTransformationMatrix);

                    contentStream.restoreGraphicsState();

@@ -133,6 +139,12 @@ public class ViewerDocumentService {
    }


+    private AffineTransform getPageTransformationMatrix(PDPage pdPage) {
+
+        return new AffineTransform(1, 0, 0, 1, pdPage.getCropBox().getLowerLeftX(), pdPage.getCropBox().getLowerLeftY());
+    }
+
+
    private static Map<ContentStreams.Identifier, PDOptionalContentGroup> addLayersToDocument(List<Visualizations> visualizations, PDDocument pdDocument) {

        Map<ContentStreams.Identifier, PDOptionalContentGroup> optionalContentGroupMap = new HashMap<>();
@@ -147,7 +159,8 @@ public class ViewerDocumentService {
    private static void drawVisualizationsToContentStream(PDDocument pdDocument,
                                                          VisualizationsOnPage visualizationsOnPage,
                                                          PDPageContentStream contentStream,
-                                                          AffineTransform textDeRotationMatrix) throws IOException {
+                                                          AffineTransform textDeRotationMatrix,
+                                                          AffineTransform pageTransformationMatrix) throws IOException {

        if (visualizationsOnPage.isMakePathsInvisible()) {
            contentStream.addRect(0, 0, 1, 1);
@@ -155,17 +168,18 @@ public class ViewerDocumentService {
        }

        for (ColoredLine coloredLine : visualizationsOnPage.getColoredLines()) {
+            Line2D line = transformLine(coloredLine.line(), pageTransformationMatrix);
            contentStream.setLineWidth(coloredLine.lineWidth());
            contentStream.setStrokingColor(coloredLine.color());
-            contentStream.moveTo((float) coloredLine.line().getX1(), (float) coloredLine.line().getY1());
-            contentStream.lineTo((float) coloredLine.line().getX2(), (float) coloredLine.line().getY2());
+            contentStream.moveTo((float) line.getX1(), (float) line.getY1());
+            contentStream.lineTo((float) line.getX2(), (float) line.getY2());
            contentStream.stroke();
        }

        for (ColoredRectangle coloredRectangle : visualizationsOnPage.getColoredRectangles()) {
            contentStream.setLineWidth(coloredRectangle.lineWidth());
            contentStream.setStrokingColor(coloredRectangle.color());
-            Rectangle2D r = coloredRectangle.rectangle2D();
+            Rectangle2D r = transformRect(coloredRectangle.rectangle2D(), pageTransformationMatrix);
            contentStream.addRect((float) r.getX(), (float) r.getY(), (float) r.getWidth(), (float) r.getHeight());
            contentStream.stroke();
        }
@@ -175,7 +189,7 @@ public class ViewerDocumentService {
            PDExtendedGraphicsState graphicsState = new PDExtendedGraphicsState();
            graphicsState.setNonStrokingAlphaConstant(filledRectangle.alpha());
            contentStream.setGraphicsStateParameters(graphicsState);
-            Rectangle2D r = filledRectangle.rectangle2D();
+            Rectangle2D r = transformRect(filledRectangle.rectangle2D(), pageTransformationMatrix);
            contentStream.addRect((float) r.getX(), (float) r.getY(), (float) r.getWidth(), (float) r.getHeight());
            contentStream.fill();
        }
@@ -190,7 +204,7 @@ public class ViewerDocumentService {
            } else {
                contentStream.setRenderingMode(RenderingMode.FILL);
            }
-            Matrix textMatrix = getTextMatrix(placedText, textDeRotationMatrix);
+            Matrix textMatrix = getTextMatrix(placedText, textDeRotationMatrix, pageTransformationMatrix);
            contentStream.setTextMatrix(textMatrix);
            contentStream.showText(placedText.text());
            contentStream.endText();
@@ -223,7 +237,7 @@ public class ViewerDocumentService {
    }


-    private static Matrix getTextMatrix(PlacedText placedText, AffineTransform textDeRotationMatrix) {
+    private static Matrix getTextMatrix(PlacedText placedText, AffineTransform textDeRotationMatrix, AffineTransform pageTransformationMatrix) {

        Matrix textMatrix;
        if (placedText.textMatrix().isEmpty()) {
@@ -236,7 +250,7 @@ public class ViewerDocumentService {
        } else {
            textMatrix = placedText.textMatrix().get();
        }
-        return textMatrix;
+        return new Matrix(pageTransformationMatrix).multiply(textMatrix);
    }


@@ -313,4 +327,16 @@ public class ViewerDocumentService {
        });
    }

+
+    private static Line2D transformLine(Line2D line, AffineTransform pageTransformation) {
+
+        return new Line2D.Double(pageTransformation.transform(line.getP1(), null), pageTransformation.transform(line.getP2(), null));
+    }
+
+
+    private static Rectangle2D transformRect(Rectangle2D r, AffineTransform pageTransformation) {
+
+        return pageTransformation.createTransformedShape(r).getBounds2D();
+    }
+
 }
Author	SHA1	Message	Date
Dominique Eifländer	ca69d3f5dc	Merge branch 'RED-9760-4.1' into 'release/0.142.x' RED-9760: Fixed missing newLines See merge request fforesight/layout-parser!185	2024-07-30 16:06:30 +02:00
Dominique Eifländer	366c12bab0	RED-9760: Fixed missing newLines	2024-07-30 15:46:40 +02:00
Andrei Isvoran	4f3f72bcbc	Merge branch 'RED-9607' into 'release/0.142.x' RED-9607 - Correctly determine text position sequence based on file rotation See merge request fforesight/layout-parser!183	2024-07-25 14:11:03 +02:00
Andrei Isvoran	6133b142cf	RED-9607 - Correctly determine text position sequence based on file rotation	2024-07-24 16:32:54 +03:00
Kilian Schüttler	c3d24393ea	Merge branch 'RED-8800-bp' into 'release/0.142.x' Red 8800 bp See merge request fforesight/layout-parser!180	2024-07-15 17:58:59 +02:00
Kilian Schüttler	d4500b879b	Red 8800 bp	2024-07-15 17:58:59 +02:00
Andrei Isvoran	44277572ab	Merge branch 'RED-9496-shutdown' into 'release/0.142.x' RED-9496 - Implement graceful shutdown See merge request fforesight/layout-parser!175	2024-07-04 14:25:58 +02:00
Andrei Isvoran	cbf809316b	RED-9496 - Implement graceful shutdown	2024-07-04 14:19:19 +03:00
Kilian Schüttler	b216f02e15	Merge branch 'RED-9194' into 'main' RED-9194: roll back single digit headline change See merge request fforesight/layout-parser!171	2024-06-21 15:13:40 +02:00
Kilian Schuettler	2e2f30ba35	RED-9194: roll back single digit headline change	2024-06-21 14:42:30 +02:00
Kilian Schuettler	9f7ed974ec	RED-9194: roll back single digit headline change	2024-06-21 14:41:30 +02:00
Kilian Schuettler	570a348a77	RED-9194: roll back single digit headline change	2024-06-21 14:39:47 +02:00
Maverick Studer	859dba2ecf	Merge branch 'RED-9374' into 'main' hotfix for table/paragraph section creation on document start before first headline See merge request fforesight/layout-parser!170	2024-06-18 17:36:04 +02:00
Maverick Studer	1c5d755111	hotfix for table/paragraph section creation on document start before first headline	2024-06-18 17:36:04 +02:00
Maverick Studer	133e06460f	Merge branch 'RED-9374' into 'main' RED-9374: Ner Entities are at wrong locations See merge request fforesight/layout-parser!169	2024-06-18 16:31:24 +02:00
Maverick Studer	da91fcff97	RED-9374: Ner Entities are at wrong locations	2024-06-18 16:31:24 +02:00
Kilian Schüttler	79795e408a	Merge branch 'RED-9194' into 'main' RED-9194: allow single digit headline identifiers See merge request fforesight/layout-parser!168	2024-06-07 09:09:25 +02:00
Kilian Schuettler	b719db86ab	RED-9194: allow single digit headline identifiers	2024-06-06 16:32:05 +02:00
Maverick Studer	797602e373	Merge branch 'thread-safe-hcs-fields' into 'main' fixed issue with thread-safety of local fields in the HeadlineClassificationService See merge request fforesight/layout-parser!167	2024-06-06 14:51:24 +02:00
maverickstuder	3d2f66cf10	fixed issue with thread-safety of local fields in the HeadlineClassificationService: * HeadlineClassificationService is no singleton anymore * instead initialize it in the ClassificationService and pass it to the classifyMethods as required	2024-06-06 14:39:23 +02:00
Kilian Schüttler	e304a9f2d7	Merge branch 'RED-7074-le' into 'main' RED-7074: Design Subsection section tree structure algorithm See merge request fforesight/layout-parser!166	2024-06-06 13:22:14 +02:00
Maverick Studer	c05f67cf44	RED-7074: Design Subsection section tree structure algorithm	2024-06-06 13:22:14 +02:00
Corina Olariu	3a2ee903af	Merge branch 'RED-9206-2' into 'main' RED-9206 - Sections are no longer correctly separated from each other in the test file See merge request fforesight/layout-parser!165	2024-06-05 13:39:15 +02:00
Corina Olariu	072a8aa3da	RED-9206 - Sections are no longer correctly separated from each other in the test file - add REDACT_MANAGER_WITHOUT_DUPLICATE_PARAGRAPH case	2024-06-05 14:26:54 +03:00
Corina Olariu	b5cfa7b63d	Merge branch 'RED-9206' into 'main' RED-9206 - Sections are no longer correctly separated from each other in the test file See merge request fforesight/layout-parser!163	2024-06-05 13:13:45 +02:00
Corina Olariu	5f5a6258c5	Merge branch 'main' into RED-9206	2024-06-05 13:34:14 +03:00
Maverick Studer	ac0e83725a	Merge branch 'RED-7074-lgs' into 'main' RED-7074: Design Subsection section tree structure algorithm See merge request fforesight/layout-parser!164	2024-06-05 12:28:00 +02:00
Maverick Studer	5d33ad570e	RED-7074: Design Subsection section tree structure algorithm	2024-06-05 12:28:00 +02:00
Corina Olariu	fd698a78fc	RED-9206 - Sections are no longer correctly separated from each other in the test file - introduce new layout parsing type: REDACT_MANAGER_WITHOUT_DUPLICATE_PARAGRAPH to include changes from REDACT_MANAGER apart from duplicate paragraph. - updated junit tests -	2024-06-04 20:55:37 +03:00
Maverick Studer	c3edeb3c7d	Merge branch 'RED-7074-test' into 'main' RED-7074: Design Subsection section tree structure algorithm See merge request fforesight/layout-parser!162	2024-06-04 15:07:40 +02:00
Maverick Studer	fc06dba2ce	RED-7074: Design Subsection section tree structure algorithm	2024-06-04 15:07:40 +02:00
Maverick Studer	b6742c1e89	Merge branch 'RED-7074_2' into 'main' RED-7074: Design Subsection section tree structure algorithm See merge request fforesight/layout-parser!160	2024-05-28 14:48:21 +02:00
Maverick Studer	efb1a748af	RED-7074: Design Subsection section tree structure algorithm	2024-05-28 14:48:21 +02:00
Maverick Studer	23985b14be	Merge branch 'RED-7074_2' into 'main' RED-7074: Design Subsection section tree structure algorithm See merge request fforesight/layout-parser!159	2024-05-24 13:30:25 +02:00
Maverick Studer	48b7a22e2b	RED-7074: Design Subsection section tree structure algorithm	2024-05-24 13:30:25 +02:00
Corina Olariu	546341ee75	Merge branch 'RED-9177' into 'main' RED-9177 - Layout parser fails to process file See merge request fforesight/layout-parser!158	2024-05-22 13:26:10 +02:00
Corina Olariu	0ed1481517	RED-9177 - Layout parser fails to process file - use originFile as viewerDocumentFile - return layoutGridOCGName in case the name is found and not check further properties	2024-05-22 13:02:42 +03:00
Andrei Isvoran	b2a47f66ae	Merge branch 'RED-9149-header' into 'main' RED-9149 - Remove header detection See merge request fforesight/layout-parser!157	2024-05-20 14:12:04 +02:00
Andrei Isvoran	3835d03036	RED-9149 - Remove header detection	2024-05-20 14:59:34 +03:00
Dominique Eifländer	b867deb9f9	Merge branch 'CLARI-hotfix' into 'main' hotifx for clarifynd See merge request fforesight/layout-parser!154	2024-05-15 14:08:07 +02:00
Kilian Schuettler	8648ed0952	hotifx for clarifynd	2024-05-15 14:02:02 +02:00