RED-10196: Backend adaptions for RM/DM unification

layoutparser-service/layoutparser-service-internal-api/build.gradle.kts

@@ -8,4 +8,9 @@ description = "layoutparser-service-internal-api"
 dependencies {
     implementation("io.swagger.core.v3:swagger-annotations:2.2.15")
     implementation("com.google.protobuf:protobuf-java-util:4.27.1")
+
+    api("com.iqser.red.service:persistence-service-shared-api-v1:2.589.1-RED10196.0") {
+        exclude("org.springframework.boot", "spring-boot-starter-security")
+        exclude("org.springframework.boot", "spring-boot-starter-validation")
+    }
 }

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/queue/LayoutParsingRequest.java

@@ -3,6 +3,8 @@ package com.knecon.fforesight.service.layoutparser.internal.api.queue;
 import java.util.Map;
 import java.util.Optional;
 
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.LayoutParsingType;
+
 import io.swagger.v3.oas.annotations.media.Schema;
 import lombok.Builder;
 import lombok.NonNull;
@@ -10,7 +12,7 @@ import lombok.NonNull;
 @Builder
 @Schema(description = "Object containing all storage paths the service needs to know.")
 public record LayoutParsingRequest(
-        @Schema(description = "Enum specifying the type of layout parsing to be performed.", allowableValues = "{RedactManager, DocuMine, TAAS}")//
+        @Schema(description = "Enum specifying the type of layout parsing to be performed.") //
         @NonNull LayoutParsingType layoutParsingType,
 
         @Schema(description = "General purpose identifiers. They are not changed by the service at all and are returned as is in the response queue.")//

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/queue/LayoutParsingType.java

@@ -1,12 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.internal.api.queue;
-
-public enum LayoutParsingType {
-    REDACT_MANAGER,
-    REDACT_MANAGER_OLD,
-    REDACT_MANAGER_PARAGRAPH_DEBUG,
-    REDACT_MANAGER_WITHOUT_DUPLICATE_PARAGRAPH,
-    DOCUMINE,
-    DOCUMINE_OLD,
-    CLARIFYND,
-    CLARIFYND_PARAGRAPH_DEBUG
-}

layoutparser-service/layoutparser-service-processor/build.gradle.kts

@@ -12,10 +12,6 @@ dependencies {
     implementation(project(":layoutparser-service-internal-api"))
     implementation(project(":viewer-doc-processor"))
 
-    implementation("com.iqser.red.service:persistence-service-shared-api-v1:2.564.0-RED9010.0") {
-        exclude("org.springframework.boot", "spring-boot-starter-security")
-        exclude("org.springframework.boot", "spring-boot-starter-validation")
-    }
     implementation("com.knecon.fforesight:tenant-commons:0.30.0") {
         exclude("com.iqser.red.commons", "storage-commons")
     }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/LayoutParsingPipeline.java

@@ -22,12 +22,10 @@ import org.apache.pdfbox.pdmodel.common.PDRectangle;
 import org.apache.pdfbox.pdmodel.documentinterchange.markedcontent.PDMarkedContent;
 import org.springframework.stereotype.Service;
 
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.LayoutParsingType;
 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeTypeProto.NodeType;
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingFinishedEvent;
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingRequest;
-import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
-import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.Character;
-import com.knecon.fforesight.service.layoutparser.processor.model.table.Ruling;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.RedTextPosition;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextDirection;
 import com.knecon.fforesight.service.layoutparser.processor.services.classification.ClassificationService;
@@ -127,8 +125,7 @@ public class LayoutParsingPipeline {
         TableServiceResponse tableServiceResponse = layoutParsingRequest.tablesFileStorageId()
                 .map(layoutParsingStorageService::getTablesFile).orElse(new TableServiceResponse());
 
-        ClassificationDocument classificationDocument = parseLayout(settings.getLayoutParsingTypeOverride() == null //
-                                                                            ? layoutParsingRequest.layoutParsingType() : settings.getLayoutParsingTypeOverride(),
+        ClassificationDocument classificationDocument = parseLayout(layoutParsingRequest.layoutParsingType(),
                                                                     originFile,
                                                                     imageServiceResponse,
                                                                     tableServiceResponse,
@@ -137,8 +134,7 @@ public class LayoutParsingPipeline {
 
         log.info("Building document graph for {}", layoutParsingRequest.identifier());
 
-        Document documentGraph = observeBuildDocumentGraph(settings.getLayoutParsingTypeOverride() == null //
-                                                                   ? layoutParsingRequest.layoutParsingType() : settings.getLayoutParsingTypeOverride(), classificationDocument);
+        Document documentGraph = observeBuildDocumentGraph(layoutParsingRequest.layoutParsingType(), classificationDocument);
 
         log.info("Creating viewer document for {}", layoutParsingRequest.identifier());
 
@@ -160,9 +156,9 @@ public class LayoutParsingPipeline {
         }
 
         if (!viewerDocumentFile.equals(originFile)) {
-            assert !viewerDocumentFile.exists() || viewerDocumentFile.delete();
+            viewerDocumentFile.delete();
         }
-        assert !originFile.exists() || originFile.delete();
+        originFile.delete();
 
         return LayoutParsingFinishedEvent.builder()
                 .identifier(layoutParsingRequest.identifier())
@@ -242,8 +238,12 @@ public class LayoutParsingPipeline {
         }
 
         List<ClassificationPage> classificationPages = new ArrayList<>();
+        OutlineObject lastProcessedOutlineObject = null;
 
-        classificationDocument.setOutlineObjectTree(outlineExtractorService.getOutlineObjectTree(originDocument));
+        // parsing the structure elements could be useful as well
+        if (layoutParsingType != LayoutParsingType.REDACT_MANAGER_OLD) {
+            classificationDocument.setOutlineObjectTree(outlineExtractorService.getOutlineObjectTree(originDocument));
+        }
 
         long pageCount = originDocument.getNumberOfPages();
 
@@ -275,15 +275,18 @@ public class LayoutParsingPipeline {
             if (layoutParsingType.equals(LayoutParsingType.DOCUMINE_OLD)) {
                 var lines = TextPositionOperations.groupByLine(new HashSet<>(words));
                 classificationDocument.getLayoutDebugLayer().addLineVisualizationsFromNestedTextPosition(lines, pageNumber);
-                words = TextPositionOperations.sortWords(lines);
+                words = TextPositionOperations.sortLines(lines);
             }
             classificationDocument.getLayoutDebugLayer().addTextVisualizations(words, pageNumber);
 
             PDRectangle pdr = pdPage.getMediaBox();
 
-            List<Ruling> rulings = stripper.getRulings();
-            classificationDocument.getLayoutDebugLayer().addRulingVisualization(rulings, pageNumber);
-            CleanRulings cleanRulings = rulingCleaningService.deduplicateAndStraightenRulings(pdfTableCells.get(pageNumber), rulings);
+            int rotation = pdPage.getRotation();
+            boolean isLandscape = pdr.getWidth() > pdr.getHeight() && (rotation == 0 || rotation == 180) || pdr.getHeight() > pdr.getWidth() && (rotation == 90 || rotation == 270);
+
+            PDRectangle cropbox = pdPage.getCropBox();
+            classificationDocument.getLayoutDebugLayer().addRulingVisualization(stripper.getRulings(), pageNumber);
+            CleanRulings cleanRulings = rulingCleaningService.deduplicateAndStraightenRulings(pdfTableCells.get(pageNumber), stripper.getRulings());
 
             PageInformation pageInformation = PageInformation.fromPDPage(pageNumber, pdPage);
             List<Cell> emptyTableCells = TableExtractionService.findCells(cleanRulings.getHorizontals(), cleanRulings.getVerticals(), pageInformation);
@@ -303,7 +306,8 @@ public class LayoutParsingPipeline {
                                     .toList());
 
             ClassificationPage classificationPage = switch (layoutParsingType) {
-                case REDACT_MANAGER_OLD -> redactManagerBlockificationService.blockify(stripper.getWords(), cleanRulings, classificationDocument.getLayoutDebugLayer());
+                case REDACT_MANAGER_OLD ->
+                        redactManagerBlockificationService.blockify(stripper.getWords(), cleanRulings, classificationDocument.getLayoutDebugLayer());
                 case DOCUMINE_OLD -> docuMineBlockificationService.blockify(words, cleanRulings);
                 case DOCUMINE, REDACT_MANAGER, REDACT_MANAGER_PARAGRAPH_DEBUG, REDACT_MANAGER_WITHOUT_DUPLICATE_PARAGRAPH ->
                         docstrumBlockificationService.blockify(words, cleanRulings, true, classificationDocument.getLayoutDebugLayer(), layoutParsingType);
@@ -311,9 +315,27 @@ public class LayoutParsingPipeline {
                         docstrumBlockificationService.blockify(words, cleanRulings, false, classificationDocument.getLayoutDebugLayer(), layoutParsingType);
             };
 
-            updateClassificationPage(pdPage, pdr, classificationPage, cleanRulings, pageNumber, pageInformation);
+            classificationPage.setCleanRulings(cleanRulings);
+            classificationPage.setRotation(rotation);
+            classificationPage.setLandscape(isLandscape);
+            classificationPage.setPageNumber(pageNumber);
+            classificationPage.setPageWidth(cropbox.getWidth());
+            classificationPage.setPageHeight(cropbox.getHeight());
 
-            blockificationPostprocessingService.findHeadlinesFromOutline(classificationDocument, pageNumber, classificationPage, pageInformation);
+            if (layoutParsingType != LayoutParsingType.REDACT_MANAGER_OLD) {
+                List<OutlineObject> outlineObjects = classificationDocument.getOutlineObjectTree().getOutlineObjectsPerPage().getOrDefault(pageNumber, new ArrayList<>());
+
+                OutlineObject notFoundOutlineObject = null;
+                if (lastProcessedOutlineObject != null && !lastProcessedOutlineObject.isFound()) {
+                    lastProcessedOutlineObject.resetPoint();
+                    notFoundOutlineObject = lastProcessedOutlineObject;
+                }
+                if (!outlineObjects.isEmpty()) {
+                    classificationPage.setOutlineObjects(outlineObjects);
+                    lastProcessedOutlineObject = blockificationPostprocessingService.sanitizeOutlineBlocks(classificationPage, notFoundOutlineObject);
+                }
+                classificationDocument.getLayoutDebugLayer().addOutlineObjects(outlineObjects, pageInformation);
+            }
 
             classificationDocument.getLayoutDebugLayer().addMarkedContentVisualizations(stripper.getMarkedContents(), pageNumber);
             // MarkedContent needs to be converted at this point, otherwise it leads to GC Problems in Pdfbox.
@@ -359,31 +381,13 @@ public class LayoutParsingPipeline {
     }
 
 
-    private static void updateClassificationPage(PDPage pdPage,
-                                                 PDRectangle pdr,
-                                                 ClassificationPage classificationPage,
-                                                 CleanRulings cleanRulings,
-                                                 int pageNumber,
-                                                 PageInformation pageInformation) {
-
-        int rotation = pdPage.getRotation();
-        boolean isLandscape = pdr.getWidth() > pdr.getHeight() && (rotation == 0 || rotation == 180) || pdr.getHeight() > pdr.getWidth() && (rotation == 90 || rotation == 270);
-        classificationPage.setCleanRulings(cleanRulings);
-        classificationPage.setRotation(rotation);
-        classificationPage.setLandscape(isLandscape);
-        classificationPage.setPageNumber(pageNumber);
-        classificationPage.setPageWidth((float) pageInformation.width());
-        classificationPage.setPageHeight((float) pageInformation.height());
-    }
-
-
     private static void rotateDirAdjExactly(List<Word> words, PDPage pdPage) {
 
         for (TextDirection dir : TextDirection.values()) {
+
             double averageRotation = words.stream()
-                    .map(Word::getCharacters)
+                    .map(Word::getTextPositions)
                     .flatMap(Collection::stream)
-                    .map(Character::getTextPosition)
                     .filter(pos -> pos.getDir().equals(dir))
                     .mapToDouble(RedTextPosition::getExactDir).average().orElse(0);
 

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/LayoutparserSettings.java

@@ -3,8 +3,6 @@ package com.knecon.fforesight.service.layoutparser.processor;
 import org.springframework.boot.context.properties.ConfigurationProperties;
 import org.springframework.context.annotation.Configuration;
 
-import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
-
 import lombok.AccessLevel;
 import lombok.Data;
 import lombok.experimental.FieldDefaults;
@@ -16,6 +14,5 @@ import lombok.experimental.FieldDefaults;
 public class LayoutparserSettings {
 
     boolean debug;
-    LayoutParsingType layoutParsingTypeOverride;
     String pdftronLicense;
 }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/docstrum/DocstrumSegmentationService.java

@@ -80,12 +80,16 @@ public class DocstrumSegmentationService {
 
     private List<Zone> computeZones(List<Word> textPositions, CleanRulings rulings, LayoutDebugLayer visualizations, TextDirection direction) {
 
-        List<Character> characters = textPositions.stream()
+        List<RedTextPosition> positions = textPositions.stream()
                 .filter(t -> t.getDir() == direction)
-                .map(Word::getCharacters)
+                .map(Word::getTextPositions)
                 .flatMap(List::stream)
                 .toList();
 
+        List<Character> characters = positions.stream()
+                .map(Character::new)
+                .collect(Collectors.toList());
+
         nearestNeighbourService.findNearestNeighbors(characters);
 
         double characterSpacing = spacingService.computeCharacterSpacing(characters);

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/docstrum/model/Line.java

@@ -36,13 +36,18 @@ public class Line extends TextBoundingBox {
     @EqualsAndHashCode.Include
     private final double y1;
 
+    private final double height;
+
     private FontStyle fontStyle;
 
-    private final List<Word> words;
+    private final List<Character> characters;
+    private final List<Word> words = new ArrayList<>();
 
 
     public Line(List<Character> characters, double wordSpacing) {
 
+        this.characters = characters;
+
         if (characters.size() >= 2) {
             // linear regression
             double sx = 0.0;
@@ -71,25 +76,13 @@ public class Line extends TextBoundingBox {
             this.y0 = character.getY() - dy;
             this.y1 = character.getY() + dy;
         }
-        this.words = new ArrayList<>();
-        computeWords(characters, wordSpacing * WORD_DISTANCE_MULTIPLIER);
+        height = computeHeight();
+        computeWords(wordSpacing * WORD_DISTANCE_MULTIPLIER);
         buildBBox();
         computeFontStyle();
     }
 
 
-    public Line(List<Word> words) {
-
-        this.words = words;
-        buildBBox();
-        x0 = getMinX();
-        y0 = getMinY();
-        x1 = getMaxX();
-        y1 = getMaxY();
-        computeFontStyle();
-    }
-
-
     private void computeFontStyle() {
 
         EnumMap<FontStyle, AtomicInteger> fontStyleCounter = new EnumMap<>(FontStyle.class);
@@ -107,7 +100,8 @@ public class Line extends TextBoundingBox {
         fontStyle = fontStyleCounter.entrySet()
                 .stream()
                 .max(Comparator.comparing(entry -> entry.getValue().get()))
-                .map(Map.Entry::getKey).orElse(FontStyle.REGULAR);
+                .map(Map.Entry::getKey)
+                .orElse(FontStyle.REGULAR);
     }
 
 
@@ -123,6 +117,14 @@ public class Line extends TextBoundingBox {
     }
 
 
+    private double computeHeight() {
+
+        return characters.stream()
+                       .map(Character::getHeight)
+                       .reduce(0d, Double::sum) / characters.size();
+    }
+
+
     public double angularDifference(Line j) {
 
         double diff = Math.abs(getAngle() - j.getAngle());
@@ -155,7 +157,7 @@ public class Line extends TextBoundingBox {
     }
 
 
-    private void computeWords(List<Character> characters, double wordSpacing) {
+    private void computeWords(double wordSpacing) {
 
         Word word = new Word();
         Character previous = null;
@@ -167,7 +169,7 @@ public class Line extends TextBoundingBox {
                     word = new Word();
                 }
             }
-            word.add(current);
+            word.getTextPositions().add(current.getTextPosition());
             previous = current;
         }
         words.add(word);
@@ -176,7 +178,9 @@ public class Line extends TextBoundingBox {
 
     private void buildBBox() {
 
-        this.setToBBoxOfComponents(words);
+        this.setToBBoxOfComponents(characters.stream()
+                                           .map(Character::getTextPosition)
+                                           .toList());
     }
 
 

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/docstrum/model/Zone.java

@@ -18,6 +18,7 @@ public class Zone extends TextBoundingBox {
     @SuppressWarnings("PMD.ConstructorCallsOverridableMethod")
     public Zone(List<Line> lines) {
 
+        lines.sort(Comparator.comparingDouble(Line::getY0));
         this.lines = lines;
         setToBBoxOfComponents(lines);
     }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/docstrum/service/LineBuilderService.java

@@ -17,7 +17,7 @@ public class LineBuilderService {
 
     private static final double CHARACTER_SPACING_DISTANCE_MULTIPLIER = 3.5;
     private static final double LINE_SPACING_THRESHOLD_MULTIPLIER = 0.67;
-    private static final double ANGLE_TOLERANCE = Math.toRadians(5);
+    private static final double ANGLE_TOLERANCE = Math.PI / 6;
 
 
     public List<Line> buildLines(List<Character> characters, double characterSpacing, double lineSpacing, CleanRulings rulings) {

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/docstrum/service/ZoneBuilderService.java

@@ -1,7 +1,9 @@
 package com.knecon.fforesight.service.layoutparser.processor.docstrum.service;
 
+import static com.knecon.fforesight.service.layoutparser.processor.model.SectionIdentifier.numericalIdentifierPattern;
+
 import java.util.ArrayList;
-import java.util.Collection;
+import java.util.Comparator;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
@@ -9,12 +11,11 @@ import java.util.stream.Collectors;
 
 import org.springframework.stereotype.Service;
 
+import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.Character;
 import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.Line;
 import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.UnionFind;
 import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.Zone;
 import com.knecon.fforesight.service.layoutparser.processor.model.table.CleanRulings;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.Word;
-import com.knecon.fforesight.service.layoutparser.processor.utils.TextPositionOperations;
 
 @Service
 public class ZoneBuilderService {
@@ -30,7 +31,7 @@ public class ZoneBuilderService {
 
     private static final double MAX_LINE_SIZE_SCALE = 2.5;
 
-    private static final double ANGLE_TOLERANCE = Math.toRadians(5);
+    private static final double ANGLE_TOLERANCE = Math.PI / 6;
 
     private static final double MAX_VERTICAL_MERGE_DISTANCE = 0.5;
 
@@ -113,14 +114,64 @@ public class ZoneBuilderService {
 
     private Zone mergeLinesInZone(List<Line> lines, double characterSpacing, double lineSpacing) {
 
-        Set<Word> words = lines.stream()
-                .map(Line::getWords)
-                .flatMap(Collection::stream)
-                .collect(Collectors.toSet());
-        Collection<Set<Word>> groupedLines = TextPositionOperations.groupByLine(words);
+        double maxHorizontalDistance = 0;
+        double minVerticalDistance = 0;
+        double maxVerticalDistance = lineSpacing * MAX_VERTICAL_MERGE_DISTANCE;
 
-        List<Line> sortedLines = TextPositionOperations.sortLines(groupedLines);
-        return new Zone(sortedLines);
+        UnionFind<Line> unionFind = new UnionFind<>(new HashSet<>(lines));
+
+        lines.forEach(outer -> {
+            lines.forEach(inner -> {
+                if (inner == outer) {
+                    return;
+                }
+
+                double horizontalDistance = outer.horizontalDistance(inner);
+                double verticalDistance = outer.verticalDistance(inner);
+
+                if (horizontalDistance <= maxHorizontalDistance && minVerticalDistance <= verticalDistance && verticalDistance <= maxVerticalDistance) {
+
+                    unionFind.union(outer, inner);
+
+                } else if (minVerticalDistance <= verticalDistance
+                           && verticalDistance <= maxVerticalDistance
+                           && Math.abs(horizontalDistance - Math.min(outer.getLength(), inner.getLength())) < 0.1) {
+
+                    boolean characterOverlap = false;
+                    int overlappingCount = 0;
+                    for (Character outerCharacter : outer.getCharacters()) {
+                        for (Character innerCharacter : inner.getCharacters()) {
+                            double characterOverlapDistance = outerCharacter.overlappingDistance(innerCharacter);
+                            if (characterOverlapDistance > 2) {
+                                characterOverlap = true;
+                            }
+                            if (characterOverlapDistance > 0) {
+                                overlappingCount++;
+                            }
+                        }
+                    }
+                    if (!characterOverlap && overlappingCount <= 2) {
+                        unionFind.union(outer, inner);
+                    }
+                }
+
+            });
+        });
+
+        List<Line> outputZone = new ArrayList<>();
+        for (Set<Line> group : unionFind.getGroups()) {
+            List<Character> characters = new ArrayList<>();
+            for (Line line : group) {
+                characters.addAll(line.getCharacters());
+            }
+            characters.sort(Comparator.comparingDouble(Character::getX));
+
+            outputZone.add(new Line(characters, characterSpacing));
+        }
+
+        return new Zone(outputZone.stream()
+                                .sorted(Comparator.comparing(Line::getY0))
+                                .collect(Collectors.toList()));
     }
 
 }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/PageBlockType.java

@@ -14,7 +14,6 @@ public enum PageBlockType {
     PARAGRAPH_ITALIC,
     PARAGRAPH_UNKNOWN,
     OTHER,
-    TABLE_OF_CONTENTS_HEADLINE,
     TABLE_OF_CONTENTS_ITEM,
     LIST_ITEM,
     TABLE;
@@ -36,7 +35,7 @@ public enum PageBlockType {
     public static int getHeadlineNumber(PageBlockType pageBlockType) {
 
         return switch (pageBlockType) {
-            case H1, TABLE_OF_CONTENTS_HEADLINE -> 1;
+            case H1 -> 1;
             case H2 -> 2;
             case H3 -> 3;
             case H4 -> 4;
@@ -48,6 +47,6 @@ public enum PageBlockType {
 
     public boolean isHeadline() {
 
-        return this.equals(H1) || this.equals(H2) || this.equals(H3) || this.equals(H4) || this.equals(H5) || this.equals(H6) || this.equals(TABLE_OF_CONTENTS_HEADLINE);
+        return this.equals(H1) || this.equals(H2) || this.equals(H3) || this.equals(H4) || this.equals(H5) || this.equals(H6);
     }
 }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/SectionIdentifier.java

@@ -16,8 +16,8 @@ import lombok.experimental.FieldDefaults;
 @FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
 public class SectionIdentifier {
 
-    public static Pattern numericalIdentifierPattern = Pattern.compile("^[\\s]?(\\d{1,2})[\\s.,;]?(\\d{1,2})?[\\s.,;]?(\\d{1,2})?[\\s.,;]?(\\d{1,2})?");
-    public static Pattern alphanumericIdentifierPattern = Pattern.compile("^[\\s]?[A-Za-z][\\s.,;]?(\\d{1,2})[\\s.,;]?(\\d{1,2})?[\\s.,;]?(\\d{1,2})?[\\s.,;]?(\\d{1,2})?[\\s.,;]?");
+    public static Pattern numericalIdentifierPattern = Pattern.compile("^[\\s]?(\\d+)[\\s.,;]?(\\d+)?[\\s.,;]?(\\d+)?[\\s.,;]?(\\d+)?");
+    public static Pattern alphanumericIdentifierPattern = Pattern.compile("^[\\s]?[A-Za-z][\\s.,;]?(\\d+)[\\s.,;]?(\\d+)?[\\s.,;]?(\\d+)?[\\s.,;]?(\\d+)?[\\s.,;]?");
 
     public enum Format {
         EMPTY,

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/AbstractNodeVisitor.java

@@ -11,8 +11,6 @@ import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Se
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.SuperSection;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Table;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.TableCell;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.TableOfContents;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.TableOfContentsItem;
 
 public abstract class AbstractNodeVisitor implements NodeVisitor {
 
@@ -85,18 +83,6 @@ public abstract class AbstractNodeVisitor implements NodeVisitor {
         visitChildren(tableCell);
     }
 
-    @Override
-    public void visit(TableOfContents toc) {
-
-        visitChildren(toc);
-    }
-
-    @Override
-    public void visit(TableOfContentsItem toci) {
-
-        visitChildren(toci);
-    }
-
 
     protected void visitChildren(SemanticNode semanticNode) {
 

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/NodeVisitor.java

@@ -10,10 +10,6 @@ import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Se
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.SuperSection;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Table;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.TableCell;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.TableOfContents;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.TableOfContentsItem;
-
-import software.amazon.awssdk.utils.builder.ToCopyableBuilder;
 
 public interface NodeVisitor {
 
@@ -46,10 +42,4 @@ public interface NodeVisitor {
 
     void visit(TableCell tableCell);
 
-
-    void visit(TableOfContents tableOfContents);
-
-
-    void visit(TableOfContentsItem tableOfContentsItem);
-
 }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/TableOfContents.java

@@ -1,41 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes;
-
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeTypeProto;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.NodeVisitor;
-
-import lombok.AccessLevel;
-import lombok.AllArgsConstructor;
-import lombok.Data;
-import lombok.EqualsAndHashCode;
-import lombok.experimental.FieldDefaults;
-import lombok.experimental.SuperBuilder;
-
-@Data
-@SuperBuilder
-@AllArgsConstructor
-@FieldDefaults(level = AccessLevel.PRIVATE)
-@EqualsAndHashCode(callSuper = true)
-public class TableOfContents extends AbstractSemanticNode {
-
-    @Override
-    public NodeTypeProto.NodeType getType() {
-
-        return NodeTypeProto.NodeType.TABLE_OF_CONTENTS;
-    }
-
-
-    public Headline getHeadline() {
-
-        return streamChildrenOfType(NodeTypeProto.NodeType.HEADLINE).map(node -> (Headline) node)
-                .findFirst()
-                .orElseGet(() -> getParent().getHeadline());
-    }
-
-
-    @Override
-    public void accept(NodeVisitor visitor) {
-
-        visitor.visit(this);
-    }
-
-}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/TableOfContentsItem.java

@@ -1,51 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes;
-
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeTypeProto;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.NodeVisitor;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlock;
-
-import lombok.AccessLevel;
-import lombok.AllArgsConstructor;
-import lombok.Data;
-import lombok.EqualsAndHashCode;
-import lombok.experimental.FieldDefaults;
-import lombok.experimental.SuperBuilder;
-
-@Data
-@SuperBuilder
-@AllArgsConstructor
-@FieldDefaults(level = AccessLevel.PRIVATE)
-@EqualsAndHashCode(callSuper = true)
-public class TableOfContentsItem extends AbstractSemanticNode {
-
-    TextBlock leafTextBlock;
-
-
-    @Override
-    public NodeTypeProto.NodeType getType() {
-
-        return NodeTypeProto.NodeType.TABLE_OF_CONTENTS_ITEM;
-    }
-
-
-    @Override
-    public boolean isLeaf() {
-
-        return true;
-    }
-
-
-    @Override
-    public void accept(NodeVisitor visitor) {
-
-        visitor.visit(this);
-    }
-
-
-    @Override
-    public TextBlock getTextBlock() {
-
-        return leafTextBlock;
-    }
-
-}

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/text/ListIdentifier.java

@@ -14,12 +14,11 @@ import lombok.experimental.FieldDefaults;
 @FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
 public class ListIdentifier {
 
-    public static final Pattern NUMBER_WITH_DOT = Pattern.compile("^\\s*([1-9]{1,4})\\.\\s+");
-    public static final Pattern NUMBER_IN_PARENTHESES = Pattern.compile("^\\s*\\(([1-9]{1,4})\\)\\s+");
+    public static final Pattern NUMBER_WITH_DOT = Pattern.compile("^\\s*([1-9]+)\\.\\s+");
+    public static final Pattern NUMBER_IN_PARENTHESES = Pattern.compile("^\\s*\\(([1-9]+)\\)\\s+");
 
     enum Format {
-        NUMBER_WITH_DOT,
-        NUMBER_IN_PARENTHESES
+        NUMBER_WITH_DOT, NUMBER_IN_PARENTHESES
     }
 
     Format format;
@@ -49,33 +48,17 @@ public class ListIdentifier {
         Matcher numberMatcher = NUMBER_WITH_DOT.matcher(text);
 
         if (numberMatcher.find()) {
-            Optional<Integer> representation = parseInteger(numberMatcher.group(1));
-            if (representation.isPresent()) {
-                return Optional.of(new ListIdentifier(Format.NUMBER_WITH_DOT, sequences.get(0), page, representation.get()));
-            }
+            return Optional.of(new ListIdentifier(Format.NUMBER_WITH_DOT, sequences.get(0), page, Integer.parseInt(numberMatcher.group(1))));
         }
 
         Matcher parenthesisMatcher = NUMBER_IN_PARENTHESES.matcher(text);
         if (parenthesisMatcher.find()) {
-            Optional<Integer> representation = parseInteger(parenthesisMatcher.group(1));
-            if (representation.isPresent()) {
-                return Optional.of(new ListIdentifier(Format.NUMBER_IN_PARENTHESES, sequences.get(0), page, representation.get()));
-            }
+            return Optional.of(new ListIdentifier(Format.NUMBER_IN_PARENTHESES, sequences.get(0), page, Integer.parseInt(parenthesisMatcher.group(1))));
         }
         return Optional.empty();
     }
 
 
-    private static Optional<Integer> parseInteger(String text) {
-
-        try {
-            return Optional.of(Integer.parseInt(text));
-        } catch (NumberFormatException e) {
-            return Optional.empty();
-        }
-    }
-
-
     public static boolean isInOrder(List<ListIdentifier> listIdentifiers) {
 
         if (listIdentifiers.size() <= 1) {

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/text/Word.java

@@ -13,7 +13,6 @@ import java.util.stream.Collectors;
 
 import org.apache.pdfbox.text.TextPosition;
 
-import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.Character;
 import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.TextBoundingBox;
 
 import lombok.AllArgsConstructor;
@@ -39,7 +38,8 @@ public class Word extends TextBoundingBox implements CharSequence {
     private int page;
 
     @Builder.Default
-    private List<Character> characters = new ArrayList<>();
+    private List<RedTextPosition> textPositions = new ArrayList<>();
+
     private boolean isParagraphStart;
     private boolean strikethrough;
     private boolean underline;
@@ -49,9 +49,8 @@ public class Word extends TextBoundingBox implements CharSequence {
 
     public Word(List<TextPosition> textPositions, int pageNumber, boolean isParagraphStart) {
 
-        this.characters = textPositions.stream()
+        this.textPositions = textPositions.stream()
                 .map(RedTextPosition::fromTextPosition)
-                .map(Character::new)
                 .collect(Collectors.toList());
         this.page = pageNumber;
         this.isParagraphStart = isParagraphStart;
@@ -66,9 +65,9 @@ public class Word extends TextBoundingBox implements CharSequence {
     }
 
 
-    public Word(List<Character> textPositions, int page) {
+    public Word(List<RedTextPosition> textPositions, int page) {
 
-        this.characters = new ArrayList<>(textPositions);
+        this.textPositions = textPositions;
         this.page = page;
         calculateBBoxAndHashcode();
     }
@@ -77,7 +76,7 @@ public class Word extends TextBoundingBox implements CharSequence {
     @Override
     public int length() {
 
-        return characters.size();
+        return textPositions.size();
     }
 
 
@@ -102,7 +101,7 @@ public class Word extends TextBoundingBox implements CharSequence {
     public Word subSequence(int start, int end) {
 
         var textPositionSequence = new Word();
-        textPositionSequence.characters = characters.subList(start, end);
+        textPositionSequence.textPositions = textPositions.subList(start, end);
         textPositionSequence.page = page;
         textPositionSequence.dir = dir;
         textPositionSequence.setToBBoxOfComponents(getTextPositions());
@@ -123,59 +122,53 @@ public class Word extends TextBoundingBox implements CharSequence {
 
     public RedTextPosition textPositionAt(int index) {
 
-        return characters.get(index).getTextPosition();
+        return textPositions.get(index);
     }
 
 
     public void add(Word word, RedTextPosition textPosition) {
 
-        this.characters.add(new Character(textPosition));
+        this.textPositions.add(textPosition);
         this.page = word.getPage();
         calculateBBoxAndHashcode();
     }
 
 
-    public void add(Character current) {
-
-        characters.add(current);
-        calculateBBoxAndHashcode();
-    }
-
-
     public void add(TextPosition textPosition) {
 
-        add(new Character(RedTextPosition.fromTextPosition(textPosition)));
+        this.textPositions.add(RedTextPosition.fromTextPosition(textPosition));
+        calculateBBoxAndHashcode();
     }
 
 
     public double getTextHeightNoPadding() {
 
-        return characters.get(0).getTextPosition().getHeightDirAdj();
+        return textPositions.get(0).getHeightDirAdj();
     }
 
 
     public double getTextHeight() {
 
-        return characters.get(0).getTextPosition().getHeightDirAdj() + HEIGHT_PADDING;
+        return textPositions.get(0).getHeightDirAdj() + HEIGHT_PADDING;
     }
 
 
     public String getFont() {
 
-        if (characters.get(0).getTextPosition().getFontName() == null) {
+        if (textPositions.get(0).getFontName() == null) {
             return "none";
         }
 
-        return FONT_CLEANER.matcher(characters.get(0).getTextPosition().getFontName().toLowerCase(Locale.ROOT)).replaceAll("");
+        return FONT_CLEANER.matcher(textPositions.get(0).getFontName().toLowerCase(Locale.ROOT)).replaceAll("");
     }
 
 
     public String getFontStyle() {
 
-        if (characters.get(0).getTextPosition().getFontName() == null) {
+        if (textPositions.get(0).getFontName() == null) {
             return STANDARD;
         }
-        String lowercaseFontName = characters.get(0).getTextPosition().getFontName().toLowerCase(Locale.ROOT);
+        String lowercaseFontName = textPositions.get(0).getFontName().toLowerCase(Locale.ROOT);
 
         if (lowercaseFontName.contains(BOLD) && lowercaseFontName.contains(ITALIC)) {
             return BOLD_ITALIC;
@@ -191,13 +184,13 @@ public class Word extends TextBoundingBox implements CharSequence {
 
     public float getFontSize() {
 
-        return characters.get(0).getTextPosition().getFontSizeInPt();
+        return textPositions.get(0).getFontSizeInPt();
     }
 
 
     public float getSpaceWidth() {
 
-        return characters.get(0).getTextPosition().getWidthOfSpace();
+        return textPositions.get(0).getWidthOfSpace();
     }
 
 
@@ -251,14 +244,6 @@ public class Word extends TextBoundingBox implements CharSequence {
     }
 
 
-    private List<RedTextPosition> getTextPositions() {
-
-        return characters.stream()
-                .map(Character::getTextPosition)
-                .toList();
-    }
-
-
     public void transform(AffineTransform rotateInstance) {
 
         for (RedTextPosition textPosition : getTextPositions()) {

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/BodyTextFrameService.java

@@ -5,9 +5,9 @@ import java.util.List;
 
 import org.springframework.stereotype.Service;
 
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.LayoutParsingType;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Point;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
-import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
 import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBlock;
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationDocument;
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationPage;

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/blockification/BlockificationPostprocessingService.java

@@ -11,14 +11,12 @@ import org.apache.commons.text.similarity.LevenshteinDistance;
 import org.springframework.stereotype.Service;
 
 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngineProto.LayoutEngine;
-import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationDocument;
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationPage;
 import com.knecon.fforesight.service.layoutparser.processor.model.PageBlockType;
 import com.knecon.fforesight.service.layoutparser.processor.model.SectionIdentifier;
 import com.knecon.fforesight.service.layoutparser.processor.model.outline.OutlineObject;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.Word;
-import com.knecon.fforesight.service.layoutparser.processor.utils.PageInformation;
 import com.knecon.fforesight.service.layoutparser.processor.utils.TextNormalizationUtilities;
 
 import lombok.Data;
@@ -29,24 +27,6 @@ public class BlockificationPostprocessingService {
     private static final float STRING_SIMILARITY_THRESHOLD = 0.1f;
 
 
-    public void findHeadlinesFromOutline(ClassificationDocument classificationDocument, int pageNumber, ClassificationPage classificationPage, PageInformation pageInformation) {
-
-        OutlineObject lastProcessedOutlineObject = null;
-        List<OutlineObject> outlineObjects = classificationDocument.getOutlineObjectTree().getOutlineObjectsPerPage().getOrDefault(pageNumber, new ArrayList<>());
-
-        OutlineObject notFoundOutlineObject = null;
-        if (lastProcessedOutlineObject != null && !lastProcessedOutlineObject.isFound()) {
-            lastProcessedOutlineObject.resetPoint();
-            notFoundOutlineObject = lastProcessedOutlineObject;
-        }
-        if (!outlineObjects.isEmpty()) {
-            classificationPage.setOutlineObjects(outlineObjects);
-            lastProcessedOutlineObject = sanitizeOutlineBlocks(classificationPage, notFoundOutlineObject);
-        }
-        classificationDocument.getLayoutDebugLayer().addOutlineObjects(outlineObjects, pageInformation);
-    }
-
-
     public OutlineObject sanitizeOutlineBlocks(ClassificationPage classificationPage, OutlineObject notFoundOutlineObject) {
 
         List<OutlineObject> outlineObjects = classificationPage.getOutlineObjects();
@@ -349,8 +329,8 @@ public class BlockificationPostprocessingService {
 
         if (index > 0) {
             in = createSubSequence(sequence, 0, index);
-        } else if (endIndex < sequence.length()) {
-            in = createSubSequence(sequence, endIndex, sequence.length());
+        } else if (endIndex < sequence.getTextPositions().size()) {
+            in = createSubSequence(sequence, endIndex, sequence.getTextPositions().size());
         }
 
         return new SplitSequenceResult(in, out);
@@ -359,7 +339,7 @@ public class BlockificationPostprocessingService {
 
     private static Word createSubSequence(Word sequence, int start, int end) {
 
-        Word newSeq = new Word(new ArrayList<>(sequence.getCharacters().subList(start, end)), sequence.getPage());
+        Word newSeq = new Word(new ArrayList<>(sequence.getTextPositions().subList(start, end)), sequence.getPage());
         newSeq.setParagraphStart(sequence.isParagraphStart());
         return newSeq;
     }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/blockification/DocstrumBlockificationService.java

@@ -6,7 +6,7 @@ import java.util.ListIterator;
 
 import org.springframework.stereotype.Service;
 
-import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.LayoutParsingType;
 import com.knecon.fforesight.service.layoutparser.processor.docstrum.DocstrumSegmentationService;
 import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.Zone;
 import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBlock;
@@ -77,7 +77,7 @@ public class DocstrumBlockificationService {
                     .forEach(line -> {
                         line.getWords()
                                 .forEach(word -> {
-                                    words.add(new Word(word.getCharacters(), word.getPage()));
+                                    words.add(new Word(word.getTextPositions(), word.getPage()));
                                 });
                     });
 

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/classification/ClarifyndClassificationService.java

@@ -81,7 +81,7 @@ public class ClarifyndClassificationService {
                    && (textBlock.getMostPopularWordStyle().equals("bold")
                        || !document.getFontStyleCounter().getCountPerValue().containsKey("bold")
                           && textBlock.getMostPopularWordFontSize() > document.getFontSizeCounter().getMostPopular() + 1)
-                   && textBlock.getWords().get(0).getFontSize()>= textBlock.getMostPopularWordFontSize()) {
+                   && textBlock.getWords().get(0).getTextPositions().get(0).getFontSizeInPt() >= textBlock.getMostPopularWordFontSize()) {
 
             PageBlockType headlineType = HeadlineClassificationService.headlineClassByFontSize(textBlock, headlineFontSizes);
             headlineClassificationService.classifyHeadline(textBlock, headlineType);
@@ -91,7 +91,7 @@ public class ClarifyndClassificationService {
                    && textBlock.getMostPopularWordStyle().equals("bold")
                    && !document.getFontStyleCounter().getMostPopular().equals("bold")
                    && PositionUtils.getApproxLineCount(textBlock) < 2.9
-                   && textBlock.getWords().get(0).getFontSize() >= textBlock.getMostPopularWordFontSize()) {
+                   && textBlock.getWords().get(0).getTextPositions().get(0).getFontSizeInPt() >= textBlock.getMostPopularWordFontSize()) {
 
             PageBlockType headlineType = HeadlineClassificationService.headlineClassByFontSize(textBlock, headlineFontSizes);
             headlineClassificationService.classifyHeadline(textBlock, headlineType);

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/classification/ClassificationService.java

@@ -4,11 +4,9 @@ import java.util.Map;
 
 import org.springframework.stereotype.Service;
 
-import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.LayoutParsingType;
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationDocument;
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationPage;
-import com.knecon.fforesight.service.layoutparser.processor.model.PageBlockType;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
 import com.knecon.fforesight.service.layoutparser.processor.services.BodyTextFrameService;
 import com.knecon.fforesight.service.layoutparser.processor.services.blockification.DocuMineBlockificationService;
 

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/classification/RedactManagerClassificationService.java

@@ -94,7 +94,7 @@ public class RedactManagerClassificationService {
                    && (textBlock.getMostPopularWordStyle().equals("bold")
                        || !document.getFontStyleCounter().getCountPerValue().containsKey("bold")
                           && textBlock.getMostPopularWordFontSize() > document.getFontSizeCounter().getMostPopular() + 1)
-                   && textBlock.getWords().get(0).getFontSize() >= textBlock.getMostPopularWordFontSize()) {
+                   && textBlock.getWords().get(0).getTextPositions().get(0).getFontSizeInPt() >= textBlock.getMostPopularWordFontSize()) {
 
             PageBlockType headlineType = HeadlineClassificationService.headlineClassByFontSize(textBlock, headlineFontSizes);
             headlineClassificationService.classifyHeadline(textBlock, headlineType);
@@ -104,7 +104,7 @@ public class RedactManagerClassificationService {
                    && textBlock.getMostPopularWordStyle().equals("bold")
                    && !document.getFontStyleCounter().getMostPopular().equals("bold")
                    && PositionUtils.getApproxLineCount(textBlock) < 2.9
-                   && textBlock.getWords().get(0).getFontSize() >= textBlock.getMostPopularWordFontSize()) {
+                   && textBlock.getWords().get(0).getTextPositions().get(0).getFontSizeInPt() >= textBlock.getMostPopularWordFontSize()) {
 
             PageBlockType headlineType = HeadlineClassificationService.headlineClassByFontSize(textBlock, headlineFontSizes);
             headlineClassificationService.classifyHeadline(textBlock, headlineType);

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/classification/TableOfContentsClassificationService.java

@@ -59,7 +59,7 @@ public class TableOfContentsClassificationService {
 
             if (end > i + 1) {
                 if (textBlock.textBlock().getClassification() == null) {
-                    textBlock.textBlock().setClassification(PageBlockType.TABLE_OF_CONTENTS_HEADLINE);
+                    textBlock.textBlock().setClassification(PageBlockType.H1);
                 }
                 i = end;
             }
@@ -71,9 +71,9 @@ public class TableOfContentsClassificationService {
 
         ClassificationPage startPage = textBlocks.get(start).page();
         List<TextBlockOnPage> initialLookAhead = textBlocks.subList(start, Math.min(start + SURROUNDING_BLOCKS_RADIUS, textBlocks.size()));
-        HashMap<Word, TextBlockOnPage> numberToBlockLookup = new HashMap<>();
-        List<Word> numbers = extractNumbers(initialLookAhead, numberToBlockLookup, document.getPages().size());
-        TocNumberFinder tocNumberFinder = new TocNumberFinder(numbers, numberToBlockLookup);
+        HashMap<Word, TextBlockOnPage> lookup = new HashMap<>();
+        List<Word> numbers = extractNumbers(initialLookAhead, lookup, document.getPages().size());
+        TocNumberFinder tocNumberFinder = new TocNumberFinder(numbers, lookup);
 
         int lastCandidate = start;
         for (int i = start; i < Math.min(lastCandidate + SURROUNDING_BLOCKS_RADIUS, textBlocks.size()); i++) {
@@ -93,7 +93,7 @@ public class TableOfContentsClassificationService {
                 break;
             }
 
-            List<Word> numbersFromBlock = extractNumbers(textBlockOnPage, numberToBlockLookup, document.getPages().size());
+            List<Word> numbersFromBlock = extractNumbers(textBlockOnPage, lookup, document.getPages().size());
 
             List<Word> currentRightmostCluster = tocNumberFinder.getCurrentRightmostCluster();
 
@@ -102,19 +102,19 @@ public class TableOfContentsClassificationService {
                 return start;
             }
 
-            if (anyIntersection(currentRightmostCluster, numbersFromBlock, numberToBlockLookup)) {
+            if (anyIntersection(currentRightmostCluster, numbersFromBlock, lookup)) {
                 lastCandidate = i;
                 numbersFromBlock.forEach(tocNumberFinder::add);
             }
         }
 
+        addVisualization(document.getLayoutDebugLayer(), tocNumberFinder, lookup);
+
         Set<TextBlockOnPage> blocksWithNumberInCluster = tocNumberFinder.getCurrentRightmostCluster()
                 .stream()
-                .map(numberToBlockLookup::get)
+                .map(lookup::get)
                 .collect(Collectors.toSet());
 
-        addVisualization(document.getLayoutDebugLayer(), tocNumberFinder, numberToBlockLookup, blocksWithNumberInCluster, textBlocks.get(start - 1));
-
         int lastConfirmed = start;
         for (int i = start; i < lastCandidate + 1; i++) {
             TextBlockOnPage textBlockOnPage = textBlocks.get(i);
@@ -132,22 +132,18 @@ public class TableOfContentsClassificationService {
     }
 
 
-    private static void addVisualization(LayoutDebugLayer layoutDebugLayer,
-                                         TocNumberFinder tocNumberFinder,
-                                         Map<Word, TextBlockOnPage> lookup,
-                                         Set<TextBlockOnPage> blocksWithNumberInCluster,
-                                         TextBlockOnPage startingHeadline) {
+    private static void addVisualization(LayoutDebugLayer layoutDebugLayer, TocNumberFinder tocNumberFinder, Map<Word, TextBlockOnPage> lookup) {
 
         tocNumberFinder.getCurrentRightmostCluster()
                 .stream()
                 .collect(Collectors.groupingBy(key -> lookup.get(key).page().getPageNumber()))
                 .forEach((pageNumber, number) -> layoutDebugLayer.addTocPages(number, pageNumber));
-        layoutDebugLayer.addTocBlocks(blocksWithNumberInCluster);
-        layoutDebugLayer.addTocBlocks(Set.of(startingHeadline));
     }
 
 
-    private static boolean anyIntersection(Collection<Word> numbers1, Collection<Word> numbers2, Map<Word, TextBlockOnPage> lookup) {
+    private static boolean anyIntersection(Collection<Word> numbers1,
+                                           Collection<Word> numbers2,
+                                           Map<Word, TextBlockOnPage> lookup) {
 
         return numbers1.stream()
                 .anyMatch(numberFromCluster -> numbers2.stream()
@@ -174,7 +170,7 @@ public class TableOfContentsClassificationService {
 
             Word word = sequences.get(i);
 
-            if (!NUMERIC.matcher(word).matches() || word.length() > 5) {
+            if (!NUMERIC.matcher(word).matches()) {
                 continue;
             }
 

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/factory/DocumentGraphFactory.java

@@ -16,14 +16,13 @@ import java.util.Optional;
 import java.util.Set;
 import java.util.stream.Collectors;
 
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.LayoutParsingType;
 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeType;
-import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
 import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBlock;
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationDocument;
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationFooter;
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationHeader;
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationPage;
-import com.knecon.fforesight.service.layoutparser.processor.model.PageBlockType;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.DocumentTree;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.AbstractSemanticNode;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
@@ -36,7 +35,6 @@ import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Im
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Page;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Paragraph;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.SemanticNode;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.TableOfContentsItem;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.AtomicTextBlock;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlock;
 import com.knecon.fforesight.service.layoutparser.processor.model.image.ClassifiedImage;
@@ -98,7 +96,6 @@ public class DocumentGraphFactory {
 
         for (TableOfContentItem tocItem : classificationDocument.getTableOfContents()) {
             GenericSemanticNode parent = tocItem.getParent() == null ? null : tocItem.getParent().getSection();
-
             Optional<GenericSemanticNode> section = SectionNodeFactory.addSection(layoutParsingType,
                                                                                   parent,
                                                                                   tocItem.getChildren().isEmpty(),
@@ -124,8 +121,6 @@ public class DocumentGraphFactory {
             node = Headline.builder().documentTree(context.getDocumentTree()).build();
         } else if (originalTextBlock.isToDuplicate() && layoutParsingType.equals(LayoutParsingType.REDACT_MANAGER)) {
             node = DuplicatedParagraph.builder().documentTree(context.getDocumentTree()).build();
-        } else if (originalTextBlock.getClassification().equals(PageBlockType.TABLE_OF_CONTENTS_ITEM)) {
-            node = TableOfContentsItem.builder().documentTree(context.getDocumentTree()).build();
         } else {
             node = Paragraph.builder().documentTree(context.getDocumentTree()).build();
         }
@@ -147,9 +142,7 @@ public class DocumentGraphFactory {
         List<Integer> treeId = context.documentTree.createNewChildEntryAndReturnId(parentNode, node);
         node.setLeafTextBlock(textBlock);
         node.setTreeId(treeId);
-        node.getEngines().
-
-                addAll(originalTextBlock.getEngines());
+        node.getEngines().addAll(originalTextBlock.getEngines());
     }
 
 

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/factory/SearchTextWithTextPositionFactory.java

@@ -10,7 +10,6 @@ import java.util.List;
 import java.util.Locale;
 import java.util.Objects;
 
-import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.Character;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.TextRange;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.RedTextPosition;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.Word;
@@ -32,19 +31,19 @@ public class SearchTextWithTextPositionFactory {
     public SearchTextWithTextPositionDto buildSearchTextToTextPositionDto(List<Word> sequences) {
 
         if (sequences.isEmpty() || sequences.stream()
-                .allMatch(sequence -> sequence.getCharacters().isEmpty())) {
+                .allMatch(sequence -> sequence.getTextPositions().isEmpty())) {
             return SearchTextWithTextPositionDto.empty();
         }
 
         Context context = new Context();
 
-        RedTextPosition currentTextPosition = sequences.get(0).getCharacters().get(0).getTextPosition();
+        RedTextPosition currentTextPosition = sequences.get(0).getTextPositions().get(0);
         RedTextPosition previousTextPosition = RedTextPosition.builder().unicode(" ").bBoxDirAdj(currentTextPosition.getBBoxDirAdj()).build();
 
         for (Word word : sequences) {
-            for (int i = 0; i < word.getCharacters().size(); ++i) {
+            for (int i = 0; i < word.getTextPositions().size(); ++i) {
 
-                currentTextPosition = word.getCharacters().get(i).getTextPosition();
+                currentTextPosition = word.getTextPositions().get(i);
                 if (isLineBreak(currentTextPosition, previousTextPosition)) {
                     removeHyphenLinebreaks(context);
                     context.lineBreaksStringIdx.add(context.stringIdx);
@@ -67,9 +66,8 @@ public class SearchTextWithTextPositionFactory {
         }
 
         List<Rectangle2D> positions = sequences.stream()
-                .map(Word::getCharacters)
+                .map(Word::getTextPositions)
                 .flatMap(Collection::stream)
-                .map(Character::getTextPosition)
                 .map(RedTextPosition::getBBoxPdf)
                 .toList();
 

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/factory/SectionNodeFactory.java

@@ -10,15 +10,13 @@ import java.util.List;
 import java.util.Optional;
 import java.util.Set;
 
-import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.LayoutParsingType;
 import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBlock;
-import com.knecon.fforesight.service.layoutparser.processor.model.PageBlockType;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.AbstractSemanticNode;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.GenericSemanticNode;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Section;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.SuperSection;
-import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.TableOfContents;
 import com.knecon.fforesight.service.layoutparser.processor.model.image.ClassifiedImage;
 import com.knecon.fforesight.service.layoutparser.processor.model.table.TablePageBlock;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
@@ -29,20 +27,6 @@ import lombok.experimental.UtilityClass;
 @UtilityClass
 public class SectionNodeFactory {
 
-    public GenericSemanticNode addTocSection(LayoutParsingType layoutParsingType, List<AbstractPageBlock> pageBlocks, DocumentGraphFactory.Context context, Document document) {
-
-        AbstractSemanticNode section = TableOfContents.builder().documentTree(context.getDocumentTree()).build();
-        context.getSections().add(section);
-        section.setTreeId(getTreeId(null, context, section));
-        for (AbstractPageBlock pageBlock : pageBlocks) {
-            if (pageBlock instanceof TextPageBlock textPageBlock) {
-                DocumentGraphFactory.addParagraphOrHeadline(section, textPageBlock, context, new ArrayList<>(), layoutParsingType);
-            }
-        }
-        return section;
-    }
-
-
     public Optional<GenericSemanticNode> addSection(LayoutParsingType layoutParsingType,
                                                     GenericSemanticNode parentNode,
                                                     boolean isLeaf,

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/factory/TableNodeFactory.java

@@ -6,7 +6,7 @@ import java.util.Collection;
 import java.util.List;
 import java.util.stream.Collectors;
 
-import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.LayoutParsingType;
 import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBlock;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.GenericSemanticNode;

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/parsing/PDFLinesTextStripper.java

@@ -248,8 +248,8 @@ public class PDFLinesTextStripper extends PDFTextStripper {
 
             if (!words.isEmpty()) {
                 previous = words.get(words.size() - 1)
-                        .getCharacters()
-                        .get(words.get(words.size() - 1).getCharacters().size() - 1).getTextPosition();
+                        .getTextPositions()
+                        .get(words.get(words.size() - 1).getTextPositions().size() - 1);
             }
 
             if (i == 0 && (textPositions.get(i).getUnicode().equals(" ") || textPositions.get(i).getUnicode().equals("\u00A0") || textPositions.get(i).getUnicode().equals("\t"))) {

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/utils/TextPositionOperations.java

@@ -11,7 +11,6 @@ import java.util.Set;
 import java.util.stream.Collectors;
 
 import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.AngleFilter;
-import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.Line;
 import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.TextBoundingBox;
 import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.UnionFind;
 import com.knecon.fforesight.service.layoutparser.processor.docstrum.utils.DoubleUtils;
@@ -55,20 +54,11 @@ public class TextPositionOperations {
 
     private List<Word> sortUsingLineDetection(Set<Word> sequences) {
 
-        return sortWords(groupByLine(sequences));
+        return sortLines(groupByLine(sequences));
     }
 
 
-    public List<Word> sortWords(Collection<Set<Word>> lines) {
-
-        return sortLines(lines).stream()
-                .map(Line::getWords)
-                .flatMap(Collection::stream)
-                .collect(Collectors.toList());
-    }
-
-
-    public List<Line> sortLines(Collection<Set<Word>> lines) {
+    public List<Word> sortLines(Collection<Set<Word>> lines) {
 
         List<List<Word>> lineBlocks = new ArrayList<>();
         for (Set<Word> line : lines) {
@@ -80,9 +70,9 @@ public class TextPositionOperations {
         // need to use old sorting, since COMPARATOR_DIR_ADJ is not transitive
         QuickSort.sort(lineBlocks, Comparator.comparing(line -> line.get(0), COMPARATOR_DIR_ADJ));
 
-        List<Line> list = new ArrayList<>();
-        for (List<Word> lineBlock : lineBlocks) {
-            list.add(new Line(lineBlock));
+        List<Word> list = new ArrayList<>();
+        for (List<Word> words : lineBlocks) {
+            list.addAll(words);
         }
         return list;
     }
@@ -105,12 +95,6 @@ public class TextPositionOperations {
                                  .map(Word::getBBoxDirAdj)
                                  .mapToDouble(RectangularShape::getWidth).average().orElse(75) * MAX_WORD_DISTANCE_FACTOR;
 
-        return groupByLine(sequences, maxLineDistance, maxXGap);
-    }
-
-
-    public Collection<Set<Word>> groupByLine(Set<Word> sequences, double maxLineDistance, double maxXGap) {
-
         UnionFind<Word> unionFind = new UnionFind<>(sequences);
 
         for (Word sequence : sequences) {

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/visualization/LayoutDebugLayer.java

@@ -27,7 +27,6 @@ import com.knecon.fforesight.service.layoutparser.processor.model.outline.Outlin
 import com.knecon.fforesight.service.layoutparser.processor.model.table.CleanRulings;
 import com.knecon.fforesight.service.layoutparser.processor.model.table.Ruling;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.ListIdentifier;
-import com.knecon.fforesight.service.layoutparser.processor.model.text.TextBlockOnPage;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.Word;
 import com.knecon.fforesight.service.layoutparser.processor.utils.CoordinateTransforms;
 import com.knecon.fforesight.service.layoutparser.processor.utils.MarkedContentUtils;
@@ -274,9 +273,7 @@ public class LayoutDebugLayer extends LayoutDebugLayerConfig {
         AtomicInteger index = new AtomicInteger(0);
         zones.forEach(zone -> zone.getLines()
                 .stream()
-                .map(Line::getWords)
-                .flatMap(Collection::stream)
-                .map(Word::getCharacters)
+                .map(Line::getCharacters)
                 .flatMap(Collection::stream)
                 .forEach(character -> {
                     Color color = getRotatingColor(index);
@@ -333,10 +330,8 @@ public class LayoutDebugLayer extends LayoutDebugLayerConfig {
 
     private void addOutlineObject(OutlineObject outlineObject, PageInformation pageInformation) {
 
-        if (!active) {
-            return;
-        }
         int rectSize = 5;
+
         Point2D point2D;
         if (outlineObject.getPoint().isPresent()) {
             point2D = outlineObject.getPoint().get();
@@ -360,25 +355,10 @@ public class LayoutDebugLayer extends LayoutDebugLayerConfig {
 
     public void addListIdentifiers(List<ListIdentifier> listIdentifiers) {
 
-        if (!active) {
-            return;
-        }
         for (ListIdentifier listIdentifier : listIdentifiers) {
             getOrCreateVisualizationsOnPage(listIdentifier.getPage(), this.listIdentifiers).getColoredRectangles()
                     .add(new ColoredRectangle(listIdentifier.getWord().getBBoxPdf(), WORDS_COLOR, LINE_WIDTH));
         }
     }
 
-
-    public void addTocBlocks(Set<TextBlockOnPage> blocksWithNumberInCluster) {
-
-        if (!active) {
-            return;
-        }
-        for (TextBlockOnPage textBlockOnPage : blocksWithNumberInCluster) {
-            getOrCreateVisualizationsOnPage(textBlockOnPage.page().getPageNumber(), this.tocBlocks).getColoredRectangles()
-                    .add(new ColoredRectangle(textBlockOnPage.textBlock().getBBoxPdf(), TOC_COLOR, LINE_WIDTH));
-        }
-    }
-
 }

layoutparser-service/layoutparser-service-server/build.gradle.kts

@@ -42,6 +42,7 @@ dependencies {
 
     implementation("com.pdftron:PDFNet:10.11.0")
 
+
     // for integration testing only
     testImplementation(project(":viewer-doc-processor"))
     testImplementation(project(":layoutparser-service-internal-api"))
@@ -49,7 +50,7 @@ dependencies {
 
     testImplementation("org.springframework.boot:spring-boot-starter-amqp:${springBootStarterVersion}")
     testImplementation("org.springframework.boot:spring-boot-starter-test:${springBootStarterVersion}")
-    testImplementation("com.iqser.red.service:persistence-service-shared-api-v1:2.564.0-RED9010.0")
+    testImplementation("com.iqser.red.service:persistence-service-shared-api-v1:2.589.1-RED10196.0")
     testImplementation("com.iqser.red.commons:jackson-commons:1.3.0")
     testImplementation("com.fasterxml.jackson.module:jackson-module-afterburner:${jacksonVersion}")
     testImplementation("com.fasterxml.jackson.datatype:jackson-datatype-jsr310:${jacksonVersion}")

layoutparser-service/layoutparser-service-server/src/main/java/com/knecon/fforesight/service/layoutparser/server/queue/MessageHandler.java

@@ -13,10 +13,10 @@ import org.springframework.amqp.rabbit.core.RabbitTemplate;
 import org.springframework.stereotype.Service;
 
 import com.fasterxml.jackson.databind.ObjectMapper;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.LayoutParsingType;
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingFinishedEvent;
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingQueueNames;
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingRequest;
-import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
 import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingPipeline;
 import com.knecon.fforesight.tenantcommons.TenantContext;
 

layoutparser-service/layoutparser-service-server/src/test/java/com/knecon/fforesight/service/layoutparser/server/BdrJsonBuildTest.java

@@ -21,8 +21,8 @@ import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.core.io.ClassPathResource;
 
 import com.fasterxml.jackson.databind.ObjectMapper;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.LayoutParsingType;
 import com.knecon.fforesight.service.layoutparser.internal.api.data.taas.ResearchDocumentData;
-import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
 import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingPipeline;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.textblock.TextBlock;

layoutparser-service/layoutparser-service-server/src/test/java/com/knecon/fforesight/service/layoutparser/server/HeadlinesGoldStandardIntegrationTest.java

@@ -27,10 +27,9 @@ import org.springframework.core.io.ClassPathResource;
 import org.springframework.test.context.junit.jupiter.SpringExtension;
 
 import com.fasterxml.jackson.databind.ObjectMapper;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ChangeType;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.LayoutParsingType;
 import com.iqser.red.storage.commons.StorageAutoConfiguration;
 import com.iqser.red.storage.commons.service.StorageService;
-import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
 import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingPipeline;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.SemanticNode;

layoutparser-service/layoutparser-service-server/src/test/java/com/knecon/fforesight/service/layoutparser/server/LayoutparserEnd2EndTest.java

@@ -14,10 +14,10 @@ import org.junit.jupiter.api.Test;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.core.io.ClassPathResource;
 
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.LayoutParsingType;
 import com.iqser.red.storage.commons.utils.FileSystemBackedStorageService;
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingFinishedEvent;
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingRequest;
-import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
 import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingPipeline;
 import com.knecon.fforesight.service.layoutparser.server.utils.AbstractTest;
 

layoutparser-service/layoutparser-service-server/src/test/java/com/knecon/fforesight/service/layoutparser/server/OutlineDetectionTest.java

@@ -10,7 +10,6 @@ import java.util.Collection;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
-import java.util.function.Predicate;
 
 import org.apache.commons.lang3.StringUtils;
 import org.junit.jupiter.api.BeforeEach;
@@ -18,8 +17,8 @@ import org.junit.jupiter.api.Test;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.core.io.ClassPathResource;
 
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.LayoutParsingType;
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingRequest;
-import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
 import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingPipeline;
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationDocument;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;

layoutparser-service/layoutparser-service-server/src/test/java/com/knecon/fforesight/service/layoutparser/server/SimplifiedTextServiceTest.java

@@ -5,15 +5,14 @@ import static org.assertj.core.api.AssertionsForClassTypes.assertThat;
 import java.io.File;
 import java.util.List;
 import java.util.Map;
-import java.util.stream.Collectors;
 
 import org.junit.jupiter.api.Test;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.core.io.ClassPathResource;
 
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.LayoutParsingType;
 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.SimplifiedSectionText;
 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.SimplifiedText;
-import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
 import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingPipeline;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.image.ImageServiceResponse;

layoutparser-service/layoutparser-service-server/src/test/java/com/knecon/fforesight/service/layoutparser/server/graph/DocumentGraphJsonWritingTest.java

@@ -12,8 +12,8 @@ import org.springframework.core.io.ClassPathResource;
 
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.iqser.red.commons.jackson.ObjectMapperFactory;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.LayoutParsingType;
 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentData;
-import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.image.ImageServiceResponse;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableServiceResponse;

layoutparser-service/layoutparser-service-server/src/test/java/com/knecon/fforesight/service/layoutparser/server/graph/DocumentReadingOrderTest.java

@@ -17,9 +17,9 @@ import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.Test;
 import org.springframework.beans.factory.annotation.Autowired;
 
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.LayoutParsingType;
 import com.iqser.red.storage.commons.service.StorageService;
 import com.iqser.red.storage.commons.utils.FileSystemBackedStorageService;
-import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
 import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBlock;
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationDocument;
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationPage;

layoutparser-service/layoutparser-service-server/src/test/java/com/knecon/fforesight/service/layoutparser/server/graph/ViewerDocumentTest.java

@@ -11,7 +11,7 @@ import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.core.io.ClassPathResource;
 
 import com.iqser.red.commons.jackson.ObjectMapperFactory;
-import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.LayoutParsingType;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.image.ImageServiceResponse;
 import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableServiceResponse;

layoutparser-service/layoutparser-service-server/src/test/java/com/knecon/fforesight/service/layoutparser/server/segmentation/PdfSegmentationServiceTest.java

@@ -22,7 +22,7 @@ import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.core.io.ClassPathResource;
 
 import com.fasterxml.jackson.databind.ObjectMapper;
-import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.LayoutParsingType;
 import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingPipeline;
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationDocument;
 import com.knecon.fforesight.service.layoutparser.processor.model.PageContents;

layoutparser-service/layoutparser-service-server/src/test/java/com/knecon/fforesight/service/layoutparser/server/services/BodyTextFrameServiceTest.java

@@ -5,7 +5,7 @@ import java.util.List;
 
 import org.junit.jupiter.api.Test;
 
-import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.LayoutParsingType;
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationDocument;
 import com.knecon.fforesight.service.layoutparser.processor.utils.RectangleTransformations;
 import com.knecon.fforesight.service.layoutparser.server.utils.BuildDocumentTest;

layoutparser-service/layoutparser-service-server/src/test/java/com/knecon/fforesight/service/layoutparser/server/services/RulingCleaningServiceTest.java

@@ -16,11 +16,11 @@ import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.Test;
 import org.springframework.core.io.ClassPathResource;
 
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.LayoutParsingType;
 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentData;
 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentStructureWrapper;
 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.EntryDataProto.EntryData;
 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeTypeProto.NodeType;
-import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
 import com.knecon.fforesight.service.layoutparser.processor.model.PageContents;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Table;
@@ -105,7 +105,7 @@ public class RulingCleaningServiceTest extends BuildDocumentTest {
     private void writeJsons(Path filename) {
 
         Document documentGraphBefore = DocumentGraphFactory.buildDocumentGraph(LayoutParsingType.REDACT_MANAGER_WITHOUT_DUPLICATE_PARAGRAPH,
-                layoutParsingPipeline.parseLayout(LayoutParsingType.REDACT_MANAGER_WITHOUT_DUPLICATE_PARAGRAPH,
+                                                                               layoutParsingPipeline.parseLayout(LayoutParsingType.REDACT_MANAGER_WITHOUT_DUPLICATE_PARAGRAPH,
                         filename.toFile(),
                         new ImageServiceResponse(),
                         new TableServiceResponse(),

layoutparser-service/layoutparser-service-server/src/test/java/com/knecon/fforesight/service/layoutparser/server/utils/AbstractTest.java

@@ -27,10 +27,10 @@ import org.springframework.core.io.ClassPathResource;
 import org.springframework.test.context.junit.jupiter.SpringExtension;
 
 import com.iqser.red.commons.jackson.ObjectMapperFactory;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.LayoutParsingType;
 import com.iqser.red.storage.commons.service.StorageService;
 import com.iqser.red.storage.commons.utils.FileSystemBackedStorageService;
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingRequest;
-import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
 import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingStorageService;
 import com.knecon.fforesight.service.layoutparser.server.Application;
 import com.knecon.fforesight.tenantcommons.TenantContext;

layoutparser-service/layoutparser-service-server/src/test/java/com/knecon/fforesight/service/layoutparser/server/utils/BuildDocumentTest.java

@@ -7,8 +7,8 @@ import java.util.Map;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.core.io.ClassPathResource;
 
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.LayoutParsingType;
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingRequest;
-import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
 import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingPipeline;
 import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationDocument;
 import com.knecon.fforesight.service.layoutparser.processor.model.graph.nodes.Document;

layoutparser-service/viewer-doc-processor/src/main/java/com/knecon/fforesight/service/viewerdoc/LayerIdentifier.java

@@ -57,7 +57,6 @@ public record LayerIdentifier(String name, String markedContentName) {
     public static final LayerIdentifier OUTLINE_OBJECTS = new LayerIdentifier("Outline Positions", "OUTLINE_OBJECTS");
     public static final LayerIdentifier SENTENCES = new LayerIdentifier("Sentences", "SENTENCES");
     public static final LayerIdentifier TOC_PAGES = new LayerIdentifier("TOC pages", "TOC_PAGES");
-    public static final LayerIdentifier TOC_BLOCKS = new LayerIdentifier("TOC blocks", "TOC_BLOCKS");
     public static final LayerIdentifier LIST_IDENTIFIERS = new LayerIdentifier("List identifiers", "LIST_IDENTIFIERS");
 
     // Visual layout parser

layoutparser-service/viewer-doc-processor/src/main/java/com/knecon/fforesight/service/viewerdoc/layers/LayoutDebugLayerConfig.java

@@ -22,7 +22,6 @@ public class LayoutDebugLayerConfig extends AbstractLayerGroup {
 
     protected static final Color WORDS_COLOR = new Color(68, 84, 147);
     protected static final Color LINES_COLOR = new Color(152, 45, 179);
-    protected static final Color TOC_COLOR = new Color(33, 159, 144);
     protected static final Color ZONES_COLOR = new Color(131, 38, 38);
 
     protected static final Color RULINGS_COLOR = new Color(21, 221, 174);
@@ -60,7 +59,6 @@ public class LayoutDebugLayerConfig extends AbstractLayerGroup {
     protected final Visualizations outlineObjects = Visualizations.builder().layer(LayerIdentifier.OUTLINE_OBJECTS).build();
     protected final Visualizations sentences = Visualizations.builder().layer(LayerIdentifier.SENTENCES).build();
     protected final Visualizations tocPages = Visualizations.builder().layer(LayerIdentifier.TOC_PAGES).build();
-    protected final Visualizations tocBlocks = Visualizations.builder().layer(LayerIdentifier.TOC_BLOCKS).build();
     protected final Visualizations listIdentifiers = Visualizations.builder().layer(LayerIdentifier.LIST_IDENTIFIERS).build();
 
 
@@ -79,7 +77,6 @@ public class LayoutDebugLayerConfig extends AbstractLayerGroup {
                        markedContent, //
                        outlineObjects, //
                        tocPages, //
-                       tocBlocks, //
                        listIdentifiers //
         );
     }