Merge branch 'RED-9975-bp' into 'release/0.159.x'

Red 9975: add outline debug layer

See merge request fforesight/layout-parser!207

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/data/redaction/SimplifiedText.java

@@ -21,14 +21,5 @@ public class SimplifiedText {
     @Schema(description = "A List of simplified Sections, which contains almost exclusively text.")
     @Builder.Default
     private List<SimplifiedSectionText> sectionTexts = new ArrayList<>();
-    @Schema(description = "A list of the main section numbers ")
-    @Builder.Default
-    private List<String> mainSectionNumbers = new ArrayList<>();
-    @Schema(description = "A list of the header section numbers ")
-    @Builder.Default
-    private List<String> headerSectionNumbers = new ArrayList<>();
-    @Schema(description = "A list of the footer section numbers ")
-    @Builder.Default
-    private List<String> footerSectionNumbers = new ArrayList<>();
 
 }

layoutparser-service/layoutparser-service-internal-api/src/main/java/com/knecon/fforesight/service/layoutparser/internal/api/queue/LayoutParsingQueueNames.java

@@ -2,9 +2,6 @@ package com.knecon.fforesight.service.layoutparser.internal.api.queue;
 
 public class LayoutParsingQueueNames {
 
-    public static final String LAYOUT_PARSING_REQUEST_QUEUE_PREFIX = "layout_parsing_request";
-    public static final String LAYOUT_PARSING_REQUEST_EXCHANGE = "layout_parsing_request_exchange";
-    public static final String LAYOUT_PARSING_RESPONSE_QUEUE_PREFIX = "layout_parsing_response";
-    public static final String LAYOUT_PARSING_RESPONSE_EXCHANGE = "layout_parsing_response_exchange";
-    public static final String LAYOUT_PARSING_DLQ = "layout_parsing_error";
+    public static final String LAYOUT_PARSING_REQUEST_QUEUE = "layout_parsing_request_queue";
+    public static final String LAYOUT_PARSING_FINISHED_EVENT_QUEUE = "layout_parsing_response_queue";
 }

layoutparser-service/layoutparser-service-processor/build.gradle.kts

@@ -16,7 +16,7 @@ dependencies {
         exclude("org.springframework.boot", "spring-boot-starter-security")
         exclude("org.springframework.boot", "spring-boot-starter-validation")
     }
-    implementation("com.knecon.fforesight:tenant-commons:0.29.0")
+    implementation("com.knecon.fforesight:tenant-commons:0.21.0")
     implementation("com.iqser.red.commons:storage-commons:2.45.0")
 
     implementation("org.apache.pdfbox:pdfbox:${pdfBoxVersion}")
@@ -29,6 +29,5 @@ dependencies {
     implementation("org.commonmark:commonmark:0.22.0")
     implementation("org.commonmark:commonmark-ext-gfm-tables:0.22.0")
     implementation("com.pdftron:PDFNet:10.11.0")
-    implementation("org.apache.commons:commons-text:1.12.0")
 
 }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/LayoutParsingPipeline.java

@@ -119,14 +119,18 @@ public class LayoutParsingPipeline {
         log.info("Starting layout parsing for {}", layoutParsingRequest.identifier());
 
         File originFile = layoutParsingStorageService.getOriginFile(layoutParsingRequest.originFileStorageId());
-        File viewerDocumentFile = layoutParsingStorageService.getViewerDocFile(layoutParsingRequest.viewerDocumentStorageId()).orElse(originFile);
+        File viewerDocumentFile = layoutParsingStorageService.getViewerDocFile(layoutParsingRequest.viewerDocumentStorageId())
+                .orElse(originFile);
 
         VisualLayoutParsingResponse visualLayoutParsingResponse = layoutParsingRequest.visualLayoutParsingFileId()
-                .map(layoutParsingStorageService::getVisualLayoutParsingFile).orElse(new VisualLayoutParsingResponse());
+                .map(layoutParsingStorageService::getVisualLayoutParsingFile)
+                .orElse(new VisualLayoutParsingResponse());
         ImageServiceResponse imageServiceResponse = layoutParsingRequest.imagesFileStorageId()
-                .map(layoutParsingStorageService::getImagesFile).orElse(new ImageServiceResponse());
+                .map(layoutParsingStorageService::getImagesFile)
+                .orElse(new ImageServiceResponse());
         TableServiceResponse tableServiceResponse = layoutParsingRequest.tablesFileStorageId()
-                .map(layoutParsingStorageService::getTablesFile).orElse(new TableServiceResponse());
+                .map(layoutParsingStorageService::getTablesFile)
+                .orElse(new TableServiceResponse());
 
         ClassificationDocument classificationDocument = parseLayout(settings.getLayoutParsingTypeOverride() == null //
                                                                             ? layoutParsingRequest.layoutParsingType() : settings.getLayoutParsingTypeOverride(),
@@ -143,13 +147,20 @@ public class LayoutParsingPipeline {
 
         log.info("Creating viewer document for {}", layoutParsingRequest.identifier());
 
-        layoutGridService.addLayoutGrid(viewerDocumentFile, documentGraph, viewerDocumentFile, false, layoutParsingRequest.visualLayoutParsingFileId().isPresent());
+        layoutGridService.addLayoutGrid(viewerDocumentFile,
+                                        documentGraph,
+                                        viewerDocumentFile,
+                                        false,
+                                        layoutParsingRequest.visualLayoutParsingFileId()
+                                                .isPresent());
 
         log.info("Storing resulting files for {}", layoutParsingRequest.identifier());
 
         layoutParsingStorageService.storeDocumentData(layoutParsingRequest, DocumentDataMapper.toDocumentData(documentGraph));
-        if (layoutParsingRequest.documentMarkdownFileStorageId().isPresent()) {
-            layoutParsingStorageService.storeMarkdownFile(layoutParsingRequest.documentMarkdownFileStorageId().get(), new MarkdownMapper().toMarkdownContent(documentGraph));
+        if (layoutParsingRequest.documentMarkdownFileStorageId()
+                .isPresent()) {
+            layoutParsingStorageService.storeMarkdownFile(layoutParsingRequest.documentMarkdownFileStorageId()
+                                                                  .get(), new MarkdownMapper().toMarkdownContent(documentGraph));
         }
         layoutParsingStorageService.storeSimplifiedText(layoutParsingRequest, simplifiedSectionTextService.toSimplifiedText(documentGraph));
         layoutParsingStorageService.storeViewerDocument(layoutParsingRequest, viewerDocumentFile);
@@ -325,18 +336,17 @@ public class LayoutParsingPipeline {
             classificationPage.setPageHeight(cropbox.getHeight());
 
             if (layoutParsingType != LayoutParsingType.REDACT_MANAGER_OLD) {
-                List<OutlineObject> outlineObjects = classificationDocument.getOutlineObjectTree().getOutlineObjectsPerPage().getOrDefault(pageNumber, new ArrayList<>());
+                List<OutlineObject> outlineObjects = classificationDocument.getOutlineObjectTree().getOutlineObjectsPerPage().getOrDefault(pageNumber - 1, new ArrayList<>());
 
                 OutlineObject notFoundOutlineObject = null;
                 if (lastProcessedOutlineObject != null && !lastProcessedOutlineObject.isFound()) {
-                    lastProcessedOutlineObject.resetPoint();
+                    lastProcessedOutlineObject.setPoint(new Point2D.Float(0, cropbox.getHeight()));
                     notFoundOutlineObject = lastProcessedOutlineObject;
                 }
                 if (!outlineObjects.isEmpty()) {
                     classificationPage.setOutlineObjects(outlineObjects);
                     lastProcessedOutlineObject = blockificationPostprocessingService.sanitizeOutlineBlocks(classificationPage, notFoundOutlineObject);
                 }
-                classificationDocument.getLayoutDebugLayer().addOutlineObjects(outlineObjects, pageInformation);
             }
 
             classificationDocument.getLayoutDebugLayer().addMarkedContentVisualizations(stripper.getMarkedContents(), pageNumber);

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/graph/nodes/Page.java

@@ -85,7 +85,7 @@ public class Page {
     private SemanticNode getHighestParentOnPage(SemanticNode node) {
 
         SemanticNode currentNode = node;
-        while (currentNode.hasParent() && currentNode.getParent().onlyOnPage(this)) {
+        while (currentNode.getParent().onlyOnPage(this)) {
             currentNode = currentNode.getParent();
         }
         return currentNode;

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/outline/OutlineExtractorService.java

@@ -1,6 +1,5 @@
 package com.knecon.fforesight.service.layoutparser.processor.model.outline;
 
-import java.awt.geom.AffineTransform;
 import java.awt.geom.Point2D;
 import java.io.IOException;
 import java.util.ArrayList;
@@ -27,9 +26,6 @@ import org.apache.pdfbox.pdmodel.interactive.documentnavigation.outline.PDDocume
 import org.apache.pdfbox.pdmodel.interactive.documentnavigation.outline.PDOutlineItem;
 import org.springframework.stereotype.Service;
 
-import com.knecon.fforesight.service.layoutparser.processor.utils.CoordinateTransforms;
-import com.knecon.fforesight.service.layoutparser.processor.utils.PageInformation;
-
 import lombok.SneakyThrows;
 import lombok.extern.slf4j.Slf4j;
 
@@ -93,13 +89,12 @@ public class OutlineExtractorService {
             if (page == null) {
                 return Optional.empty();
             }
-        } catch (IOException e) {
+        }catch (IOException e){
             log.info(String.format("Error occurred during position resolution for outline item with title %s: " + e, title));
             return Optional.empty();
         }
 
-        int pageNumber = document.getPages().indexOf(page) + 1;
-        AffineTransform userSpaceToPageCoords = CoordinateTransforms.calculateInitialUserSpaceCoordsToPageCoords(PageInformation.fromPDPage(pageNumber, page));
+        int pageNumber = document.getPages().indexOf(page);
 
         Optional<Point2D> outlinePosition = Optional.empty();
 
@@ -128,15 +123,8 @@ public class OutlineExtractorService {
             log.info(String.format("Error occurred during position resolution for outline item on page %s with title %s: " + e, pageNumber, title));
         }
 
-        return Optional.of(new OutlineObjectTreeNode(new OutlineObject(title,
-                                                                       pageNumber,
-                                                                       transformPointToPageCoords(outlinePosition, userSpaceToPageCoords), depth)));
-    }
+        return Optional.of(new OutlineObjectTreeNode(new OutlineObject(title, pageNumber, outlinePosition.orElse(new Point2D.Float(0, 0)), depth)));
 
-
-    private static Point2D transformPointToPageCoords(Optional<Point2D> outlinePosition, AffineTransform userSpaceToPageCoords) {
-
-        return outlinePosition.map(point -> userSpaceToPageCoords.transform(point, null)).orElse(null);
     }
 
 

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/outline/OutlineObject.java

@@ -1,34 +1,27 @@
 package com.knecon.fforesight.service.layoutparser.processor.model.outline;
 
 import java.awt.geom.Point2D;
-import java.util.Optional;
 
-import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.BoundingBox;
-
-import lombok.Getter;
-import lombok.Setter;
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.RequiredArgsConstructor;
 
+@Data
+@RequiredArgsConstructor
+@AllArgsConstructor
 public class OutlineObject {
 
-    @Getter
     private final String title;
-    @Getter
     private final int pageNumber;
-    @Getter
+    private Point2D point;
     private final int treeDepth;
 
-    private Point2D point; // java coordinates, (0, 0) is always top left
-
-    @Getter
-    @Setter
     private boolean found;
 
 
     public OutlineObject(String title, int pageNumber, Point2D point2D, int depth) {
 
-        this.title = title;
-        this.pageNumber = pageNumber;
-        this.treeDepth = depth;
+        this(title, pageNumber, depth);
         this.point = point2D;
     }
 
@@ -39,39 +32,4 @@ public class OutlineObject {
         return "OutlineObject{" + "title='" + title + '\'' + '}';
     }
 
-
-    public Optional<Point2D> getPoint() {
-
-        return Optional.ofNullable(point);
-    }
-
-
-    public boolean isAbove(BoundingBox boundingBox) {
-
-        if (point == null) {
-            return true;
-        }
-        return point.getY() <= boundingBox.getMaxY();
-    }
-
-
-    public double distance(BoundingBox boundingBox) {
-
-        if (point == null) {
-            return 0;
-        }
-        if (boundingBox.getBBox().contains(point)) {
-            return 0;
-        }
-        double deltaX = Math.min(Math.abs(boundingBox.getMinX() - point.getX()), Math.abs(boundingBox.getMaxX() - point.getX()));
-        double deltaY = Math.min(Math.abs(boundingBox.getMinY() - point.getY()), Math.abs(boundingBox.getMaxY() - point.getY()));
-        return Math.sqrt(deltaX * deltaX + deltaY * deltaY);
-    }
-
-
-    public void resetPoint() {
-
-        this.point = null;
-    }
-
 }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/outline/OutlineObjectTree.java

@@ -39,28 +39,4 @@ public class OutlineObjectTree {
         }
     }
 
-    @Override
-    public String toString() {
-
-        StringBuilder sb = new StringBuilder();
-        sb.append("OutlineObjectTree(\n");
-        for (OutlineObjectTreeNode node : rootNodes) {
-            buildString(node, sb, 1);
-        }
-        sb.append(")");
-        return sb.toString();
-    }
-
-    private void buildString(OutlineObjectTreeNode node, StringBuilder sb, int depth) {
-
-        for (int i = 0; i < depth; i++) {
-            sb.append("  ");
-        }
-        sb.append(node.getOutlineObject().getTitle()).append("\n");
-
-        for (OutlineObjectTreeNode child : node.getChildren()) {
-            buildString(child, sb, depth + 1);
-        }
-    }
-
 }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/table/Cell.java

@@ -87,7 +87,7 @@ public class Cell extends BoundingBox {
 
         }
 
-        return TextNormalizationUtilities.cleanString(sb.toString());
+        return TextNormalizationUtilities.removeHyphenLineBreaks(sb.toString()).replaceAll("\n", " ").replaceAll(" {2}", " ");
     }
 
 

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/text/SearchableText.java

@@ -2,7 +2,6 @@ package com.knecon.fforesight.service.layoutparser.processor.model.text;
 
 import java.util.ArrayList;
 import java.util.List;
-
 import com.knecon.fforesight.service.layoutparser.processor.utils.TextNormalizationUtilities;
 
 import lombok.Getter;
@@ -39,7 +38,11 @@ public class SearchableText {
             sb.append(word);
             sb.append(' ');
         }
-        return TextNormalizationUtilities.cleanString(sb.toString());
+        String text = sb.toString();
+        text = TextNormalizationUtilities.removeHyphenLineBreaks(text);
+        text = TextNormalizationUtilities.removeLineBreaks(text);
+        text = TextNormalizationUtilities.removeRepeatingWhitespaces(text);
+        return text;
     }
 
 }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/text/TextPageBlock.java

@@ -213,7 +213,7 @@ public class TextPageBlock extends AbstractPageBlock {
             previous = word;
         }
 
-        return TextNormalizationUtilities.removeHyphenLinebreaks(sb.toString());
+        return TextNormalizationUtilities.removeHyphenLineBreaks(sb.toString());
     }
 
 

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/SimplifiedSectionTextService.java

@@ -34,22 +34,7 @@ public class SimplifiedSectionTextService {
         List<SimplifiedSectionText> simplifiedText = Stream.of(simplifiedMainSectionsList, simplifiedHeadersList, simplifiedFootersList)
                 .flatMap(List::stream)
                 .collect(Collectors.toList());
-        return SimplifiedText.builder()
-                .numberOfPages(document.getNumberOfPages())
-                .sectionTexts(simplifiedText)
-                .mainSectionNumbers(document.getAllSections()
-                                            .stream()
-                                            .map(this::getSectionNumber)
-                                            .toList())
-                .headerSectionNumbers(document.getHeaders()
-                                              .stream()
-                                              .map(this::getSectionNumber)
-                                              .toList())
-                .footerSectionNumbers(document.getFooters()
-                                              .stream()
-                                              .map(this::getSectionNumber)
-                                              .toList())
-                .build();
+        return SimplifiedText.builder().numberOfPages(document.getNumberOfPages()).sectionTexts(simplifiedText).build();
     }
 
 
@@ -64,13 +49,4 @@ public class SimplifiedSectionTextService {
                 .build();
     }
 
-
-    private String getSectionNumber(SemanticNode semanticNode) {
-
-        return semanticNode.getTreeId()
-                .stream()
-                .map(String::valueOf)
-                .collect(Collectors.joining("."));
-    }
-
 }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/blockification/BlockificationPostprocessingService.java

@@ -8,7 +8,6 @@ import java.util.ListIterator;
 import java.util.Locale;
 
 import org.apache.commons.lang3.StringUtils;
-import org.apache.commons.text.similarity.LevenshteinDistance;
 import org.springframework.stereotype.Service;
 
 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
@@ -24,7 +23,7 @@ import lombok.Data;
 @Service
 public class BlockificationPostprocessingService {
 
-    private static final float STRING_SIMILARITY_THRESHOLD = 0.1f;
+    private static final float BLOCK_TO_OUTLINE_DISTANCE_THRESHOLD = 5.0f;
 
 
     public OutlineObject sanitizeOutlineBlocks(ClassificationPage classificationPage, OutlineObject notFoundOutlineObject) {
@@ -35,36 +34,38 @@ public class BlockificationPostprocessingService {
             return null;
         }
 
+        float pageHeight = classificationPage.getPageHeight();
+
         ListIterator<OutlineObject> outlineObjectListIterator = outlineObjects.listIterator();
 
         if (notFoundOutlineObject != null) {
             OutlineProcessionContext notFoundOutlineObjectProcessionContext = new OutlineProcessionContext(notFoundOutlineObject);
-            processTextBlocks(getTextPageBlocks(classificationPage), notFoundOutlineObjectProcessionContext);
+            processTextBlocks(getTextPageBlocks(classificationPage), pageHeight, notFoundOutlineObjectProcessionContext);
 
             OutlineObject firstOutlineObject = null;
             OutlineProcessionContext firstOutlineObjectProcessionContext = null;
             if (outlineObjectListIterator.hasNext()) {
                 firstOutlineObject = outlineObjectListIterator.next();
                 firstOutlineObjectProcessionContext = new OutlineProcessionContext(firstOutlineObject);
-                processTextBlocks(getTextPageBlocks(classificationPage), firstOutlineObjectProcessionContext);
+                processTextBlocks(getTextPageBlocks(classificationPage), pageHeight, firstOutlineObjectProcessionContext);
             }
 
             if (!contextsOverlap(notFoundOutlineObjectProcessionContext, firstOutlineObjectProcessionContext)) {
-                notFoundOutlineObject.setFound(selectMatch(classificationPage, notFoundOutlineObjectProcessionContext));
+                notFoundOutlineObject.setFound(selectMatch(classificationPage, notFoundOutlineObjectProcessionContext, pageHeight));
             }
             if (firstOutlineObject != null) {
                 // re-create the context for the updated blocks
                 firstOutlineObjectProcessionContext = new OutlineProcessionContext(firstOutlineObject);
-                processTextBlocks(getTextPageBlocks(classificationPage), firstOutlineObjectProcessionContext);
-                firstOutlineObject.setFound(selectMatch(classificationPage, firstOutlineObjectProcessionContext));
+                processTextBlocks(getTextPageBlocks(classificationPage), pageHeight, firstOutlineObjectProcessionContext);
+                firstOutlineObject.setFound(selectMatch(classificationPage, firstOutlineObjectProcessionContext, pageHeight));
             }
 
         }
 
         outlineObjectListIterator.forEachRemaining(outlineObject -> {
             OutlineProcessionContext outlineObjectProcessionContext = new OutlineProcessionContext(outlineObject);
-            processTextBlocks(getTextPageBlocks(classificationPage), outlineObjectProcessionContext);
-            outlineObject.setFound(selectMatch(classificationPage, outlineObjectProcessionContext));
+            processTextBlocks(getTextPageBlocks(classificationPage), pageHeight, outlineObjectProcessionContext);
+            outlineObject.setFound(selectMatch(classificationPage, outlineObjectProcessionContext, pageHeight));
         });
 
         if (!outlineObjects.isEmpty()) {
@@ -103,7 +104,8 @@ public class BlockificationPostprocessingService {
 
         double maxYFirst = blocksOfFirstOutline.stream()
                 .mapToDouble(TextPageBlock::getPdfMaxY)
-                .max().orElse(Double.NEGATIVE_INFINITY);
+                .max()
+                .orElse(Double.NEGATIVE_INFINITY);
 
         return blocksOfNotFoundOutline.stream()
                 .mapToDouble(TextPageBlock::getPdfMaxY)
@@ -125,13 +127,13 @@ public class BlockificationPostprocessingService {
     }
 
 
-    private void processTextBlocks(List<TextPageBlock> textBlocks, OutlineProcessionContext context) {
+    private void processTextBlocks(List<TextPageBlock> textBlocks, float pageHeight, OutlineProcessionContext context) {
 
         OutlineObject outlineObject = context.getOutlineObject();
         ListIterator<TextPageBlock> iterator = textBlocks.listIterator();
         while (iterator.hasNext()) {
             TextPageBlock pageBlock = iterator.next();
-            if (outlineObject.isAbove(pageBlock)) {
+            if (pageHeight - outlineObject.getPoint().getY() - BLOCK_TO_OUTLINE_DISTANCE_THRESHOLD <= pageBlock.getMaxY()) {
                 break;
             }
         }
@@ -146,7 +148,7 @@ public class BlockificationPostprocessingService {
     }
 
 
-    private boolean selectMatch(ClassificationPage classificationPage, OutlineProcessionContext context) {
+    private boolean selectMatch(ClassificationPage classificationPage, OutlineProcessionContext context, float pageHeight) {
 
         OutlineObject outlineObject = context.outlineObject;
         TextPageBlock directMatch = context.directMatch;
@@ -154,8 +156,8 @@ public class BlockificationPostprocessingService {
         TextPageBlock splitCandidate = context.splitCandidate;
         PageBlockType headlineType = PageBlockType.getHeadlineType(outlineObject.getTreeDepth());
 
-        double distanceToDirectMatch = directMatch != null ? calculateDistance(outlineObject, directMatch) : Double.MAX_VALUE;
-        double distanceToSplitCandidate = splitCandidate != null ? calculateDistance(outlineObject, splitCandidate) : Double.MAX_VALUE;
+        double distanceToDirectMatch = directMatch != null ? calculateDistance(outlineObject, directMatch, pageHeight) : Double.MAX_VALUE;
+        double distanceToSplitCandidate = splitCandidate != null ? calculateDistance(outlineObject, splitCandidate, pageHeight) : Double.MAX_VALUE;
 
         double distanceToBestMergeCandidates = Double.MAX_VALUE;
         List<TextPageBlock> bestMergeCandidateCombination = new ArrayList<>();
@@ -175,8 +177,9 @@ public class BlockificationPostprocessingService {
 
             for (List<TextPageBlock> combination : combinations) {
                 double averageDistance = combination.stream()
-                        .map(block -> calculateDistance(outlineObject, block))
-                        .mapToDouble(Double::doubleValue).average().orElse(Double.MAX_VALUE);
+                        .map(block -> calculateDistance(outlineObject, block, pageHeight))
+                        .mapToDouble(Double::doubleValue).average()
+                        .orElse(Double.MAX_VALUE);
                 if (distanceToBestMergeCandidates > averageDistance) {
                     distanceToBestMergeCandidates = averageDistance;
                     bestMergeCandidateCombination = combination;
@@ -403,9 +406,11 @@ public class BlockificationPostprocessingService {
     }
 
 
-    private double calculateDistance(OutlineObject outlineObject, TextPageBlock pageBlock) {
+    private double calculateDistance(OutlineObject outlineObject, TextPageBlock pageBlock, float pageHeight) {
 
-        return outlineObject.distance(pageBlock);
+        double deltaX = outlineObject.getPoint().getX() - pageBlock.getMinX();
+        double deltaY = pageHeight - outlineObject.getPoint().getY() - pageBlock.getMinY();
+        return Math.sqrt(deltaX * deltaX + deltaY * deltaY);
     }
 
 
@@ -422,13 +427,6 @@ public class BlockificationPostprocessingService {
         String blockText = sanitizeString(pageBlock.getText());
         String outlineTitle = sanitizeString(outlineObject.getTitle());
 
-        int threshold = (int) (Math.min(blockText.length(), outlineTitle.length()) * STRING_SIMILARITY_THRESHOLD) + 1;
-        int distance = new LevenshteinDistance(threshold).apply(blockText, outlineTitle);
-        if (distance >= 0 && distance < threshold) {
-            context.directMatch = pageBlock;
-            return true;
-        }
-
         boolean blockTextContainsOutlineTitle = blockText.contains(outlineTitle);
         boolean outlineTitleContainsBlockText = outlineTitle.contains(blockText);
 

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/utils/CoordinateTransforms.java

@@ -32,6 +32,7 @@ public class CoordinateTransforms {
     }
 
 
+
     @SneakyThrows
     public AffineTransform calculateInitialUserSpaceCoordsToImageCoords(PageInformation pageInformation, double scalingFactor) {
 
@@ -39,19 +40,6 @@ public class CoordinateTransforms {
     }
 
 
-    public AffineTransform calculatePageCoordsToInitialUserSpaceCoords(PageInformation pageInformation) {
-
-        return calculateImageCoordsToInitialUserSpaceCoords(pageInformation, 1);
-    }
-
-
-    @SneakyThrows
-    public AffineTransform calculateInitialUserSpaceCoordsToPageCoords(PageInformation pageInformation) {
-
-        return calculatePageCoordsToInitialUserSpaceCoords(pageInformation).createInverse();
-    }
-
-
     public double calculateScalingFactor(PageInformation pageInformation, double imageWidth) {
 
         // PDFBox always returns page height and width based on rotation

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/utils/TextNormalizationUtilities.java

@@ -1,40 +1,31 @@
 package com.knecon.fforesight.service.layoutparser.processor.utils;
 
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
 import lombok.experimental.UtilityClass;
 
 @UtilityClass
 public final class TextNormalizationUtilities {
 
-    public static final Pattern hyphenLineBreaks = Pattern.compile("[-~‐‒⁻−﹣゠⁓‑\\u00AD][\\r\\n]+");
-    public static final Pattern linebreaks = Pattern.compile("[\\r\\n]+");
-    public static final Pattern doubleWhitespaces = Pattern.compile("\\s{2,}");
+    /**
+     * Revert hyphenation due to line breaks.
+     *
+     * @param text Text to be processed.
+     * @return Text without line-break hyphenation.
+     */
+    public static String removeHyphenLineBreaks(String text) {
 
-
-    public String cleanString(String value) {
-
-        String noHyphenLinebreaks = removeHyphenLinebreaks(value);
-        String noLinebreaks = removeLinebreaks(noHyphenLinebreaks);
-        return removeMultipleWhitespaces(noLinebreaks);
+        return text.replaceAll("([^\\s\\d\\-]{2,500})[\\-\\u00AD]\\R", "$1");
     }
 
 
-    public String removeHyphenLinebreaks(String value) {
+    public static String removeLineBreaks(String text) {
 
-        return hyphenLineBreaks.matcher(value).replaceAll("");
+        return text.replaceAll("\n", " ");
     }
 
 
-    private String removeMultipleWhitespaces(String value) {
+    public static String removeRepeatingWhitespaces(String text) {
 
-        return doubleWhitespaces.matcher(value).replaceAll(" ");
+        return text.replaceAll(" {2}", " ");
     }
 
-
-    private String removeLinebreaks(String value) {
-
-        return linebreaks.matcher(value).replaceAll(" ");
-    }
 }

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/visualization/LayoutDebugLayer.java

@@ -1,14 +1,11 @@
 package com.knecon.fforesight.service.layoutparser.processor.visualization;
 
 import java.awt.Color;
-import java.awt.geom.AffineTransform;
 import java.awt.geom.Line2D;
 import java.awt.geom.Point2D;
 import java.awt.geom.Rectangle2D;
 import java.util.Collection;
-import java.util.HashMap;
 import java.util.List;
-import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.atomic.AtomicInteger;
 
@@ -18,19 +15,15 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlo
 import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.BoundingBox;
 import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.Line;
 import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.Zone;
-import com.knecon.fforesight.service.layoutparser.processor.model.outline.OutlineObject;
 import com.knecon.fforesight.service.layoutparser.processor.model.table.CleanRulings;
 import com.knecon.fforesight.service.layoutparser.processor.model.table.Ruling;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
 import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPositionSequence;
-import com.knecon.fforesight.service.layoutparser.processor.utils.CoordinateTransforms;
 import com.knecon.fforesight.service.layoutparser.processor.utils.MarkedContentUtils;
-import com.knecon.fforesight.service.layoutparser.processor.utils.PageInformation;
 import com.knecon.fforesight.service.layoutparser.processor.utils.RectangleTransformations;
 import com.knecon.fforesight.service.viewerdoc.layers.LayoutDebugLayerConfig;
 import com.knecon.fforesight.service.viewerdoc.model.ColoredLine;
 import com.knecon.fforesight.service.viewerdoc.model.ColoredRectangle;
-import com.knecon.fforesight.service.viewerdoc.model.FilledRectangle;
 import com.knecon.fforesight.service.viewerdoc.model.PlacedText;
 import com.knecon.fforesight.service.viewerdoc.model.VisualizationsOnPage;
 
@@ -50,8 +43,6 @@ public class LayoutDebugLayer extends LayoutDebugLayerConfig {
 
     boolean active;
 
-    Map<Integer, AtomicInteger> outlineObjectsWithoutPointsPerPage = new HashMap<>();
-
 
     public void addTextVisualizations(List<TextPositionSequence> textPositionSequences, int pageNumber) {
 
@@ -160,6 +151,7 @@ public class LayoutDebugLayer extends LayoutDebugLayerConfig {
     }
 
 
+
     public void addLineVisualizationsFromNestedTextPosition(Collection<Set<TextPositionSequence>> lines, int pageNumber) {
 
         if (!active) {
@@ -176,7 +168,6 @@ public class LayoutDebugLayer extends LayoutDebugLayerConfig {
 
     }
 
-
     public void addTextBlockVisualizations(List<TextPageBlock> textPageBlocks, int page) {
 
         if (!active) {
@@ -263,40 +254,4 @@ public class LayoutDebugLayer extends LayoutDebugLayerConfig {
     }
 
 
-    public void addOutlineObjects(List<OutlineObject> outlineObjects, PageInformation pageInformation) {
-
-        if (!active) {
-            return;
-        }
-
-        for (OutlineObject outlineObject : outlineObjects) {
-            addOutlineObject(outlineObject, pageInformation);
-        }
-    }
-
-
-    private void addOutlineObject(OutlineObject outlineObject, PageInformation pageInformation) {
-
-        int rectSize = 5;
-
-        Point2D point2D;
-        if (outlineObject.getPoint().isPresent()) {
-            point2D = outlineObject.getPoint().get();
-        } else {
-            int numberOfOutlineObjectsWithoutPoints = outlineObjectsWithoutPointsPerPage.computeIfAbsent(outlineObject.getPageNumber(), a -> new AtomicInteger(0))
-                    .getAndIncrement();
-            point2D = new Point2D.Double(10, 10 + numberOfOutlineObjectsWithoutPoints * (10 + rectSize * 2));
-        }
-
-        Point2D textPoint = new Point2D.Double(point2D.getX() + 2 * rectSize, point2D.getY() + rectSize);
-        AffineTransform pageToUserSpaceTransform = CoordinateTransforms.calculatePageCoordsToInitialUserSpaceCoords(pageInformation);
-        pageToUserSpaceTransform.transform(point2D, point2D);
-        pageToUserSpaceTransform.transform(textPoint, textPoint);
-
-        VisualizationsOnPage visualizationsOnPage = getOrCreateVisualizationsOnPage(outlineObject.getPageNumber(), outlineObjects);
-        visualizationsOnPage.getFilledRectangles()
-                .add(new FilledRectangle(new Rectangle2D.Double(point2D.getX() - rectSize, point2D.getY() - rectSize, rectSize * 2, rectSize * 2), OUTLINE_OBJECT_COLOR, 1));
-        visualizationsOnPage.getPlacedTexts().add(PlacedText.textFacingUp(outlineObject.getTitle(), textPoint, 10, outlineObject.isFound() ? Color.BLACK : Color.RED, FONT));
-    }
-
 }

layoutparser-service/layoutparser-service-server/build.gradle.kts

@@ -30,7 +30,7 @@ dependencies {
     implementation(project(":layoutparser-service-internal-api"))
 
     implementation("com.iqser.red.commons:storage-commons:2.45.0")
-    implementation("com.knecon.fforesight:tenant-commons:0.29.0")
+    implementation("com.knecon.fforesight:tenant-commons:0.21.0")
     implementation("com.knecon.fforesight:tracing-commons:0.5.0")
     implementation("com.knecon.fforesight:lifecycle-commons:0.6.0")
     implementation("org.springframework.boot:spring-boot-starter-actuator:${springBootStarterVersion}")

layoutparser-service/layoutparser-service-server/src/main/java/com/knecon/fforesight/service/layoutparser/server/configuration/MessagingConfiguration.java

@@ -1,29 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.server.configuration;
-
-import static com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingQueueNames.LAYOUT_PARSING_DLQ;
-
-import org.springframework.amqp.core.DirectExchange;
-import org.springframework.amqp.core.Queue;
-import org.springframework.amqp.core.QueueBuilder;
-import org.springframework.context.annotation.Bean;
-import org.springframework.context.annotation.Configuration;
-
-import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingQueueNames;
-
-@Configuration
-public class MessagingConfiguration {
-
-    @Bean
-    public DirectExchange layoutParsingRequestExchange() {
-
-        return new DirectExchange(LayoutParsingQueueNames.LAYOUT_PARSING_REQUEST_EXCHANGE);
-    }
-
-
-    @Bean
-    public Queue layoutParsingDLQ() {
-
-        return QueueBuilder.durable(LAYOUT_PARSING_DLQ).build();
-    }
-
-}

layoutparser-service/layoutparser-service-server/src/main/java/com/knecon/fforesight/service/layoutparser/server/configuration/TenantMessagingConfigurationImpl.java

@@ -1,11 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.server.configuration;
-
-import org.springframework.context.annotation.Configuration;
-
-import com.knecon.fforesight.tenantcommons.queue.TenantMessagingConfiguration;
-
-@Configuration
-public class TenantMessagingConfigurationImpl extends TenantMessagingConfiguration {
-
-
-}

layoutparser-service/layoutparser-service-server/src/main/java/com/knecon/fforesight/service/layoutparser/server/queue/MessageHandler.java

@@ -18,7 +18,6 @@ import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsi
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingRequest;
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
 import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingPipeline;
-import com.knecon.fforesight.tenantcommons.TenantContext;
 
 import lombok.RequiredArgsConstructor;
 import lombok.SneakyThrows;
@@ -29,17 +28,15 @@ import lombok.extern.slf4j.Slf4j;
 @RequiredArgsConstructor
 public class MessageHandler {
 
-    public static final String LAYOUT_PARSING_REQUEST_LISTENER_ID = "layout-parsing-request-listener";
-
     private final LayoutParsingPipeline layoutParsingPipeline;
     private final ObjectMapper objectMapper;
     private final RabbitTemplate rabbitTemplate;
     private final static String X_PIPELINE_PREFIX = "X-PIPE-";
 
 
-    @SneakyThrows
     @RabbitHandler
-    @RabbitListener(id = LAYOUT_PARSING_REQUEST_LISTENER_ID)
+    @RabbitListener(queues = LayoutParsingQueueNames.LAYOUT_PARSING_REQUEST_QUEUE)
+    @SneakyThrows
     public void receiveLayoutParsingRequest(Message message) {
 
         LayoutParsingRequest layoutParsingRequest = objectMapper.readValue(message.getBody(), LayoutParsingRequest.class);
@@ -60,7 +57,7 @@ public class MessageHandler {
     public void sendLayoutParsingFinishedEvent(LayoutParsingFinishedEvent layoutParsingFinishedEvent, Message message) {
 
         Arrays.stream(layoutParsingFinishedEvent.message().split("\n")).forEach(log::info);
-        rabbitTemplate.convertAndSend(LayoutParsingQueueNames.LAYOUT_PARSING_RESPONSE_EXCHANGE, TenantContext.getTenantId(), layoutParsingFinishedEvent, m -> {
+        rabbitTemplate.convertAndSend(LayoutParsingQueueNames.LAYOUT_PARSING_FINISHED_EVENT_QUEUE, layoutParsingFinishedEvent, m -> {
             var forwardHeaders = message.getMessageProperties()
                     .getHeaders()
                     .entrySet()

layoutparser-service/layoutparser-service-server/src/main/java/com/knecon/fforesight/service/layoutparser/server/queue/TenantExchangeMessageReceiverImpl.java

@@ -1,64 +0,0 @@
-package com.knecon.fforesight.service.layoutparser.server.queue;
-
-import java.util.Set;
-
-import org.springframework.amqp.rabbit.annotation.RabbitHandler;
-import org.springframework.amqp.rabbit.annotation.RabbitListener;
-import org.springframework.boot.context.event.ApplicationReadyEvent;
-import org.springframework.context.event.EventListener;
-import org.springframework.stereotype.Service;
-
-import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingQueueNames;
-import com.knecon.fforesight.tenantcommons.TenantProvider;
-import com.knecon.fforesight.tenantcommons.model.TenantCreatedEvent;
-import com.knecon.fforesight.tenantcommons.model.TenantQueueConfiguration;
-import com.knecon.fforesight.tenantcommons.model.TenantResponse;
-import com.knecon.fforesight.tenantcommons.queue.RabbitQueueFromExchangeService;
-import com.knecon.fforesight.tenantcommons.queue.TenantExchangeMessageReceiver;
-
-@Service
-public class TenantExchangeMessageReceiverImpl extends TenantExchangeMessageReceiver {
-
-    public TenantExchangeMessageReceiverImpl(RabbitQueueFromExchangeService rabbitQueueService, TenantProvider tenantProvider) {
-
-        super(rabbitQueueService, tenantProvider);
-    }
-
-
-    @Override
-    protected Set<TenantQueueConfiguration> getTenantQueueConfigs() {
-
-        return Set.of(TenantQueueConfiguration.builder()
-                              .listenerId(MessageHandler.LAYOUT_PARSING_REQUEST_LISTENER_ID)
-                              .exchangeName(LayoutParsingQueueNames.LAYOUT_PARSING_REQUEST_EXCHANGE)
-                              .queuePrefix(LayoutParsingQueueNames.LAYOUT_PARSING_REQUEST_QUEUE_PREFIX)
-                              .dlqName(LayoutParsingQueueNames.LAYOUT_PARSING_DLQ)
-                              .build());
-    }
-
-
-    @EventListener(ApplicationReadyEvent.class)
-    public void onApplicationReady() {
-
-        System.out.println("application ready invoked");
-        super.initializeQueues();
-    }
-
-
-    @RabbitHandler
-    @RabbitListener(queues = "#{tenantMessagingConfigurationImpl.getTenantCreatedQueueName()}")
-    public void reactToTenantCreation(TenantCreatedEvent tenantCreatedEvent) {
-
-        super.reactToTenantCreation(tenantCreatedEvent);
-    }
-
-
-    @RabbitHandler
-    @RabbitListener(queues = "#{tenantMessagingConfigurationImpl.getTenantDeletedQueueName()}")
-    public void reactToTenantDeletion(TenantResponse tenantResponse) {
-
-        super.reactToTenantDeletion(tenantResponse);
-
-    }
-
-}

layoutparser-service/layoutparser-service-server/src/test/java/com/knecon/fforesight/service/layoutparser/server/HeadlinesGoldStandardIntegrationTest.java

@@ -10,9 +10,7 @@ import java.util.stream.Collectors;
 import org.assertj.core.api.Assertions;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.extension.ExtendWith;
-import org.springframework.amqp.rabbit.core.RabbitAdmin;
 import org.springframework.amqp.rabbit.core.RabbitTemplate;
-import org.springframework.amqp.rabbit.listener.RabbitListenerEndpointRegistry;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
 import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration;
@@ -57,12 +55,6 @@ public class HeadlinesGoldStandardIntegrationTest {
     @MockBean
     private RabbitTemplate rabbitTemplate;
 
-    @MockBean
-    private RabbitAdmin rabbitAdmin;
-
-    @MockBean
-    private RabbitListenerEndpointRegistry rabbitListenerEndpointRegistry;
-
     @Autowired
     private ObjectMapper objectMapper;
 
@@ -78,18 +70,13 @@ public class HeadlinesGoldStandardIntegrationTest {
 
         List<Metrics> metrics = new ArrayList<>();
         metrics.add(getMetrics("files/syngenta/CustomerFiles/01 - CGA100251 - Acute Oral Toxicity (Up and Down Procedure) - Rat (1).pdf",
-                               "files/headlineTest/01 - CGA100251 - Acute Oral Toxicity (Up and Down Procedure) - Rat (1)_REDACTION_LOG.json"));
+                "files/headlineTest/01 - CGA100251 - Acute Oral Toxicity (Up and Down Procedure) - Rat (1)_REDACTION_LOG.json"));
         metrics.add(getMetrics("files/syngenta/CustomerFiles/91 Trinexapac-ethyl_RAR_01_Volume_1_2018-02-23.pdf",
-                               "files/headlineTest/91 Trinexapac-ethyl_RAR_01_Volume_1_2018-02-23_REDACTION_LOG.json"));
-        metrics.add(getMetrics("files/syngenta/CustomerFiles/S-Metolachlor_RAR_01_Volume_1_2018-09-06.pdf",
-                               "files/headlineTest/S-Metolachlor_RAR_01_Volume_1_2018-09-06_REDACTION_LOG.json"));
+                "files/headlineTest/91 Trinexapac-ethyl_RAR_01_Volume_1_2018-02-23_REDACTION_LOG.json"));
+        metrics.add(getMetrics("files/syngenta/CustomerFiles/S-Metolachlor_RAR_01_Volume_1_2018-09-06.pdf", "files/headlineTest/S-Metolachlor_RAR_01_Volume_1_2018-09-06_REDACTION_LOG.json"));
 
-        double precision = metrics.stream()
-                .mapToDouble(Metrics::getPrecision).average()
-                .orElse(1.0);
-        double recall = metrics.stream()
-                .mapToDouble(Metrics::getRecall).average()
-                .orElse(1.0);
+        double precision = metrics.stream().mapToDouble(Metrics::getPrecision).average().orElse(1.0);
+        double recall = metrics.stream().mapToDouble(Metrics::getRecall).average().orElse(1.0);
 
         System.out.println("Precision is: " + precision + " recall is: " + recall);
 
@@ -106,28 +93,21 @@ public class HeadlinesGoldStandardIntegrationTest {
 
         Set<Headline> goldStandardHeadlines = new HashSet<>();
         var goldStandardLog = objectMapper.readValue(redactionLogResource.getInputStream(), RedactionLog.class);
-        goldStandardLog.getRedactionLogEntry()
-                .removeIf(r -> !r.isRedacted() || r.getChanges()
-                        .get(r.getChanges().size() - 1).getType().equals(ChangeType.REMOVED));
-        goldStandardLog.getRedactionLogEntry()
-                .forEach(e -> goldStandardHeadlines.add(new Headline(e.getPositions()
-                                                                             .get(0).getPage(), e.getValue())));
+        goldStandardLog.getRedactionLogEntry().removeIf(r -> !r.isRedacted() || r.getChanges().get(r.getChanges().size() - 1).getType().equals(ChangeType.REMOVED));
+        goldStandardLog.getRedactionLogEntry().forEach(e -> goldStandardHeadlines.add(new Headline(e.getPositions().get(0).getPage(), e.getValue())));
 
         Document documentGraph = DocumentGraphFactory.buildDocumentGraph(LayoutParsingType.DOCUMINE,
                                                                          layoutParsingPipeline.parseLayout(LayoutParsingType.DOCUMINE,
-                                                                                                           pdfFileResource.getFile(),
-                                                                                                           new ImageServiceResponse(),
-                                                                                                           new TableServiceResponse(),
-                                                                                                           new VisualLayoutParsingResponse(),
-                                                                                                           Map.of("file", filePath)));
+                        pdfFileResource.getFile(),
+                        new ImageServiceResponse(),
+                        new TableServiceResponse(),
+                        new VisualLayoutParsingResponse(),
+                        Map.of("file",filePath)));
 
         var foundHeadlines = documentGraph.streamAllSubNodes()
                 .map(SemanticNode::getHeadline)
                 .distinct()
-                .map(headlineNode -> new Headline(headlineNode.getPages()
-                                                          .stream()
-                                                          .findFirst()
-                                                          .get().getNumber(), headlineNode.getTextBlock().getSearchText().stripTrailing()))
+                .map(headlineNode -> new Headline(headlineNode.getPages().stream().findFirst().get().getNumber(), headlineNode.getTextBlock().getSearchText().stripTrailing()))
                 .toList();
 
         Set<Headline> correct = new HashSet<>();
@@ -141,9 +121,7 @@ public class HeadlinesGoldStandardIntegrationTest {
             }
         }
 
-        missing = goldStandardHeadlines.stream()
-                .filter(h -> !correct.contains(h))
-                .collect(Collectors.toSet());
+        missing = goldStandardHeadlines.stream().filter(h -> !correct.contains(h)).collect(Collectors.toSet());
 
         float precision = (float) correct.size() / (float) foundHeadlines.size();
         float recall = (float) correct.size() / ((float) correct.size() + (float) missing.size());

layoutparser-service/layoutparser-service-server/src/test/java/com/knecon/fforesight/service/layoutparser/server/OutlineDetectionTest.java

@@ -84,17 +84,17 @@ public class OutlineDetectionTest extends AbstractTest {
 
         OutlineObjectTree outlineObjectTree = classificationDocument.getOutlineObjectTree();
         assertEquals(outlineObjectTree.getRootNodes().size(), 8);
-        assertEquals(outlineObjectTree.getOutlineObjectsPerPage().get(2).size(), 1);
+        assertEquals(outlineObjectTree.getOutlineObjectsPerPage().get(1).size(), 1);
+        assertEquals(outlineObjectTree.getOutlineObjectsPerPage().get(3).size(), 1);
         assertEquals(outlineObjectTree.getOutlineObjectsPerPage().get(4).size(), 1);
         assertEquals(outlineObjectTree.getOutlineObjectsPerPage().get(5).size(), 1);
-        assertEquals(outlineObjectTree.getOutlineObjectsPerPage().get(6).size(), 1);
-        assertEquals(outlineObjectTree.getOutlineObjectsPerPage().get(7).size(), 2);
-        assertEquals(outlineObjectTree.getOutlineObjectsPerPage().get(8).size(), 3);
-        assertEquals(outlineObjectTree.getOutlineObjectsPerPage().get(9).size(), 2);
-        assertEquals(outlineObjectTree.getOutlineObjectsPerPage().get(11).size(), 1);
-        assertEquals(outlineObjectTree.getOutlineObjectsPerPage().get(12).size(), 4);
-        assertEquals(outlineObjectTree.getOutlineObjectsPerPage().get(13).size(), 1);
-        assertEquals(outlineObjectTree.getOutlineObjectsPerPage().get(14).size(), 2);
+        assertEquals(outlineObjectTree.getOutlineObjectsPerPage().get(6).size(), 2);
+        assertEquals(outlineObjectTree.getOutlineObjectsPerPage().get(7).size(), 3);
+        assertEquals(outlineObjectTree.getOutlineObjectsPerPage().get(8).size(), 2);
+        assertEquals(outlineObjectTree.getOutlineObjectsPerPage().get(10).size(), 1);
+        assertEquals(outlineObjectTree.getOutlineObjectsPerPage().get(11).size(), 4);
+        assertEquals(outlineObjectTree.getOutlineObjectsPerPage().get(12).size(), 1);
+        assertEquals(outlineObjectTree.getOutlineObjectsPerPage().get(13).size(), 2);
         assertTrue(outlineObjectTree.getOutlineObjectsPerPage().values()
                            .stream()
                            .flatMap(Collection::stream)

layoutparser-service/layoutparser-service-server/src/test/java/com/knecon/fforesight/service/layoutparser/server/utils/AbstractTest.java

@@ -10,9 +10,7 @@ import java.util.Optional;
 import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.extension.ExtendWith;
-import org.springframework.amqp.rabbit.core.RabbitAdmin;
 import org.springframework.amqp.rabbit.core.RabbitTemplate;
-import org.springframework.amqp.rabbit.listener.RabbitListenerEndpointRegistry;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
 import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration;
@@ -55,11 +53,6 @@ public abstract class AbstractTest {
     @MockBean
     private RabbitTemplate rabbitTemplate;
 
-    @MockBean
-    private RabbitAdmin rabbitAdmin;
-
-    @MockBean
-    private RabbitListenerEndpointRegistry rabbitListenerEndpointRegistry;
     protected final static String ORIGIN_FILE_ID = "origin";
 
     protected final static String VISUAL_LAYOUT_FILE = "visual";

layoutparser-service/layoutparser-service-server/src/test/resources/application.yml

@@ -39,5 +39,3 @@ management:
     prometheus.enabled: ${monitoring.enabled:false}
     health.enabled: true
   endpoints.web.exposure.include: prometheus, health
-
-POD_NAME: layoutparser-service

layoutparser-service/viewer-doc-processor/src/main/java/com/knecon/fforesight/service/viewerdoc/LayerIdentifier.java

@@ -40,7 +40,6 @@ public record LayerIdentifier(String name, String markedContentName) {
     public static final LayerIdentifier KNECON_LAYOUT_FIGURES = new LayerIdentifier("Figures", "LAYOUT_FIGURES");
     public static final LayerIdentifier KNECON_LAYOUT_IMAGES = new LayerIdentifier("Images", "LAYOUT_IMAGES");
     public static final LayerIdentifier KNECON_LAYOUT_TREE_IDs = new LayerIdentifier("Tree IDs", "LAYOUT_TREE_IDs");
-    public static final LayerIdentifier OUTLINE_HEADLINES = new LayerIdentifier("Outline Headlines", "OUTLINE_HEADLINES");
 
     //layout grid debug
     public static final LayerIdentifier KNECON_LAYOUT_DEBUG = new LayerIdentifier("Layout elements", "DEBUG_LAYOUT");
@@ -54,7 +53,7 @@ public record LayerIdentifier(String name, String markedContentName) {
     public static final LayerIdentifier MARKED_CONTENT = new LayerIdentifier("Marked content", "MARKED_CONTENT");
     public static final LayerIdentifier NEIGHBOURS = new LayerIdentifier("Neighbours", "NEIGHBOURS");
     public static final LayerIdentifier CHARACTERS = new LayerIdentifier("Characters", "CHARACTERS");
-    public static final LayerIdentifier OUTLINE_OBJECTS = new LayerIdentifier("Outline Positions", "OUTLINE_OBJECTS");
+    public static final LayerIdentifier OUTLINE_HEADLINES = new LayerIdentifier("Outline Headlines", "OUTLINE_HEADLINES");
 
     public static final LayerIdentifier KNECON_VISUAL_PARSING = new LayerIdentifier("Visual Layout Parser", "VISUAL_PARSING");
 

layoutparser-service/viewer-doc-processor/src/main/java/com/knecon/fforesight/service/viewerdoc/layers/LayoutDebugLayerConfig.java

@@ -30,7 +30,6 @@ public class LayoutDebugLayerConfig extends AbstractLayerGroup {
     protected static final Color STRIKETROUGH_RULING_COLOR = new Color(171, 6, 6);
 
     protected static final Color CELLS_COLOR = new Color(31, 214, 27);
-    protected static final Color OUTLINE_OBJECT_COLOR = new Color(214, 27, 183);
 
     protected static final Color MAIN_BODY_COLOR = new Color(171, 131, 6);
     protected static final Color MARKED_CONTENT_COLOR = new Color(171, 131, 6);
@@ -54,7 +53,6 @@ public class LayoutDebugLayerConfig extends AbstractLayerGroup {
     protected final Visualizations markedContent = Visualizations.builder().layer(LayerIdentifier.MARKED_CONTENT).build();
     protected final Visualizations neighbours = Visualizations.builder().layer(LayerIdentifier.NEIGHBOURS).build();
     protected final Visualizations characters = Visualizations.builder().layer(LayerIdentifier.CHARACTERS).build();
-    protected final Visualizations outlineObjects = Visualizations.builder().layer(LayerIdentifier.OUTLINE_OBJECTS).build();
 
 
     public List<Visualizations> getVisualizations() {
@@ -68,8 +66,7 @@ public class LayoutDebugLayerConfig extends AbstractLayerGroup {
                        clean_rulings, //
                        cells, //
                        mainBody, //
-                       markedContent, //
-                       outlineObjects //
+                       markedContent //
         );
     }
 

layoutparser-service/viewer-doc-processor/src/test/java/com/knecon/fforesight/service/viewerdoc/service/PageContentCleanerTest.java

@@ -44,8 +44,8 @@ class PageContentCleanerTest {
     @SneakyThrows
     public void testContentCleaning() {
 
-        Path file = Path.of("/home/kschuettler/Downloads/ITEM 23_Absorção cutanea.pdf");
-        File tmpFile = new File("/tmp/ITEM 23_Absorção cutanea.pdf");
+        Path file = Path.of("/tmp/OCR_TEST/402Study.pdf/viewerDocument.pdf");
+        File tmpFile = new File("/tmp/cleaned.pdf");
         try (var in = new FileInputStream(file.toFile());//
              var doc = new PDFDoc(in);//
              var out = new FileOutputStream(tmpFile);//
@@ -58,7 +58,7 @@ class PageContentCleanerTest {
                     .writer(pageWriter)
                     .reader(reader)
                     .elementBuilder(builder)
-                    .markedContentToRemove(Set.of(LayerIdentifier.KNECON_LAYOUT.markedContentName()))
+                    .markedContentToRemove(Set.of(LayerIdentifier.KNECON_OCR_DEBUG.markedContentName()))
                     .build();
 
             try (PageIterator iterator = doc.getPageIterator()) {