ignore mc files

CLARI-30 - forward analysis headers

See merge request fforesight/layout-parser!112

.gitignore

@@ -42,3 +42,5 @@ gradlew.bat
 gradlew
 gradle.properties
 gradle/
+.DS_Store
+.DS_Store/

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/LayoutParsingPipeline.java

@@ -47,6 +47,7 @@ import com.knecon.fforesight.service.layoutparser.processor.services.TableExtrac
 import com.knecon.fforesight.service.layoutparser.processor.services.blockification.DocstrumBlockificationService;
 import com.knecon.fforesight.service.layoutparser.processor.services.blockification.DocuMineBlockificationService;
 import com.knecon.fforesight.service.layoutparser.processor.services.blockification.RedactManagerBlockificationService;
+import com.knecon.fforesight.service.layoutparser.processor.services.classification.ClarifyndClassificationService;
 import com.knecon.fforesight.service.layoutparser.processor.services.classification.DocuMineClassificationService;
 import com.knecon.fforesight.service.layoutparser.processor.services.classification.RedactManagerClassificationService;
 import com.knecon.fforesight.service.layoutparser.processor.services.factory.DocumentGraphFactory;
@@ -88,6 +89,7 @@ public class LayoutParsingPipeline {
     LayoutGridService layoutGridService;
     ObservationRegistry observationRegistry;
     VisualLayoutParsingAdapter visualLayoutParsingAdapter;
+    ClarifyndClassificationService clarifyndClassificationService;
 
 
     public LayoutParsingFinishedEvent parseLayoutAndSaveFilesToStorage(LayoutParsingRequest layoutParsingRequest) throws IOException {
@@ -118,7 +120,7 @@ public class LayoutParsingPipeline {
                 imageServiceResponse,
                 tableServiceResponse,
                 visualLayoutParsingResponse,
-                layoutParsingRequest.identifier().toString());
+                layoutParsingRequest.identifier());
 
         log.info("Building document graph for {}", layoutParsingRequest.identifier());
 
@@ -207,7 +209,7 @@ public class LayoutParsingPipeline {
                                               ImageServiceResponse imageServiceResponse,
                                               TableServiceResponse tableServiceResponse,
                                               VisualLayoutParsingResponse visualLayoutParsingResponse,
-                                              String identifier) {
+                                              Map<String, String> identifier) {
 
         PDDocument originDocument = openDocument(originFile);
         addNumberOfPagesToTrace(originDocument.getNumberOfPages(), Files.size(originFile.toPath()));
@@ -291,7 +293,7 @@ public class LayoutParsingPipeline {
             if (layoutParsingType == LayoutParsingType.REDACT_MANAGER) {
                 docstrumBlockificationService.combineBlocks(classificationPage);
             } else if (layoutParsingType == LayoutParsingType.CLARIFYND) {
-                docstrumBlockificationService.mergeIntersectingBlocks(classificationPage.getTextBlocks());
+                docstrumBlockificationService.mergeIntersectingBlocks(classificationPage.getTextBlocks(), 0, 6.5f);
             }
 
             buildPageStatistics(classificationPage);
@@ -306,9 +308,10 @@ public class LayoutParsingPipeline {
         bodyTextFrameService.setBodyTextFrames(classificationDocument, layoutParsingType);
         log.info("Classify TextBlocks for {}", identifier);
         switch (layoutParsingType) {
-            case REDACT_MANAGER, REDACT_MANAGER_PARAGRAPH_DEBUG, REDACT_MANAGER_OLD, CLARIFYND, CLARIFYND_PARAGRAPH_DEBUG ->
+            case REDACT_MANAGER, REDACT_MANAGER_PARAGRAPH_DEBUG, REDACT_MANAGER_OLD, CLARIFYND_PARAGRAPH_DEBUG ->
                     redactManagerClassificationService.classifyDocument(classificationDocument);
             case DOCUMINE -> docuMineClassificationService.classifyDocument(classificationDocument);
+            case CLARIFYND -> clarifyndClassificationService.classifyDocument(classificationDocument);
         }
 
         log.info("Building Sections for {}", identifier);

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/blockification/DocstrumBlockificationService.java

@@ -80,7 +80,7 @@ public class DocstrumBlockificationService {
 
     public void combineBlocks(ClassificationPage page) {
 
-        mergeIntersectingBlocks(page.getTextBlocks());
+        mergeIntersectingBlocks(page.getTextBlocks(), 0, 0);
 
         TextPageBlock previous = new TextPageBlock();
         ListIterator<AbstractPageBlock> itty = page.getTextBlocks().listIterator();
@@ -123,7 +123,7 @@ public class DocstrumBlockificationService {
             previous = current;
         }
 
-        mergeIntersectingBlocks(page.getTextBlocks());
+        mergeIntersectingBlocks(page.getTextBlocks(), 0, 0);
     }
 
 
@@ -203,7 +203,7 @@ public class DocstrumBlockificationService {
     }
 
 
-    public void mergeIntersectingBlocks(List<AbstractPageBlock> blocks) {
+    public void mergeIntersectingBlocks(List<AbstractPageBlock> blocks, float xThreshold, float yThreshold) {
 
         ListIterator<AbstractPageBlock> itty = blocks.listIterator();
         Set<AbstractPageBlock> toRemove = new HashSet<>();
@@ -237,7 +237,7 @@ public class DocstrumBlockificationService {
                     continue;
                 }
 
-                if (current.getDir() == inner.getDir() && current.almostIntersects(inner, 0, 0)) {
+                if (current.getDir() == inner.getDir() && current.almostIntersects(inner, yThreshold, xThreshold)) {
 
                     current.getSequences().addAll(inner.getSequences());
                     QuickSort.sort(current.getSequences(), new TextPositionSequenceComparator());

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/classification/ClarifyndClassificationService.java

@@ -0,0 +1,114 @@
+package com.knecon.fforesight.service.layoutparser.processor.services.classification;
+
+import java.util.List;
+import java.util.regex.Pattern;
+
+import org.springframework.stereotype.Service;
+
+import com.knecon.fforesight.service.layoutparser.processor.model.AbstractPageBlock;
+import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationDocument;
+import com.knecon.fforesight.service.layoutparser.processor.model.ClassificationPage;
+import com.knecon.fforesight.service.layoutparser.processor.model.PageBlockType;
+import com.knecon.fforesight.service.layoutparser.processor.model.text.TextPageBlock;
+import com.knecon.fforesight.service.layoutparser.processor.utils.MarkedContentUtils;
+import com.knecon.fforesight.service.layoutparser.processor.utils.PositionUtils;
+
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+@Service
+@RequiredArgsConstructor
+public class ClarifyndClassificationService {
+
+    public void classifyDocument(ClassificationDocument document) {
+
+        List<Float> headlineFontSizes = document.getFontSizeCounter().getHighterThanMostPopular();
+
+        log.debug("Document FontSize counters are: {}", document.getFontSizeCounter().getCountPerValue());
+
+        for (ClassificationPage page : document.getPages()) {
+            classifyPage(page, document, headlineFontSizes);
+        }
+    }
+
+
+    private void classifyPage(ClassificationPage page, ClassificationDocument document, List<Float> headlineFontSizes) {
+
+        for (AbstractPageBlock textBlock : page.getTextBlocks()) {
+            if (textBlock instanceof TextPageBlock) {
+                classifyBlock((TextPageBlock) textBlock, page, document, headlineFontSizes);
+            }
+        }
+    }
+
+
+    private void classifyBlock(TextPageBlock textBlock, ClassificationPage page, ClassificationDocument document, List<Float> headlineFontSizes) {
+
+        var bodyTextFrame = page.getBodyTextFrame();
+
+        if (document.getFontSizeCounter().getMostPopular() == null) {
+            textBlock.setClassification(PageBlockType.PARAGRAPH);
+            return;
+        }
+        if (MarkedContentUtils.intersects(textBlock, page.getMarkedContentBboxPerType(), MarkedContentUtils.HEADER) || PositionUtils.isOverBodyTextFrame(bodyTextFrame,
+                textBlock,
+                page.getRotation()) && (document.getFontSizeCounter().getMostPopular() == null || textBlock.getHighestFontSize() <= document.getFontSizeCounter()
+                .getMostPopular())) {
+            textBlock.setClassification(PageBlockType.PARAGRAPH);
+
+        } else if (MarkedContentUtils.intersects(textBlock, page.getMarkedContentBboxPerType(), MarkedContentUtils.FOOTER) || PositionUtils.isUnderBodyTextFrame(bodyTextFrame,
+                textBlock,
+                page.getRotation()) && (document.getFontSizeCounter().getMostPopular() == null || textBlock.getHighestFontSize() <= document.getFontSizeCounter()
+                .getMostPopular())) {
+            textBlock.setClassification(PageBlockType.PARAGRAPH);
+        } else if (page.getPageNumber() == 1 && (PositionUtils.getHeightDifferenceBetweenChunkWordAndDocumentWord(textBlock,
+                document.getTextHeightCounter().getMostPopular()) > 2.5 && textBlock.getHighestFontSize() > document.getFontSizeCounter().getMostPopular() || page.getTextBlocks()
+                .size() == 1)) {
+            if (!Pattern.matches("[0-9]+", textBlock.toString())) {
+                textBlock.setClassification(PageBlockType.TITLE);
+            }
+        } else if (textBlock.getMostPopularWordFontSize() > document.getFontSizeCounter()
+                .getMostPopular() && PositionUtils.getApproxLineCount(textBlock) < 4.9 && (textBlock.getMostPopularWordStyle().equals("bold") || !document.getFontStyleCounter()
+                .getCountPerValue()
+                .containsKey("bold") && textBlock.getMostPopularWordFontSize() > document.getFontSizeCounter().getMostPopular() + 1) && textBlock.getSequences()
+                .get(0)
+                .getTextPositions()
+                .get(0)
+                .getFontSizeInPt() >= textBlock.getMostPopularWordFontSize()) {
+
+            for (int i = 1; i <= headlineFontSizes.size(); i++) {
+                if (textBlock.getMostPopularWordFontSize() == headlineFontSizes.get(i - 1)) {
+                    textBlock.setClassification(PageBlockType.getHeadlineType(i));
+                    document.setHeadlines(true);
+                }
+            }
+        } else if (!textBlock.getText().startsWith("Figure ") && PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordStyle()
+                .equals("bold") && !document.getFontStyleCounter().getMostPopular().equals("bold") && PositionUtils.getApproxLineCount(textBlock) < 2.9 && textBlock.getSequences()
+                .get(0)
+                .getTextPositions()
+                .get(0)
+                .getFontSizeInPt() >= textBlock.getMostPopularWordFontSize()) {
+            textBlock.setClassification(PageBlockType.getHeadlineType(headlineFontSizes.size() + 1));
+            document.setHeadlines(true);
+        } else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordFontSize() == document.getFontSizeCounter()
+                .getMostPopular() && textBlock.getMostPopularWordStyle().equals("bold") && !document.getFontStyleCounter().getMostPopular().equals("bold")) {
+            textBlock.setClassification(PageBlockType.PARAGRAPH_BOLD);
+        } else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordFont()
+                .equals(document.getFontCounter().getMostPopular()) && textBlock.getMostPopularWordStyle()
+                .equals(document.getFontStyleCounter().getMostPopular()) && textBlock.getMostPopularWordFontSize() == document.getFontSizeCounter().getMostPopular()) {
+            textBlock.setClassification(PageBlockType.PARAGRAPH);
+        } else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordFontSize() == document.getFontSizeCounter()
+                .getMostPopular() && textBlock.getMostPopularWordStyle().equals("italic") && !document.getFontStyleCounter()
+                .getMostPopular()
+                .equals("italic") && PositionUtils.getApproxLineCount(textBlock) < 2.9) {
+            textBlock.setClassification(PageBlockType.PARAGRAPH_ITALIC);
+        } else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock)) {
+            textBlock.setClassification(PageBlockType.PARAGRAPH_UNKNOWN);
+        } else {
+            textBlock.setClassification(PageBlockType.PARAGRAPH);
+        }
+    }
+
+}
+

layoutparser-service/layoutparser-service-server/src/main/java/com/knecon/fforesight/service/layoutparser/server/queue/MessageHandler.java

@@ -1,6 +1,9 @@
 package com.knecon.fforesight.service.layoutparser.server.queue;
 
 import java.util.Arrays;
+import java.util.Locale;
+import java.util.Map;
+import java.util.stream.Collectors;
 
 import org.springframework.amqp.AmqpRejectAndDontRequeueException;
 import org.springframework.amqp.core.Message;
@@ -28,6 +31,7 @@ public class MessageHandler {
     private final LayoutParsingPipeline layoutParsingPipeline;
     private final ObjectMapper objectMapper;
     private final RabbitTemplate rabbitTemplate;
+    private final static String X_PIPELINE_PREFIX = "X-PIPE-";
 
 
     @RabbitHandler
@@ -46,14 +50,24 @@ public class MessageHandler {
                     layoutParsingRequest.identifier()));
         }
         LayoutParsingFinishedEvent layoutParsingFinishedEvent = layoutParsingPipeline.parseLayoutAndSaveFilesToStorage(layoutParsingRequest);
-        sendLayoutParsingFinishedEvent(layoutParsingFinishedEvent);
+        sendLayoutParsingFinishedEvent(layoutParsingFinishedEvent, message);
     }
 
 
-    public void sendLayoutParsingFinishedEvent(LayoutParsingFinishedEvent layoutParsingFinishedEvent) {
+    public void sendLayoutParsingFinishedEvent(LayoutParsingFinishedEvent layoutParsingFinishedEvent, Message message) {
 
         Arrays.stream(layoutParsingFinishedEvent.message().split("\n")).forEach(log::info);
-        rabbitTemplate.convertAndSend(LayoutParsingQueueNames.LAYOUT_PARSING_FINISHED_EVENT_QUEUE, layoutParsingFinishedEvent);
+        rabbitTemplate.convertAndSend(LayoutParsingQueueNames.LAYOUT_PARSING_FINISHED_EVENT_QUEUE, layoutParsingFinishedEvent, m -> {
+            var forwardHeaders = message.getMessageProperties()
+                    .getHeaders()
+                    .entrySet()
+                    .stream()
+                    .filter(e -> e.getKey().toUpperCase(Locale.ROOT).startsWith(X_PIPELINE_PREFIX))
+                    .collect(Collectors.toMap(Map.Entry::getKey,
+                            Map.Entry::getValue));
+            m.getMessageProperties().getHeaders().putAll(forwardHeaders);
+            return m;
+        });
     }
 
 }

layoutparser-service/layoutparser-service-server/src/test/java/com/knecon/fforesight/service/layoutparser/server/BdrJsonBuildTest.java

@@ -11,6 +11,7 @@ import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.time.Duration;
 import java.util.List;
+import java.util.Map;
 
 import org.apache.pdfbox.Loader;
 import org.apache.pdfbox.pdmodel.PDDocument;
@@ -54,7 +55,7 @@ public class BdrJsonBuildTest extends AbstractTest {
                         new ImageServiceResponse(),
                         new TableServiceResponse(),
                         new VisualLayoutParsingResponse(),
-                        file.toString()));
+                        Map.of("file",file.toString())));
     }
 
 

layoutparser-service/layoutparser-service-server/src/test/java/com/knecon/fforesight/service/layoutparser/server/HeadlinesGoldStandardIntegrationTest.java

@@ -3,6 +3,7 @@ package com.knecon.fforesight.service.layoutparser.server;
 import java.util.ArrayList;
 import java.util.HashSet;
 import java.util.List;
+import java.util.Map;
 import java.util.Set;
 import java.util.stream.Collectors;
 
@@ -101,7 +102,7 @@ public class HeadlinesGoldStandardIntegrationTest {
                         new ImageServiceResponse(),
                         new TableServiceResponse(),
                         new VisualLayoutParsingResponse(),
-                        filePath));
+                        Map.of("file",filePath)));
 
         var foundHeadlines = documentGraph.streamAllSubNodes()
                 .map(SemanticNode::getHeadline)

layoutparser-service/layoutparser-service-server/src/test/java/com/knecon/fforesight/service/layoutparser/server/graph/DocumentGraphJsonWritingTest.java

@@ -4,6 +4,7 @@ import java.io.File;
 import java.io.FileOutputStream;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.util.Map;
 
 import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.Test;
@@ -61,7 +62,7 @@ public class DocumentGraphJsonWritingTest extends BuildDocumentTest {
                         new ImageServiceResponse(),
                         new TableServiceResponse(),
                         new VisualLayoutParsingResponse(),
-                        filename.toFile().toString()));
+                        Map.of("file",filename.toFile().toString())));
 
         DocumentData documentData = DocumentDataMapper.toDocumentData(documentGraph);
         ObjectMapper mapper = ObjectMapperFactory.create();

layoutparser-service/layoutparser-service-server/src/test/java/com/knecon/fforesight/service/layoutparser/server/graph/ViewerDocumentTest.java

@@ -2,6 +2,7 @@ package com.knecon.fforesight.service.layoutparser.server.graph;
 
 import java.io.File;
 import java.nio.file.Path;
+import java.util.Map;
 
 import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.Test;
@@ -26,7 +27,7 @@ public class ViewerDocumentTest extends BuildDocumentTest {
     @SneakyThrows
     public void testViewerDocument() {
 
-        String fileName = "files/S-Metolachlor_RAR_01_Volume_1_2018-09-06.pdf";
+        String fileName = "files/bdr/notMergedParagraphs.pdf";
         String tmpFileName = "/tmp/" + Path.of(fileName).getFileName() + "_VIEWER.pdf";
 
         var documentFile = new ClassPathResource(fileName).getFile();
@@ -34,7 +35,7 @@ public class ViewerDocumentTest extends BuildDocumentTest {
         LayoutGridService layoutGridService = new LayoutGridService(viewerDocumentService);
 
         long start = System.currentTimeMillis();
-        Document document = buildGraph(fileName, LayoutParsingType.REDACT_MANAGER);
+        Document document = buildGraph(fileName, LayoutParsingType.CLARIFYND);
         layoutGridService.addLayoutGrid(documentFile, document, new File(tmpFileName), true);
         System.out.printf("Total time: %.2fs%n", ((float) (System.currentTimeMillis() - start)) / 1000);
     }
@@ -58,7 +59,7 @@ public class ViewerDocumentTest extends BuildDocumentTest {
                 new ImageServiceResponse(),
                 tableResponse,
                 new VisualLayoutParsingResponse(),
-                Path.of(fileName).getFileName().toFile().toString());
+                Map.of("file",Path.of(fileName).getFileName().toFile().toString()));
         ViewerDocumentService viewerDocumentService = new ViewerDocumentService(null);
         LayoutGridService layoutGridService = new LayoutGridService(viewerDocumentService);
         Document document = DocumentGraphFactory.buildDocumentGraph(LayoutParsingType.DOCUMINE, classificationDocument);

layoutparser-service/layoutparser-service-server/src/test/java/com/knecon/fforesight/service/layoutparser/server/segmentation/PdfSegmentationServiceTest.java

@@ -61,7 +61,7 @@ public class PdfSegmentationServiceTest extends AbstractTest {
                 new ImageServiceResponse(),
                 tableServiceResponse,
                 new VisualLayoutParsingResponse(),
-                "document");
+                Map.of("file","document"));
 
         redactManagerClassificationService.classifyDocument(classificationDocument);
 

layoutparser-service/layoutparser-service-server/src/test/java/com/knecon/fforesight/service/layoutparser/server/services/RulingCleaningServiceTest.java

@@ -7,6 +7,7 @@ import java.nio.file.Path;
 import java.util.Collections;
 import java.util.LinkedList;
 import java.util.List;
+import java.util.Map;
 import java.util.stream.Collectors;
 
 import org.apache.pdfbox.Loader;
@@ -105,14 +106,14 @@ public class RulingCleaningServiceTest extends BuildDocumentTest {
                         new ImageServiceResponse(),
                         new TableServiceResponse(),
                         new VisualLayoutParsingResponse(),
-                        filename.toFile().toString()));
+                        Map.of("file",filename.toFile().toString())));
         Document documentGraphAfter = DocumentGraphFactory.buildDocumentGraph(LayoutParsingType.REDACT_MANAGER_OLD,
                 layoutParsingPipeline.parseLayout(LayoutParsingType.REDACT_MANAGER_OLD,
                         filename.toFile(),
                         new ImageServiceResponse(),
                         new TableServiceResponse(),
                         new VisualLayoutParsingResponse(),
-                        filename.toFile().toString()));
+                        Map.of("file",filename.toFile().toString())));
         DocumentData documentDataBefore = DocumentDataMapper.toDocumentData(documentGraphBefore);
         DocumentData documentDataAfter = DocumentDataMapper.toDocumentData(documentGraphAfter);
         if (!compareStructures(documentDataBefore.getDocumentStructure(), documentDataAfter.getDocumentStructure())) {

layoutparser-service/layoutparser-service-server/src/test/java/com/knecon/fforesight/service/layoutparser/server/utils/BuildDocumentTest.java

@@ -1,6 +1,7 @@
 package com.knecon.fforesight.service.layoutparser.server.utils;
 
 import java.io.File;
+import java.util.Map;
 
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.core.io.ClassPathResource;
@@ -31,7 +32,7 @@ public abstract class BuildDocumentTest extends AbstractTest {
                 layoutParsingStorageService.getImagesFile(IMAGE_FILE_ID),
                 new TableServiceResponse(),
                 new VisualLayoutParsingResponse(),
-                filename);
+                Map.of("file",filename));
     }