Merge branch 'clari-30' into 'main'

CLARI-30 - forward analysis headers

See merge request fforesight/layout-parser!112

layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/LayoutParsingPipeline.java

@@ -118,7 +118,7 @@ public class LayoutParsingPipeline {
                 imageServiceResponse,
                 tableServiceResponse,
                 visualLayoutParsingResponse,
-                layoutParsingRequest.identifier().toString());
+                layoutParsingRequest.identifier());
 
         log.info("Building document graph for {}", layoutParsingRequest.identifier());
 
@@ -207,7 +207,7 @@ public class LayoutParsingPipeline {
                                               ImageServiceResponse imageServiceResponse,
                                               TableServiceResponse tableServiceResponse,
                                               VisualLayoutParsingResponse visualLayoutParsingResponse,
-                                              String identifier) {
+                                              Map<String, String> identifier) {
 
         PDDocument originDocument = openDocument(originFile);
         addNumberOfPagesToTrace(originDocument.getNumberOfPages(), Files.size(originFile.toPath()));

layoutparser-service/layoutparser-service-server/src/main/java/com/knecon/fforesight/service/layoutparser/server/queue/MessageHandler.java

@@ -1,6 +1,9 @@
 package com.knecon.fforesight.service.layoutparser.server.queue;
 
 import java.util.Arrays;
+import java.util.Locale;
+import java.util.Map;
+import java.util.stream.Collectors;
 
 import org.springframework.amqp.AmqpRejectAndDontRequeueException;
 import org.springframework.amqp.core.Message;
@@ -28,6 +31,7 @@ public class MessageHandler {
     private final LayoutParsingPipeline layoutParsingPipeline;
     private final ObjectMapper objectMapper;
     private final RabbitTemplate rabbitTemplate;
+    private final static String X_PIPELINE_PREFIX = "X-PIPE-";
 
 
     @RabbitHandler
@@ -46,14 +50,24 @@ public class MessageHandler {
                     layoutParsingRequest.identifier()));
         }
         LayoutParsingFinishedEvent layoutParsingFinishedEvent = layoutParsingPipeline.parseLayoutAndSaveFilesToStorage(layoutParsingRequest);
-        sendLayoutParsingFinishedEvent(layoutParsingFinishedEvent);
+        sendLayoutParsingFinishedEvent(layoutParsingFinishedEvent, message);
     }
 
 
-    public void sendLayoutParsingFinishedEvent(LayoutParsingFinishedEvent layoutParsingFinishedEvent) {
+    public void sendLayoutParsingFinishedEvent(LayoutParsingFinishedEvent layoutParsingFinishedEvent, Message message) {
 
         Arrays.stream(layoutParsingFinishedEvent.message().split("\n")).forEach(log::info);
-        rabbitTemplate.convertAndSend(LayoutParsingQueueNames.LAYOUT_PARSING_FINISHED_EVENT_QUEUE, layoutParsingFinishedEvent);
+        rabbitTemplate.convertAndSend(LayoutParsingQueueNames.LAYOUT_PARSING_FINISHED_EVENT_QUEUE, layoutParsingFinishedEvent, m -> {
+            var forwardHeaders = message.getMessageProperties()
+                    .getHeaders()
+                    .entrySet()
+                    .stream()
+                    .filter(e -> e.getKey().toUpperCase(Locale.ROOT).startsWith(X_PIPELINE_PREFIX))
+                    .collect(Collectors.toMap(Map.Entry::getKey,
+                            Map.Entry::getValue));
+            m.getMessageProperties().getHeaders().putAll(forwardHeaders);
+            return m;
+        });
     }
 
 }

layoutparser-service/layoutparser-service-server/src/test/java/com/knecon/fforesight/service/layoutparser/server/BdrJsonBuildTest.java

@@ -11,6 +11,7 @@ import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.time.Duration;
 import java.util.List;
+import java.util.Map;
 
 import org.apache.pdfbox.Loader;
 import org.apache.pdfbox.pdmodel.PDDocument;
@@ -54,7 +55,7 @@ public class BdrJsonBuildTest extends AbstractTest {
                         new ImageServiceResponse(),
                         new TableServiceResponse(),
                         new VisualLayoutParsingResponse(),
-                        file.toString()));
+                        Map.of("file",file.toString())));
     }
 
 

layoutparser-service/layoutparser-service-server/src/test/java/com/knecon/fforesight/service/layoutparser/server/HeadlinesGoldStandardIntegrationTest.java

@@ -3,6 +3,7 @@ package com.knecon.fforesight.service.layoutparser.server;
 import java.util.ArrayList;
 import java.util.HashSet;
 import java.util.List;
+import java.util.Map;
 import java.util.Set;
 import java.util.stream.Collectors;
 
@@ -101,7 +102,7 @@ public class HeadlinesGoldStandardIntegrationTest {
                         new ImageServiceResponse(),
                         new TableServiceResponse(),
                         new VisualLayoutParsingResponse(),
-                        filePath));
+                        Map.of("file",filePath)));
 
         var foundHeadlines = documentGraph.streamAllSubNodes()
                 .map(SemanticNode::getHeadline)

layoutparser-service/layoutparser-service-server/src/test/java/com/knecon/fforesight/service/layoutparser/server/graph/DocumentGraphJsonWritingTest.java

@@ -4,6 +4,7 @@ import java.io.File;
 import java.io.FileOutputStream;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.util.Map;
 
 import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.Test;
@@ -61,7 +62,7 @@ public class DocumentGraphJsonWritingTest extends BuildDocumentTest {
                         new ImageServiceResponse(),
                         new TableServiceResponse(),
                         new VisualLayoutParsingResponse(),
-                        filename.toFile().toString()));
+                        Map.of("file",filename.toFile().toString())));
 
         DocumentData documentData = DocumentDataMapper.toDocumentData(documentGraph);
         ObjectMapper mapper = ObjectMapperFactory.create();

layoutparser-service/layoutparser-service-server/src/test/java/com/knecon/fforesight/service/layoutparser/server/graph/ViewerDocumentTest.java

@@ -2,6 +2,7 @@ package com.knecon.fforesight.service.layoutparser.server.graph;
 
 import java.io.File;
 import java.nio.file.Path;
+import java.util.Map;
 
 import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.Test;
@@ -58,7 +59,7 @@ public class ViewerDocumentTest extends BuildDocumentTest {
                 new ImageServiceResponse(),
                 tableResponse,
                 new VisualLayoutParsingResponse(),
-                Path.of(fileName).getFileName().toFile().toString());
+                Map.of("file",Path.of(fileName).getFileName().toFile().toString()));
         ViewerDocumentService viewerDocumentService = new ViewerDocumentService(null);
         LayoutGridService layoutGridService = new LayoutGridService(viewerDocumentService);
         Document document = DocumentGraphFactory.buildDocumentGraph(LayoutParsingType.DOCUMINE, classificationDocument);

layoutparser-service/layoutparser-service-server/src/test/java/com/knecon/fforesight/service/layoutparser/server/segmentation/PdfSegmentationServiceTest.java

@@ -61,7 +61,7 @@ public class PdfSegmentationServiceTest extends AbstractTest {
                 new ImageServiceResponse(),
                 tableServiceResponse,
                 new VisualLayoutParsingResponse(),
-                "document");
+                Map.of("file","document"));
 
         redactManagerClassificationService.classifyDocument(classificationDocument);
 

layoutparser-service/layoutparser-service-server/src/test/java/com/knecon/fforesight/service/layoutparser/server/services/RulingCleaningServiceTest.java

@@ -7,6 +7,7 @@ import java.nio.file.Path;
 import java.util.Collections;
 import java.util.LinkedList;
 import java.util.List;
+import java.util.Map;
 import java.util.stream.Collectors;
 
 import org.apache.pdfbox.Loader;
@@ -105,14 +106,14 @@ public class RulingCleaningServiceTest extends BuildDocumentTest {
                         new ImageServiceResponse(),
                         new TableServiceResponse(),
                         new VisualLayoutParsingResponse(),
-                        filename.toFile().toString()));
+                        Map.of("file",filename.toFile().toString())));
         Document documentGraphAfter = DocumentGraphFactory.buildDocumentGraph(LayoutParsingType.REDACT_MANAGER_OLD,
                 layoutParsingPipeline.parseLayout(LayoutParsingType.REDACT_MANAGER_OLD,
                         filename.toFile(),
                         new ImageServiceResponse(),
                         new TableServiceResponse(),
                         new VisualLayoutParsingResponse(),
-                        filename.toFile().toString()));
+                        Map.of("file",filename.toFile().toString())));
         DocumentData documentDataBefore = DocumentDataMapper.toDocumentData(documentGraphBefore);
         DocumentData documentDataAfter = DocumentDataMapper.toDocumentData(documentGraphAfter);
         if (!compareStructures(documentDataBefore.getDocumentStructure(), documentDataAfter.getDocumentStructure())) {

layoutparser-service/layoutparser-service-server/src/test/java/com/knecon/fforesight/service/layoutparser/server/utils/BuildDocumentTest.java

@@ -1,6 +1,7 @@
 package com.knecon.fforesight.service.layoutparser.server.utils;
 
 import java.io.File;
+import java.util.Map;
 
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.core.io.ClassPathResource;
@@ -31,7 +32,7 @@ public abstract class BuildDocumentTest extends AbstractTest {
                 layoutParsingStorageService.getImagesFile(IMAGE_FILE_ID),
                 new TableServiceResponse(),
                 new VisualLayoutParsingResponse(),
-                filename);
+                Map.of("file",filename));
     }