Merge remote-tracking branch 'origin/RED-10200-2' into RED-10200-2

# Conflicts:
#	redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/TextEntity.java
#	redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/EntityDroolsExecutionService.java
#	redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AnalysisEnd2EndTest.java

redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/TextEntity.java

@@ -71,6 +71,8 @@ public class TextEntity implements IEntity {
     List<SemanticNode> intersectingNodes = new LinkedList<>();
     SemanticNode deepestFullyContainingNode;
 
+    List<Relation> relations = new LinkedList<>();
+
 
     public static TextEntity initialEntityNode(TextRange textRange, String type, EntityType entityType, SemanticNode node) {
 
@@ -335,5 +337,4 @@ public class TextEntity implements IEntity {
         return Optional.empty();
     }
 
-
 }

redaction-service-v1/redaction-service-server-v1/build.gradle.kts

@@ -12,7 +12,7 @@ plugins {
 description = "redaction-service-server-v1"
 
 
-val layoutParserVersion = "0.191.0"
+val layoutParserVersion = "0.193.0"
 val jacksonVersion = "2.15.2"
 val droolsVersion = "9.44.0.Final"
 val pdfBoxVersion = "3.0.0"

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/RedactionServiceSettings.java

@@ -40,7 +40,7 @@ public class RedactionServiceSettings {
 
     private boolean annotationMode;
 
-    private boolean droolsDebug;
+    private boolean droolsDebug = true;
 
     private boolean protobufJsonFallback = true;
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/DictionarySearchService.java

@@ -7,7 +7,6 @@ import org.springframework.stereotype.Service;
 
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine;
 import com.iqser.red.service.redaction.v1.server.model.dictionary.Dictionary;
-import com.iqser.red.service.redaction.v1.server.model.dictionary.SearchImplementation;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
 import com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService;

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityCreationService.java

@@ -1505,18 +1505,19 @@ public class EntityCreationService {
 
         documentTree.getRoot().getNode().addThisToEntityIfIntersects(entity);
 
-        if (!nodesInKieSession.isEmpty() && entity.getIntersectingNodes()
+//        if (!nodesInKieSession.isEmpty() && entity.getIntersectingNodes()
-                .stream()
+//                .stream()
-                .anyMatch(node -> !nodesInKieSession.contains(node))) {
+//                .anyMatch(node -> !nodesInKieSession.contains(node))) {
-            entity.removeFromGraph();
+//            entity.removeFromGraph();
-            return false;
+//            return false;
-        }
+//        }
 
         TextBlock textBlock = entity.getDeepestFullyContainingNode().getTextBlock();
         entityEnrichmentService.enrichEntity(entity, textBlock);
 
         addToPages(entity);
         addEntityToNodeEntitySets(entity);
+        // compute relationships, by looping through deepestFullyContainingNode's entities
         return true;
     }
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/EntityDroolsExecutionService.java

@@ -2,7 +2,6 @@ package com.iqser.red.service.redaction.v1.server.service.drools;
 
 import static com.iqser.red.service.redaction.v1.server.service.drools.ComponentDroolsExecutionService.RULES_LOGGER_GLOBAL;
 
-import java.util.Collections;
 import java.util.HashSet;
 import java.util.LinkedList;
 import java.util.List;
@@ -118,21 +117,18 @@ public class EntityDroolsExecutionService {
             kieSession.setGlobal(RULES_LOGGER_GLOBAL, logger);
         }
 
-        kieSession.insert(document);
+        nodesInKieSession.add(document);
+        nodesInKieSession.addAll(sectionsToAnalyze);
+        nodesInKieSession.addAll(sectionsToAnalyze.stream()
+                                         .flatMap(SemanticNode::streamAllSubNodes)
+                                         .toList());
+        nodesInKieSession.forEach(kieSession::insert);
         System.out.println("after document insert : " + kieSession.getFactCount());
 
         document.getEntities()
                 .forEach(kieSession::insert);
         System.out.println("after getEntities insert : " + kieSession.getFactCount());
 
-        sectionsToAnalyze.forEach(kieSession::insert);
-        System.out.println("after sectionsToAnalyze insert : " + kieSession.getFactCount());
-//
-//        sectionsToAnalyze.stream()
-//                .flatMap(SemanticNode::streamAllSubNodes)
-//                .forEach(kieSession::insert);
-//        System.out.println("after SemanticNode insert : " + kieSession.getFactCount());
-
         document.getPages()
                 .forEach(kieSession::insert);
         System.out.println("after getPages insert : " + kieSession.getFactCount());
@@ -162,24 +158,24 @@ public class EntityDroolsExecutionService {
 
         try {
             completableFuture.get(settings.getDroolsExecutionTimeoutSecs(document.getNumberOfPages()), TimeUnit.SECONDS);
-
+//
-            nodesInKieSession = sectionsToAnalyze.size() == document.streamAllSubNodes()
+//            nodesInKieSession = sectionsToAnalyze.size() == document.streamAllSubNodes()
-                    .count() ? Collections.emptySet() : buildSet(sectionsToAnalyze, document);
+//                    .count() ? Collections.emptySet() : buildSet(sectionsToAnalyze, document);
-
+//
-            kieSessionUpdater = new KieSessionUpdater(nodesInKieSession, kieSession);
+//            kieSessionUpdater = new KieSessionUpdater(nodesInKieSession, kieSession);
-
+//
-            document.setKieSessionUpdater(kieSessionUpdater);
+//            document.setKieSessionUpdater(kieSessionUpdater);
-
+//
-            kieSession.insert(document);
+//            kieSession.insert(document);
-
+//
-            kieSession.setGlobal("entityCreationService", entityCreationService);
+//            kieSession.setGlobal("entityCreationService", entityCreationService);
-
+//
-            sectionsToAnalyze.stream()
+//            sectionsToAnalyze.stream()
-                    .flatMap(SemanticNode::streamAllSubNodes)
+//                    .flatMap(SemanticNode::streamAllSubNodes)
-                    .forEach(kieSession::insert);
+//                    .forEach(kieSession::insert);
-            System.out.println("after SemanticNode insert : " + kieSession.getFactCount());
+//            System.out.println("after SemanticNode insert : " + kieSession.getFactCount());
-
+//
-            completableFuture.get(settings.getDroolsExecutionTimeoutSecs(document.getNumberOfPages()), TimeUnit.SECONDS);
+//            completableFuture.get(settings.getDroolsExecutionTimeoutSecs(document.getNumberOfPages()), TimeUnit.SECONDS);
 
         } catch (ExecutionException e) {
             logger.error(e, "Exception during rule execution");

redaction-service-v1/redaction-service-server-v1/src/main/resources/logback-spring.xml

@@ -14,4 +14,5 @@
         <appender-ref ref="${logType}"/>
     </root>
     <logger name="org.drools.mvel" level="ERROR"/>
+    <logger name="org.springframework.web.socket.config" level="WARN"/>
 </configuration>

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AnalysisEnd2EndTest.java

@@ -121,7 +121,7 @@ import lombok.extern.slf4j.Slf4j;
                                                               FileType.DOCUMENT_POSITION,
                                                               FileType.DOCUMENT_STRUCTURE,
                                                               FileType.DOCUMENT_TEXT);
-    Path dossierTemplateToUse = Path.of("/Users/maverickstuder/Downloads/DossierTemplate_mod"); // Add your dossier-template here
+    Path dossierTemplateToUse = Path.of("/home/kschuettler/iqser/business-logic/redactmanager/prod-cp-eu-reg/EFSA_sanitisation_pre_GFL_v1"); // Add your dossier-template here
     ObjectMapper mapper = ObjectMapperFactory.create();
     final String TENANT_ID = "tenant";
     TestDossierTemplate testDossierTemplate;
@@ -172,11 +172,12 @@ import lombok.extern.slf4j.Slf4j;
         }
     }
 
+
     @Test
     @SneakyThrows
     public void runAnalysisEnd2End() {
 
-        String folder = "/Users/maverickstuder/Downloads/11-26-2024-10-26-32_files_list"; // Should contain all files from minio directly, still zipped. Can contain multiple files.
+        String folder = "/home/kschuettler/Dokumente/analysisend2end/file0"; // Should contain all files from minio directly, still zipped. Can contain multiple files.
 
         Path absoluteFolderPath;
         if (folder.startsWith("files")) { // if it starts with "files" it is most likely in the resources folder, else it should be an absolute path
@@ -350,30 +351,7 @@ import lombok.extern.slf4j.Slf4j;
         Set<FileType> missingFileTypes = Sets.difference(REQUIRED_FILES, uploadedFileTypes);
 
         if (!missingFileTypes.isEmpty() && !missingFileTypes.contains(FileType.ORIGIN)) {
-            var cvServiceResponse = "files/cv_service_empty_response.json";
+            runLayoutParsingAndSaveFilesToFolder(folder, uploadedFileTypes, request);
-            var imageServiceResponse = "files/empty_image_response.json";
-            ClassPathResource cvServiceResponseFileResource = new ClassPathResource(cvServiceResponse);
-            ClassPathResource imageServiceResponseFileResource = new ClassPathResource(imageServiceResponse);
-
-            storageService.storeObject(TenantContext.getTenantId(),
-                                       RedactionStorageService.StorageIdUtils.getStorageId(request.getDossierId(), request.getFileId(), FileType.TABLES),
-                                       cvServiceResponseFileResource.getInputStream());
-            storageService.storeObject(TenantContext.getTenantId(),
-                                       RedactionStorageService.StorageIdUtils.getStorageId(request.getDossierId(), request.getFileId(), FileType.IMAGE_INFO),
-                                       imageServiceResponseFileResource.getInputStream());
-
-            LayoutParsingRequest layoutParsingRequest = LayoutParsingRequestProvider.build(LayoutParsingType.REDACT_MANAGER_WITHOUT_DUPLICATE_PARAGRAPH, request);
-            layoutParsingPipeline.parseLayoutAndSaveFilesToStorage(layoutParsingRequest);
-
-//            try {
-//                storeFileFromStorage(TENANT_ID, layoutParsingRequest.structureFileStorageId(), folder);
-//                storeFileFromStorage(TENANT_ID, layoutParsingRequest.textBlockFileStorageId(), folder);
-//                storeFileFromStorage(TENANT_ID, layoutParsingRequest.positionBlockFileStorageId(), folder);
-//                storeFileFromStorage(TENANT_ID, layoutParsingRequest.pageFileStorageId(), folder);
-//            } catch (IOException e) {
-//                log.error("Failed to store files from storage to folder {}", folder, e);
-//                return Optional.empty();
-//            }
         }
 
 //        if (!missingFileTypes.isEmpty()) {
@@ -388,11 +366,48 @@ import lombok.extern.slf4j.Slf4j;
     }
 
 
+    private void runLayoutParsingAndSaveFilesToFolder(Path folder, Set<FileType> uploadedFileTypes, AnalyzeRequest request) throws IOException {
+
+        uploadImageAndTableFilesIfMissing(uploadedFileTypes, request);
+
+        LayoutParsingRequest layoutParsingRequest = LayoutParsingRequestProvider.build(LayoutParsingType.DOCUMINE_OLD, request);
+        layoutParsingPipeline.parseLayoutAndSaveFilesToStorage(layoutParsingRequest);
+
+        try {
+            storeFileFromStorage(TENANT_ID, layoutParsingRequest.structureFileStorageId(), folder);
+            storeFileFromStorage(TENANT_ID, layoutParsingRequest.textBlockFileStorageId(), folder);
+            storeFileFromStorage(TENANT_ID, layoutParsingRequest.positionBlockFileStorageId(), folder);
+            storeFileFromStorage(TENANT_ID, layoutParsingRequest.pageFileStorageId(), folder);
+        } catch (IOException e) {
+            log.error("Failed to store files from storage to folder {}", folder, e);
+        }
+    }
+
+
+    private void uploadImageAndTableFilesIfMissing(Set<FileType> uploadedFileTypes, AnalyzeRequest request) throws IOException {
+
+        if (!uploadedFileTypes.contains(FileType.TABLES)) {
+            var cvServiceResponse = "files/cv_service_empty_response.json";
+            ClassPathResource cvServiceResponseFileResource = new ClassPathResource(cvServiceResponse);
+            storageService.storeObject(TenantContext.getTenantId(),
+                                       RedactionStorageService.StorageIdUtils.getStorageId(request.getDossierId(), request.getFileId(), FileType.TABLES),
+                                       cvServiceResponseFileResource.getInputStream());
+        }
+        if (!uploadedFileTypes.contains(FileType.IMAGE_INFO)) {
+            var imageServiceResponse = "files/empty_image_response.json";
+            ClassPathResource imageServiceResponseFileResource = new ClassPathResource(imageServiceResponse);
+            storageService.storeObject(TenantContext.getTenantId(),
+                                       RedactionStorageService.StorageIdUtils.getStorageId(request.getDossierId(), request.getFileId(), FileType.IMAGE_INFO),
+                                       imageServiceResponseFileResource.getInputStream());
+        }
+    }
+
+
     private void storeFileFromStorage(String tenantId, String storageId, Path folder) throws IOException {
 
         var inputStream = storageService.getObject(tenantId, storageId);
-        try (FileOutputStream fileOut = new FileOutputStream(folder.toString() + "/" + storageId.split("/")[1]); ObjectOutputStream out = new ObjectOutputStream(fileOut)) {
+        try (FileOutputStream fileOut = new FileOutputStream(folder.toString() + "/" + storageId.split("/")[1])) {
-            out.writeObject(inputStream);
+            fileOut.write(inputStream.getContentAsByteArray());
         } catch (IOException e) {
             e.printStackTrace();
         }

redaction-service-v1/redaction-service-server-v1/src/test/resources/files/syngenta

@@ -1 +1 @@
-Subproject commit 57e6e0dd3c08a3a65ec59b5dfb70f0f77ebcc7c7
+Subproject commit 5705cc0782605fdca5dfff134b436f7143c9e421

redaction-service-v1/redaction-service-server-v1/src/test/resources/logback-spring.xml

@@ -16,5 +16,7 @@
 
     <logger name="org.apache.fontbox.ttf" level="ERROR"/>
     <logger name="org.drools.mvel" level="ERROR"/>
+    <logger name="org.springframework.web.socket.config" level="WARN"/>
+    <logger name="org.mongodb.driver.client" level="ERROR"/>
 
 </configuration>