diff --git a/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/TextEntity.java b/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/TextEntity.java index 61c4e926..d196b335 100644 --- a/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/TextEntity.java +++ b/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/TextEntity.java @@ -71,6 +71,8 @@ public class TextEntity implements IEntity { List intersectingNodes = new LinkedList<>(); SemanticNode deepestFullyContainingNode; + List relations = new LinkedList<>(); + public static TextEntity initialEntityNode(TextRange textRange, String type, EntityType entityType, SemanticNode node) { @@ -335,5 +337,4 @@ public class TextEntity implements IEntity { return Optional.empty(); } - } diff --git a/redaction-service-v1/redaction-service-server-v1/build.gradle.kts b/redaction-service-v1/redaction-service-server-v1/build.gradle.kts index fb7c3dd6..ec968189 100644 --- a/redaction-service-v1/redaction-service-server-v1/build.gradle.kts +++ b/redaction-service-v1/redaction-service-server-v1/build.gradle.kts @@ -12,7 +12,7 @@ plugins { description = "redaction-service-server-v1" -val layoutParserVersion = "0.191.0" +val layoutParserVersion = "0.193.0" val jacksonVersion = "2.15.2" val droolsVersion = "9.44.0.Final" val pdfBoxVersion = "3.0.0" diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/RedactionServiceSettings.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/RedactionServiceSettings.java index 21e7c63b..b774d3dc 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/RedactionServiceSettings.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/RedactionServiceSettings.java @@ -40,7 +40,7 @@ public class RedactionServiceSettings { private boolean annotationMode; - private boolean droolsDebug; + private boolean droolsDebug = true; private boolean protobufJsonFallback = true; diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/DictionarySearchService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/DictionarySearchService.java index 1690c19c..478c4a6f 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/DictionarySearchService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/DictionarySearchService.java @@ -7,7 +7,6 @@ import org.springframework.stereotype.Service; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine; import com.iqser.red.service.redaction.v1.server.model.dictionary.Dictionary; -import com.iqser.red.service.redaction.v1.server.model.dictionary.SearchImplementation; import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType; import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode; import com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService; diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityCreationService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityCreationService.java index 345ef08d..7519e221 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityCreationService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityCreationService.java @@ -1505,18 +1505,19 @@ public class EntityCreationService { documentTree.getRoot().getNode().addThisToEntityIfIntersects(entity); - if (!nodesInKieSession.isEmpty() && entity.getIntersectingNodes() - .stream() - .anyMatch(node -> !nodesInKieSession.contains(node))) { - entity.removeFromGraph(); - return false; - } +// if (!nodesInKieSession.isEmpty() && entity.getIntersectingNodes() +// .stream() +// .anyMatch(node -> !nodesInKieSession.contains(node))) { +// entity.removeFromGraph(); +// return false; +// } TextBlock textBlock = entity.getDeepestFullyContainingNode().getTextBlock(); entityEnrichmentService.enrichEntity(entity, textBlock); addToPages(entity); addEntityToNodeEntitySets(entity); + // compute relationships, by looping through deepestFullyContainingNode's entities return true; } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/EntityDroolsExecutionService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/EntityDroolsExecutionService.java index 55c9b3bf..cfce5fa4 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/EntityDroolsExecutionService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/EntityDroolsExecutionService.java @@ -2,7 +2,6 @@ package com.iqser.red.service.redaction.v1.server.service.drools; import static com.iqser.red.service.redaction.v1.server.service.drools.ComponentDroolsExecutionService.RULES_LOGGER_GLOBAL; -import java.util.Collections; import java.util.HashSet; import java.util.LinkedList; import java.util.List; @@ -118,21 +117,18 @@ public class EntityDroolsExecutionService { kieSession.setGlobal(RULES_LOGGER_GLOBAL, logger); } - kieSession.insert(document); + nodesInKieSession.add(document); + nodesInKieSession.addAll(sectionsToAnalyze); + nodesInKieSession.addAll(sectionsToAnalyze.stream() + .flatMap(SemanticNode::streamAllSubNodes) + .toList()); + nodesInKieSession.forEach(kieSession::insert); System.out.println("after document insert : " + kieSession.getFactCount()); document.getEntities() .forEach(kieSession::insert); System.out.println("after getEntities insert : " + kieSession.getFactCount()); - sectionsToAnalyze.forEach(kieSession::insert); - System.out.println("after sectionsToAnalyze insert : " + kieSession.getFactCount()); -// -// sectionsToAnalyze.stream() -// .flatMap(SemanticNode::streamAllSubNodes) -// .forEach(kieSession::insert); -// System.out.println("after SemanticNode insert : " + kieSession.getFactCount()); - document.getPages() .forEach(kieSession::insert); System.out.println("after getPages insert : " + kieSession.getFactCount()); @@ -162,24 +158,24 @@ public class EntityDroolsExecutionService { try { completableFuture.get(settings.getDroolsExecutionTimeoutSecs(document.getNumberOfPages()), TimeUnit.SECONDS); - - nodesInKieSession = sectionsToAnalyze.size() == document.streamAllSubNodes() - .count() ? Collections.emptySet() : buildSet(sectionsToAnalyze, document); - - kieSessionUpdater = new KieSessionUpdater(nodesInKieSession, kieSession); - - document.setKieSessionUpdater(kieSessionUpdater); - - kieSession.insert(document); - - kieSession.setGlobal("entityCreationService", entityCreationService); - - sectionsToAnalyze.stream() - .flatMap(SemanticNode::streamAllSubNodes) - .forEach(kieSession::insert); - System.out.println("after SemanticNode insert : " + kieSession.getFactCount()); - - completableFuture.get(settings.getDroolsExecutionTimeoutSecs(document.getNumberOfPages()), TimeUnit.SECONDS); +// +// nodesInKieSession = sectionsToAnalyze.size() == document.streamAllSubNodes() +// .count() ? Collections.emptySet() : buildSet(sectionsToAnalyze, document); +// +// kieSessionUpdater = new KieSessionUpdater(nodesInKieSession, kieSession); +// +// document.setKieSessionUpdater(kieSessionUpdater); +// +// kieSession.insert(document); +// +// kieSession.setGlobal("entityCreationService", entityCreationService); +// +// sectionsToAnalyze.stream() +// .flatMap(SemanticNode::streamAllSubNodes) +// .forEach(kieSession::insert); +// System.out.println("after SemanticNode insert : " + kieSession.getFactCount()); +// +// completableFuture.get(settings.getDroolsExecutionTimeoutSecs(document.getNumberOfPages()), TimeUnit.SECONDS); } catch (ExecutionException e) { logger.error(e, "Exception during rule execution"); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/resources/logback-spring.xml b/redaction-service-v1/redaction-service-server-v1/src/main/resources/logback-spring.xml index 02d4f3ec..3233828b 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/resources/logback-spring.xml +++ b/redaction-service-v1/redaction-service-server-v1/src/main/resources/logback-spring.xml @@ -14,4 +14,5 @@ + \ No newline at end of file diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AnalysisEnd2EndTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AnalysisEnd2EndTest.java index 79666f30..84ed4c3b 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AnalysisEnd2EndTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AnalysisEnd2EndTest.java @@ -121,7 +121,7 @@ import lombok.extern.slf4j.Slf4j; FileType.DOCUMENT_POSITION, FileType.DOCUMENT_STRUCTURE, FileType.DOCUMENT_TEXT); - Path dossierTemplateToUse = Path.of("/Users/maverickstuder/Downloads/DossierTemplate_mod"); // Add your dossier-template here + Path dossierTemplateToUse = Path.of("/home/kschuettler/iqser/business-logic/redactmanager/prod-cp-eu-reg/EFSA_sanitisation_pre_GFL_v1"); // Add your dossier-template here ObjectMapper mapper = ObjectMapperFactory.create(); final String TENANT_ID = "tenant"; TestDossierTemplate testDossierTemplate; @@ -172,11 +172,12 @@ import lombok.extern.slf4j.Slf4j; } } + @Test @SneakyThrows public void runAnalysisEnd2End() { - String folder = "/Users/maverickstuder/Downloads/11-26-2024-10-26-32_files_list"; // Should contain all files from minio directly, still zipped. Can contain multiple files. + String folder = "/home/kschuettler/Dokumente/analysisend2end/file0"; // Should contain all files from minio directly, still zipped. Can contain multiple files. Path absoluteFolderPath; if (folder.startsWith("files")) { // if it starts with "files" it is most likely in the resources folder, else it should be an absolute path @@ -350,30 +351,7 @@ import lombok.extern.slf4j.Slf4j; Set missingFileTypes = Sets.difference(REQUIRED_FILES, uploadedFileTypes); if (!missingFileTypes.isEmpty() && !missingFileTypes.contains(FileType.ORIGIN)) { - var cvServiceResponse = "files/cv_service_empty_response.json"; - var imageServiceResponse = "files/empty_image_response.json"; - ClassPathResource cvServiceResponseFileResource = new ClassPathResource(cvServiceResponse); - ClassPathResource imageServiceResponseFileResource = new ClassPathResource(imageServiceResponse); - - storageService.storeObject(TenantContext.getTenantId(), - RedactionStorageService.StorageIdUtils.getStorageId(request.getDossierId(), request.getFileId(), FileType.TABLES), - cvServiceResponseFileResource.getInputStream()); - storageService.storeObject(TenantContext.getTenantId(), - RedactionStorageService.StorageIdUtils.getStorageId(request.getDossierId(), request.getFileId(), FileType.IMAGE_INFO), - imageServiceResponseFileResource.getInputStream()); - - LayoutParsingRequest layoutParsingRequest = LayoutParsingRequestProvider.build(LayoutParsingType.REDACT_MANAGER_WITHOUT_DUPLICATE_PARAGRAPH, request); - layoutParsingPipeline.parseLayoutAndSaveFilesToStorage(layoutParsingRequest); - -// try { -// storeFileFromStorage(TENANT_ID, layoutParsingRequest.structureFileStorageId(), folder); -// storeFileFromStorage(TENANT_ID, layoutParsingRequest.textBlockFileStorageId(), folder); -// storeFileFromStorage(TENANT_ID, layoutParsingRequest.positionBlockFileStorageId(), folder); -// storeFileFromStorage(TENANT_ID, layoutParsingRequest.pageFileStorageId(), folder); -// } catch (IOException e) { -// log.error("Failed to store files from storage to folder {}", folder, e); -// return Optional.empty(); -// } + runLayoutParsingAndSaveFilesToFolder(folder, uploadedFileTypes, request); } // if (!missingFileTypes.isEmpty()) { @@ -388,11 +366,48 @@ import lombok.extern.slf4j.Slf4j; } + private void runLayoutParsingAndSaveFilesToFolder(Path folder, Set uploadedFileTypes, AnalyzeRequest request) throws IOException { + + uploadImageAndTableFilesIfMissing(uploadedFileTypes, request); + + LayoutParsingRequest layoutParsingRequest = LayoutParsingRequestProvider.build(LayoutParsingType.DOCUMINE_OLD, request); + layoutParsingPipeline.parseLayoutAndSaveFilesToStorage(layoutParsingRequest); + + try { + storeFileFromStorage(TENANT_ID, layoutParsingRequest.structureFileStorageId(), folder); + storeFileFromStorage(TENANT_ID, layoutParsingRequest.textBlockFileStorageId(), folder); + storeFileFromStorage(TENANT_ID, layoutParsingRequest.positionBlockFileStorageId(), folder); + storeFileFromStorage(TENANT_ID, layoutParsingRequest.pageFileStorageId(), folder); + } catch (IOException e) { + log.error("Failed to store files from storage to folder {}", folder, e); + } + } + + + private void uploadImageAndTableFilesIfMissing(Set uploadedFileTypes, AnalyzeRequest request) throws IOException { + + if (!uploadedFileTypes.contains(FileType.TABLES)) { + var cvServiceResponse = "files/cv_service_empty_response.json"; + ClassPathResource cvServiceResponseFileResource = new ClassPathResource(cvServiceResponse); + storageService.storeObject(TenantContext.getTenantId(), + RedactionStorageService.StorageIdUtils.getStorageId(request.getDossierId(), request.getFileId(), FileType.TABLES), + cvServiceResponseFileResource.getInputStream()); + } + if (!uploadedFileTypes.contains(FileType.IMAGE_INFO)) { + var imageServiceResponse = "files/empty_image_response.json"; + ClassPathResource imageServiceResponseFileResource = new ClassPathResource(imageServiceResponse); + storageService.storeObject(TenantContext.getTenantId(), + RedactionStorageService.StorageIdUtils.getStorageId(request.getDossierId(), request.getFileId(), FileType.IMAGE_INFO), + imageServiceResponseFileResource.getInputStream()); + } + } + + private void storeFileFromStorage(String tenantId, String storageId, Path folder) throws IOException { var inputStream = storageService.getObject(tenantId, storageId); - try (FileOutputStream fileOut = new FileOutputStream(folder.toString() + "/" + storageId.split("/")[1]); ObjectOutputStream out = new ObjectOutputStream(fileOut)) { - out.writeObject(inputStream); + try (FileOutputStream fileOut = new FileOutputStream(folder.toString() + "/" + storageId.split("/")[1])) { + fileOut.write(inputStream.getContentAsByteArray()); } catch (IOException e) { e.printStackTrace(); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/files/syngenta b/redaction-service-v1/redaction-service-server-v1/src/test/resources/files/syngenta index 57e6e0dd..5705cc07 160000 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/files/syngenta +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/files/syngenta @@ -1 +1 @@ -Subproject commit 57e6e0dd3c08a3a65ec59b5dfb70f0f77ebcc7c7 +Subproject commit 5705cc0782605fdca5dfff134b436f7143c9e421 diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/logback-spring.xml b/redaction-service-v1/redaction-service-server-v1/src/test/resources/logback-spring.xml index 8e3e4cf2..03547171 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/logback-spring.xml +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/logback-spring.xml @@ -16,5 +16,7 @@ + + \ No newline at end of file