From e744c488ef5b9f70c64cd1d1beded3fe77a388d7 Mon Sep 17 00:00:00 2001 From: Kilian Schuettler Date: Mon, 31 Jul 2023 15:47:40 +0200 Subject: [PATCH] add simplifiedText --- .../build.gradle.kts | 4 +- .../AbstractRedactionIntegrationTest.java | 26 ++----------- .../redaction/v1/server/RulesTest.java | 26 ++----------- .../utils/LayoutParsingRequestProvider.java | 38 +++++++++++++++++++ 4 files changed, 47 insertions(+), 47 deletions(-) create mode 100644 redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/utils/LayoutParsingRequestProvider.java diff --git a/redaction-service-v1/redaction-service-server-v1/build.gradle.kts b/redaction-service-v1/redaction-service-server-v1/build.gradle.kts index 8b522a53..f8c12bb5 100644 --- a/redaction-service-v1/redaction-service-server-v1/build.gradle.kts +++ b/redaction-service-v1/redaction-service-server-v1/build.gradle.kts @@ -16,7 +16,7 @@ extra["layoutparserVersion"] = "0.8.0" dependencies { implementation(project(":redaction-service-api-v1")) implementation("com.iqser.red.service:persistence-service-internal-api-v1:RED-6725") - implementation("com.knecon.fforesight:layoutparser-service-internal-api:0.12.0") + implementation("com.knecon.fforesight:layoutparser-service-internal-api:0.16.0") implementation("com.iqser.red.commons:spring-commons:2.1.0") { exclude(group = "org.springframework.boot", module = "spring-boot-starter-log4j2") } implementation("com.iqser.red.commons:metric-commons:2.1.0") { exclude(group = "org.springframework.boot", module = "spring-boot-starter-log4j2") } @@ -43,7 +43,7 @@ dependencies { implementation("org.springframework.cloud:spring-cloud-starter-openfeign:4.0.1") implementation("org.springframework.boot:spring-boot-starter-amqp:3.0.4") testImplementation("org.springframework.boot:spring-boot-starter-test:3.0.4") - testImplementation("com.knecon.fforesight:layoutparser-service-processor:0.12.0") { exclude(group = "com.iqser.red.service", module = "persistence-service-shared-api-v1") } + testImplementation("com.knecon.fforesight:layoutparser-service-processor:0.16.0") { exclude(group = "com.iqser.red.service", module = "persistence-service-shared-api-v1") } } tasks.test { diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AbstractRedactionIntegrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AbstractRedactionIntegrationTest.java index 912d6ebe..b0b3d1f5 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AbstractRedactionIntegrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AbstractRedactionIntegrationTest.java @@ -13,7 +13,6 @@ import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; @@ -29,7 +28,6 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.iqser.red.service.dictionarymerge.commons.DictionaryEntry; import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.configuration.Colors; - import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type; import com.iqser.red.service.redaction.v1.server.annotate.AnnotationService; @@ -42,9 +40,9 @@ import com.iqser.red.service.redaction.v1.server.redaction.service.ManualRedacti import com.iqser.red.service.redaction.v1.server.redaction.utils.ResourceLoader; import com.iqser.red.service.redaction.v1.server.redaction.utils.TextNormalizationUtilities; import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService; +import com.iqser.red.service.redaction.v1.server.utils.LayoutParsingRequestProvider; import com.iqser.red.storage.commons.service.StorageService; import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingFinishedEvent; -import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingRequest; import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType; import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingPipeline; import com.knecon.fforesight.tenantcommons.TenantContext; @@ -449,26 +447,8 @@ public abstract class AbstractRedactionIntegrationTest { @SneakyThrows protected LayoutParsingFinishedEvent analyzeDocumentStructure(LayoutParsingType layoutParsingType, AnalyzeRequest request) { - var originId = RedactionStorageService.StorageIdUtils.getStorageId(request.getDossierId(), request.getFileId(), FileType.ORIGIN); - var tableId = RedactionStorageService.StorageIdUtils.getStorageId(request.getDossierId(), request.getFileId(), FileType.TABLES); - var imageId = RedactionStorageService.StorageIdUtils.getStorageId(request.getDossierId(), request.getFileId(), FileType.IMAGE_INFO); - var sectionGridId = RedactionStorageService.StorageIdUtils.getStorageId(request.getDossierId(), request.getFileId(), FileType.SECTION_GRID); - var structureId = RedactionStorageService.StorageIdUtils.getStorageId(request.getDossierId(), request.getFileId(), FileType.DOCUMENT_STRUCTURE); - var textId = RedactionStorageService.StorageIdUtils.getStorageId(request.getDossierId(), request.getFileId(), FileType.DOCUMENT_TEXT); - var positionID = RedactionStorageService.StorageIdUtils.getStorageId(request.getDossierId(), request.getFileId(), FileType.DOCUMENT_POSITION); - var pagesID = RedactionStorageService.StorageIdUtils.getStorageId(request.getDossierId(), request.getFileId(), FileType.DOCUMENT_PAGES); - var layoutParsingRequest = LayoutParsingRequest.builder() - .layoutParsingType(layoutParsingType) - .originFileStorageId(originId) - .tablesFileStorageId(Optional.of(tableId)) - .imagesFileStorageId(Optional.of(imageId)) - .structureFileStorageId(structureId) - .textBlockFileStorageId(textId) - .positionBlockFileStorageId(positionID) - .pageFileStorageId(pagesID) - .sectionGridStorageId(sectionGridId) - .build(); - return layoutParsingPipeline.parseLayoutAndSaveFilesToStorage(layoutParsingRequest); + + return layoutParsingPipeline.parseLayoutAndSaveFilesToStorage(LayoutParsingRequestProvider.build(layoutParsingType, request)); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RulesTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RulesTest.java index 32cd98b3..3b22caa0 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RulesTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RulesTest.java @@ -24,7 +24,6 @@ import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; @@ -82,10 +81,10 @@ import com.iqser.red.service.redaction.v1.server.redaction.service.ManualRedacti import com.iqser.red.service.redaction.v1.server.redaction.utils.ResourceLoader; import com.iqser.red.service.redaction.v1.server.redaction.utils.TextNormalizationUtilities; import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService; +import com.iqser.red.service.redaction.v1.server.utils.LayoutParsingRequestProvider; import com.iqser.red.storage.commons.StorageAutoConfiguration; import com.iqser.red.storage.commons.service.StorageService; import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingFinishedEvent; -import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingRequest; import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType; import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingPipeline; import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingServiceProcessorConfiguration; @@ -821,29 +820,12 @@ public class RulesTest { } } + @SneakyThrows protected LayoutParsingFinishedEvent analyzeDocumentStructure(LayoutParsingType layoutParsingType) { - var originId = RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.ORIGIN); - var tableId = RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.TABLES); - var imageId = RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.IMAGE_INFO); - var sectionGridId = RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.SECTION_GRID); - var structureId = RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.DOCUMENT_STRUCTURE); - var textId = RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.DOCUMENT_TEXT); - var positionID = RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.DOCUMENT_POSITION); - var pagesID = RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.DOCUMENT_PAGES); - var layoutParsingRequest = LayoutParsingRequest.builder() - .layoutParsingType(layoutParsingType) - .originFileStorageId(originId) - .tablesFileStorageId(Optional.of(tableId)) - .imagesFileStorageId(Optional.of(imageId)) - .structureFileStorageId(structureId) - .textBlockFileStorageId(textId) - .positionBlockFileStorageId(positionID) - .pageFileStorageId(pagesID) - .sectionGridStorageId(sectionGridId) - .build(); - return layoutParsingPipeline.parseLayoutAndSaveFilesToStorage(layoutParsingRequest); + return layoutParsingPipeline.parseLayoutAndSaveFilesToStorage(LayoutParsingRequestProvider.build(layoutParsingType, + AnalyzeRequest.builder().dossierId(TEST_DOSSIER_ID).fileId(TEST_FILE_ID).build())); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/utils/LayoutParsingRequestProvider.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/utils/LayoutParsingRequestProvider.java new file mode 100644 index 00000000..ced43956 --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/utils/LayoutParsingRequestProvider.java @@ -0,0 +1,38 @@ +package com.iqser.red.service.redaction.v1.server.utils; + +import java.util.Optional; + +import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType; +import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService; +import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingRequest; +import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType; + +public class LayoutParsingRequestProvider { + + public static LayoutParsingRequest build(LayoutParsingType layoutParsingType, AnalyzeRequest request) { + + var originId = RedactionStorageService.StorageIdUtils.getStorageId(request.getDossierId(), request.getFileId(), FileType.ORIGIN); + var tableId = RedactionStorageService.StorageIdUtils.getStorageId(request.getDossierId(), request.getFileId(), FileType.TABLES); + var imageId = RedactionStorageService.StorageIdUtils.getStorageId(request.getDossierId(), request.getFileId(), FileType.IMAGE_INFO); + var sectionGridId = RedactionStorageService.StorageIdUtils.getStorageId(request.getDossierId(), request.getFileId(), FileType.SECTION_GRID); + var structureId = RedactionStorageService.StorageIdUtils.getStorageId(request.getDossierId(), request.getFileId(), FileType.DOCUMENT_STRUCTURE); + var textId = RedactionStorageService.StorageIdUtils.getStorageId(request.getDossierId(), request.getFileId(), FileType.DOCUMENT_TEXT); + var positionID = RedactionStorageService.StorageIdUtils.getStorageId(request.getDossierId(), request.getFileId(), FileType.DOCUMENT_POSITION); + var pagesID = RedactionStorageService.StorageIdUtils.getStorageId(request.getDossierId(), request.getFileId(), FileType.DOCUMENT_PAGES); + var simplifiedTextID = RedactionStorageService.StorageIdUtils.getStorageId(request.getDossierId(), request.getFileId(), FileType.SIMPLIFIED_TEXT); + return LayoutParsingRequest.builder() + .layoutParsingType(layoutParsingType) + .originFileStorageId(originId) + .tablesFileStorageId(Optional.of(tableId)) + .imagesFileStorageId(Optional.of(imageId)) + .structureFileStorageId(structureId) + .textBlockFileStorageId(textId) + .positionBlockFileStorageId(positionID) + .pageFileStorageId(pagesID) + .sectionGridStorageId(sectionGridId) + .simplifiedTextStorageId(simplifiedTextID) + .build(); + } + +}