Marmelator/redaction-service
1
0
Fork 0
Code Issues Pull Requests 14 Actions Packages Projects Releases 622 Wiki Activity

add simplifiedText

Browse Source
This commit is contained in:
Kilian Schuettler 2023-07-31 15:47:40 +02:00
parent 72fd542a25
commit e744c488ef
4 changed files with 47 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
redaction-service-v1/redaction-service-server-v1/build.gradle.kts
View File

@ -16,7 +16,7 @@ extra["layoutparserVersion"] = "0.8.0"
dependencies {
implementation(project(":redaction-service-api-v1"))
implementation("com.iqser.red.service:persistence-service-internal-api-v1:RED-6725")
implementation("com.knecon.fforesight:layoutparser-service-internal-api:0.12.0")
implementation("com.knecon.fforesight:layoutparser-service-internal-api:0.16.0")
implementation("com.iqser.red.commons:spring-commons:2.1.0") { exclude(group = "org.springframework.boot", module = "spring-boot-starter-log4j2") }
implementation("com.iqser.red.commons:metric-commons:2.1.0") { exclude(group = "org.springframework.boot", module = "spring-boot-starter-log4j2") }
@ -43,7 +43,7 @@ dependencies {
implementation("org.springframework.cloud:spring-cloud-starter-openfeign:4.0.1")
implementation("org.springframework.boot:spring-boot-starter-amqp:3.0.4")
testImplementation("org.springframework.boot:spring-boot-starter-test:3.0.4")
testImplementation("com.knecon.fforesight:layoutparser-service-processor:0.12.0") { exclude(group = "com.iqser.red.service", module = "persistence-service-shared-api-v1") }
testImplementation("com.knecon.fforesight:layoutparser-service-processor:0.16.0") { exclude(group = "com.iqser.red.service", module = "persistence-service-shared-api-v1") }
}
tasks.test {

26
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AbstractRedactionIntegrationTest.java
View File

@ -13,7 +13,6 @@ import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
@ -29,7 +28,6 @@ import com.fasterxml.jackson.databind.ObjectMapper;
import com.iqser.red.service.dictionarymerge.commons.DictionaryEntry;
import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.configuration.Colors;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type;
import com.iqser.red.service.redaction.v1.server.annotate.AnnotationService;
@ -42,9 +40,9 @@ import com.iqser.red.service.redaction.v1.server.redaction.service.ManualRedacti
import com.iqser.red.service.redaction.v1.server.redaction.utils.ResourceLoader;
import com.iqser.red.service.redaction.v1.server.redaction.utils.TextNormalizationUtilities;
import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
import com.iqser.red.service.redaction.v1.server.utils.LayoutParsingRequestProvider;
import com.iqser.red.storage.commons.service.StorageService;
import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingFinishedEvent;
import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingRequest;
import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingPipeline;
import com.knecon.fforesight.tenantcommons.TenantContext;
@ -449,26 +447,8 @@ public abstract class AbstractRedactionIntegrationTest {
@SneakyThrows
protected LayoutParsingFinishedEvent analyzeDocumentStructure(LayoutParsingType layoutParsingType, AnalyzeRequest request) {
var originId = RedactionStorageService.StorageIdUtils.getStorageId(request.getDossierId(), request.getFileId(), FileType.ORIGIN);
var tableId = RedactionStorageService.StorageIdUtils.getStorageId(request.getDossierId(), request.getFileId(), FileType.TABLES);
var imageId = RedactionStorageService.StorageIdUtils.getStorageId(request.getDossierId(), request.getFileId(), FileType.IMAGE_INFO);
var sectionGridId = RedactionStorageService.StorageIdUtils.getStorageId(request.getDossierId(), request.getFileId(), FileType.SECTION_GRID);
var structureId = RedactionStorageService.StorageIdUtils.getStorageId(request.getDossierId(), request.getFileId(), FileType.DOCUMENT_STRUCTURE);
var textId = RedactionStorageService.StorageIdUtils.getStorageId(request.getDossierId(), request.getFileId(), FileType.DOCUMENT_TEXT);
var positionID = RedactionStorageService.StorageIdUtils.getStorageId(request.getDossierId(), request.getFileId(), FileType.DOCUMENT_POSITION);
var pagesID = RedactionStorageService.StorageIdUtils.getStorageId(request.getDossierId(), request.getFileId(), FileType.DOCUMENT_PAGES);
var layoutParsingRequest = LayoutParsingRequest.builder()
.layoutParsingType(layoutParsingType)
.originFileStorageId(originId)
.tablesFileStorageId(Optional.of(tableId))
.imagesFileStorageId(Optional.of(imageId))
.structureFileStorageId(structureId)
.textBlockFileStorageId(textId)
.positionBlockFileStorageId(positionID)
.pageFileStorageId(pagesID)
.sectionGridStorageId(sectionGridId)
.build();
return layoutParsingPipeline.parseLayoutAndSaveFilesToStorage(layoutParsingRequest);
return layoutParsingPipeline.parseLayoutAndSaveFilesToStorage(LayoutParsingRequestProvider.build(layoutParsingType, request));
}

26
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RulesTest.java
View File

@ -24,7 +24,6 @@ import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
@ -82,10 +81,10 @@ import com.iqser.red.service.redaction.v1.server.redaction.service.ManualRedacti
import com.iqser.red.service.redaction.v1.server.redaction.utils.ResourceLoader;
import com.iqser.red.service.redaction.v1.server.redaction.utils.TextNormalizationUtilities;
import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
import com.iqser.red.service.redaction.v1.server.utils.LayoutParsingRequestProvider;
import com.iqser.red.storage.commons.StorageAutoConfiguration;
import com.iqser.red.storage.commons.service.StorageService;
import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingFinishedEvent;
import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingRequest;
import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingPipeline;
import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingServiceProcessorConfiguration;
@ -821,29 +820,12 @@ public class RulesTest {
}
}
@SneakyThrows
protected LayoutParsingFinishedEvent analyzeDocumentStructure(LayoutParsingType layoutParsingType) {
var originId = RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.ORIGIN);
var tableId = RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.TABLES);
var imageId = RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.IMAGE_INFO);
var sectionGridId = RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.SECTION_GRID);
var structureId = RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.DOCUMENT_STRUCTURE);
var textId = RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.DOCUMENT_TEXT);
var positionID = RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.DOCUMENT_POSITION);
var pagesID = RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.DOCUMENT_PAGES);
var layoutParsingRequest = LayoutParsingRequest.builder()
.layoutParsingType(layoutParsingType)
.originFileStorageId(originId)
.tablesFileStorageId(Optional.of(tableId))
.imagesFileStorageId(Optional.of(imageId))
.structureFileStorageId(structureId)
.textBlockFileStorageId(textId)
.positionBlockFileStorageId(positionID)
.pageFileStorageId(pagesID)
.sectionGridStorageId(sectionGridId)
.build();
return layoutParsingPipeline.parseLayoutAndSaveFilesToStorage(layoutParsingRequest);
return layoutParsingPipeline.parseLayoutAndSaveFilesToStorage(LayoutParsingRequestProvider.build(layoutParsingType,
AnalyzeRequest.builder().dossierId(TEST_DOSSIER_ID).fileId(TEST_FILE_ID).build()));
}

38
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/utils/LayoutParsingRequestProvider.java Normal file
View File

@ -0,0 +1,38 @@
package com.iqser.red.service.redaction.v1.server.utils;
import java.util.Optional;
import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingRequest;
import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
public class LayoutParsingRequestProvider {
public static LayoutParsingRequest build(LayoutParsingType layoutParsingType, AnalyzeRequest request) {
var originId = RedactionStorageService.StorageIdUtils.getStorageId(request.getDossierId(), request.getFileId(), FileType.ORIGIN);
var tableId = RedactionStorageService.StorageIdUtils.getStorageId(request.getDossierId(), request.getFileId(), FileType.TABLES);
var imageId = RedactionStorageService.StorageIdUtils.getStorageId(request.getDossierId(), request.getFileId(), FileType.IMAGE_INFO);
var sectionGridId = RedactionStorageService.StorageIdUtils.getStorageId(request.getDossierId(), request.getFileId(), FileType.SECTION_GRID);
var structureId = RedactionStorageService.StorageIdUtils.getStorageId(request.getDossierId(), request.getFileId(), FileType.DOCUMENT_STRUCTURE);
var textId = RedactionStorageService.StorageIdUtils.getStorageId(request.getDossierId(), request.getFileId(), FileType.DOCUMENT_TEXT);
var positionID = RedactionStorageService.StorageIdUtils.getStorageId(request.getDossierId(), request.getFileId(), FileType.DOCUMENT_POSITION);
var pagesID = RedactionStorageService.StorageIdUtils.getStorageId(request.getDossierId(), request.getFileId(), FileType.DOCUMENT_PAGES);
var simplifiedTextID = RedactionStorageService.StorageIdUtils.getStorageId(request.getDossierId(), request.getFileId(), FileType.SIMPLIFIED_TEXT);
return LayoutParsingRequest.builder()
.layoutParsingType(layoutParsingType)
.originFileStorageId(originId)
.tablesFileStorageId(Optional.of(tableId))
.imagesFileStorageId(Optional.of(imageId))
.structureFileStorageId(structureId)
.textBlockFileStorageId(textId)
.positionBlockFileStorageId(positionID)
.pageFileStorageId(pagesID)
.sectionGridStorageId(sectionGridId)
.simplifiedTextStorageId(simplifiedTextID)
.build();
}
}