Marmelator/redaction-service
1
0
Fork 0
Code Issues Pull Requests 14 Actions Packages Projects Releases 622 Wiki Activity

Merge remote-tracking branch 'origin/RED-10200-2' into RED-10200-2

Browse Source 
# Conflicts:
#	redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/TextEntity.java
#	redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/EntityDroolsExecutionService.java
#	redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AnalysisEnd2EndTest.java
This commit is contained in:
maverickstuder 2024-11-26 15:50:16 +01:00
commit 3429563079
10 changed files with 82 additions and 67 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/TextEntity.java
View File

@ -71,6 +71,8 @@ public class TextEntity implements IEntity {
List<SemanticNode> intersectingNodes = new LinkedList<>();
SemanticNode deepestFullyContainingNode;
List<Relation> relations = new LinkedList<>();
public static TextEntity initialEntityNode(TextRange textRange, String type, EntityType entityType, SemanticNode node) {
@ -335,5 +337,4 @@ public class TextEntity implements IEntity {
return Optional.empty();
}
}

2
redaction-service-v1/redaction-service-server-v1/build.gradle.kts
View File

@ -12,7 +12,7 @@ plugins {
description = "redaction-service-server-v1"
val layoutParserVersion = "0.191.0"
val layoutParserVersion = "0.193.0"
val jacksonVersion = "2.15.2"
val droolsVersion = "9.44.0.Final"
val pdfBoxVersion = "3.0.0"

2
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/RedactionServiceSettings.java
View File

@ -40,7 +40,7 @@ public class RedactionServiceSettings {
private boolean annotationMode;
private boolean droolsDebug;
private boolean droolsDebug = true;
private boolean protobufJsonFallback = true;

1
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/DictionarySearchService.java
View File

@ -7,7 +7,6 @@ import org.springframework.stereotype.Service;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine;
import com.iqser.red.service.redaction.v1.server.model.dictionary.Dictionary;
import com.iqser.red.service.redaction.v1.server.model.dictionary.SearchImplementation;
import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
import com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService;

13
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityCreationService.java
View File

@ -1505,18 +1505,19 @@ public class EntityCreationService {
documentTree.getRoot().getNode().addThisToEntityIfIntersects(entity);
if (!nodesInKieSession.isEmpty() && entity.getIntersectingNodes()
.stream()
.anyMatch(node -> !nodesInKieSession.contains(node))) {
entity.removeFromGraph();
return false;
}
// if (!nodesInKieSession.isEmpty() && entity.getIntersectingNodes()
// .stream()
// .anyMatch(node -> !nodesInKieSession.contains(node))) {
// entity.removeFromGraph();
// return false;
// }
TextBlock textBlock = entity.getDeepestFullyContainingNode().getTextBlock();
entityEnrichmentService.enrichEntity(entity, textBlock);
addToPages(entity);
addEntityToNodeEntitySets(entity);
// compute relationships, by looping through deepestFullyContainingNode's entities
return true;
}

52
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/EntityDroolsExecutionService.java
View File

@ -2,7 +2,6 @@ package com.iqser.red.service.redaction.v1.server.service.drools;
import static com.iqser.red.service.redaction.v1.server.service.drools.ComponentDroolsExecutionService.RULES_LOGGER_GLOBAL;
import java.util.Collections;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
@ -118,21 +117,18 @@ public class EntityDroolsExecutionService {
kieSession.setGlobal(RULES_LOGGER_GLOBAL, logger);
}
kieSession.insert(document);
nodesInKieSession.add(document);
nodesInKieSession.addAll(sectionsToAnalyze);
nodesInKieSession.addAll(sectionsToAnalyze.stream()
.flatMap(SemanticNode::streamAllSubNodes)
.toList());
nodesInKieSession.forEach(kieSession::insert);
System.out.println("after document insert : " + kieSession.getFactCount());
document.getEntities()
.forEach(kieSession::insert);
System.out.println("after getEntities insert : " + kieSession.getFactCount());
sectionsToAnalyze.forEach(kieSession::insert);
System.out.println("after sectionsToAnalyze insert : " + kieSession.getFactCount());
//
// sectionsToAnalyze.stream()
// .flatMap(SemanticNode::streamAllSubNodes)
// .forEach(kieSession::insert);
// System.out.println("after SemanticNode insert : " + kieSession.getFactCount());
document.getPages()
.forEach(kieSession::insert);
System.out.println("after getPages insert : " + kieSession.getFactCount());
@ -162,24 +158,24 @@ public class EntityDroolsExecutionService {
try {
completableFuture.get(settings.getDroolsExecutionTimeoutSecs(document.getNumberOfPages()), TimeUnit.SECONDS);
nodesInKieSession = sectionsToAnalyze.size() == document.streamAllSubNodes()
.count() ? Collections.emptySet() : buildSet(sectionsToAnalyze, document);
kieSessionUpdater = new KieSessionUpdater(nodesInKieSession, kieSession);
document.setKieSessionUpdater(kieSessionUpdater);
kieSession.insert(document);
kieSession.setGlobal("entityCreationService", entityCreationService);
sectionsToAnalyze.stream()
.flatMap(SemanticNode::streamAllSubNodes)
.forEach(kieSession::insert);
System.out.println("after SemanticNode insert : " + kieSession.getFactCount());
completableFuture.get(settings.getDroolsExecutionTimeoutSecs(document.getNumberOfPages()), TimeUnit.SECONDS);
//
// nodesInKieSession = sectionsToAnalyze.size() == document.streamAllSubNodes()
// .count() ? Collections.emptySet() : buildSet(sectionsToAnalyze, document);
//
// kieSessionUpdater = new KieSessionUpdater(nodesInKieSession, kieSession);
//
// document.setKieSessionUpdater(kieSessionUpdater);
//
// kieSession.insert(document);
//
// kieSession.setGlobal("entityCreationService", entityCreationService);
//
// sectionsToAnalyze.stream()
// .flatMap(SemanticNode::streamAllSubNodes)
// .forEach(kieSession::insert);
// System.out.println("after SemanticNode insert : " + kieSession.getFactCount());
//
// completableFuture.get(settings.getDroolsExecutionTimeoutSecs(document.getNumberOfPages()), TimeUnit.SECONDS);
} catch (ExecutionException e) {
logger.error(e, "Exception during rule execution");

1
redaction-service-v1/redaction-service-server-v1/src/main/resources/logback-spring.xml
View File

@ -14,4 +14,5 @@
<appender-ref ref="${logType}"/>
</root>
<logger name="org.drools.mvel" level="ERROR"/>
<logger name="org.springframework.web.socket.config" level="WARN"/>
</configuration>

71
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AnalysisEnd2EndTest.java
View File

@ -121,7 +121,7 @@ import lombok.extern.slf4j.Slf4j;
FileType.DOCUMENT_POSITION,
FileType.DOCUMENT_STRUCTURE,
FileType.DOCUMENT_TEXT);
Path dossierTemplateToUse = Path.of("/Users/maverickstuder/Downloads/DossierTemplate_mod"); // Add your dossier-template here
Path dossierTemplateToUse = Path.of("/home/kschuettler/iqser/business-logic/redactmanager/prod-cp-eu-reg/EFSA_sanitisation_pre_GFL_v1"); // Add your dossier-template here
ObjectMapper mapper = ObjectMapperFactory.create();
final String TENANT_ID = "tenant";
TestDossierTemplate testDossierTemplate;
@ -172,11 +172,12 @@ import lombok.extern.slf4j.Slf4j;
}
}
@Test
@SneakyThrows
public void runAnalysisEnd2End() {
String folder = "/Users/maverickstuder/Downloads/11-26-2024-10-26-32_files_list"; // Should contain all files from minio directly, still zipped. Can contain multiple files.
String folder = "/home/kschuettler/Dokumente/analysisend2end/file0"; // Should contain all files from minio directly, still zipped. Can contain multiple files.
Path absoluteFolderPath;
if (folder.startsWith("files")) { // if it starts with "files" it is most likely in the resources folder, else it should be an absolute path
@ -350,30 +351,7 @@ import lombok.extern.slf4j.Slf4j;
Set<FileType> missingFileTypes = Sets.difference(REQUIRED_FILES, uploadedFileTypes);
if (!missingFileTypes.isEmpty() && !missingFileTypes.contains(FileType.ORIGIN)) {
var cvServiceResponse = "files/cv_service_empty_response.json";
var imageServiceResponse = "files/empty_image_response.json";
ClassPathResource cvServiceResponseFileResource = new ClassPathResource(cvServiceResponse);
ClassPathResource imageServiceResponseFileResource = new ClassPathResource(imageServiceResponse);
storageService.storeObject(TenantContext.getTenantId(),
RedactionStorageService.StorageIdUtils.getStorageId(request.getDossierId(), request.getFileId(), FileType.TABLES),
cvServiceResponseFileResource.getInputStream());
storageService.storeObject(TenantContext.getTenantId(),
RedactionStorageService.StorageIdUtils.getStorageId(request.getDossierId(), request.getFileId(), FileType.IMAGE_INFO),
imageServiceResponseFileResource.getInputStream());
LayoutParsingRequest layoutParsingRequest = LayoutParsingRequestProvider.build(LayoutParsingType.REDACT_MANAGER_WITHOUT_DUPLICATE_PARAGRAPH, request);
layoutParsingPipeline.parseLayoutAndSaveFilesToStorage(layoutParsingRequest);
// try {
// storeFileFromStorage(TENANT_ID, layoutParsingRequest.structureFileStorageId(), folder);
// storeFileFromStorage(TENANT_ID, layoutParsingRequest.textBlockFileStorageId(), folder);
// storeFileFromStorage(TENANT_ID, layoutParsingRequest.positionBlockFileStorageId(), folder);
// storeFileFromStorage(TENANT_ID, layoutParsingRequest.pageFileStorageId(), folder);
// } catch (IOException e) {
// log.error("Failed to store files from storage to folder {}", folder, e);
// return Optional.empty();
// }
runLayoutParsingAndSaveFilesToFolder(folder, uploadedFileTypes, request);
}
// if (!missingFileTypes.isEmpty()) {
@ -388,11 +366,48 @@ import lombok.extern.slf4j.Slf4j;
}
private void runLayoutParsingAndSaveFilesToFolder(Path folder, Set<FileType> uploadedFileTypes, AnalyzeRequest request) throws IOException {
uploadImageAndTableFilesIfMissing(uploadedFileTypes, request);
LayoutParsingRequest layoutParsingRequest = LayoutParsingRequestProvider.build(LayoutParsingType.DOCUMINE_OLD, request);
layoutParsingPipeline.parseLayoutAndSaveFilesToStorage(layoutParsingRequest);
try {
storeFileFromStorage(TENANT_ID, layoutParsingRequest.structureFileStorageId(), folder);
storeFileFromStorage(TENANT_ID, layoutParsingRequest.textBlockFileStorageId(), folder);
storeFileFromStorage(TENANT_ID, layoutParsingRequest.positionBlockFileStorageId(), folder);
storeFileFromStorage(TENANT_ID, layoutParsingRequest.pageFileStorageId(), folder);
} catch (IOException e) {
log.error("Failed to store files from storage to folder {}", folder, e);
}
}
private void uploadImageAndTableFilesIfMissing(Set<FileType> uploadedFileTypes, AnalyzeRequest request) throws IOException {
if (!uploadedFileTypes.contains(FileType.TABLES)) {
var cvServiceResponse = "files/cv_service_empty_response.json";
ClassPathResource cvServiceResponseFileResource = new ClassPathResource(cvServiceResponse);
storageService.storeObject(TenantContext.getTenantId(),
RedactionStorageService.StorageIdUtils.getStorageId(request.getDossierId(), request.getFileId(), FileType.TABLES),
cvServiceResponseFileResource.getInputStream());
}
if (!uploadedFileTypes.contains(FileType.IMAGE_INFO)) {
var imageServiceResponse = "files/empty_image_response.json";
ClassPathResource imageServiceResponseFileResource = new ClassPathResource(imageServiceResponse);
storageService.storeObject(TenantContext.getTenantId(),
RedactionStorageService.StorageIdUtils.getStorageId(request.getDossierId(), request.getFileId(), FileType.IMAGE_INFO),
imageServiceResponseFileResource.getInputStream());
}
}
private void storeFileFromStorage(String tenantId, String storageId, Path folder) throws IOException {
var inputStream = storageService.getObject(tenantId, storageId);
try (FileOutputStream fileOut = new FileOutputStream(folder.toString() + "/" + storageId.split("/")[1]); ObjectOutputStream out = new ObjectOutputStream(fileOut)) {
out.writeObject(inputStream);
try (FileOutputStream fileOut = new FileOutputStream(folder.toString() + "/" + storageId.split("/")[1])) {
fileOut.write(inputStream.getContentAsByteArray());
} catch (IOException e) {
e.printStackTrace();
}

2
redaction-service-v1/redaction-service-server-v1/src/test/resources/files/syngenta

@ -1 +1 @@
Subproject commit 57e6e0dd3c08a3a65ec59b5dfb70f0f77ebcc7c7
Subproject commit 5705cc0782605fdca5dfff134b436f7143c9e421

2
redaction-service-v1/redaction-service-server-v1/src/test/resources/logback-spring.xml
View File

@ -16,5 +16,7 @@
<logger name="org.apache.fontbox.ttf" level="ERROR"/>
<logger name="org.drools.mvel" level="ERROR"/>
<logger name="org.springframework.web.socket.config" level="WARN"/>
<logger name="org.mongodb.driver.client" level="ERROR"/>
</configuration>