Marmelator/redaction-service
1
0
Fork 0
Code Issues Pull Requests 14 Actions Packages Projects Releases 622 Wiki Activity

RED-9123: Improve performance of re-analysis (Spike)

Browse Source
This commit is contained in:
Maverick Studer 2024-10-08 09:24:18 +02:00
parent daed4e07ef
commit d32c56e101
48 changed files with 934 additions and 408 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
redaction-service-v1/redaction-service-api-v1/build.gradle.kts
View File

@ -4,7 +4,7 @@ plugins {
} }
description = "redaction-service-api-v1" description = "redaction-service-api-v1"
val persistenceServiceVersion = "2.580.0" val persistenceServiceVersion = "2.581.0"
dependencies { dependencies {
implementation("org.springframework:spring-web:6.0.12") implementation("org.springframework:spring-web:6.0.12")

13
redaction-service-v1/redaction-service-server-v1/build.gradle.kts
View File

@ -12,11 +12,11 @@ plugins {
description = "redaction-service-server-v1" description = "redaction-service-server-v1"
val layoutParserVersion = "0.174.0" val layoutParserVersion = "0.181.0"
val jacksonVersion = "2.15.2" val jacksonVersion = "2.15.2"
val droolsVersion = "9.44.0.Final" val droolsVersion = "9.44.0.Final"
val pdfBoxVersion = "3.0.0" val pdfBoxVersion = "3.0.0"
val persistenceServiceVersion = "2.580.0" val persistenceServiceVersion = "2.581.0"
val llmServiceVersion = "1.11.0" val llmServiceVersion = "1.11.0"
val springBootStarterVersion = "3.1.5" val springBootStarterVersion = "3.1.5"
val springCloudVersion = "4.0.4" val springCloudVersion = "4.0.4"
@ -31,6 +31,12 @@ configurations {
} }
} }
configurations.all {
resolutionStrategy {
force("com.google.protobuf:protobuf-java:4.27.1")
}
}
dependencies { dependencies {
implementation(project(":redaction-service-api-v1")) { exclude(group = "com.iqser.red.service", module = "persistence-service-internal-api-v1") } implementation(project(":redaction-service-api-v1")) { exclude(group = "com.iqser.red.service", module = "persistence-service-internal-api-v1") }
@ -45,7 +51,7 @@ dependencies {
implementation("com.iqser.red.commons:metric-commons:2.3.0") implementation("com.iqser.red.commons:metric-commons:2.3.0")
implementation("com.iqser.red.commons:dictionary-merge-commons:1.5.0") implementation("com.iqser.red.commons:dictionary-merge-commons:1.5.0")
implementation("com.iqser.red.commons:storage-commons:2.45.0") implementation("com.iqser.red.commons:storage-commons:2.50.0")
implementation("com.knecon.fforesight:tenant-commons:0.30.0") implementation("com.knecon.fforesight:tenant-commons:0.30.0")
implementation("com.knecon.fforesight:keycloak-commons:0.30.0") { implementation("com.knecon.fforesight:keycloak-commons:0.30.0") {
exclude(group = "com.knecon.fforesight", module = "tenant-commons") exclude(group = "com.knecon.fforesight", module = "tenant-commons")
@ -61,6 +67,7 @@ dependencies {
implementation("org.drools:drools-engine:${droolsVersion}") implementation("org.drools:drools-engine:${droolsVersion}")
implementation("org.drools:drools-mvel:${droolsVersion}") implementation("org.drools:drools-mvel:${droolsVersion}")
implementation("org.kie:kie-spring:7.74.1.Final") implementation("org.kie:kie-spring:7.74.1.Final")
implementation("com.google.protobuf:protobuf-java:4.27.1")
implementation("org.locationtech.jts:jts-core:1.19.0") implementation("org.locationtech.jts:jts-core:1.19.0")

2
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/RedactionServiceSettings.java
View File

@ -40,6 +40,8 @@ public class RedactionServiceSettings {
private boolean droolsDebug; private boolean droolsDebug;
private boolean protobufJsonFallback = true;
public int getDroolsExecutionTimeoutSecs(int numberOfPages) { public int getDroolsExecutionTimeoutSecs(int numberOfPages) {

24
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/DocumentData.java
View File

@ -1,11 +1,13 @@
package com.iqser.red.service.redaction.v1.server.model.document; package com.iqser.red.service.redaction.v1.server.model.document;
import static com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentStructureProto.DocumentStructure;
import java.io.Serializable; import java.io.Serializable;
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPage; import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPageProto.AllDocumentPages;
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPositionData; import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPositionDataProto.AllDocumentPositionData;
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentStructure; import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentStructureWrapper;
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentTextData; import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentTextDataProto.AllDocumentTextData;
import lombok.AccessLevel; import lombok.AccessLevel;
import lombok.AllArgsConstructor; import lombok.AllArgsConstructor;
@ -21,9 +23,15 @@ import lombok.experimental.FieldDefaults;
@FieldDefaults(level = AccessLevel.PRIVATE) @FieldDefaults(level = AccessLevel.PRIVATE)
public class DocumentData implements Serializable { public class DocumentData implements Serializable {
DocumentPage[] documentPages; AllDocumentPages documentPages;
DocumentTextData[] documentTextData; AllDocumentTextData documentTextData;
DocumentPositionData[] documentPositionData; AllDocumentPositionData documentPositionData;
DocumentStructure documentStructure; DocumentStructureWrapper documentStructureWrapper;
public DocumentStructure getDocumentStructure() {
return documentStructureWrapper.getDocumentStructure();
}
} }

2
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/AbstractSemanticNode.java
View File

@ -9,7 +9,7 @@ import java.util.Set;
import com.iqser.red.service.redaction.v1.server.model.document.DocumentTree; import com.iqser.red.service.redaction.v1.server.model.document.DocumentTree;
import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity; import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock; import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine; import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngineProto.LayoutEngine;
import lombok.AccessLevel; import lombok.AccessLevel;
import lombok.AllArgsConstructor; import lombok.AllArgsConstructor;

10
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Footer.java
View File

@ -1,19 +1,9 @@
package com.iqser.red.service.redaction.v1.server.model.document.nodes; package com.iqser.red.service.redaction.v1.server.model.document.nodes;
import java.awt.geom.Rectangle2D;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import com.iqser.red.service.redaction.v1.server.model.document.DocumentTree;
import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock; import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
import lombok.AccessLevel; import lombok.AccessLevel;
import lombok.AllArgsConstructor; import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data; import lombok.Data;
import lombok.EqualsAndHashCode; import lombok.EqualsAndHashCode;
import lombok.NoArgsConstructor; import lombok.NoArgsConstructor;

10
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Header.java
View File

@ -1,19 +1,9 @@
package com.iqser.red.service.redaction.v1.server.model.document.nodes; package com.iqser.red.service.redaction.v1.server.model.document.nodes;
import java.awt.geom.Rectangle2D;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
import com.iqser.red.service.redaction.v1.server.model.document.DocumentTree;
import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock; import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
import lombok.AccessLevel; import lombok.AccessLevel;
import lombok.AllArgsConstructor; import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data; import lombok.Data;
import lombok.EqualsAndHashCode; import lombok.EqualsAndHashCode;
import lombok.NoArgsConstructor; import lombok.NoArgsConstructor;

10
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Headline.java
View File

@ -1,20 +1,10 @@
package com.iqser.red.service.redaction.v1.server.model.document.nodes; package com.iqser.red.service.redaction.v1.server.model.document.nodes;
import java.awt.geom.Rectangle2D;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import com.iqser.red.service.redaction.v1.server.model.document.DocumentTree;
import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
import com.iqser.red.service.redaction.v1.server.model.document.textblock.AtomicTextBlock; import com.iqser.red.service.redaction.v1.server.model.document.textblock.AtomicTextBlock;
import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock; import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
import lombok.AccessLevel; import lombok.AccessLevel;
import lombok.AllArgsConstructor; import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data; import lombok.Data;
import lombok.EqualsAndHashCode; import lombok.EqualsAndHashCode;
import lombok.NoArgsConstructor; import lombok.NoArgsConstructor;

6
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Image.java
View File

@ -3,22 +3,16 @@ package com.iqser.red.service.redaction.v1.server.model.document.nodes;
import java.awt.geom.Rectangle2D; import java.awt.geom.Rectangle2D;
import java.util.Collections; import java.util.Collections;
import java.util.HashMap; import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Locale; import java.util.Locale;
import java.util.Map; import java.util.Map;
import java.util.PriorityQueue; import java.util.PriorityQueue;
import java.util.Set; import java.util.Set;
import com.iqser.red.service.redaction.v1.server.model.document.DocumentTree;
import com.iqser.red.service.redaction.v1.server.model.document.TextRange; import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
import com.iqser.red.service.redaction.v1.server.model.document.entity.IEntity; import com.iqser.red.service.redaction.v1.server.model.document.entity.IEntity;
import com.iqser.red.service.redaction.v1.server.model.document.entity.ManualChangeOverwrite; import com.iqser.red.service.redaction.v1.server.model.document.entity.ManualChangeOverwrite;
import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule; import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule;
import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock; import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlockCollector;
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
import lombok.AccessLevel; import lombok.AccessLevel;
import lombok.AllArgsConstructor; import lombok.AllArgsConstructor;

10
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Paragraph.java
View File

@ -1,19 +1,9 @@
package com.iqser.red.service.redaction.v1.server.model.document.nodes; package com.iqser.red.service.redaction.v1.server.model.document.nodes;
import java.awt.geom.Rectangle2D;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import com.iqser.red.service.redaction.v1.server.model.document.DocumentTree;
import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock; import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
import lombok.AccessLevel; import lombok.AccessLevel;
import lombok.AllArgsConstructor; import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data; import lombok.Data;
import lombok.EqualsAndHashCode; import lombok.EqualsAndHashCode;
import lombok.experimental.FieldDefaults; import lombok.experimental.FieldDefaults;

14
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Section.java
View File

@ -1,23 +1,9 @@
package com.iqser.red.service.redaction.v1.server.model.document.nodes; package com.iqser.red.service.redaction.v1.server.model.document.nodes;
import java.awt.geom.Rectangle2D;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
import com.iqser.red.service.redaction.v1.server.model.document.DocumentTree;
import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlockCollector;
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
import lombok.AccessLevel; import lombok.AccessLevel;
import lombok.AllArgsConstructor; import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data; import lombok.Data;
import lombok.EqualsAndHashCode; import lombok.EqualsAndHashCode;
import lombok.RequiredArgsConstructor;
import lombok.experimental.FieldDefaults; import lombok.experimental.FieldDefaults;
import lombok.experimental.SuperBuilder; import lombok.experimental.SuperBuilder;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;

2
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/SemanticNode.java
View File

@ -25,7 +25,7 @@ import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBl
import com.iqser.red.service.redaction.v1.server.service.document.NodeVisitor; import com.iqser.red.service.redaction.v1.server.service.document.NodeVisitor;
import com.iqser.red.service.redaction.v1.server.utils.RectangleTransformations; import com.iqser.red.service.redaction.v1.server.utils.RectangleTransformations;
import com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility; import com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility;
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine; import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngineProto.LayoutEngine;
public interface SemanticNode { public interface SemanticNode {

2
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Table.java
View File

@ -17,7 +17,7 @@ import com.iqser.red.service.redaction.v1.server.model.document.DocumentTree;
import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity; import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock; import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlockCollector; import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlockCollector;
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine; import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngineProto.LayoutEngine;
import lombok.AccessLevel; import lombok.AccessLevel;
import lombok.AllArgsConstructor; import lombok.AllArgsConstructor;

20
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/textblock/AtomicTextBlock.java
View File

@ -1,12 +1,12 @@
package com.iqser.red.service.redaction.v1.server.model.document.textblock; package com.iqser.red.service.redaction.v1.server.model.document.textblock;
import static com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPositionDataProto.DocumentPositionData;
import static java.lang.String.format; import static java.lang.String.format;
import java.awt.geom.Rectangle2D; import java.awt.geom.Rectangle2D;
import java.lang.ref.SoftReference; import java.lang.ref.SoftReference;
import java.text.BreakIterator; import java.text.BreakIterator;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection; import java.util.Collection;
import java.util.Collections; import java.util.Collections;
import java.util.HashMap; import java.util.HashMap;
@ -20,8 +20,8 @@ import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Page; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Page;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode; import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
import com.iqser.red.service.redaction.v1.server.utils.RectangleTransformations; import com.iqser.red.service.redaction.v1.server.utils.RectangleTransformations;
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPositionData; import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPositionDataProto.DocumentPositionData.Position;
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentTextData; import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentTextDataProto.DocumentTextData;
import lombok.AccessLevel; import lombok.AccessLevel;
import lombok.AllArgsConstructor; import lombok.AllArgsConstructor;
@ -89,20 +89,18 @@ public class AtomicTextBlock implements TextBlock {
.page(page) .page(page)
.textRange(new TextRange(atomicTextBlockData.getStart(), atomicTextBlockData.getEnd())) .textRange(new TextRange(atomicTextBlockData.getStart(), atomicTextBlockData.getEnd()))
.searchText(atomicTextBlockData.getSearchText()) .searchText(atomicTextBlockData.getSearchText())
.lineBreaks(Arrays.stream(atomicTextBlockData.getLineBreaks()).boxed() .lineBreaks(atomicTextBlockData.getLineBreaksList())
.toList()) .stringIdxToPositionIdx(atomicPositionBlockData.getStringIdxToPositionIdxList())
.stringIdxToPositionIdx(Arrays.stream(atomicPositionBlockData.getStringIdxToPositionIdx()).boxed() .positions(toRectangle2DList(atomicPositionBlockData.getPositionsList()))
.toList())
.positions(toRectangle2DList(atomicPositionBlockData.getPositions()))
.parent(parent) .parent(parent)
.build(); .build();
} }
private static List<Rectangle2D> toRectangle2DList(float[][] positions) { private static List<Rectangle2D> toRectangle2DList(List<Position> positions) {
return Arrays.stream(positions) return positions.stream()
.map(floatArr -> (Rectangle2D) new Rectangle2D.Float(floatArr[0], floatArr[1], floatArr[2], floatArr[3])) .map(pos -> (Rectangle2D) new Rectangle2D.Float(pos.getValue(0), pos.getValue(1), pos.getValue(2), pos.getValue(3)))
.toList(); .toList();
} }

145
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalysisFinalizationService.java Normal file
View File

@ -0,0 +1,145 @@
package com.iqser.red.service.redaction.v1.server.service;
import java.util.List;
import java.util.Optional;
import java.util.Set;
import java.util.concurrent.CompletableFuture;
import org.springframework.stereotype.Service;
import com.iqser.gin4.commons.metrics.meters.FunctionTimerValues;
import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeResult;
import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentLog;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogChanges;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
import com.iqser.red.service.redaction.v1.server.RedactionServiceSettings;
import com.iqser.red.service.redaction.v1.server.logger.Context;
import com.iqser.red.service.redaction.v1.server.model.KieWrapper;
import com.iqser.red.service.redaction.v1.server.model.component.Component;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
import com.iqser.red.service.redaction.v1.server.service.components.ComponentLogCreatorService;
import com.iqser.red.service.redaction.v1.server.service.drools.ComponentDroolsExecutionService;
import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
import io.micrometer.core.annotation.Timed;
import io.micrometer.observation.annotation.Observed;
import lombok.AccessLevel;
import lombok.RequiredArgsConstructor;
import lombok.experimental.FieldDefaults;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@Service
@RequiredArgsConstructor
@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
public class AnalysisFinalizationService {
RedactionStorageService redactionStorageService;
FunctionTimerValues redactmanagerAnalyzePagewiseValues;
ComponentLogCreatorService componentLogCreatorService;
RedactionServiceSettings redactionServiceSettings;
ComponentDroolsExecutionService componentDroolsExecutionService;
@Timed("redactmanager_finalizeAnalysis")
@Observed(name = "AnalysisFinalizationService", contextualName = "finalize-analysis")
public AnalyzeResult finalizeAnalysis(AnalyzeRequest analyzeRequest,
long startTime,
KieWrapper kieWrapperComponentRules,
EntityLogChanges entityLogChanges,
Document document,
int numberOfPages,
boolean isReanalysis,
Set<FileAttribute> addedFileAttributes,
Context context) {
EntityLog entityLog = entityLogChanges.getEntityLog();
// as workaround for duplicate key exceptions occurring due to simultaneous analyses and reanalyses save instead of insert is used
// also analysis numbers should be incremented in every follow-up request, so checking if the log exists is not needed
if (!redactionStorageService.entityLogExists(analyzeRequest.getDossierId(), analyzeRequest.getFileId())) {
redactionStorageService.saveEntityLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), entityLog);
} else {
redactionStorageService.updateEntityLogWithoutEntries(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), entityLog);
if (!entityLogChanges.getNewEntityLogEntries().isEmpty()) {
redactionStorageService.saveEntityLogEntries(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), entityLogChanges.getNewEntityLogEntries());
}
if (!entityLogChanges.getUpdatedEntityLogEntries().isEmpty()) {
redactionStorageService.updateEntityLogEntries(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), entityLogChanges.getUpdatedEntityLogEntries());
}
}
log.info("Created entity log for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
context.setRuleVersion(kieWrapperComponentRules.rulesVersion());
Optional<ComponentLog> componentLog = computeComponentsWhenRulesArePresent(analyzeRequest, kieWrapperComponentRules, document, addedFileAttributes, entityLog, context);
long duration = System.currentTimeMillis() - startTime;
redactmanagerAnalyzePagewiseValues.increase(numberOfPages, duration);
return AnalyzeResult.builder()
.dossierId(analyzeRequest.getDossierId())
.fileId(analyzeRequest.getFileId())
.duration(duration)
.numberOfPages(numberOfPages)
.hasUpdates(entityLogChanges.hasChanges())
.analysisVersion(redactionServiceSettings.getAnalysisVersion())
.analysisNumber(analyzeRequest.getAnalysisNumber())
.rulesVersion(entityLog.getRulesVersion())
.componentRulesVersion(kieWrapperComponentRules.rulesVersion())
.dateFormatsVersion(componentLog.map(ComponentLog::getDateFormatsVersion)
.orElse(-1L))
.dictionaryVersion(entityLog.getDictionaryVersion())
.legalBasisVersion(entityLog.getLegalBasisVersion())
.dossierDictionaryVersion(entityLog.getDossierDictionaryVersion())
.wasReanalyzed(isReanalysis)
.manualRedactions(analyzeRequest.getManualRedactions())
.addedFileAttributes(addedFileAttributes)
.usedComponentMappings(analyzeRequest.getComponentMappings())
.build();
}
private Optional<ComponentLog> computeComponentsWhenRulesArePresent(AnalyzeRequest analyzeRequest,
KieWrapper kieWrapperComponentRules,
Document document,
Set<FileAttribute> addedFileAttributes,
EntityLog entityLog,
Context context) {
if (!kieWrapperComponentRules.isPresent()) {
return Optional.empty();
}
// We need the latest EntityLog entries for components rules execution
entityLog.setEntityLogEntry(redactionStorageService.getEntityLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId()).getEntityLogEntry());
List<Component> components = componentDroolsExecutionService.executeRules(kieWrapperComponentRules.container(),
entityLog,
document,
addedFileAttributes,
analyzeRequest.getComponentMappings(),
context);
log.info("Finished component rule execution for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
ComponentLog componentLog = componentLogCreatorService.buildComponentLog(analyzeRequest.getAnalysisNumber(),
components,
kieWrapperComponentRules.rulesVersion(),
analyzeRequest.getComponentDefinitions(),
context.getDateFormatsVersion());
redactionStorageService.saveComponentLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), componentLog);
log.info("Stored component log for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
return Optional.of(componentLog);
}
}

18
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalysisPreparationService.java
View File

@ -40,6 +40,8 @@ import com.iqser.red.service.redaction.v1.server.service.drools.KieContainerCrea
import com.iqser.red.service.redaction.v1.server.storage.ObservedStorageService; import com.iqser.red.service.redaction.v1.server.storage.ObservedStorageService;
import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService; import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
import io.micrometer.core.annotation.Timed;
import io.micrometer.observation.annotation.Observed;
import lombok.AccessLevel; import lombok.AccessLevel;
import lombok.SneakyThrows; import lombok.SneakyThrows;
import lombok.experimental.FieldDefaults; import lombok.experimental.FieldDefaults;
@ -84,6 +86,8 @@ public class AnalysisPreparationService {
@SneakyThrows @SneakyThrows
@Timed("redactmanager_getAnalysisData")
@Observed(name = "AnalysisPreparationService", contextualName = "get-analysis-data")
public AnalysisData getAnalysisData(AnalyzeRequest analyzeRequest) { public AnalysisData getAnalysisData(AnalyzeRequest analyzeRequest) {
CompletableFuture<KieWrapper> kieWrapperComponentRulesFuture = CompletableFuture.supplyAsync(() -> getKieWrapper(analyzeRequest, RuleFileType.COMPONENT), taskExecutor); CompletableFuture<KieWrapper> kieWrapperComponentRulesFuture = CompletableFuture.supplyAsync(() -> getKieWrapper(analyzeRequest, RuleFileType.COMPONENT), taskExecutor);
@ -121,6 +125,8 @@ public class AnalysisPreparationService {
@SneakyThrows @SneakyThrows
@Timed("redactmanager_getReanalysisSetupData")
@Observed(name = "AnalysisPreparationService", contextualName = "get-reanalysis-setup-data")
public ReanalysisSetupData getReanalysisSetupData(AnalyzeRequest analyzeRequest) { public ReanalysisSetupData getReanalysisSetupData(AnalyzeRequest analyzeRequest) {
return getReanalysisSetupData(analyzeRequest, () -> getEntityLogWithoutEntries(analyzeRequest)); return getReanalysisSetupData(analyzeRequest, () -> getEntityLogWithoutEntries(analyzeRequest));
@ -147,6 +153,8 @@ public class AnalysisPreparationService {
@SneakyThrows @SneakyThrows
@Timed("redactmanager_getReanalysisInitialProcessingData")
@Observed(name = "AnalysisPreparationService", contextualName = "get-reanalysis-initial-processing-data")
public ReanalysisInitialProcessingData getReanalysisInitialProcessingData(AnalyzeRequest analyzeRequest, ReanalysisSetupData reanalysisSetupData) { public ReanalysisInitialProcessingData getReanalysisInitialProcessingData(AnalyzeRequest analyzeRequest, ReanalysisSetupData reanalysisSetupData) {
CompletableFuture<ImportedRedactions> importedRedactionsFuture = CompletableFuture.supplyAsync(() -> getImportedRedactions(analyzeRequest), taskExecutor); CompletableFuture<ImportedRedactions> importedRedactionsFuture = CompletableFuture.supplyAsync(() -> getImportedRedactions(analyzeRequest), taskExecutor);
@ -173,6 +181,8 @@ public class AnalysisPreparationService {
@SneakyThrows @SneakyThrows
@Timed("redactmanager_getReanalysisFinalProcessingData")
@Observed(name = "AnalysisPreparationService", contextualName = "get-reanalysis-final-processing-data")
public ReanalysisFinalProcessingData getReanalysisFinalProcessingData(AnalyzeRequest analyzeRequest, public ReanalysisFinalProcessingData getReanalysisFinalProcessingData(AnalyzeRequest analyzeRequest,
ReanalysisSetupData reanalysisSetupData, ReanalysisSetupData reanalysisSetupData,
ReanalysisInitialProcessingData reanalysisInitialProcessingData) { ReanalysisInitialProcessingData reanalysisInitialProcessingData) {
@ -209,7 +219,7 @@ public class AnalysisPreparationService {
} }
public Document getDocument(AnalyzeRequest analyzeRequest) { private Document getDocument(AnalyzeRequest analyzeRequest) {
Document document = DocumentGraphMapper.toDocumentGraph(observedStorageService.getDocumentData(analyzeRequest.getDossierId(), analyzeRequest.getFileId())); Document document = DocumentGraphMapper.toDocumentGraph(observedStorageService.getDocumentData(analyzeRequest.getDossierId(), analyzeRequest.getFileId()));
log.info("Loaded Document Graph for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); log.info("Loaded Document Graph for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
@ -217,7 +227,7 @@ public class AnalysisPreparationService {
} }
public ImportedRedactions getImportedRedactions(AnalyzeRequest analyzeRequest) { private ImportedRedactions getImportedRedactions(AnalyzeRequest analyzeRequest) {
ImportedRedactions importedRedactions = redactionStorageService.getImportedRedactions(analyzeRequest.getDossierId(), analyzeRequest.getFileId()); ImportedRedactions importedRedactions = redactionStorageService.getImportedRedactions(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
log.info("Loaded Imported Redactions for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); log.info("Loaded Imported Redactions for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
@ -225,7 +235,7 @@ public class AnalysisPreparationService {
} }
public NerEntities getNerEntities(AnalyzeRequest analyzeRequest, Document document) { private NerEntities getNerEntities(AnalyzeRequest analyzeRequest, Document document) {
NerEntities nerEntities = getEntityRecognitionEntities(analyzeRequest, document); NerEntities nerEntities = getEntityRecognitionEntities(analyzeRequest, document);
log.info("Loaded Ner Entities for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); log.info("Loaded Ner Entities for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
@ -233,7 +243,7 @@ public class AnalysisPreparationService {
} }
public NerEntities getNerEntitiesFiltered(AnalyzeRequest analyzeRequest, Document document, Set<Integer> sectionsToReanalyseIds) { private NerEntities getNerEntitiesFiltered(AnalyzeRequest analyzeRequest, Document document, Set<Integer> sectionsToReanalyseIds) {
NerEntities nerEntities = getEntityRecognitionEntitiesFilteredBySectionIds(analyzeRequest, document, sectionsToReanalyseIds); NerEntities nerEntities = getEntityRecognitionEntitiesFilteredBySectionIds(analyzeRequest, document, sectionsToReanalyseIds);
log.info("Loaded Ner Entities for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); log.info("Loaded Ner Entities for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());

117
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalyzeService.java
View File

@ -6,7 +6,6 @@ import static com.iqser.red.service.redaction.v1.server.service.AnalysisPreparat
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Collection; import java.util.Collection;
import java.util.Collections;
import java.util.HashSet; import java.util.HashSet;
import java.util.List; import java.util.List;
import java.util.Optional; import java.util.Optional;
@ -26,8 +25,6 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogChanges; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogChanges;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedLegalBases; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedLegalBases;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedactions;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.mapper.ImportedLegalBasisMapper; import com.iqser.red.service.persistence.service.v1.api.shared.model.mapper.ImportedLegalBasisMapper;
import com.iqser.red.service.persistence.service.v1.api.shared.mongo.service.EntityLogMongoService; import com.iqser.red.service.persistence.service.v1.api.shared.mongo.service.EntityLogMongoService;
import com.iqser.red.service.redaction.v1.server.RedactionServiceSettings; import com.iqser.red.service.redaction.v1.server.RedactionServiceSettings;
@ -57,17 +54,14 @@ import lombok.extern.slf4j.Slf4j;
public class AnalyzeService { public class AnalyzeService {
EntityDroolsExecutionService entityDroolsExecutionService; EntityDroolsExecutionService entityDroolsExecutionService;
ComponentDroolsExecutionService componentDroolsExecutionService;
DictionarySearchService dictionarySearchService; DictionarySearchService dictionarySearchService;
EntityLogCreatorService entityLogCreatorService; EntityLogCreatorService entityLogCreatorService;
EntityLogMongoService entityLogMongoService; EntityLogMongoService entityLogMongoService;
ComponentLogCreatorService componentLogCreatorService;
RedactionStorageService redactionStorageService; RedactionStorageService redactionStorageService;
RedactionServiceSettings redactionServiceSettings;
NotFoundImportedEntitiesService notFoundImportedEntitiesService; NotFoundImportedEntitiesService notFoundImportedEntitiesService;
FunctionTimerValues redactmanagerAnalyzePagewiseValues;
ImportedLegalBasisMapper importedLegalBasisMapper = ImportedLegalBasisMapper.INSTANCE;
AnalysisPreparationService analysisPreparationService; AnalysisPreparationService analysisPreparationService;
AnalysisFinalizationService analysisFinalizationService;
ImportedLegalBasisMapper importedLegalBasisMapper = ImportedLegalBasisMapper.INSTANCE;
@Timed("redactmanager_reanalyze") @Timed("redactmanager_reanalyze")
@ -100,7 +94,7 @@ public class AnalyzeService {
new ArrayList<>(), new ArrayList<>(),
new ArrayList<>()); new ArrayList<>());
return finalizeAnalysis(analyzeRequest, return analysisFinalizationService.finalizeAnalysis(analyzeRequest,
startTime, startTime,
initialProcessingData.kieWrapperComponentRules(), initialProcessingData.kieWrapperComponentRules(),
entityLogChanges, entityLogChanges,
@ -141,7 +135,7 @@ public class AnalyzeService {
notFoundImportedEntitiesService.processEntityLog(entityLogChanges.getEntityLog(), analyzeRequest, finalProcessingData.notFoundImportedEntries()); notFoundImportedEntitiesService.processEntityLog(entityLogChanges.getEntityLog(), analyzeRequest, finalProcessingData.notFoundImportedEntries());
return finalizeAnalysis(analyzeRequest, return analysisFinalizationService.finalizeAnalysis(analyzeRequest,
startTime, startTime,
initialProcessingData.kieWrapperComponentRules(), initialProcessingData.kieWrapperComponentRules(),
entityLogChanges, entityLogChanges,
@ -184,7 +178,7 @@ public class AnalyzeService {
.entityLog(entityLog.get()) .entityLog(entityLog.get())
.build(); .build();
return finalizeAnalysis(analyzeRequest, return analysisFinalizationService.finalizeAnalysis(analyzeRequest,
startTime, startTime,
kieWrapperComponentRules, kieWrapperComponentRules,
entityLogChanges, entityLogChanges,
@ -237,7 +231,7 @@ public class AnalyzeService {
notFoundImportedEntitiesService.processEntityLog(entityLogChanges.getEntityLog(), analyzeRequest, analysisData.notFoundImportedEntries()); notFoundImportedEntitiesService.processEntityLog(entityLogChanges.getEntityLog(), analyzeRequest, analysisData.notFoundImportedEntries());
return finalizeAnalysis(analyzeRequest, return analysisFinalizationService.finalizeAnalysis(analyzeRequest,
startTime, startTime,
analysisData.kieWrapperComponentRules(), analysisData.kieWrapperComponentRules(),
entityLogChanges, entityLogChanges,
@ -284,7 +278,7 @@ public class AnalyzeService {
notFoundImportedEntitiesService.processEntityLog(entityLogChanges.getEntityLog(), analyzeRequest, analysisData.notFoundImportedEntries()); notFoundImportedEntitiesService.processEntityLog(entityLogChanges.getEntityLog(), analyzeRequest, analysisData.notFoundImportedEntries());
return finalizeAnalysis(analyzeRequest, return analysisFinalizationService.finalizeAnalysis(analyzeRequest,
startTime, startTime,
analysisData.kieWrapperComponentRules(), analysisData.kieWrapperComponentRules(),
entityLogChanges, entityLogChanges,
@ -295,101 +289,4 @@ public class AnalyzeService {
context); context);
} }
private AnalyzeResult finalizeAnalysis(AnalyzeRequest analyzeRequest,
long startTime,
KieWrapper kieWrapperComponentRules,
EntityLogChanges entityLogChanges,
Document document,
int numberOfPages,
boolean isReanalysis,
Set<FileAttribute> addedFileAttributes,
Context context) {
EntityLog entityLog = entityLogChanges.getEntityLog();
// as workaround for duplicate key exceptions occurring due to simultaneous analyses and reanalyses save instead of insert is used
// also analysis numbers should be incremented in every follow-up request, so checking if the log exists is not needed
if (!redactionStorageService.entityLogExists(analyzeRequest.getDossierId(), analyzeRequest.getFileId())) {
redactionStorageService.saveEntityLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), entityLog);
} else {
redactionStorageService.updateEntityLogWithoutEntries(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), entityLog);
if (!entityLogChanges.getNewEntityLogEntries().isEmpty()) {
redactionStorageService.saveEntityLogEntries(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), entityLogChanges.getNewEntityLogEntries());
}
if (!entityLogChanges.getUpdatedEntityLogEntries().isEmpty()) {
redactionStorageService.updateEntityLogEntries(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), entityLogChanges.getUpdatedEntityLogEntries());
}
}
log.info("Created entity log for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
context.setRuleVersion(kieWrapperComponentRules.rulesVersion());
Optional<ComponentLog> componentLog = computeComponentsWhenRulesArePresent(analyzeRequest, kieWrapperComponentRules, document, addedFileAttributes, entityLog, context);
long duration = System.currentTimeMillis() - startTime;
redactmanagerAnalyzePagewiseValues.increase(numberOfPages, duration);
return AnalyzeResult.builder()
.dossierId(analyzeRequest.getDossierId())
.fileId(analyzeRequest.getFileId())
.duration(duration)
.numberOfPages(numberOfPages)
.hasUpdates(entityLogChanges.hasChanges())
.analysisVersion(redactionServiceSettings.getAnalysisVersion())
.analysisNumber(analyzeRequest.getAnalysisNumber())
.rulesVersion(entityLog.getRulesVersion())
.componentRulesVersion(kieWrapperComponentRules.rulesVersion())
.dateFormatsVersion(componentLog.map(ComponentLog::getDateFormatsVersion)
.orElse(-1L))
.dictionaryVersion(entityLog.getDictionaryVersion())
.legalBasisVersion(entityLog.getLegalBasisVersion())
.dossierDictionaryVersion(entityLog.getDossierDictionaryVersion())
.wasReanalyzed(isReanalysis)
.manualRedactions(analyzeRequest.getManualRedactions())
.addedFileAttributes(addedFileAttributes)
.usedComponentMappings(analyzeRequest.getComponentMappings())
.build();
}
private Optional<ComponentLog> computeComponentsWhenRulesArePresent(AnalyzeRequest analyzeRequest,
KieWrapper kieWrapperComponentRules,
Document document,
Set<FileAttribute> addedFileAttributes,
EntityLog entityLog,
Context context) {
if (!kieWrapperComponentRules.isPresent()) {
return Optional.empty();
}
// We need the latest EntityLog entries for components rules execution
entityLog.setEntityLogEntry(redactionStorageService.getEntityLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId()).getEntityLogEntry());
List<Component> components = componentDroolsExecutionService.executeRules(kieWrapperComponentRules.container(),
entityLog,
document,
addedFileAttributes,
analyzeRequest.getComponentMappings(),
context);
log.info("Finished component rule execution for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
ComponentLog componentLog = componentLogCreatorService.buildComponentLog(analyzeRequest.getAnalysisNumber(),
components,
kieWrapperComponentRules.rulesVersion(),
analyzeRequest.getComponentDefinitions(),
context.getDateFormatsVersion());
redactionStorageService.saveComponentLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), componentLog);
log.info("Stored component log for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
return Optional.of(componentLog);
}
} }

19
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/DictionarySearchService.java
View File

@ -39,14 +39,27 @@ public class DictionarySearchService {
@Observed(name = "DictionarySearchService", contextualName = "add-dictionary-entries") @Observed(name = "DictionarySearchService", contextualName = "add-dictionary-entries")
public void addDictionaryEntities(Dictionary dictionary, SemanticNode node) { public void addDictionaryEntities(Dictionary dictionary, SemanticNode node) {
for (DictionaryModel model : dictionary.getDictionaryModels()) { dictionary.getDictionaryModels()
bySearchImplementationAsDictionary(model.getEntriesSearch(), model.getType(), model.isHint() ? EntityType.HINT : EntityType.ENTITY, node, model.isDossierDictionary()); .stream()
.parallel()
.forEach(model -> {
synchronized (node) {
bySearchImplementationAsDictionary(model.getEntriesSearch(),
model.getType(),
model.isHint() ? EntityType.HINT : EntityType.ENTITY,
node,
model.isDossierDictionary());
bySearchImplementationAsDictionary(model.getFalsePositiveSearch(), model.getType(), EntityType.FALSE_POSITIVE, node, model.isDossierDictionary()); bySearchImplementationAsDictionary(model.getFalsePositiveSearch(), model.getType(), EntityType.FALSE_POSITIVE, node, model.isDossierDictionary());
bySearchImplementationAsDictionary(model.getFalseRecommendationsSearch(), model.getType(), EntityType.FALSE_RECOMMENDATION, node, model.isDossierDictionary()); bySearchImplementationAsDictionary(model.getFalseRecommendationsSearch(),
model.getType(),
EntityType.FALSE_RECOMMENDATION,
node,
model.isDossierDictionary());
if (model.isDossierDictionary()) { if (model.isDossierDictionary()) {
bySearchImplementationAsDictionary(model.getDeletionEntriesSearch(), model.getType(), EntityType.DICTIONARY_REMOVAL, node, model.isDossierDictionary()); bySearchImplementationAsDictionary(model.getDeletionEntriesSearch(), model.getType(), EntityType.DICTIONARY_REMOVAL, node, model.isDossierDictionary());
} }
} }
});
} }

3
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/DocumentSearchService.java
View File

@ -1,5 +1,6 @@
package com.iqser.red.service.redaction.v1.server.service; package com.iqser.red.service.redaction.v1.server.service;
import java.util.Collections;
import java.util.List; import java.util.List;
import java.util.Set; import java.util.Set;
@ -53,7 +54,7 @@ public class DocumentSearchService {
} }
List<FoundTerm> foundTerms = possibleEntities.stream() List<FoundTerm> foundTerms = possibleEntities.stream()
.flatMap(entity -> entityLogCreatorService.toEntityLogEntries(entity, 0, "", "") .flatMap(entity -> entityLogCreatorService.toEntityLogEntries(entity, 0, Collections.emptyMap())
.stream()) .stream())
.map(entityLogEntry -> new FoundTerm(entityLogEntry.getPositions(), entityLogEntry.getValue())) .map(entityLogEntry -> new FoundTerm(entityLogEntry.getPositions(), entityLogEntry.getValue()))
.toList(); .toList();

2
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityChangeLogService.java
View File

@ -22,6 +22,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
import com.iqser.red.service.redaction.v1.server.utils.EntityLogEntryDiffChecker; import com.iqser.red.service.redaction.v1.server.utils.EntityLogEntryDiffChecker;
import io.micrometer.core.annotation.Timed; import io.micrometer.core.annotation.Timed;
import io.micrometer.observation.annotation.Observed;
import lombok.AccessLevel; import lombok.AccessLevel;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
import lombok.experimental.FieldDefaults; import lombok.experimental.FieldDefaults;
@ -34,6 +35,7 @@ import lombok.extern.slf4j.Slf4j;
public class EntityChangeLogService { public class EntityChangeLogService {
@Timed("redactmanager_computeChanges") @Timed("redactmanager_computeChanges")
@Observed(name = "EntityChangeLogService", contextualName = "compute-changes")
public EntryChanges computeChanges(List<EntityLogEntry> previousEntityLogEntries, List<EntityLogEntry> newEntityLogEntries, int analysisNumber) { public EntryChanges computeChanges(List<EntityLogEntry> previousEntityLogEntries, List<EntityLogEntry> newEntityLogEntries, int analysisNumber) {
var now = OffsetDateTime.now(); var now = OffsetDateTime.now();

86
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java
View File

@ -2,12 +2,16 @@ package com.iqser.red.service.redaction.v1.server.service;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Collections; import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet; import java.util.HashSet;
import java.util.List; import java.util.List;
import java.util.Locale; import java.util.Locale;
import java.util.Map;
import java.util.Optional; import java.util.Optional;
import java.util.Set; import java.util.Set;
import java.util.concurrent.atomic.AtomicReference;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import java.util.stream.Stream;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
@ -43,6 +47,10 @@ import com.iqser.red.service.redaction.v1.server.service.EntityChangeLogService.
import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService; import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
import com.iqser.red.service.redaction.v1.server.utils.ManualChangesUtils; import com.iqser.red.service.redaction.v1.server.utils.ManualChangesUtils;
import io.micrometer.core.annotation.Timed;
import io.micrometer.observation.Observation;
import io.micrometer.observation.ObservationRegistry;
import io.micrometer.observation.annotation.Observed;
import lombok.AccessLevel; import lombok.AccessLevel;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
import lombok.experimental.FieldDefaults; import lombok.experimental.FieldDefaults;
@ -60,6 +68,7 @@ public class EntityLogCreatorService {
EntityChangeLogService entityChangeLogService; EntityChangeLogService entityChangeLogService;
RedactionStorageService redactionStorageService; RedactionStorageService redactionStorageService;
EntityLogMongoService entityLogMongoService; EntityLogMongoService entityLogMongoService;
ObservationRegistry observationRegistry;
private static boolean notFalsePositiveOrFalseRecommendationOrRemoval(TextEntity textEntity) { private static boolean notFalsePositiveOrFalseRecommendationOrRemoval(TextEntity textEntity) {
@ -70,6 +79,8 @@ public class EntityLogCreatorService {
} }
@Timed("redactmanager_createInitialEntityLog")
@Observed(name = "EntityLogCreatorService", contextualName = "create-initial-entity-log")
public EntityLogChanges createInitialEntityLog(AnalyzeRequest analyzeRequest, public EntityLogChanges createInitialEntityLog(AnalyzeRequest analyzeRequest,
Document document, Document document,
List<PrecursorEntity> notFoundEntities, List<PrecursorEntity> notFoundEntities,
@ -109,6 +120,8 @@ public class EntityLogCreatorService {
} }
@Timed("redactmanager_updateVersionsAndReturnChanges")
@Observed(name = "EntityLogCreatorService", contextualName = "update-versions-and-return-changes")
public EntityLogChanges updateVersionsAndReturnChanges(EntityLog entityLog, public EntityLogChanges updateVersionsAndReturnChanges(EntityLog entityLog,
DictionaryVersion dictionaryVersion, DictionaryVersion dictionaryVersion,
AnalyzeRequest analyzeRequest, AnalyzeRequest analyzeRequest,
@ -126,6 +139,8 @@ public class EntityLogCreatorService {
} }
@Timed("redactmanager_updatePreviousEntityLog")
@Observed(name = "EntityLogCreatorService", contextualName = "update-previous-entity-log")
public EntityLogChanges updatePreviousEntityLog(AnalyzeRequest analyzeRequest, public EntityLogChanges updatePreviousEntityLog(AnalyzeRequest analyzeRequest,
Document document, Document document,
EntityLog entityLogWithoutEntries, EntityLog entityLogWithoutEntries,
@ -133,16 +148,23 @@ public class EntityLogCreatorService {
Set<Integer> sectionsToReanalyseIds, Set<Integer> sectionsToReanalyseIds,
DictionaryVersion dictionaryVersion) { DictionaryVersion dictionaryVersion) {
List<EntityLogEntry> newEntityLogEntries = createEntityLogEntries(document, analyzeRequest, notFoundEntries, analyzeRequest.getAnalysisNumber()).stream() AtomicReference<List<EntityLogEntry>> newEntityLogEntriesReference = new AtomicReference<>();
.filter(entry -> entry.getContainingNodeId().isEmpty() || sectionsToReanalyseIds.contains(entry.getContainingNodeId()
Observation.createNotStarted("EntityLogCreatorService", observationRegistry)
.contextualName("create-new-entity-log-entries")
.observe(() -> newEntityLogEntriesReference.set(createEntityLogEntries(document, analyzeRequest, notFoundEntries, analyzeRequest.getAnalysisNumber()).stream()
.filter(entry -> entry.getContainingNodeId().isEmpty()
|| sectionsToReanalyseIds.contains(entry.getContainingNodeId()
.get(0))) .get(0)))
.collect(Collectors.toList()); .collect(Collectors.toList())));
List<EntityLogEntry> previousEntriesFromReAnalyzedSections = redactionStorageService.findEntriesContainedBySectionsOrNotContained(analyzeRequest.getDossierId(), List<EntityLogEntry> previousEntriesFromReAnalyzedSections = redactionStorageService.findEntriesContainedBySectionsOrNotContained(analyzeRequest.getDossierId(),
analyzeRequest.getFileId(), analyzeRequest.getFileId(),
sectionsToReanalyseIds); sectionsToReanalyseIds);
EntryChanges entryChanges = entityChangeLogService.computeChanges(previousEntriesFromReAnalyzedSections, newEntityLogEntries, analyzeRequest.getAnalysisNumber()); EntryChanges entryChanges = entityChangeLogService.computeChanges(previousEntriesFromReAnalyzedSections,
newEntityLogEntriesReference.get(),
analyzeRequest.getAnalysisNumber());
return updateVersionsAndReturnChanges(entityLogWithoutEntries, dictionaryVersion, analyzeRequest, entryChanges.inserted(), entryChanges.updated()); return updateVersionsAndReturnChanges(entityLogWithoutEntries, dictionaryVersion, analyzeRequest, entryChanges.inserted(), entryChanges.updated());
} }
@ -154,26 +176,47 @@ public class EntityLogCreatorService {
List<EntityLogEntry> entries = new ArrayList<>(); List<EntityLogEntry> entries = new ArrayList<>();
document.getEntities() List<TextEntity> textEntities = document.getEntities()
.stream() .stream()
.filter(entity -> !entity.getValue().isEmpty()) .filter(entity -> !entity.getValue().isEmpty())
.filter(EntityLogCreatorService::notFalsePositiveOrFalseRecommendationOrRemoval) .filter(EntityLogCreatorService::notFalsePositiveOrFalseRecommendationOrRemoval)
.filter(entity -> !entity.removed()) .filter(entity -> !entity.removed())
.forEach(entityNode -> entries.addAll(toEntityLogEntries(entityNode, analysisNumber, analyzeRequest.getDossierId(), analyzeRequest.getFileId()))); .toList();
document.streamAllImages() List<Image> images = document.streamAllImages()
.filter(entity -> !entity.removed()) .filter(entity -> !entity.removed())
.forEach(imageNode -> entries.add(createEntityLogEntry(imageNode, dossierTemplateId, analysisNumber, analyzeRequest.getDossierId(), analyzeRequest.getFileId()))); .toList();
notFoundPrecursorEntries.stream() List<PrecursorEntity> notFoundPrecursorEntities = notFoundPrecursorEntries.stream()
.filter(entity -> !entity.removed()) .filter(entity -> !entity.removed())
.forEach(precursorEntity -> entries.add(createEntityLogEntry(precursorEntity, analysisNumber, analyzeRequest.getDossierId(), analyzeRequest.getFileId()))); .toList();
List<String> allIds = new ArrayList<>();
allIds.addAll(textEntities.stream()
.flatMap(entity -> entity.getPositionsOnPagePerPage()
.stream()
.map(PositionOnPage::getId))
.toList());
allIds.addAll(images.stream()
.map(Image::getId)
.toList());
allIds.addAll(notFoundPrecursorEntities.stream()
.map(PrecursorEntity::getId)
.toList());
Map<String, List<ManualChange>> manualChangesMap = getManualChangesByEntityLogIds(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), allIds);
textEntities.forEach(entityNode -> entries.addAll(toEntityLogEntries(entityNode, analysisNumber, manualChangesMap)));
images.forEach(imageNode -> entries.add(createEntityLogEntry(imageNode, dossierTemplateId, analysisNumber, manualChangesMap.getOrDefault(imageNode.getId(), new ArrayList<>()))));
notFoundPrecursorEntities.forEach(precursorEntity -> entries.add(createEntityLogEntry(precursorEntity, analysisNumber, manualChangesMap.getOrDefault(precursorEntity.getId(), new ArrayList<>()))));
return entries; return entries;
} }
public List<EntityLogEntry> toEntityLogEntries(TextEntity textEntity, int analysisNumber, String dossierId, String fileId) { public List<EntityLogEntry> toEntityLogEntries(TextEntity textEntity, int analysisNumber, Map<String, List<ManualChange>> existingManualChangesMap) {
List<EntityLogEntry> entityLogEntries = new ArrayList<>(); List<EntityLogEntry> entityLogEntries = new ArrayList<>();
@ -185,7 +228,7 @@ public class EntityLogCreatorService {
.map(rectangle2D -> new Position(rectangle2D, positionOnPage.getPage().getNumber())) .map(rectangle2D -> new Position(rectangle2D, positionOnPage.getPage().getNumber()))
.toList(); .toList();
EntityLogEntry entityLogEntry = createEntityLogEntry(textEntity, analysisNumber, positionOnPage.getId(), dossierId, fileId); EntityLogEntry entityLogEntry = createEntityLogEntry(textEntity, analysisNumber, existingManualChangesMap.getOrDefault(positionOnPage.getId(), new ArrayList<>()));
// set the ID from the positions, since it might contain a "-" with the page number if the entity is split across multiple pages // set the ID from the positions, since it might contain a "-" with the page number if the entity is split across multiple pages
entityLogEntry.setId(positionOnPage.getId()); entityLogEntry.setId(positionOnPage.getId());
@ -197,12 +240,11 @@ public class EntityLogCreatorService {
} }
private EntityLogEntry createEntityLogEntry(Image image, String dossierTemplateId, int analysisNumber, String dossierId, String fileId) { private EntityLogEntry createEntityLogEntry(Image image, String dossierTemplateId, int analysisNumber, List<ManualChange> existingManualChanges) {
String imageType = image.getImageType().equals(ImageType.OTHER) ? "image" : image.getImageType().toString().toLowerCase(Locale.ENGLISH); String imageType = image.getImageType().equals(ImageType.OTHER) ? "image" : image.getImageType().toString().toLowerCase(Locale.ENGLISH);
boolean isHint = dictionaryService.isHint(imageType, dossierTemplateId); boolean isHint = dictionaryService.isHint(imageType, dossierTemplateId);
List<ManualChange> existingManualChanges = getManualChangesByEntityLogId(dossierId, fileId, image.getId());
List<ManualChange> allManualChanges = ManualChangeFactory.toLocalManualChangeList(image.getManualOverwrite().getManualChangeLog(), true, analysisNumber); List<ManualChange> allManualChanges = ManualChangeFactory.toLocalManualChangeList(image.getManualOverwrite().getManualChangeLog(), true, analysisNumber);
return EntityLogEntry.builder() return EntityLogEntry.builder()
@ -230,12 +272,11 @@ public class EntityLogCreatorService {
} }
private EntityLogEntry createEntityLogEntry(PrecursorEntity precursorEntity, int analysisNumber, String dossierId, String fileId) { private EntityLogEntry createEntityLogEntry(PrecursorEntity precursorEntity, int analysisNumber, List<ManualChange> existingManualChanges) {
String type = precursorEntity.getManualOverwrite().getType() String type = precursorEntity.getManualOverwrite().getType()
.orElse(precursorEntity.getType()); .orElse(precursorEntity.getType());
List<ManualChange> existingManualChanges = getManualChangesByEntityLogId(dossierId, fileId, precursorEntity.getId());
List<ManualChange> allManualChanges = ManualChangeFactory.toLocalManualChangeList(precursorEntity.getManualOverwrite().getManualChangeLog(), true, analysisNumber); List<ManualChange> allManualChanges = ManualChangeFactory.toLocalManualChangeList(precursorEntity.getManualOverwrite().getManualChangeLog(), true, analysisNumber);
return EntityLogEntry.builder() return EntityLogEntry.builder()
@ -273,7 +314,7 @@ public class EntityLogCreatorService {
} }
private EntityLogEntry createEntityLogEntry(TextEntity entity, int analysisNumber, String id, String dossierId, String fileId) { private EntityLogEntry createEntityLogEntry(TextEntity entity, int analysisNumber, List<ManualChange> existingManualChanges) {
Set<String> referenceIds = new HashSet<>(); Set<String> referenceIds = new HashSet<>();
entity.references() entity.references()
@ -284,7 +325,6 @@ public class EntityLogCreatorService {
EntryType entryType = buildEntryType(entity); EntryType entryType = buildEntryType(entity);
List<ManualChange> existingManualChanges = getManualChangesByEntityLogId(dossierId, fileId, id);
List<ManualChange> allManualChanges = ManualChangeFactory.toLocalManualChangeList(entity.getManualOverwrite().getManualChangeLog(), true, analysisNumber); List<ManualChange> allManualChanges = ManualChangeFactory.toLocalManualChangeList(entity.getManualOverwrite().getManualChangeLog(), true, analysisNumber);
return EntityLogEntry.builder() return EntityLogEntry.builder()
@ -415,16 +455,16 @@ public class EntityLogCreatorService {
} }
public List<ManualChange> getManualChangesByEntityLogId(String dossierId, String fileId, String id) { public Map<String, List<ManualChange>> getManualChangesByEntityLogIds(String dossierId, String fileId, List<String> ids) {
List<ManualChange> manualChanges = new ArrayList<>(); Map<String, List<ManualChange>> manualChangesMap = new HashMap<>();
List<EntityLogEntry> entityLogEntries = entityLogMongoService.findEntityLogEntriesByIds(dossierId, fileId, List.of(id)); List<EntityLogEntry> entityLogEntries = entityLogMongoService.findEntityLogEntriesByIds(dossierId, fileId, ids);
for (EntityLogEntry entry : entityLogEntries) { for (EntityLogEntry entry : entityLogEntries) {
manualChanges.addAll(entry.getManualChanges()); manualChangesMap.put(entry.getId(), entry.getManualChanges());
} }
return manualChanges; return manualChangesMap;
} }
} }

12
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/NotFoundImportedEntitiesService.java
View File

@ -9,8 +9,9 @@ import java.util.Map;
import java.util.Set; import java.util.Set;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import org.springframework.stereotype.Service;
import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest; import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Change;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ChangeType; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ChangeType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
@ -20,11 +21,9 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ChangeFactory; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ChangeFactory;
import com.iqser.red.service.redaction.v1.server.model.PrecursorEntity; import com.iqser.red.service.redaction.v1.server.model.PrecursorEntity;
import com.iqser.red.service.redaction.v1.server.model.RectangleWithPage; import com.iqser.red.service.redaction.v1.server.model.RectangleWithPage;
import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
import org.springframework.stereotype.Service;
import io.micrometer.core.annotation.Timed; import io.micrometer.core.annotation.Timed;
import io.micrometer.observation.annotation.Observed;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
@ -37,6 +36,7 @@ public class NotFoundImportedEntitiesService {
@Timed("redactmanager_processEntityLog") @Timed("redactmanager_processEntityLog")
@Observed(name = "NotFoundImportedEntitiesService", contextualName = "process-entity-log")
public void processEntityLog(EntityLog entityLog, AnalyzeRequest analyzeRequest, List<PrecursorEntity> notFoundEntities) { public void processEntityLog(EntityLog entityLog, AnalyzeRequest analyzeRequest, List<PrecursorEntity> notFoundEntities) {
// recreate imported redactions with manual changes // recreate imported redactions with manual changes
if (notFoundEntities == null || notFoundEntities.isEmpty()) { if (notFoundEntities == null || notFoundEntities.isEmpty()) {
@ -65,7 +65,7 @@ public class NotFoundImportedEntitiesService {
} }
public Map<Integer, List<PrecursorEntity>> mapImportedRedactionsOnPage(List<PrecursorEntity> importedEntities) { private Map<Integer, List<PrecursorEntity>> mapImportedRedactionsOnPage(List<PrecursorEntity> importedEntities) {
Map<Integer, List<PrecursorEntity>> importedRedactionsMap = new HashMap<>(); Map<Integer, List<PrecursorEntity>> importedRedactionsMap = new HashMap<>();
Set<Integer> pageNumbers = importedEntities.stream() Set<Integer> pageNumbers = importedEntities.stream()
@ -82,7 +82,7 @@ public class NotFoundImportedEntitiesService {
} }
public void addIntersections(EntityLogEntry entityLogEntry, Map<Integer, List<PrecursorEntity>> importedEntitiesMap, int analysisNumber) { private void addIntersections(EntityLogEntry entityLogEntry, Map<Integer, List<PrecursorEntity>> importedEntitiesMap, int analysisNumber) {
for (Position rectangle : entityLogEntry.getPositions()) { for (Position rectangle : entityLogEntry.getPositions()) {
if (importedEntitiesMap.containsKey(rectangle.getPageNumber())) { if (importedEntitiesMap.containsKey(rectangle.getPageNumber())) {

74
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/DocumentGraphMapper.java
View File

@ -1,8 +1,11 @@
package com.iqser.red.service.redaction.v1.server.service.document; package com.iqser.red.service.redaction.v1.server.service.document;
import static com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPageProto.DocumentPage;
import static com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPositionDataProto.AllDocumentPositionData;
import static com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentTextDataProto.AllDocumentTextData;
import static com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.EntryDataProto.EntryData;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashSet; import java.util.HashSet;
import java.util.LinkedList; import java.util.LinkedList;
import java.util.List; import java.util.List;
@ -26,10 +29,6 @@ import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell;
import com.iqser.red.service.redaction.v1.server.model.document.textblock.AtomicTextBlock; import com.iqser.red.service.redaction.v1.server.model.document.textblock.AtomicTextBlock;
import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock; import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlockCollector; import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlockCollector;
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPage;
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPositionData;
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentStructure;
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentTextData;
import lombok.experimental.UtilityClass; import lombok.experimental.UtilityClass;
@ -42,27 +41,29 @@ public class DocumentGraphMapper {
DocumentTree documentTree = new DocumentTree(document); DocumentTree documentTree = new DocumentTree(document);
Context context = new Context(documentData, documentTree); Context context = new Context(documentData, documentTree);
context.pageData.addAll(Arrays.stream(documentData.getDocumentPages()) context.pageData.addAll(documentData.getDocumentPages().getDocumentPagesList()
.stream()
.map(DocumentGraphMapper::buildPage) .map(DocumentGraphMapper::buildPage)
.toList()); .toList());
context.documentTree.getRoot().getChildren().addAll(buildEntries(documentData.getDocumentStructure().getRoot().getChildren(), context)); context.documentTree.getRoot().getChildren().addAll(buildEntries(documentData.getDocumentStructure().getRoot().getChildrenList(), context));
document.setDocumentTree(context.documentTree); document.setDocumentTree(context.documentTree);
document.setPages(new HashSet<>(context.pageData)); document.setPages(new HashSet<>(context.pageData));
document.setNumberOfPages(documentData.getDocumentPages().length); document.setNumberOfPages(documentData.getDocumentPages().getDocumentPagesCount());
document.setTextBlock(document.getTextBlock()); document.setTextBlock(document.getTextBlock());
return document; return document;
} }
private List<DocumentTree.Entry> buildEntries(List<DocumentStructure.EntryData> entries, Context context) { private List<DocumentTree.Entry> buildEntries(List<EntryData> entries, Context context) {
List<DocumentTree.Entry> newEntries = new ArrayList<>(entries.size()); List<DocumentTree.Entry> newEntries = new ArrayList<>(entries.size());
for (DocumentStructure.EntryData entryData : entries) { for (EntryData entryData : entries) {
List<Page> pages = Arrays.stream(entryData.getPageNumbers()) List<Page> pages = entryData.getPageNumbersList()
.stream()
.map(context::getPage) .map(context::getPage)
.toList(); .toList();
@ -75,12 +76,12 @@ public class DocumentGraphMapper {
case FOOTER -> buildFooter(context); case FOOTER -> buildFooter(context);
case TABLE -> buildTable(context, entryData.getProperties()); case TABLE -> buildTable(context, entryData.getProperties());
case TABLE_CELL -> buildTableCell(context, entryData.getProperties()); case TABLE_CELL -> buildTableCell(context, entryData.getProperties());
case IMAGE -> buildImage(context, entryData.getProperties(), entryData.getPageNumbers()); case IMAGE -> buildImage(context, entryData.getProperties(), entryData.getPageNumbersList());
default -> throw new UnsupportedOperationException("Not yet implemented for type " + entryData.getType()); default -> throw new UnsupportedOperationException("Not yet implemented for type " + entryData.getType());
}; };
if (entryData.getAtomicBlockIds().length > 0) { if (entryData.getAtomicBlockIdsCount() > 0) {
TextBlock textBlock = toTextBlock(entryData.getAtomicBlockIds(), context, node); TextBlock textBlock = toTextBlock(entryData.getAtomicBlockIdsList(), context, node);
node.setLeafTextBlock(textBlock); node.setLeafTextBlock(textBlock);
switch (entryData.getType()) { switch (entryData.getType()) {
@ -92,20 +93,13 @@ public class DocumentGraphMapper {
} }
} }
List<Integer> treeId = Arrays.stream(entryData.getTreeId()).boxed() List<Integer> treeId = entryData.getTreeIdList();
.toList(); entryData.getEnginesList()
if (entryData.getEngines() != null) {
entryData.getEngines()
.forEach(node::addEngine); .forEach(node::addEngine);
} else {
entryData.setEngines(Collections.emptySet());
}
node.setTreeId(treeId); node.setTreeId(treeId);
newEntries.add(DocumentTree.Entry.builder().treeId(treeId).children(buildEntries(entryData.getChildren(), context)).node(node).build()); newEntries.add(DocumentTree.Entry.builder().treeId(treeId).children(buildEntries(entryData.getChildrenList(), context)).node(node).build());
} return newEntries;
}
return newEntries;
} }
@ -115,10 +109,10 @@ public class DocumentGraphMapper {
} }
private Image buildImage(Context context, Map<String, String> properties, Long[] pageNumbers) { private Image buildImage(Context context, Map<String, String> properties, List<Long> pageNumbers) {
assert pageNumbers.length == 1; assert pageNumbers.size() == 1;
Page page = context.getPage(pageNumbers[0]); Page page = context.getPage(pageNumbers.get(0));
var builder = Image.builder(); var builder = Image.builder();
PropertiesMapper.parseImageProperties(properties, builder); PropertiesMapper.parseImageProperties(properties, builder);
return builder.documentTree(context.documentTree).page(page).build(); return builder.documentTree(context.documentTree).page(page).build();
@ -171,7 +165,7 @@ public class DocumentGraphMapper {
DuplicatedParagraph duplicatedParagraph = DuplicatedParagraph.builder().documentTree(context.documentTree).build(); DuplicatedParagraph duplicatedParagraph = DuplicatedParagraph.builder().documentTree(context.documentTree).build();
Long[] unsortedTextblockIds = PropertiesMapper.getUnsortedTextblockIds(properties); var unsortedTextblockIds = PropertiesMapper.getUnsortedTextblockIds(properties);
duplicatedParagraph.setUnsortedLeafTextBlock(toTextBlock(unsortedTextblockIds, context, duplicatedParagraph)); duplicatedParagraph.setUnsortedLeafTextBlock(toTextBlock(unsortedTextblockIds, context, duplicatedParagraph));
return duplicatedParagraph; return duplicatedParagraph;
} }
@ -180,9 +174,9 @@ public class DocumentGraphMapper {
} }
private TextBlock toTextBlock(Long[] atomicTextBlockIds, Context context, SemanticNode parent) { private TextBlock toTextBlock(List<Long> atomicTextBlockIds, Context context, SemanticNode parent) {
return Arrays.stream(atomicTextBlockIds) return atomicTextBlockIds.stream()
.map(atomicTextBlockId -> getAtomicTextBlock(context, parent, atomicTextBlockId)) .map(atomicTextBlockId -> getAtomicTextBlock(context, parent, atomicTextBlockId))
.collect(new TextBlockCollector()); .collect(new TextBlockCollector());
} }
@ -190,10 +184,10 @@ public class DocumentGraphMapper {
private AtomicTextBlock getAtomicTextBlock(Context context, SemanticNode parent, Long atomicTextBlockId) { private AtomicTextBlock getAtomicTextBlock(Context context, SemanticNode parent, Long atomicTextBlockId) {
return AtomicTextBlock.fromAtomicTextBlockData(context.documentTextData.get(Math.toIntExact(atomicTextBlockId)), return AtomicTextBlock.fromAtomicTextBlockData(context.documentTextData.getDocumentTextData(Math.toIntExact(atomicTextBlockId)),
context.documentPositionData.get(Math.toIntExact(atomicTextBlockId)), context.documentPositionData.getDocumentPositionData(Math.toIntExact(atomicTextBlockId)),
parent, parent,
context.getPage(context.documentTextData.get(Math.toIntExact(atomicTextBlockId)).getPage())); context.getPage(context.documentTextData.getDocumentTextData(Math.toIntExact(atomicTextBlockId)).getPage()));
} }
@ -207,18 +201,16 @@ public class DocumentGraphMapper {
private final DocumentTree documentTree; private final DocumentTree documentTree;
private final List<Page> pageData; private final List<Page> pageData;
private final List<DocumentTextData> documentTextData; private final AllDocumentTextData documentTextData;
private final List<DocumentPositionData> documentPositionData; private final AllDocumentPositionData documentPositionData;
Context(DocumentData documentData, DocumentTree documentTree) { Context(DocumentData documentData, DocumentTree documentTree) {
this.documentTree = documentTree; this.documentTree = documentTree;
this.pageData = new ArrayList<>(); this.pageData = new ArrayList<>();
this.documentTextData = Arrays.stream(documentData.getDocumentTextData()) this.documentTextData = documentData.getDocumentTextData();
.toList(); this.documentPositionData = documentData.getDocumentPositionData();
this.documentPositionData = Arrays.stream(documentData.getDocumentPositionData())
.toList();
} }

34
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/PropertiesMapper.java
View File

@ -9,7 +9,7 @@ import com.iqser.red.service.redaction.v1.server.model.document.nodes.Image;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.ImageType; import com.iqser.red.service.redaction.v1.server.model.document.nodes.ImageType;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Table; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Table;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell; import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell;
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentStructure; import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentStructureWrapper;
import lombok.experimental.UtilityClass; import lombok.experimental.UtilityClass;
@ -18,32 +18,32 @@ public class PropertiesMapper {
public void parseImageProperties(Map<String, String> properties, Image.ImageBuilder builder) { public void parseImageProperties(Map<String, String> properties, Image.ImageBuilder builder) {
builder.imageType(ImageType.fromString(properties.get(DocumentStructure.ImageProperties.IMAGE_TYPE))); builder.imageType(ImageType.fromString(properties.get(DocumentStructureWrapper.ImageProperties.IMAGE_TYPE)));
builder.transparent(Boolean.parseBoolean(properties.get(DocumentStructure.ImageProperties.TRANSPARENT))); builder.transparent(Boolean.parseBoolean(properties.get(DocumentStructureWrapper.ImageProperties.TRANSPARENT)));
builder.position(parseRectangle2D(properties.get(DocumentStructure.ImageProperties.POSITION))); builder.position(parseRectangle2D(properties.get(DocumentStructureWrapper.ImageProperties.POSITION)));
builder.id(properties.get(DocumentStructure.ImageProperties.ID)); builder.id(properties.get(DocumentStructureWrapper.ImageProperties.ID));
} }
public void parseTableCellProperties(Map<String, String> properties, TableCell.TableCellBuilder builder) { public void parseTableCellProperties(Map<String, String> properties, TableCell.TableCellBuilder builder) {
builder.row(Integer.parseInt(properties.get(DocumentStructure.TableCellProperties.ROW))); builder.row(Integer.parseInt(properties.get(DocumentStructureWrapper.TableCellProperties.ROW)));
builder.col(Integer.parseInt(properties.get(DocumentStructure.TableCellProperties.COL))); builder.col(Integer.parseInt(properties.get(DocumentStructureWrapper.TableCellProperties.COL)));
builder.header(Boolean.parseBoolean(properties.get(DocumentStructure.TableCellProperties.HEADER))); builder.header(Boolean.parseBoolean(properties.get(DocumentStructureWrapper.TableCellProperties.HEADER)));
builder.bBox(parseRectangle2D(properties.get(DocumentStructure.TableCellProperties.B_BOX))); builder.bBox(parseRectangle2D(properties.get(DocumentStructureWrapper.TableCellProperties.B_BOX)));
} }
public void parseTableProperties(Map<String, String> properties, Table.TableBuilder builder) { public void parseTableProperties(Map<String, String> properties, Table.TableBuilder builder) {
builder.numberOfRows(Integer.parseInt(properties.get(DocumentStructure.TableProperties.NUMBER_OF_ROWS))); builder.numberOfRows(Integer.parseInt(properties.get(DocumentStructureWrapper.TableProperties.NUMBER_OF_ROWS)));
builder.numberOfCols(Integer.parseInt(properties.get(DocumentStructure.TableProperties.NUMBER_OF_COLS))); builder.numberOfCols(Integer.parseInt(properties.get(DocumentStructureWrapper.TableProperties.NUMBER_OF_COLS)));
} }
private Rectangle2D parseRectangle2D(String bBox) { private Rectangle2D parseRectangle2D(String bBox) {
List<Float> floats = Arrays.stream(bBox.split(DocumentStructure.RECTANGLE_DELIMITER)) List<Float> floats = Arrays.stream(bBox.split(DocumentStructureWrapper.RECTANGLE_DELIMITER))
.map(Float::parseFloat) .map(Float::parseFloat)
.toList(); .toList();
return new Rectangle2D.Float(floats.get(0), floats.get(1), floats.get(2), floats.get(3)); return new Rectangle2D.Float(floats.get(0), floats.get(1), floats.get(2), floats.get(3));
@ -52,19 +52,19 @@ public class PropertiesMapper {
public static boolean isDuplicateParagraph(Map<String, String> properties) { public static boolean isDuplicateParagraph(Map<String, String> properties) {
return properties.containsKey(DocumentStructure.DuplicateParagraphProperties.UNSORTED_TEXTBLOCK_ID); return properties.containsKey(DocumentStructureWrapper.DuplicateParagraphProperties.UNSORTED_TEXTBLOCK_ID);
} }
public static Long[] getUnsortedTextblockIds(Map<String, String> properties) { public static List<Long> getUnsortedTextblockIds(Map<String, String> properties) {
return toLongArray(properties.get(DocumentStructure.DuplicateParagraphProperties.UNSORTED_TEXTBLOCK_ID)); return toLongList(properties.get(DocumentStructureWrapper.DuplicateParagraphProperties.UNSORTED_TEXTBLOCK_ID));
} }
public static Long[] toLongArray(String ids) { public static List<Long> toLongList(String ids) {
return Arrays.stream(ids.substring(1, ids.length() - 1).trim().split(",")).map(Long::valueOf).toArray(Long[]::new); return Arrays.stream(ids.substring(1, ids.length() - 1).trim().split(",")).map(Long::valueOf).toList();
} }

236
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/DocumentDataFallbackService.java Normal file
View File

@ -0,0 +1,236 @@
package com.iqser.red.service.redaction.v1.server.storage;
import java.io.BufferedInputStream;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.nio.file.StandardOpenOption;
import java.util.Arrays;
import java.util.stream.Collectors;
import org.springframework.stereotype.Service;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.primitives.Floats;
import com.google.protobuf.Message;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
import com.iqser.red.storage.commons.service.StorageService;
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPage;
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPageProto;
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPositionData;
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPositionDataProto;
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentStructure;
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentStructureProto;
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentTextData;
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentTextDataProto;
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.EntryDataProto;
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeTypeProto;
import com.knecon.fforesight.tenantcommons.TenantContext;
import lombok.RequiredArgsConstructor;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
@Service
@Slf4j
@RequiredArgsConstructor
public class DocumentDataFallbackService {
private final ObjectMapper objectMapper;
private final StorageService storageService;
private <T> T getOldData(String dossierId, String fileId, FileType fileType, Class<T> valueType) {
String oldStorageId = RedactionStorageService.StorageIdUtils.getStorageId(dossierId, fileId, fileType.name(), ".json");
try (InputStream inputStream = getObject(TenantContext.getTenantId(), oldStorageId)) {
return objectMapper.readValue(inputStream, valueType);
} catch (IOException e) {
log.error("Could not read JSON for " + fileType.name() + ", error was: " + e);
return null;
}
}
private void storeNewData(String dossierId, String fileId, FileType fileType, Message protoObject) {
String newStorageId = RedactionStorageService.StorageIdUtils.getStorageId(dossierId, fileId, fileType);
storageService.storeProtoObject(TenantContext.getTenantId(), newStorageId, protoObject);
}
public DocumentStructureProto.DocumentStructure readOldDocumentStructureAndConvert(String dossierId, String fileId) {
DocumentStructure oldDocumentStructure = getOldData(dossierId, fileId, FileType.DOCUMENT_STRUCTURE, DocumentStructure.class);
if (oldDocumentStructure == null) {
return null;
}
DocumentStructureProto.DocumentStructure documentStructure = convertDocumentStructure(oldDocumentStructure);
//storeNewData(dossierId, fileId, FileType.DOCUMENT_STRUCTURE, documentStructure);
return documentStructure;
}
public DocumentPageProto.AllDocumentPages readOldDocumentPagesAndConvert(String dossierId, String fileId) {
DocumentPage[] oldDocumentPages = getOldData(dossierId, fileId, FileType.DOCUMENT_PAGES, DocumentPage[].class);
if (oldDocumentPages == null) {
return null;
}
DocumentPageProto.AllDocumentPages allDocumentPages = convertAllDocumentPages(oldDocumentPages);
//storeNewData(dossierId, fileId, FileType.DOCUMENT_PAGES, allDocumentPages);
return allDocumentPages;
}
public DocumentPositionDataProto.AllDocumentPositionData readOldDocumentPositionDataAndConvert(String dossierId, String fileId) {
DocumentPositionData[] oldDocumentPositionData = getOldData(dossierId, fileId, FileType.DOCUMENT_POSITION, DocumentPositionData[].class);
if (oldDocumentPositionData == null) {
return null;
}
DocumentPositionDataProto.AllDocumentPositionData allDocumentPositionData = convertAllDocumentPositionData(oldDocumentPositionData);
//storeNewData(dossierId, fileId, FileType.DOCUMENT_POSITION, allDocumentPositionData);
return allDocumentPositionData;
}
public DocumentTextDataProto.AllDocumentTextData readOldDocumentTextDataAndConvert(String dossierId, String fileId) {
DocumentTextData[] oldDocumentTextData = getOldData(dossierId, fileId, FileType.DOCUMENT_TEXT, DocumentTextData[].class);
if (oldDocumentTextData == null) {
return null;
}
DocumentTextDataProto.AllDocumentTextData allDocumentTextData = convertAllDocumentTextData(oldDocumentTextData);
//storeNewData(dossierId, fileId, FileType.DOCUMENT_TEXT, allDocumentTextData);
return allDocumentTextData;
}
private static EntryDataProto.EntryData convertEntryData(DocumentStructure.EntryData oldEntryData) {
EntryDataProto.EntryData.Builder builder = EntryDataProto.EntryData.newBuilder();
builder.setType(NodeTypeProto.NodeType.valueOf(oldEntryData.getType().name()));
builder.addAllTreeId(Arrays.stream(oldEntryData.getTreeId()).boxed()
.collect(Collectors.toList()));
builder.addAllAtomicBlockIds(Arrays.asList(oldEntryData.getAtomicBlockIds()));
builder.addAllPageNumbers(Arrays.asList(oldEntryData.getPageNumbers()));
builder.putAllProperties(oldEntryData.getProperties());
if (oldEntryData.getChildren() != null) {
oldEntryData.getChildren()
.forEach(child -> builder.addChildren(convertEntryData(child)));
}
return builder.build();
}
private static DocumentStructureProto.DocumentStructure convertDocumentStructure(DocumentStructure oldStructure) {
DocumentStructureProto.DocumentStructure.Builder newBuilder = DocumentStructureProto.DocumentStructure.newBuilder();
if (oldStructure.getRoot() != null) {
newBuilder.setRoot(convertEntryData(oldStructure.getRoot()));
}
return newBuilder.build();
}
private static DocumentPageProto.DocumentPage convertDocumentPage(DocumentPage oldPage) {
return DocumentPageProto.DocumentPage.newBuilder()
.setNumber(oldPage.getNumber())
.setHeight(oldPage.getHeight())
.setWidth(oldPage.getWidth())
.setRotation(oldPage.getRotation())
.build();
}
private static DocumentPageProto.AllDocumentPages convertAllDocumentPages(DocumentPage[] oldPages) {
DocumentPageProto.AllDocumentPages.Builder allPagesBuilder = DocumentPageProto.AllDocumentPages.newBuilder();
for (DocumentPage oldPage : oldPages) {
DocumentPageProto.DocumentPage newPage = convertDocumentPage(oldPage);
allPagesBuilder.addDocumentPages(newPage);
}
return allPagesBuilder.build();
}
private static DocumentPositionDataProto.DocumentPositionData convertDocumentPositionData(DocumentPositionData oldData) {
DocumentPositionDataProto.DocumentPositionData.Builder builder = DocumentPositionDataProto.DocumentPositionData.newBuilder()
.setId(oldData.getId())
.addAllStringIdxToPositionIdx(Arrays.stream(oldData.getStringIdxToPositionIdx()).boxed()
.collect(Collectors.toList()));
for (float[] pos : oldData.getPositions()) {
DocumentPositionDataProto.DocumentPositionData.Position position = DocumentPositionDataProto.DocumentPositionData.Position.newBuilder()
.addAllValue(Floats.asList(pos))
.build();
builder.addPositions(position);
}
return builder.build();
}
private static DocumentPositionDataProto.AllDocumentPositionData convertAllDocumentPositionData(DocumentPositionData[] oldDataList) {
DocumentPositionDataProto.AllDocumentPositionData.Builder allDataBuilder = DocumentPositionDataProto.AllDocumentPositionData.newBuilder();
for (DocumentPositionData oldData : oldDataList) {
allDataBuilder.addDocumentPositionData(convertDocumentPositionData(oldData));
}
return allDataBuilder.build();
}
private static DocumentTextDataProto.DocumentTextData convertDocumentTextData(DocumentTextData oldData) {
DocumentTextDataProto.DocumentTextData.Builder builder = DocumentTextDataProto.DocumentTextData.newBuilder()
.setId(oldData.getId())
.setPage(oldData.getPage())
.setSearchText(oldData.getSearchText())
.setNumberOnPage(oldData.getNumberOnPage())
.setStart(oldData.getStart())
.setEnd(oldData.getEnd())
.addAllLineBreaks(Arrays.stream(oldData.getLineBreaks()).boxed()
.collect(Collectors.toList()));
return builder.build();
}
private static DocumentTextDataProto.AllDocumentTextData convertAllDocumentTextData(DocumentTextData[] oldDataList) {
DocumentTextDataProto.AllDocumentTextData.Builder allDataBuilder = DocumentTextDataProto.AllDocumentTextData.newBuilder();
for (DocumentTextData oldData : oldDataList) {
allDataBuilder.addDocumentTextData(convertDocumentTextData(oldData));
}
return allDataBuilder.build();
}
@SneakyThrows
private InputStream getObject(String tenantId, String storageId) {
File tempFile = File.createTempFile("temp", ".data");
storageService.downloadTo(tenantId, storageId, tempFile);
return new BufferedInputStream(Files.newInputStream(Paths.get(tempFile.getPath()), StandardOpenOption.DELETE_ON_CLOSE));
}
}

125
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/RedactionStorageService.java
View File

@ -1,5 +1,6 @@
package com.iqser.red.service.redaction.v1.server.storage; package com.iqser.red.service.redaction.v1.server.storage;
import static com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPositionDataProto.AllDocumentPositionData;
import static org.springframework.boot.autoconfigure.task.TaskExecutionAutoConfiguration.APPLICATION_TASK_EXECUTOR_BEAN_NAME; import static org.springframework.boot.autoconfigure.task.TaskExecutionAutoConfiguration.APPLICATION_TASK_EXECUTOR_BEAN_NAME;
import java.io.File; import java.io.File;
@ -26,16 +27,21 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
import com.iqser.red.service.persistence.service.v1.api.shared.mongo.service.ComponentLogMongoService; import com.iqser.red.service.persistence.service.v1.api.shared.mongo.service.ComponentLogMongoService;
import com.iqser.red.service.persistence.service.v1.api.shared.mongo.service.EntityLogMongoService; import com.iqser.red.service.persistence.service.v1.api.shared.mongo.service.EntityLogMongoService;
import com.iqser.red.service.redaction.v1.server.RedactionServiceSettings;
import com.iqser.red.service.redaction.v1.server.client.model.NerEntitiesModel; import com.iqser.red.service.redaction.v1.server.client.model.NerEntitiesModel;
import com.iqser.red.service.redaction.v1.server.model.document.DocumentData; import com.iqser.red.service.redaction.v1.server.model.document.DocumentData;
import com.iqser.red.service.redaction.v1.server.utils.exception.NotFoundException; import com.iqser.red.service.redaction.v1.server.utils.exception.NotFoundException;
import com.iqser.red.storage.commons.exception.StorageException;
import com.iqser.red.storage.commons.exception.StorageObjectDoesNotExist; import com.iqser.red.storage.commons.exception.StorageObjectDoesNotExist;
import com.iqser.red.storage.commons.service.StorageService; import com.iqser.red.storage.commons.service.StorageService;
import com.knecon.fforesight.llm.service.LlmNerEntities; import com.knecon.fforesight.llm.service.LlmNerEntities;
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPage; import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPageProto;
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPositionData; import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPageProto.AllDocumentPages;
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentStructure; import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPositionDataProto;
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentTextData; import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentStructureProto.DocumentStructure;
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentStructureWrapper;
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentTextDataProto;
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentTextDataProto.AllDocumentTextData;
import com.knecon.fforesight.tenantcommons.TenantContext; import com.knecon.fforesight.tenantcommons.TenantContext;
import io.micrometer.core.annotation.Timed; import io.micrometer.core.annotation.Timed;
@ -48,23 +54,29 @@ import lombok.extern.slf4j.Slf4j;
@Service @Service
public class RedactionStorageService { public class RedactionStorageService {
private final RedactionServiceSettings settings;
private final StorageService storageService; private final StorageService storageService;
private final EntityLogMongoService entityLogMongoService; private final EntityLogMongoService entityLogMongoService;
private final DocumentDataFallbackService documentDataFallbackService;
private final ComponentLogMongoService componentLogMongoService; private final ComponentLogMongoService componentLogMongoService;
private final TaskExecutor taskExecutor; private final TaskExecutor taskExecutor;
public RedactionStorageService(StorageService storageService, public RedactionStorageService(RedactionServiceSettings settings,
StorageService storageService,
EntityLogMongoService entityLogMongoService, EntityLogMongoService entityLogMongoService,
ComponentLogMongoService componentLogMongoService, ComponentLogMongoService componentLogMongoService,
DocumentDataFallbackService documentDataFallbackService,
@Qualifier(APPLICATION_TASK_EXECUTOR_BEAN_NAME) TaskExecutor taskExecutor) { @Qualifier(APPLICATION_TASK_EXECUTOR_BEAN_NAME) TaskExecutor taskExecutor) {
this.settings = settings;
this.storageService = storageService; this.storageService = storageService;
this.entityLogMongoService = entityLogMongoService; this.entityLogMongoService = entityLogMongoService;
this.componentLogMongoService = componentLogMongoService; this.componentLogMongoService = componentLogMongoService;
this.documentDataFallbackService = documentDataFallbackService;
this.taskExecutor = taskExecutor; this.taskExecutor = taskExecutor;
} }
@ -257,32 +269,15 @@ public class RedactionStorageService {
public DocumentData getDocumentData(String dossierId, String fileId) { public DocumentData getDocumentData(String dossierId, String fileId) {
try { try {
Supplier<DocumentStructure> documentStructureSupplier = () -> storageService.readJSONObject(TenantContext.getTenantId(), CompletableFuture<DocumentStructure> documentStructureFuture = fetchDocumentStructure(dossierId, fileId);
StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_STRUCTURE), CompletableFuture<AllDocumentTextData> documentTextDataFuture = fetchDocumentTextData(dossierId, fileId);
DocumentStructure.class); CompletableFuture<AllDocumentPositionData> documentPositionDataFuture = fetchDocumentPositionData(dossierId, fileId);
CompletableFuture<DocumentStructure> documentStructureFuture = CompletableFuture.supplyAsync(documentStructureSupplier, taskExecutor); CompletableFuture<AllDocumentPages> documentPagesFuture = fetchDocumentPages(dossierId, fileId);
Supplier<DocumentTextData[]> documentTextDataSupplier = () -> storageService.readJSONObject(TenantContext.getTenantId(),
StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_TEXT),
DocumentTextData[].class);
CompletableFuture<DocumentTextData[]> documentTextDataFuture = CompletableFuture.supplyAsync(documentTextDataSupplier, taskExecutor);
Supplier<DocumentPositionData[]> documentPositionDataSupplier = () -> storageService.readJSONObject(TenantContext.getTenantId(),
StorageIdUtils.getStorageId(dossierId,
fileId,
FileType.DOCUMENT_POSITION),
DocumentPositionData[].class);
CompletableFuture<DocumentPositionData[]> documentPositionDataFuture = CompletableFuture.supplyAsync(documentPositionDataSupplier, taskExecutor);
Supplier<DocumentPage[]> documentPageSupplier = () -> storageService.readJSONObject(TenantContext.getTenantId(),
StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_PAGES),
DocumentPage[].class);
CompletableFuture<DocumentPage[]> documentPagesFuture = CompletableFuture.supplyAsync(documentPageSupplier, taskExecutor);
CompletableFuture.allOf(documentStructureFuture, documentTextDataFuture, documentPositionDataFuture, documentPagesFuture).join(); CompletableFuture.allOf(documentStructureFuture, documentTextDataFuture, documentPositionDataFuture, documentPagesFuture).join();
return DocumentData.builder() return DocumentData.builder()
.documentStructure(documentStructureFuture.get()) .documentStructureWrapper(new DocumentStructureWrapper(documentStructureFuture.get()))
.documentTextData(documentTextDataFuture.get()) .documentTextData(documentTextDataFuture.get())
.documentPositionData(documentPositionDataFuture.get()) .documentPositionData(documentPositionDataFuture.get())
.documentPages(documentPagesFuture.get()) .documentPages(documentPagesFuture.get())
@ -297,6 +292,74 @@ public class RedactionStorageService {
} }
private CompletableFuture<DocumentStructure> fetchDocumentStructure(String dossierId, String fileId) {
Supplier<DocumentStructure> documentStructureSupplier = () -> {
DocumentStructure documentStructure = null;
String storageId = StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_STRUCTURE);
if (storageService.objectExists(TenantContext.getTenantId(), storageId)) {
documentStructure = storageService.readProtoObject(TenantContext.getTenantId(), storageId, DocumentStructure.parser());
}
if (documentStructure == null && settings.isProtobufJsonFallback()) {
documentStructure = documentDataFallbackService.readOldDocumentStructureAndConvert(dossierId, fileId);
}
return documentStructure;
};
return CompletableFuture.supplyAsync(documentStructureSupplier, taskExecutor);
}
private CompletableFuture<DocumentTextDataProto.AllDocumentTextData> fetchDocumentTextData(String dossierId, String fileId) {
Supplier<DocumentTextDataProto.AllDocumentTextData> documentTextDataSupplier = () -> {
DocumentTextDataProto.AllDocumentTextData documentTextData = null;
String storageId = StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_TEXT);
if (storageService.objectExists(TenantContext.getTenantId(), storageId)) {
documentTextData = storageService.readProtoObject(TenantContext.getTenantId(), storageId, DocumentTextDataProto.AllDocumentTextData.parser());
}
if (documentTextData == null && settings.isProtobufJsonFallback()) {
documentTextData = documentDataFallbackService.readOldDocumentTextDataAndConvert(dossierId, fileId);
}
return documentTextData;
};
return CompletableFuture.supplyAsync(documentTextDataSupplier, taskExecutor);
}
private CompletableFuture<DocumentPositionDataProto.AllDocumentPositionData> fetchDocumentPositionData(String dossierId, String fileId) {
Supplier<DocumentPositionDataProto.AllDocumentPositionData> documentPositionDataSupplier = () -> {
DocumentPositionDataProto.AllDocumentPositionData documentPositionData = null;
String storageId = StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_POSITION);
if (storageService.objectExists(TenantContext.getTenantId(), storageId)) {
documentPositionData = storageService.readProtoObject(TenantContext.getTenantId(), storageId, DocumentPositionDataProto.AllDocumentPositionData.parser());
}
if (documentPositionData == null && settings.isProtobufJsonFallback()) {
documentPositionData = documentDataFallbackService.readOldDocumentPositionDataAndConvert(dossierId, fileId);
}
return documentPositionData;
};
return CompletableFuture.supplyAsync(documentPositionDataSupplier, taskExecutor);
}
private CompletableFuture<DocumentPageProto.AllDocumentPages> fetchDocumentPages(String dossierId, String fileId) {
Supplier<DocumentPageProto.AllDocumentPages> documentPagesSupplier = () -> {
DocumentPageProto.AllDocumentPages documentPages = null;
String storageId = StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_PAGES);
if (storageService.objectExists(TenantContext.getTenantId(), storageId)) {
documentPages = storageService.readProtoObject(TenantContext.getTenantId(), storageId, AllDocumentPages.parser());
}
if (documentPages == null && settings.isProtobufJsonFallback()) {
documentPages = documentDataFallbackService.readOldDocumentPagesAndConvert(dossierId, fileId);
}
return documentPages;
};
return CompletableFuture.supplyAsync(documentPagesSupplier, taskExecutor);
}
@Timed("redactmanager_getNerEntities") @Timed("redactmanager_getNerEntities")
public NerEntitiesModel getNerEntities(String dossierId, String fileId) { public NerEntitiesModel getNerEntities(String dossierId, String fileId) {
@ -347,6 +410,12 @@ public class RedactionStorageService {
return dossierId + "/" + fileId + "." + fileType.name() + fileType.getExtension(); return dossierId + "/" + fileId + "." + fileType.name() + fileType.getExtension();
} }
public static String getStorageId(String dossierId, String fileId, String fileName, String fileExtension) {
return dossierId + "/" + fileId + "." + fileName + fileExtension;
}
} }
} }

2
redaction-service-v1/redaction-service-server-v1/src/main/resources/drools/all_rules_documine.drl
View File

@ -47,7 +47,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange;
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine; import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngineProto.LayoutEngine;
global Document document global Document document
global EntityCreationService entityCreationService global EntityCreationService entityCreationService

57
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AnalysisEnd2EndTest.java
View File

@ -9,10 +9,14 @@ import static org.mockito.Mockito.when;
import java.io.File; import java.io.File;
import java.io.FileInputStream; import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException; import java.io.IOException;
import java.io.ObjectInputStream;
import java.io.ObjectOutputStream;
import java.nio.file.FileVisitOption; import java.nio.file.FileVisitOption;
import java.nio.file.Files; import java.nio.file.Files;
import java.nio.file.Path; import java.nio.file.Path;
import java.time.OffsetDateTime;
import java.util.Arrays; import java.util.Arrays;
import java.util.Collections; import java.util.Collections;
import java.util.HashMap; import java.util.HashMap;
@ -41,13 +45,17 @@ import org.springframework.core.io.ClassPathResource;
import org.springframework.data.redis.listener.RedisMessageListenerContainer; import org.springframework.data.redis.listener.RedisMessageListenerContainer;
import org.springframework.test.context.junit.jupiter.SpringExtension; import org.springframework.test.context.junit.jupiter.SpringExtension;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.collect.Sets; import com.google.common.collect.Sets;
import com.iqser.red.commons.jackson.ObjectMapperFactory; import com.iqser.red.commons.jackson.ObjectMapperFactory;
import com.iqser.red.service.dictionarymerge.commons.DictionaryEntryModel; import com.iqser.red.service.dictionarymerge.commons.DictionaryEntryModel;
import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest; import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType; import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive; import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
import com.iqser.red.service.redaction.v1.server.client.DictionaryClient; import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
@ -105,7 +113,6 @@ import lombok.extern.slf4j.Slf4j;
FileType.DOCUMENT_POSITION, FileType.DOCUMENT_POSITION,
FileType.DOCUMENT_STRUCTURE, FileType.DOCUMENT_STRUCTURE,
FileType.DOCUMENT_TEXT); FileType.DOCUMENT_TEXT);
Path dossierTemplateToUse = Path.of("/home/kschuettler/Downloads/New Folder/DOSSIER_TEMPLATE"); // Add your dossier-template here Path dossierTemplateToUse = Path.of("/home/kschuettler/Downloads/New Folder/DOSSIER_TEMPLATE"); // Add your dossier-template here
ObjectMapper mapper = ObjectMapperFactory.create(); ObjectMapper mapper = ObjectMapperFactory.create();
final String TENANT_ID = "tenant"; final String TENANT_ID = "tenant";
@ -114,6 +121,8 @@ import lombok.extern.slf4j.Slf4j;
@Autowired @Autowired
StorageService storageService; StorageService storageService;
@Autowired @Autowired
RedactionStorageService redactionStorageService;
@Autowired
protected AnalyzeService analyzeService; protected AnalyzeService analyzeService;
@Autowired @Autowired
protected MongoTestConfig mongoTestConfig; protected MongoTestConfig mongoTestConfig;
@ -171,6 +180,52 @@ import lombok.extern.slf4j.Slf4j;
} }
private static void addManualRedactionEntryToFile(AnalyzeRequest analyzeRequest) {
ManualRedactionEntry manualRedactionEntry = new ManualRedactionEntry();
manualRedactionEntry.setAnnotationId("yourAnnotationId");
manualRedactionEntry.setFileId("fileId");
manualRedactionEntry.setType("CBI_author");
manualRedactionEntry.setValue("7232");
manualRedactionEntry.setReason(
"(Regulations (EU) 2016/679 and (EU) 2018/1725 shall apply to the processing of personal data carried out pursuant to this Regulation. Any personal data made public pursuant to Article 38 of this Regulation and this Article shall only be used to ensure the transparency of the risk assessment under this Regulation and shall not be further processed in a manner that is incompatible with these purposes, in accordance with point (b) of Article 5(1) of Regulation (EU) 2016/679 and point (b) of Article 4(1) of Regulation (EU) 2018/1725, as the case may be)");
manualRedactionEntry.setLegalBasis("Article 39(e)(3) of Regulation (EC) No 178/2002");
manualRedactionEntry.setProcessedDate(OffsetDateTime.now());
manualRedactionEntry.setRequestDate(OffsetDateTime.now());
manualRedactionEntry.setPositions(List.of(Rectangle.builder().topLeftX(332.134f).topLeftY(689.72f).width(26.688f).height(13.872f).page(1).build()));
analyzeRequest.setManualRedactions(ManualRedactions.builder().entriesToAdd(Set.of(manualRedactionEntry)).build());
}
private void restoreEntityLogFromTempFile(ObjectMapper objectMapper, AnalyzeRequest analyzeRequest) throws JsonProcessingException {
//Deserialize entityLog from a file
String entityLogString = null;
try (FileInputStream fileIn = new FileInputStream("/tmp/entityLog.ser"); ObjectInputStream in = new ObjectInputStream(fileIn)) {
entityLogString = (String) in.readObject();
} catch (IOException | ClassNotFoundException e) {
e.printStackTrace();
}
var entityLog = objectMapper.readValue(entityLogString, EntityLog.class);
redactionStorageService.saveEntityLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), entityLog);
}
private void storeEntityLogAsTempFile(AnalyzeRequest analyzeRequest, ObjectMapper objectMapper) throws JsonProcessingException {
EntityLog entityLog = redactionStorageService.getEntityLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
// Serialize
String jsonString = objectMapper.writeValueAsString(entityLog);
// Serialize entityLog to a file
try (FileOutputStream fileOut = new FileOutputStream("/tmp/entityLog.ser"); ObjectOutputStream out = new ObjectOutputStream(fileOut)) {
out.writeObject(jsonString);
} catch (IOException e) {
e.printStackTrace();
}
}
@BeforeEach @BeforeEach
public void setup() { public void setup() {

17
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/MapEntityLogToDocumentDataTest.java
View File

@ -56,10 +56,11 @@ import com.iqser.red.service.redaction.v1.server.service.document.EntityFindingU
import com.iqser.red.storage.commons.StorageAutoConfiguration; import com.iqser.red.storage.commons.StorageAutoConfiguration;
import com.iqser.red.storage.commons.service.StorageService; import com.iqser.red.storage.commons.service.StorageService;
import com.iqser.red.storage.commons.utils.FileSystemBackedStorageService; import com.iqser.red.storage.commons.utils.FileSystemBackedStorageService;
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPage; import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPageProto;
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPositionData; import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPositionDataProto;
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentStructure; import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentStructureProto;
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentTextData; import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentStructureWrapper;
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentTextDataProto;
import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingServiceProcessorConfiguration; import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingServiceProcessorConfiguration;
import com.knecon.fforesight.tenantcommons.TenantContext; import com.knecon.fforesight.tenantcommons.TenantContext;
@ -368,16 +369,16 @@ public class MapEntityLogToDocumentDataTest extends AbstractRedactionIntegration
var documentDataBuilder = DocumentData.builder(); var documentDataBuilder = DocumentData.builder();
try (var in = new FileInputStream(documentStructurePath.toFile())) { try (var in = new FileInputStream(documentStructurePath.toFile())) {
documentDataBuilder.documentStructure(mapper.readValue(in, DocumentStructure.class)); documentDataBuilder.documentStructureWrapper(new DocumentStructureWrapper(mapper.readValue(in, DocumentStructureProto.DocumentStructure.class)));
} }
try (var in = new FileInputStream(documentTextPath.toFile())) { try (var in = new FileInputStream(documentTextPath.toFile())) {
documentDataBuilder.documentTextData(mapper.readValue(in, DocumentTextData[].class)); documentDataBuilder.documentTextData(mapper.readValue(in, DocumentTextDataProto.AllDocumentTextData.class));
} }
try (var in = new FileInputStream(documentPositionsPath.toFile())) { try (var in = new FileInputStream(documentPositionsPath.toFile())) {
documentDataBuilder.documentPositionData(mapper.readValue(in, DocumentPositionData[].class)); documentDataBuilder.documentPositionData(mapper.readValue(in, DocumentPositionDataProto.AllDocumentPositionData.class));
} }
try (var in = new FileInputStream(documentPagesPath.toFile())) { try (var in = new FileInputStream(documentPagesPath.toFile())) {
documentDataBuilder.documentPages(mapper.readValue(in, DocumentPage[].class)); documentDataBuilder.documentPages(mapper.readValue(in, DocumentPageProto.AllDocumentPages.class));
} }
return DocumentGraphMapper.toDocumentGraph(documentDataBuilder.build()); return DocumentGraphMapper.toDocumentGraph(documentDataBuilder.build());

77
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java
View File

@ -1,13 +1,18 @@
package com.iqser.red.service.redaction.v1.server; package com.iqser.red.service.redaction.v1.server;
import static com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService.StorageIdUtils;
import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when; import static org.mockito.Mockito.when;
import java.io.BufferedWriter;
import java.io.FileOutputStream; import java.io.FileOutputStream;
import java.io.FileWriter;
import java.io.IOException; import java.io.IOException;
import java.net.URI; import java.net.URI;
import java.net.URL; import java.net.URL;
@ -32,6 +37,7 @@ import org.junit.jupiter.api.Order;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.TestMethodOrder; import org.junit.jupiter.api.TestMethodOrder;
import org.junit.jupiter.api.extension.ExtendWith; import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.Mock;
import org.springframework.boot.autoconfigure.EnableAutoConfiguration; import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration; import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration;
import org.springframework.boot.test.context.SpringBootTest; import org.springframework.boot.test.context.SpringBootTest;
@ -71,17 +77,19 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemp
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Point; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Point;
import com.iqser.red.service.redaction.v1.server.annotate.AnnotateRequest; import com.iqser.red.service.redaction.v1.server.annotate.AnnotateRequest;
import com.iqser.red.service.redaction.v1.server.annotate.AnnotateResponse; import com.iqser.red.service.redaction.v1.server.annotate.AnnotateResponse;
import com.iqser.red.service.redaction.v1.server.model.document.DocumentData;
import com.iqser.red.service.redaction.v1.server.model.document.TextRange; import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Section; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Section;
import com.iqser.red.service.redaction.v1.server.redaction.utils.OsUtils; import com.iqser.red.service.redaction.v1.server.redaction.utils.OsUtils;
import com.iqser.red.service.redaction.v1.server.rules.RulesIntegrationTest; import com.iqser.red.service.redaction.v1.server.rules.RulesIntegrationTest;
import com.iqser.red.service.redaction.v1.server.service.document.DocumentGraphMapper; import com.iqser.red.service.redaction.v1.server.service.document.DocumentGraphMapper;
import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService; import com.iqser.red.service.redaction.v1.server.storage.DocumentDataFallbackService;
import com.iqser.red.storage.commons.StorageAutoConfiguration; import com.iqser.red.storage.commons.StorageAutoConfiguration;
import com.iqser.red.storage.commons.service.StorageService; import com.iqser.red.storage.commons.service.StorageService;
import com.iqser.red.storage.commons.utils.FileSystemBackedStorageService; import com.iqser.red.storage.commons.utils.FileSystemBackedStorageService;
import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType; import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingServiceProcessorConfiguration; import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingServiceProcessorConfiguration;
import com.knecon.fforesight.service.layoutparser.processor.utils.ProtobufUtil;
import com.knecon.fforesight.tenantcommons.TenantContext; import com.knecon.fforesight.tenantcommons.TenantContext;
import lombok.SneakyThrows; import lombok.SneakyThrows;
@ -94,6 +102,7 @@ public class RedactionIntegrationTest extends RulesIntegrationTest {
private static final String RULES = loadFromClassPath("drools/rules.drl"); private static final String RULES = loadFromClassPath("drools/rules.drl");
@Configuration @Configuration
@EnableAutoConfiguration(exclude = {RabbitAutoConfiguration.class}) @EnableAutoConfiguration(exclude = {RabbitAutoConfiguration.class})
@Import({LayoutParsingServiceProcessorConfiguration.class}) @Import({LayoutParsingServiceProcessorConfiguration.class})
@ -287,7 +296,7 @@ public class RedactionIntegrationTest extends RulesIntegrationTest {
ClassPathResource importedRedactionClasspathResource = new ClassPathResource( ClassPathResource importedRedactionClasspathResource = new ClassPathResource(
"files/ImportedRedactions/18 Chlorothalonil RAR 08 Volume 3CA B 6a Oct 2017.IMPORTED_REDACTIONS.json"); "files/ImportedRedactions/18 Chlorothalonil RAR 08 Volume 3CA B 6a Oct 2017.IMPORTED_REDACTIONS.json");
storageService.storeObject(TenantContext.getTenantId(), storageService.storeObject(TenantContext.getTenantId(),
RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.IMPORTED_REDACTIONS), StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.IMPORTED_REDACTIONS),
importedRedactionClasspathResource.getInputStream()); importedRedactionClasspathResource.getInputStream());
AnalyzeRequest request = uploadFileToStorage("files/syngenta/CustomerFiles/18 Chlorothalonil RAR 08 Volume 3CA B 6a Oct 2017.pdf"); AnalyzeRequest request = uploadFileToStorage("files/syngenta/CustomerFiles/18 Chlorothalonil RAR 08 Volume 3CA B 6a Oct 2017.pdf");
@ -605,9 +614,7 @@ public class RedactionIntegrationTest extends RulesIntegrationTest {
String outputFileName = OsUtils.getTemporaryDirectory() + "/Annotated.pdf"; String outputFileName = OsUtils.getTemporaryDirectory() + "/Annotated.pdf";
ClassPathResource responseJson = new ClassPathResource("files/crafted_document.NER_ENTITIES.json"); ClassPathResource responseJson = new ClassPathResource("files/crafted_document.NER_ENTITIES.json");
storageService.storeObject(TenantContext.getTenantId(), storageService.storeObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.NER_ENTITIES), responseJson.getInputStream());
RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.NER_ENTITIES),
responseJson.getInputStream());
long start = System.currentTimeMillis(); long start = System.currentTimeMillis();
AnalyzeRequest request = uploadFileToStorage(fileName); AnalyzeRequest request = uploadFileToStorage(fileName);
@ -630,8 +637,8 @@ public class RedactionIntegrationTest extends RulesIntegrationTest {
System.out.println("first analysis duration: " + (end - start)); System.out.println("first analysis duration: " + (end - start));
try (FileOutputStream fileOutputStream = new FileOutputStream(OsUtils.getTemporaryDirectory() + "/Test.json")) { try (BufferedWriter writer = new BufferedWriter(new FileWriter(OsUtils.getTemporaryDirectory() + "/Test.json"))) {
fileOutputStream.write(objectMapper.writeValueAsBytes(redactionStorageService.getDocumentData(TEST_DOSSIER_ID, TEST_FILE_ID))); writer.write(getDocumentDataString(redactionStorageService.getDocumentData(TEST_DOSSIER_ID, TEST_FILE_ID)));
} }
List<String> valuesInDocument = entityLog.getEntityLogEntry() List<String> valuesInDocument = entityLog.getEntityLogEntry()
@ -710,6 +717,17 @@ public class RedactionIntegrationTest extends RulesIntegrationTest {
} }
@SneakyThrows
private String getDocumentDataString(DocumentData documentData) {
String result = ProtobufUtil.toJson(documentData.getDocumentPages());
result += ",\n" + ProtobufUtil.toJson(documentData.getDocumentTextData());
result += ",\n" + ProtobufUtil.toJson(documentData.getDocumentPositionData());
result += ",\n" + ProtobufUtil.toJson(documentData.getDocumentStructure());
return result;
}
@Test @Test
public void testRemovePublishedInformations() throws IOException { public void testRemovePublishedInformations() throws IOException {
@ -1186,7 +1204,7 @@ public class RedactionIntegrationTest extends RulesIntegrationTest {
AnalyzeRequest request = uploadFileToStorage("files/ImportedRedactions/RotateTestFile_without_highlights.pdf"); AnalyzeRequest request = uploadFileToStorage("files/ImportedRedactions/RotateTestFile_without_highlights.pdf");
storageService.storeObject(TenantContext.getTenantId(), storageService.storeObject(TenantContext.getTenantId(),
RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.IMPORTED_REDACTIONS), StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.IMPORTED_REDACTIONS),
importedRedactions.getInputStream()); importedRedactions.getInputStream());
analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request); analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request);
@ -1258,7 +1276,7 @@ public class RedactionIntegrationTest extends RulesIntegrationTest {
ClassPathResource imageServiceResponseFileResource = new ClassPathResource("files/new/SYNGENTA_EFSA_sanitisation_GFL_v1 (1).IMAGE_INFO.json"); ClassPathResource imageServiceResponseFileResource = new ClassPathResource("files/new/SYNGENTA_EFSA_sanitisation_GFL_v1 (1).IMAGE_INFO.json");
storageService.storeObject(TenantContext.getTenantId(), storageService.storeObject(TenantContext.getTenantId(),
RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.IMAGE_INFO), StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.IMAGE_INFO),
imageServiceResponseFileResource.getInputStream()); imageServiceResponseFileResource.getInputStream());
System.out.println("Start Full integration test"); System.out.println("Start Full integration test");
@ -1299,7 +1317,7 @@ public class RedactionIntegrationTest extends RulesIntegrationTest {
ClassPathResource imageServiceResponseFileResource = new ClassPathResource("files/new/SYNGENTA_EFSA_sanitisation_GFL_v1 (1).IMAGE_INFO.json"); ClassPathResource imageServiceResponseFileResource = new ClassPathResource("files/new/SYNGENTA_EFSA_sanitisation_GFL_v1 (1).IMAGE_INFO.json");
storageService.storeObject(TenantContext.getTenantId(), storageService.storeObject(TenantContext.getTenantId(),
RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.IMAGE_INFO), StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.IMAGE_INFO),
imageServiceResponseFileResource.getInputStream()); imageServiceResponseFileResource.getInputStream());
System.out.println("Start Full integration test"); System.out.println("Start Full integration test");
@ -1738,9 +1756,7 @@ public class RedactionIntegrationTest extends RulesIntegrationTest {
String fileName = "files/new/crafted document.pdf"; String fileName = "files/new/crafted document.pdf";
ClassPathResource responseJson = new ClassPathResource("files/crafted_document.NER_ENTITIES.json"); ClassPathResource responseJson = new ClassPathResource("files/crafted_document.NER_ENTITIES.json");
storageService.storeObject(TenantContext.getTenantId(), storageService.storeObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.NER_ENTITIES), responseJson.getInputStream());
RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.NER_ENTITIES),
responseJson.getInputStream());
long start = System.currentTimeMillis(); long start = System.currentTimeMillis();
AnalyzeRequest request = uploadFileToStorage(fileName); AnalyzeRequest request = uploadFileToStorage(fileName);
@ -2198,6 +2214,41 @@ public class RedactionIntegrationTest extends RulesIntegrationTest {
} }
@Test
@SneakyThrows
public void testDocumentDataFallback() {
AnalyzeRequest request = uploadFileToStorage("files/new/crafted document.pdf");
analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request);
String oldDocumentStructureStorageId = StorageIdUtils.getStorageId(request.getDossierId(), request.getFileId(), FileType.DOCUMENT_STRUCTURE.name(), ".json");
String newDocumentStructureStorageId = StorageIdUtils.getStorageId(request.getDossierId(), request.getFileId(), FileType.DOCUMENT_STRUCTURE);
String oldDocumentPagesStorageId = StorageIdUtils.getStorageId(request.getDossierId(), request.getFileId(), FileType.DOCUMENT_PAGES.name(), ".json");
String newDocumentPagesStorageId = StorageIdUtils.getStorageId(request.getDossierId(), request.getFileId(), FileType.DOCUMENT_PAGES);
String oldDocumentPositionDataStorageId = StorageIdUtils.getStorageId(request.getDossierId(), request.getFileId(), FileType.DOCUMENT_POSITION.name(), ".json");
String newDocumentPositionDataStorageId = StorageIdUtils.getStorageId(request.getDossierId(), request.getFileId(), FileType.DOCUMENT_POSITION);
String oldDocumentTextDataStorageId = StorageIdUtils.getStorageId(request.getDossierId(), request.getFileId(), FileType.DOCUMENT_TEXT.name(), ".json");
String newDocumentTextDataStorageId = StorageIdUtils.getStorageId(request.getDossierId(), request.getFileId(), FileType.DOCUMENT_TEXT);
storageService.deleteObject(TenantContext.getTenantId(), newDocumentStructureStorageId);
storageService.deleteObject(TenantContext.getTenantId(), newDocumentPagesStorageId);
storageService.deleteObject(TenantContext.getTenantId(), newDocumentPositionDataStorageId);
storageService.deleteObject(TenantContext.getTenantId(), newDocumentTextDataStorageId);
ClassPathResource oldDocumentStructureClassPath = new ClassPathResource("files/oldDocumentData/old.DOCUMENT_STRUCTURE.json");
storageService.storeObject(TenantContext.getTenantId(), oldDocumentStructureStorageId, oldDocumentStructureClassPath.getInputStream());
ClassPathResource oldDocumentPagesClassPath = new ClassPathResource("files/oldDocumentData/old.DOCUMENT_PAGES.json");
storageService.storeObject(TenantContext.getTenantId(), oldDocumentPagesStorageId, oldDocumentPagesClassPath.getInputStream());
ClassPathResource oldDocumentPositionDataClassPath = new ClassPathResource("files/oldDocumentData/old.DOCUMENT_POSITION.json");
storageService.storeObject(TenantContext.getTenantId(), oldDocumentPositionDataStorageId, oldDocumentPositionDataClassPath.getInputStream());
ClassPathResource oldDocumentTextDataClassPath = new ClassPathResource("files/oldDocumentData/old.DOCUMENT_TEXT.json");
storageService.storeObject(TenantContext.getTenantId(), oldDocumentTextDataStorageId, oldDocumentTextDataClassPath.getInputStream());
analyzeService.analyze(request);
}
@Test @Test
public void testFileWithImagesAndNoText() { public void testFileWithImagesAndNoText() {

4
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentIEntityInsertionIntegrationTest.java
View File

@ -287,7 +287,7 @@ public class DocumentIEntityInsertionIntegrationTest extends BuildDocumentIntegr
assertEquals("1 Statement of subject matter and purpose for which this report has been prepared and background information on the application ", assertEquals("1 Statement of subject matter and purpose for which this report has been prepared and background information on the application ",
textEntity.getDeepestFullyContainingNode().getHeadline().getTextBlock().getSearchText()); textEntity.getDeepestFullyContainingNode().getHeadline().getTextBlock().getSearchText());
assertEquals(searchTerm, textEntity.getValue()); assertEquals(searchTerm, textEntity.getValue());
assertEquals(4, textEntity.getIntersectingNodes().size()); assertEquals(3, textEntity.getIntersectingNodes().size());
assertEquals(1, textEntity.getDeepestFullyContainingNode().getNumberOnPage()); assertEquals(1, textEntity.getDeepestFullyContainingNode().getNumberOnPage());
assertTrue(textEntity.getPages() assertTrue(textEntity.getPages()
.stream() .stream()
@ -308,7 +308,7 @@ public class DocumentIEntityInsertionIntegrationTest extends BuildDocumentIntegr
assertEquals("2-[(2-(1-hydroxy-ethyl)-6methyl-phenyl-amino]propan-1-ol (", textEntity.getTextBefore()); assertEquals("2-[(2-(1-hydroxy-ethyl)-6methyl-phenyl-amino]propan-1-ol (", textEntity.getTextBefore());
assertEquals(" of metabolite of", textEntity.getTextAfter()); assertEquals(" of metabolite of", textEntity.getTextAfter());
assertEquals(searchTerm, textEntity.getValue()); assertEquals(searchTerm, textEntity.getValue());
assertEquals(8, textEntity.getIntersectingNodes().size()); assertEquals(7, textEntity.getIntersectingNodes().size());
assertEquals("Table 2.7-1: List of substances and metabolites and related structural formula ", assertEquals("Table 2.7-1: List of substances and metabolites and related structural formula ",
textEntity.getDeepestFullyContainingNode().getHeadline().getTextBlock().getSearchText()); textEntity.getDeepestFullyContainingNode().getHeadline().getTextBlock().getSearchText());
assertTrue(textEntity.getPages() assertTrue(textEntity.getPages()

2
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentPerformanceIntegrationTest.java
View File

@ -184,7 +184,7 @@ public class DocumentPerformanceIntegrationTest extends BuildDocumentIntegration
durationMillis = ((float) (System.currentTimeMillis() - start)); durationMillis = ((float) (System.currentTimeMillis() - start));
System.out.printf("%d calls of buildTextBlock() on section took %f s, average is %f ms\n", n, durationMillis / 1000, durationMillis / n); System.out.printf("%d calls of buildTextBlock() on section took %f s, average is %f ms\n", n, durationMillis / 1000, durationMillis / n);
SemanticNode paragraph = document.getDocumentTree().getEntryById(List.of(3, 2, 4)).getNode(); SemanticNode paragraph = document.getDocumentTree().getEntryById(List.of(8, 1)).getNode();
start = System.currentTimeMillis(); start = System.currentTimeMillis();
for (int i = 0; i < n; i++) { for (int i = 0; i < n; i++) {
paragraph.getTextBlock(); paragraph.getTextBlock();

10
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/service/document/UnprocessedChangesServiceTest.java
View File

@ -158,7 +158,7 @@ public class UnprocessedChangesServiceTest extends AbstractRedactionIntegrationT
UnprocessedManualEntity unprocessedManualEntity = optionalUnprocessedManualEntity.get(); UnprocessedManualEntity unprocessedManualEntity = optionalUnprocessedManualEntity.get();
assertEquals(unprocessedManualEntity.getTextBefore(), "was above the "); assertEquals(unprocessedManualEntity.getTextBefore(), "was above the ");
assertEquals(unprocessedManualEntity.getTextAfter(), " without PPE (34%"); assertEquals(unprocessedManualEntity.getTextAfter(), " without PPE (34%");
assertEquals(unprocessedManualEntity.getSection(), "[1, 1]: Paragraph: A9396G containing 960 g/L"); assertEquals(unprocessedManualEntity.getSection(), "[1, 1, 0]: Paragraph: A9396G containing 960 g/L");
assertEquals(unprocessedManualEntity.getPositions() assertEquals(unprocessedManualEntity.getPositions()
.get(0).x(), 355.53775f); .get(0).x(), 355.53775f);
assertEquals(unprocessedManualEntity.getPositions() assertEquals(unprocessedManualEntity.getPositions()
@ -205,7 +205,7 @@ public class UnprocessedChangesServiceTest extends AbstractRedactionIntegrationT
assertEquals(unprocessedManualEntities.get(0).getAnnotationId(), aoelId); assertEquals(unprocessedManualEntities.get(0).getAnnotationId(), aoelId);
assertEquals(unprocessedManualEntities.get(0).getTextAfter(), " without PPE (34%"); assertEquals(unprocessedManualEntities.get(0).getTextAfter(), " without PPE (34%");
assertEquals(unprocessedManualEntities.get(0).getTextBefore(), "to EFSA guidance "); assertEquals(unprocessedManualEntities.get(0).getTextBefore(), "to EFSA guidance ");
assertEquals(unprocessedManualEntities.get(0).getSection(), "[1, 1]: Paragraph: A9396G containing 960 g/L"); assertEquals(unprocessedManualEntities.get(0).getSection(), "[1, 1, 0]: Paragraph: A9396G containing 960 g/L");
assertEquals(unprocessedManualEntities.get(0).getPositions() assertEquals(unprocessedManualEntities.get(0).getPositions()
.get(0).x(), positions.get(0).getTopLeftX()); .get(0).x(), positions.get(0).getTopLeftX());
assertEquals(unprocessedManualEntities.get(0).getPositions() assertEquals(unprocessedManualEntities.get(0).getPositions()
@ -288,7 +288,7 @@ public class UnprocessedChangesServiceTest extends AbstractRedactionIntegrationT
assertTrue(resizedAoel.isPresent()); assertTrue(resizedAoel.isPresent());
assertEquals(resizedAoel.get().getTextAfter(), " (max. 43% of"); assertEquals(resizedAoel.get().getTextAfter(), " (max. 43% of");
assertEquals(resizedAoel.get().getTextBefore(), "is below the "); assertEquals(resizedAoel.get().getTextBefore(), "is below the ");
assertEquals(resizedAoel.get().getSection(), "[1, 1]: Paragraph: A9396G containing 960 g/L"); assertEquals(resizedAoel.get().getSection(), "[1, 1, 0]: Paragraph: A9396G containing 960 g/L");
assertEquals(resizedAoel.get().getPositions() assertEquals(resizedAoel.get().getPositions()
.get(0).x(), positions.get(0).getTopLeftX()); .get(0).x(), positions.get(0).getTopLeftX());
assertEquals(resizedAoel.get().getPositions() assertEquals(resizedAoel.get().getPositions()
@ -320,7 +320,7 @@ public class UnprocessedChangesServiceTest extends AbstractRedactionIntegrationT
assertTrue(a9Resized.isPresent()); assertTrue(a9Resized.isPresent());
assertEquals(a9Resized.get().getTextAfter(), " were obtained from"); assertEquals(a9Resized.get().getTextAfter(), " were obtained from");
assertEquals(a9Resized.get().getTextBefore(), "data for S"); assertEquals(a9Resized.get().getTextBefore(), "data for S");
assertEquals(a9Resized.get().getSection(), "[1, 1]: Paragraph: A9396G containing 960 g/L"); assertEquals(a9Resized.get().getSection(), "[1, 1, 0]: Paragraph: A9396G containing 960 g/L");
assertEquals(a9Resized.get().getPositions() assertEquals(a9Resized.get().getPositions()
.get(0).x(), positions3.get(0).getTopLeftX()); .get(0).x(), positions3.get(0).getTopLeftX());
assertEquals(a9Resized.get().getPositions() assertEquals(a9Resized.get().getPositions()
@ -370,7 +370,7 @@ public class UnprocessedChangesServiceTest extends AbstractRedactionIntegrationT
assertEquals(unprocessedManualEntities.get(0).getAnnotationId(), aoelId); assertEquals(unprocessedManualEntities.get(0).getAnnotationId(), aoelId);
assertEquals(unprocessedManualEntities.get(0).getTextAfter(), " (max. 43% of"); assertEquals(unprocessedManualEntities.get(0).getTextAfter(), " (max. 43% of");
assertEquals(unprocessedManualEntities.get(0).getTextBefore(), "is below the "); assertEquals(unprocessedManualEntities.get(0).getTextBefore(), "is below the ");
assertEquals(unprocessedManualEntities.get(0).getSection(), "[1, 1]: Paragraph: A9396G containing 960 g/L"); assertEquals(unprocessedManualEntities.get(0).getSection(), "[1, 1, 0]: Paragraph: A9396G containing 960 g/L");
assertEquals(unprocessedManualEntities.get(0).getPositions() assertEquals(unprocessedManualEntities.get(0).getPositions()
.get(0).x(), positions.get(0).getTopLeftX()); .get(0).x(), positions.get(0).getTopLeftX());
assertEquals(unprocessedManualEntities.get(0).getPositions() assertEquals(unprocessedManualEntities.get(0).getPositions()

2
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/acceptance_rules.drl
View File

@ -55,7 +55,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange;
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine; import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngineProto.LayoutEngine;
global Document document global Document document
global EntityCreationService entityCreationService global EntityCreationService entityCreationService

2
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/all_redact_manager_rules.drl
View File

@ -55,7 +55,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange;
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine; import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngineProto.LayoutEngine;
global Document document global Document document
global EntityCreationService entityCreationService global EntityCreationService entityCreationService

2
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora.drl
View File

@ -47,7 +47,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange;
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine; import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngineProto.LayoutEngine;
global Document document global Document document
global EntityCreationService entityCreationService global EntityCreationService entityCreationService

2
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/efsa_sanitisation.drl
View File

@ -55,7 +55,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange;
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine; import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngineProto.LayoutEngine;
global Document document global Document document
global EntityCreationService entityCreationService global EntityCreationService entityCreationService

2
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/manual_redaction_rules.drl
View File

@ -55,7 +55,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange;
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine; import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngineProto.LayoutEngine;
global Document document global Document document
global EntityCreationService entityCreationService global EntityCreationService entityCreationService

2
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl
View File

@ -55,7 +55,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange;
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine; import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngineProto.LayoutEngine;
global Document document global Document document
global EntityCreationService entityCreationService global EntityCreationService entityCreationService

2
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_v2.drl
View File

@ -55,7 +55,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange;
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine; import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngineProto.LayoutEngine;
global Document document global Document document
global EntityCreationService entityCreationService global EntityCreationService entityCreationService

2
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/table_demo.drl
View File

@ -55,7 +55,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange;
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine; import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngineProto.LayoutEngine;
global Document document global Document document
global EntityCreationService entityCreationService global EntityCreationService entityCreationService

2
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/test_rules.drl
View File

@ -55,7 +55,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange;
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine; import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngineProto.LayoutEngine;
global Document document global Document document
global EntityCreationService entityCreationService global EntityCreationService entityCreationService

56
redaction-service-v1/redaction-service-server-v1/src/test/resources/files/oldDocumentData/old.DOCUMENT_PAGES.json Normal file
View File

@ -0,0 +1,56 @@
[
{
"number": 1,
"height": 792,
"width": 612,
"rotation": 0
},
{
"number": 2,
"height": 792,
"width": 612,
"rotation": 0
},
{
"number": 3,
"height": 792,
"width": 612,
"rotation": 0
},
{
"number": 4,
"height": 792,
"width": 612,
"rotation": 0
},
{
"number": 5,
"height": 792,
"width": 612,
"rotation": 0
},
{
"number": 6,
"height": 792,
"width": 612,
"rotation": 0
},
{
"number": 7,
"height": 792,
"width": 612,
"rotation": 0
},
{
"number": 8,
"height": 792,
"width": 612,
"rotation": 0
},
{
"number": 9,
"height": 792,
"width": 612,
"rotation": 0
}
]

1
redaction-service-v1/redaction-service-server-v1/src/test/resources/files/oldDocumentData/old.DOCUMENT_POSITION.json Normal file
View File

File diff suppressed because one or more lines are too long

1
redaction-service-v1/redaction-service-server-v1/src/test/resources/files/oldDocumentData/old.DOCUMENT_STRUCTURE.json Normal file
View File

File diff suppressed because one or more lines are too long

1
redaction-service-v1/redaction-service-server-v1/src/test/resources/files/oldDocumentData/old.DOCUMENT_TEXT.json Normal file
View File

File diff suppressed because one or more lines are too long

2
redaction-service-v1/rules-management/src/main/resources/all_redact_manager_rules.drl
View File

@ -55,7 +55,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange;
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine; import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngineProto.LayoutEngine;
global Document document global Document document
global EntityCreationService entityCreationService global EntityCreationService entityCreationService

2
redaction-service-v1/rules-management/src/main/resources/all_rules_documine.drl
View File

@ -47,7 +47,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange;
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine; import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngineProto.LayoutEngine;
global Document document global Document document
global EntityCreationService entityCreationService global EntityCreationService entityCreationService