evaluation whether ids stay the same when switching from json to protobuf

* improved mongodb save/delete logic

redaction-service-v1/redaction-service-api-v1/build.gradle.kts

@@ -4,7 +4,7 @@ plugins {
 }
 
 description = "redaction-service-api-v1"
-val persistenceServiceVersion = "2.473.0"
+val persistenceServiceVersion = "RED-9123"
 
 dependencies {
     implementation("org.springframework:spring-web:6.0.12")

redaction-service-v1/redaction-service-server-v1/build.gradle.kts

@@ -12,11 +12,11 @@ plugins {
 description = "redaction-service-server-v1"
 
 
-val layoutParserVersion = "0.141.0"
+val layoutParserVersion = "RED-9123"
 val jacksonVersion = "2.15.2"
 val droolsVersion = "9.44.0.Final"
 val pdfBoxVersion = "3.0.0"
-val persistenceServiceVersion = "2.473.0"
+val persistenceServiceVersion = "RED-9123"
 val springBootStarterVersion = "3.1.5"
 val springCloudVersion = "4.0.4"
 val testContainersVersion = "1.19.7"
@@ -30,6 +30,12 @@ configurations {
     }
 }
 
+configurations.all {
+    resolutionStrategy {
+        force("com.google.protobuf:protobuf-java:4.27.1")
+    }
+}
+
 dependencies {
 
     implementation(project(":redaction-service-api-v1")) { exclude(group = "com.iqser.red.service", module = "persistence-service-internal-api-v1") }
@@ -41,7 +47,7 @@ dependencies {
     implementation("com.iqser.red.commons:metric-commons:2.3.0")
 
     implementation("com.iqser.red.commons:dictionary-merge-commons:1.5.0")
-    implementation("com.iqser.red.commons:storage-commons:2.45.0")
+    implementation("com.iqser.red.commons:storage-commons:RED-9123")
     implementation("com.knecon.fforesight:keycloak-commons:0.29.0")
     implementation("com.knecon.fforesight:tenant-commons:0.25.0")
     implementation("com.knecon.fforesight:tracing-commons:0.5.0")
@@ -54,6 +60,7 @@ dependencies {
     implementation("org.drools:drools-engine:${droolsVersion}")
     implementation("org.drools:drools-mvel:${droolsVersion}")
     implementation("org.kie:kie-spring:7.74.1.Final")
+    implementation("com.google.protobuf:protobuf-java:4.27.1")
 
     implementation("org.locationtech.jts:jts-core:1.19.0")
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/DocumentData.java

@@ -1,11 +1,13 @@
 package com.iqser.red.service.redaction.v1.server.model.document;
 
+import static com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentStructureProto.DocumentStructure;
+
 import java.io.Serializable;
 
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPage;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPageProto.AllDocumentPages;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPositionData;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPositionDataProto.AllDocumentPositionData;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentStructure;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentStructureWrapper;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentTextData;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentTextDataProto.AllDocumentTextData;
 
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
@@ -21,9 +23,15 @@ import lombok.experimental.FieldDefaults;
 @FieldDefaults(level = AccessLevel.PRIVATE)
 public class DocumentData implements Serializable {
 
-    DocumentPage[] documentPages;
+    AllDocumentPages documentPages;
-    DocumentTextData[] documentTextData;
+    AllDocumentTextData documentTextData;
-    DocumentPositionData[] documentPositionData;
+    AllDocumentPositionData documentPositionData;
-    DocumentStructure documentStructure;
+    DocumentStructureWrapper documentStructureWrapper;
+
+
+    public DocumentStructure getDocumentStructure() {
+
+        return documentStructureWrapper.getDocumentStructure();
+    }
 
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/AbstractSemanticNode.java

@@ -9,7 +9,7 @@ import java.util.Set;
 import com.iqser.red.service.redaction.v1.server.model.document.DocumentTree;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngineProto.LayoutEngine;
 
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Footer.java

@@ -1,19 +1,9 @@
 package com.iqser.red.service.redaction.v1.server.model.document.nodes;
 
-import java.awt.geom.Rectangle2D;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-import com.iqser.red.service.redaction.v1.server.model.document.DocumentTree;
-import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
 
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
-import lombok.Builder;
 import lombok.Data;
 import lombok.EqualsAndHashCode;
 import lombok.NoArgsConstructor;

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Header.java

@@ -1,19 +1,9 @@
 package com.iqser.red.service.redaction.v1.server.model.document.nodes;
 
-import java.awt.geom.Rectangle2D;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
-import com.iqser.red.service.redaction.v1.server.model.document.DocumentTree;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
 
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
-import lombok.Builder;
 import lombok.Data;
 import lombok.EqualsAndHashCode;
 import lombok.NoArgsConstructor;

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Headline.java

@@ -1,20 +1,10 @@
 package com.iqser.red.service.redaction.v1.server.model.document.nodes;
 
-import java.awt.geom.Rectangle2D;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-import com.iqser.red.service.redaction.v1.server.model.document.DocumentTree;
-import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.AtomicTextBlock;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
 
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
-import lombok.Builder;
 import lombok.Data;
 import lombok.EqualsAndHashCode;
 import lombok.NoArgsConstructor;

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Image.java

@@ -3,22 +3,16 @@ package com.iqser.red.service.redaction.v1.server.model.document.nodes;
 import java.awt.geom.Rectangle2D;
 import java.util.Collections;
 import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
 import java.util.Locale;
 import java.util.Map;
 import java.util.PriorityQueue;
 import java.util.Set;
 
-import com.iqser.red.service.redaction.v1.server.model.document.DocumentTree;
 import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.IEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.ManualChangeOverwrite;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule;
-import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
-import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlockCollector;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
 
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Paragraph.java

@@ -1,19 +1,9 @@
 package com.iqser.red.service.redaction.v1.server.model.document.nodes;
 
-import java.awt.geom.Rectangle2D;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-import com.iqser.red.service.redaction.v1.server.model.document.DocumentTree;
-import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
 
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
-import lombok.Builder;
 import lombok.Data;
 import lombok.EqualsAndHashCode;
 import lombok.experimental.FieldDefaults;

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Section.java

@@ -1,23 +1,9 @@
 package com.iqser.red.service.redaction.v1.server.model.document.nodes;
 
-import java.awt.geom.Rectangle2D;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
-import com.iqser.red.service.redaction.v1.server.model.document.DocumentTree;
-import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
-import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlockCollector;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
-
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
-import lombok.Builder;
 import lombok.Data;
 import lombok.EqualsAndHashCode;
-import lombok.RequiredArgsConstructor;
 import lombok.experimental.FieldDefaults;
 import lombok.experimental.SuperBuilder;
 import lombok.extern.slf4j.Slf4j;

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/SemanticNode.java

@@ -23,7 +23,7 @@ import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBl
 import com.iqser.red.service.redaction.v1.server.service.document.NodeVisitor;
 import com.iqser.red.service.redaction.v1.server.utils.RectangleTransformations;
 import com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngineProto.LayoutEngine;
 
 public interface SemanticNode {
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Table.java

@@ -17,7 +17,7 @@ import com.iqser.red.service.redaction.v1.server.model.document.DocumentTree;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlockCollector;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngineProto.LayoutEngine;
 
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/TableCell.java

@@ -2,20 +2,13 @@ package com.iqser.red.service.redaction.v1.server.model.document.nodes;
 
 import java.awt.geom.Rectangle2D;
 import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
 import java.util.Map;
-import java.util.Set;
 
-import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
-import com.iqser.red.service.redaction.v1.server.model.document.DocumentTree;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlockCollector;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
 
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
-import lombok.Builder;
 import lombok.Data;
 import lombok.EqualsAndHashCode;
 import lombok.experimental.FieldDefaults;

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/textblock/AtomicTextBlock.java

@@ -1,11 +1,11 @@
 package com.iqser.red.service.redaction.v1.server.model.document.textblock;
 
+import static com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPositionDataProto.DocumentPositionData;
 import static java.lang.String.format;
 
 import java.awt.geom.Rectangle2D;
 import java.text.BreakIterator;
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
@@ -19,8 +19,8 @@ import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Page;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
 import com.iqser.red.service.redaction.v1.server.utils.RectangleTransformations;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPositionData;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPositionDataProto.DocumentPositionData.Position;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentTextData;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentTextDataProto.DocumentTextData;
 
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
@@ -86,20 +86,18 @@ public class AtomicTextBlock implements TextBlock {
                 .page(page)
                 .textRange(new TextRange(atomicTextBlockData.getStart(), atomicTextBlockData.getEnd()))
                 .searchText(atomicTextBlockData.getSearchText())
-                .lineBreaks(Arrays.stream(atomicTextBlockData.getLineBreaks()).boxed()
+                .lineBreaks(atomicTextBlockData.getLineBreaksList())
-                                    .toList())
+                .stringIdxToPositionIdx(atomicPositionBlockData.getStringIdxToPositionIdxList())
-                .stringIdxToPositionIdx(Arrays.stream(atomicPositionBlockData.getStringIdxToPositionIdx()).boxed()
+                .positions(toRectangle2DList(atomicPositionBlockData.getPositionsList()))
-                                                .toList())
-                .positions(toRectangle2DList(atomicPositionBlockData.getPositions()))
                 .parent(parent)
                 .build();
     }
 
 
-    private static List<Rectangle2D> toRectangle2DList(float[][] positions) {
+    private static List<Rectangle2D> toRectangle2DList(List<Position> positions) {
 
-        return Arrays.stream(positions)
+        return positions.stream()
-                .map(floatArr -> (Rectangle2D) new Rectangle2D.Float(floatArr[0], floatArr[1], floatArr[2], floatArr[3]))
+                .map(pos -> (Rectangle2D) new Rectangle2D.Float(pos.getX(), pos.getY(), pos.getWidth(), pos.getHeight()))
                 .toList();
     }
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalysisFinalizationService.java

@@ -0,0 +1,136 @@
+package com.iqser.red.service.redaction.v1.server.service;
+
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.CompletableFuture;
+
+import org.springframework.stereotype.Service;
+
+import com.iqser.gin4.commons.metrics.meters.FunctionTimerValues;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeResult;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentLog;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogChanges;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
+import com.iqser.red.service.redaction.v1.server.RedactionServiceSettings;
+import com.iqser.red.service.redaction.v1.server.logger.Context;
+import com.iqser.red.service.redaction.v1.server.model.KieWrapper;
+import com.iqser.red.service.redaction.v1.server.model.component.Component;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
+import com.iqser.red.service.redaction.v1.server.service.components.ComponentLogCreatorService;
+import com.iqser.red.service.redaction.v1.server.service.drools.ComponentDroolsExecutionService;
+import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
+
+import io.micrometer.core.annotation.Timed;
+import io.micrometer.observation.annotation.Observed;
+import lombok.AccessLevel;
+import lombok.RequiredArgsConstructor;
+import lombok.experimental.FieldDefaults;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+@Service
+@RequiredArgsConstructor
+@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
+public class AnalysisFinalizationService {
+
+    RedactionStorageService redactionStorageService;
+    FunctionTimerValues redactmanagerAnalyzePagewiseValues;
+    ComponentLogCreatorService componentLogCreatorService;
+    RedactionServiceSettings redactionServiceSettings;
+    ComponentDroolsExecutionService componentDroolsExecutionService;
+
+
+    @Timed("redactmanager_finalizeAnalysis")
+    @Observed(name = "AnalysisFinalizationService", contextualName = "finalize-analysis")
+    public AnalyzeResult finalizeAnalysis(AnalyzeRequest analyzeRequest,
+                                          long startTime,
+                                          KieWrapper kieWrapperComponentRules,
+                                          EntityLogChanges entityLogChanges,
+                                          Document document,
+                                          int numberOfPages,
+                                          boolean isReanalysis,
+                                          Set<FileAttribute> addedFileAttributes,
+                                          Context context) {
+
+        EntityLog entityLog = entityLogChanges.getEntityLog();
+
+        // as workaround for duplicate key exceptions occurring due to simultaneous analyses and reanalyses save instead of insert is used
+        // also analysis numbers should be incremented in every follow-up request, so checking if the log exists is not needed
+        if (!redactionStorageService.entityLogExists(analyzeRequest.getDossierId(), analyzeRequest.getFileId())) {
+            redactionStorageService.saveEntityLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), entityLog);
+
+        } else {
+            redactionStorageService.updateEntityLogWithoutEntries(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), entityLog);
+
+            if (!entityLogChanges.getNewEntityLogEntries().isEmpty()) {
+                redactionStorageService.saveEntityLogEntries(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), entityLogChanges.getNewEntityLogEntries());
+            }
+            if (!entityLogChanges.getUpdatedEntityLogEntries().isEmpty()) {
+                redactionStorageService.updateEntityLogEntries(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), entityLogChanges.getUpdatedEntityLogEntries());
+            }
+        }
+
+        log.info("Created entity log for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
+
+
+        computeComponentsWhenRulesArePresent(analyzeRequest, kieWrapperComponentRules, document, addedFileAttributes, entityLog, context);
+
+        long duration = System.currentTimeMillis() - startTime;
+
+        redactmanagerAnalyzePagewiseValues.increase(numberOfPages, duration);
+
+        return AnalyzeResult.builder()
+                .dossierId(analyzeRequest.getDossierId())
+                .fileId(analyzeRequest.getFileId())
+                .duration(duration)
+                .numberOfPages(numberOfPages)
+                .hasUpdates(entityLogChanges.hasChanges())
+                .analysisVersion(redactionServiceSettings.getAnalysisVersion())
+                .analysisNumber(analyzeRequest.getAnalysisNumber())
+                .rulesVersion(entityLog.getRulesVersion())
+                .componentRulesVersion(kieWrapperComponentRules.rulesVersion())
+                .dictionaryVersion(entityLog.getDictionaryVersion())
+                .legalBasisVersion(entityLog.getLegalBasisVersion())
+                .dossierDictionaryVersion(entityLog.getDossierDictionaryVersion())
+                .wasReanalyzed(isReanalysis)
+                .manualRedactions(analyzeRequest.getManualRedactions())
+                .addedFileAttributes(addedFileAttributes)
+                .usedComponentMappings(analyzeRequest.getComponentMappings())
+                .build();
+    }
+
+
+    private void computeComponentsWhenRulesArePresent(AnalyzeRequest analyzeRequest,
+                                                      KieWrapper kieWrapperComponentRules,
+                                                      Document document,
+                                                      Set<FileAttribute> addedFileAttributes,
+                                                      EntityLog entityLog,
+                                                      Context context) {
+
+        if (!kieWrapperComponentRules.isPresent()) {
+            return;
+        }
+
+        // We need the latest EntityLog entries for components rules execution
+        entityLog.setEntityLogEntry(redactionStorageService.getEntityLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId()).getEntityLogEntry());
+
+        List<Component> components = componentDroolsExecutionService.executeRules(kieWrapperComponentRules.container(),
+                                                                                  entityLog,
+                                                                                  document,
+                                                                                  addedFileAttributes,
+                                                                                  analyzeRequest.getComponentMappings(),
+                                                                                  context);
+
+        log.info("Finished component rule execution for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
+
+        ComponentLog componentLog = componentLogCreatorService.buildComponentLog(analyzeRequest.getAnalysisNumber(), components, kieWrapperComponentRules.rulesVersion());
+
+        redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.COMPONENT_LOG, componentLog);
+
+        log.info("Stored component log for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalysisPreparationService.java

@@ -37,6 +37,8 @@ import com.iqser.red.service.redaction.v1.server.service.drools.KieContainerCrea
 import com.iqser.red.service.redaction.v1.server.storage.ObservedStorageService;
 import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
 
+import io.micrometer.core.annotation.Timed;
+import io.micrometer.observation.annotation.Observed;
 import lombok.AccessLevel;
 import lombok.SneakyThrows;
 import lombok.experimental.FieldDefaults;
@@ -81,6 +83,8 @@ public class AnalysisPreparationService {
 
 
     @SneakyThrows
+    @Timed("redactmanager_getAnalysisData")
+    @Observed(name = "AnalysisPreparationService", contextualName = "get-analysis-data")
     public AnalysisData getAnalysisData(AnalyzeRequest analyzeRequest) {
 
         CompletableFuture<KieWrapper> kieWrapperComponentRulesFuture = CompletableFuture.supplyAsync(() -> getKieWrapper(analyzeRequest, RuleFileType.COMPONENT), taskExecutor);
@@ -118,6 +122,8 @@ public class AnalysisPreparationService {
 
 
     @SneakyThrows
+    @Timed("redactmanager_getReanalysisSetupData")
+    @Observed(name = "AnalysisPreparationService", contextualName = "get-reanalysis-setup-data")
     public ReanalysisSetupData getReanalysisSetupData(AnalyzeRequest analyzeRequest) {
 
         CompletableFuture<EntityLog> entityLogFuture = CompletableFuture.supplyAsync(() -> getEntityLog(analyzeRequest), taskExecutor);
@@ -131,6 +137,8 @@ public class AnalysisPreparationService {
 
 
     @SneakyThrows
+    @Timed("redactmanager_getReanalysisInitialProcessingData")
+    @Observed(name = "AnalysisPreparationService", contextualName = "get-reanalysis-initial-processing-data")
     public ReanalysisInitialProcessingData getReanalysisInitialProcessingData(AnalyzeRequest analyzeRequest, ReanalysisSetupData reanalysisSetupData) {
 
         CompletableFuture<ImportedRedactions> importedRedactionsFuture = CompletableFuture.supplyAsync(() -> getImportedRedactions(analyzeRequest), taskExecutor);
@@ -157,6 +165,8 @@ public class AnalysisPreparationService {
 
 
     @SneakyThrows
+    @Timed("redactmanager_getReanalysisFinalProcessingData")
+    @Observed(name = "AnalysisPreparationService", contextualName = "get-reanalysis-final-processing-data")
     public ReanalysisFinalProcessingData getReanalysisFinalProcessingData(AnalyzeRequest analyzeRequest,
                                                                           ReanalysisSetupData reanalysisSetupData,
                                                                           ReanalysisInitialProcessingData reanalysisInitialProcessingData) {
@@ -181,7 +191,7 @@ public class AnalysisPreparationService {
     }
 
 
-    public KieWrapper getKieWrapper(AnalyzeRequest analyzeRequest, RuleFileType ruleFileType) {
+    private KieWrapper getKieWrapper(AnalyzeRequest analyzeRequest, RuleFileType ruleFileType) {
 
         KieWrapper kieWrapperComponentRules = kieContainerCreationService.getLatestKieContainer(analyzeRequest.getDossierTemplateId(), ruleFileType);
         log.info("Updated {} Rules to Version {} for file {} in dossier {}",
@@ -193,7 +203,7 @@ public class AnalysisPreparationService {
     }
 
 
-    public Document getDocument(AnalyzeRequest analyzeRequest) {
+    private Document getDocument(AnalyzeRequest analyzeRequest) {
 
         Document document = DocumentGraphMapper.toDocumentGraph(observedStorageService.getDocumentData(analyzeRequest.getDossierId(), analyzeRequest.getFileId()));
         log.info("Loaded Document Graph for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
@@ -201,7 +211,7 @@ public class AnalysisPreparationService {
     }
 
 
-    public ImportedRedactions getImportedRedactions(AnalyzeRequest analyzeRequest) {
+    private ImportedRedactions getImportedRedactions(AnalyzeRequest analyzeRequest) {
 
         ImportedRedactions importedRedactions = redactionStorageService.getImportedRedactions(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
         log.info("Loaded Imported Redactions for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
@@ -209,7 +219,7 @@ public class AnalysisPreparationService {
     }
 
 
-    public NerEntities getNerEntities(AnalyzeRequest analyzeRequest, Document document) {
+    private NerEntities getNerEntities(AnalyzeRequest analyzeRequest, Document document) {
 
         NerEntities nerEntities = getEntityRecognitionEntities(analyzeRequest, document);
         log.info("Loaded Ner Entities for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
@@ -217,7 +227,7 @@ public class AnalysisPreparationService {
     }
 
 
-    public NerEntities getNerEntitiesFiltered(AnalyzeRequest analyzeRequest, Document document, Set<Integer> sectionsToReanalyseIds) {
+    private NerEntities getNerEntitiesFiltered(AnalyzeRequest analyzeRequest, Document document, Set<Integer> sectionsToReanalyseIds) {
 
         NerEntities nerEntities = getEntityRecognitionEntitiesFilteredBySectionIds(analyzeRequest, document, sectionsToReanalyseIds);
         log.info("Loaded Ner Entities for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalyzeService.java

@@ -9,31 +9,19 @@ import java.util.Collection;
 import java.util.Collections;
 import java.util.HashSet;
 import java.util.List;
-import java.util.Set;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
 import org.springframework.stereotype.Service;
 import org.springframework.web.bind.annotation.RequestBody;
 
-import com.iqser.gin4.commons.metrics.meters.FunctionTimerValues;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeResult;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentLog;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogChanges;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
-import com.iqser.red.service.redaction.v1.server.RedactionServiceSettings;
 import com.iqser.red.service.redaction.v1.server.logger.Context;
-import com.iqser.red.service.redaction.v1.server.model.KieWrapper;
-import com.iqser.red.service.redaction.v1.server.model.component.Component;
-import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
 import com.iqser.red.service.redaction.v1.server.service.AnalysisPreparationService.AnalysisData;
-import com.iqser.red.service.redaction.v1.server.service.components.ComponentLogCreatorService;
-import com.iqser.red.service.redaction.v1.server.service.drools.ComponentDroolsExecutionService;
 import com.iqser.red.service.redaction.v1.server.service.drools.EntityDroolsExecutionService;
-import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
 import com.knecon.fforesight.tenantcommons.TenantContext;
 
 import io.micrometer.core.annotation.Timed;
@@ -51,15 +39,11 @@ import lombok.extern.slf4j.Slf4j;
 public class AnalyzeService {
 
     EntityDroolsExecutionService entityDroolsExecutionService;
-    ComponentDroolsExecutionService componentDroolsExecutionService;
     DictionarySearchService dictionarySearchService;
     EntityLogCreatorService entityLogCreatorService;
-    ComponentLogCreatorService componentLogCreatorService;
-    RedactionStorageService redactionStorageService;
-    RedactionServiceSettings redactionServiceSettings;
     NotFoundImportedEntitiesService notFoundImportedEntitiesService;
-    FunctionTimerValues redactmanagerAnalyzePagewiseValues;
     AnalysisPreparationService analysisPreparationService;
+    AnalysisFinalizationService analysisFinalizationService;
 
 
     @Timed("redactmanager_reanalyze")
@@ -92,7 +76,7 @@ public class AnalyzeService {
                                                                                                        new ArrayList<>(),
                                                                                                        new ArrayList<>());
 
-            return finalizeAnalysis(analyzeRequest,
+            return analysisFinalizationService.finalizeAnalysis(analyzeRequest,
                                                                 startTime,
                                                                 initialProcessingData.kieWrapperComponentRules(),
                                                                 entityLogChanges,
@@ -134,7 +118,7 @@ public class AnalyzeService {
 
         notFoundImportedEntitiesService.processEntityLog(entityLogChanges.getEntityLog(), analyzeRequest, finalProcessingData.notFoundImportedEntries());
 
-        return finalizeAnalysis(analyzeRequest,
+        return analysisFinalizationService.finalizeAnalysis(analyzeRequest,
                                                             startTime,
                                                             initialProcessingData.kieWrapperComponentRules(),
                                                             entityLogChanges,
@@ -187,7 +171,7 @@ public class AnalyzeService {
 
         notFoundImportedEntitiesService.processEntityLog(entityLogChanges.getEntityLog(), analyzeRequest, analysisData.notFoundImportedEntries());
 
-        return finalizeAnalysis(analyzeRequest,
+        return analysisFinalizationService.finalizeAnalysis(analyzeRequest,
                                                             startTime,
                                                             analysisData.kieWrapperComponentRules(),
                                                             entityLogChanges,
@@ -198,92 +182,4 @@ public class AnalyzeService {
                                                             context);
     }
 
-
-    private AnalyzeResult finalizeAnalysis(AnalyzeRequest analyzeRequest,
-                                           long startTime,
-                                           KieWrapper kieWrapperComponentRules,
-                                           EntityLogChanges entityLogChanges,
-                                           Document document,
-                                           int numberOfPages,
-                                           boolean isReanalysis,
-                                           Set<FileAttribute> addedFileAttributes,
-                                           Context context) {
-
-        EntityLog entityLog = entityLogChanges.getEntityLog();
-
-        // as workaround for duplicate key exceptions occurring due to simultaneous analyses and reanalyses save instead of insert is used
-        // also analysis numbers should be incremented in every follow-up request, so checking if the log exists is not needed
-        if (!redactionStorageService.entityLogExists(analyzeRequest.getDossierId(), analyzeRequest.getFileId())) {
-            redactionStorageService.saveEntityLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), entityLog);
-
-        } else {
-            redactionStorageService.updateEntityLogWithoutEntries(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), entityLog);
-
-            if (!entityLogChanges.getNewEntityLogEntries().isEmpty()) {
-                redactionStorageService.saveEntityLogEntries(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), entityLogChanges.getNewEntityLogEntries());
-            }
-            if (!entityLogChanges.getUpdatedEntityLogEntries().isEmpty()) {
-                redactionStorageService.updateEntityLogEntries(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), entityLogChanges.getUpdatedEntityLogEntries());
-            }
-        }
-
-        log.info("Created entity log for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
-
-        computeComponentsWhenRulesArePresent(analyzeRequest, kieWrapperComponentRules, document, addedFileAttributes, entityLog, context);
-
-        long duration = System.currentTimeMillis() - startTime;
-
-        redactmanagerAnalyzePagewiseValues.increase(numberOfPages, duration);
-
-        return AnalyzeResult.builder()
-                .dossierId(analyzeRequest.getDossierId())
-                .fileId(analyzeRequest.getFileId())
-                .duration(duration)
-                .numberOfPages(numberOfPages)
-                .hasUpdates(entityLogChanges.hasChanges())
-                .analysisVersion(redactionServiceSettings.getAnalysisVersion())
-                .analysisNumber(analyzeRequest.getAnalysisNumber())
-                .rulesVersion(entityLog.getRulesVersion())
-                .componentRulesVersion(kieWrapperComponentRules.rulesVersion())
-                .dictionaryVersion(entityLog.getDictionaryVersion())
-                .legalBasisVersion(entityLog.getLegalBasisVersion())
-                .dossierDictionaryVersion(entityLog.getDossierDictionaryVersion())
-                .wasReanalyzed(isReanalysis)
-                .manualRedactions(analyzeRequest.getManualRedactions())
-                .addedFileAttributes(addedFileAttributes)
-                .usedComponentMappings(analyzeRequest.getComponentMappings())
-                .build();
-    }
-
-
-    private void computeComponentsWhenRulesArePresent(AnalyzeRequest analyzeRequest,
-                                                      KieWrapper kieWrapperComponentRules,
-                                                      Document document,
-                                                      Set<FileAttribute> addedFileAttributes,
-                                                      EntityLog entityLog,
-                                                      Context context) {
-
-        if (!kieWrapperComponentRules.isPresent()) {
-            return;
-        }
-
-        // We need the latest EntityLog entries for components rules execution
-        entityLog.setEntityLogEntry(redactionStorageService.getEntityLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId()).getEntityLogEntry());
-
-        List<Component> components = componentDroolsExecutionService.executeRules(kieWrapperComponentRules.container(),
-                                                                                  entityLog,
-                                                                                  document,
-                                                                                  addedFileAttributes,
-                                                                                  analyzeRequest.getComponentMappings(),
-                                                                                  context);
-
-        log.info("Finished component rule execution for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
-
-        ComponentLog componentLog = componentLogCreatorService.buildComponentLog(analyzeRequest.getAnalysisNumber(), components, kieWrapperComponentRules.rulesVersion());
-
-        redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.COMPONENT_LOG, componentLog);
-
-        log.info("Stored component log for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
-    }
-
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/DictionarySearchService.java

@@ -39,14 +39,27 @@ public class DictionarySearchService {
     @Observed(name = "DictionarySearchService", contextualName = "add-dictionary-entries")
     public void addDictionaryEntities(Dictionary dictionary, SemanticNode node) {
 
-        for (DictionaryModel model : dictionary.getDictionaryModels()) {
+        dictionary.getDictionaryModels()
-            bySearchImplementationAsDictionary(model.getEntriesSearch(), model.getType(), model.isHint() ? EntityType.HINT : EntityType.ENTITY, node, model.isDossierDictionary());
+                .stream()
+                .parallel()
+                .forEach(model -> {
+                    synchronized (node) {
+                        bySearchImplementationAsDictionary(model.getEntriesSearch(),
+                                                           model.getType(),
+                                                           model.isHint() ? EntityType.HINT : EntityType.ENTITY,
+                                                           node,
+                                                           model.isDossierDictionary());
                         bySearchImplementationAsDictionary(model.getFalsePositiveSearch(), model.getType(), EntityType.FALSE_POSITIVE, node, model.isDossierDictionary());
-            bySearchImplementationAsDictionary(model.getFalseRecommendationsSearch(), model.getType(), EntityType.FALSE_RECOMMENDATION, node, model.isDossierDictionary());
+                        bySearchImplementationAsDictionary(model.getFalseRecommendationsSearch(),
+                                                           model.getType(),
+                                                           EntityType.FALSE_RECOMMENDATION,
+                                                           node,
+                                                           model.isDossierDictionary());
                         if (model.isDossierDictionary()) {
                             bySearchImplementationAsDictionary(model.getDeletionEntriesSearch(), model.getType(), EntityType.DICTIONARY_REMOVAL, node, model.isDossierDictionary());
                         }
                     }
+                });
     }
 
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityChangeLogService.java

@@ -22,6 +22,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
 import com.iqser.red.service.redaction.v1.server.utils.EntityLogEntryDiffChecker;
 
 import io.micrometer.core.annotation.Timed;
+import io.micrometer.observation.annotation.Observed;
 import lombok.AccessLevel;
 import lombok.RequiredArgsConstructor;
 import lombok.experimental.FieldDefaults;
@@ -34,6 +35,7 @@ import lombok.extern.slf4j.Slf4j;
 public class EntityChangeLogService {
 
     @Timed("redactmanager_computeChanges")
+    @Observed(name = "EntityChangeLogService", contextualName = "compute-changes")
     public EntryChanges computeChanges(List<EntityLogEntry> previousEntityLogEntries, List<EntityLogEntry> newEntityLogEntries, int analysisNumber) {
 
         var now = OffsetDateTime.now();

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java

@@ -2,12 +2,16 @@ package com.iqser.red.service.redaction.v1.server.service;
 
 import java.util.ArrayList;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Locale;
+import java.util.Map;
 import java.util.Optional;
 import java.util.Set;
+import java.util.concurrent.atomic.AtomicReference;
 import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
 import org.springframework.stereotype.Service;
 
@@ -43,6 +47,10 @@ import com.iqser.red.service.redaction.v1.server.service.EntityChangeLogService.
 import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
 import com.iqser.red.service.redaction.v1.server.utils.ManualChangesUtils;
 
+import io.micrometer.core.annotation.Timed;
+import io.micrometer.observation.Observation;
+import io.micrometer.observation.ObservationRegistry;
+import io.micrometer.observation.annotation.Observed;
 import lombok.AccessLevel;
 import lombok.RequiredArgsConstructor;
 import lombok.experimental.FieldDefaults;
@@ -60,6 +68,7 @@ public class EntityLogCreatorService {
     EntityChangeLogService entityChangeLogService;
     RedactionStorageService redactionStorageService;
     EntityLogMongoService entityLogMongoService;
+    ObservationRegistry observationRegistry;
 
 
     private static boolean notFalsePositiveOrFalseRecommendationOrRemoval(TextEntity textEntity) {
@@ -70,6 +79,8 @@ public class EntityLogCreatorService {
     }
 
 
+    @Timed("redactmanager_createInitialEntityLog")
+    @Observed(name = "EntityLogCreatorService", contextualName = "create-initial-entity-log")
     public EntityLogChanges createInitialEntityLog(AnalyzeRequest analyzeRequest,
                                                    Document document,
                                                    List<PrecursorEntity> notFoundEntities,
@@ -109,6 +120,8 @@ public class EntityLogCreatorService {
     }
 
 
+    @Timed("redactmanager_updateVersionsAndReturnChanges")
+    @Observed(name = "EntityLogCreatorService", contextualName = "update-versions-and-return-changes")
     public EntityLogChanges updateVersionsAndReturnChanges(EntityLog entityLog,
                                                            DictionaryVersion dictionaryVersion,
                                                            AnalyzeRequest analyzeRequest,
@@ -126,6 +139,8 @@ public class EntityLogCreatorService {
     }
 
 
+    @Timed("redactmanager_updatePreviousEntityLog")
+    @Observed(name = "EntityLogCreatorService", contextualName = "update-previous-entity-log")
     public EntityLogChanges updatePreviousEntityLog(AnalyzeRequest analyzeRequest,
                                                     Document document,
                                                     EntityLog entityLogWithoutEntries,
@@ -133,16 +148,23 @@ public class EntityLogCreatorService {
                                                     Set<Integer> sectionsToReanalyseIds,
                                                     DictionaryVersion dictionaryVersion) {
 
-        List<EntityLogEntry> newEntityLogEntries = createEntityLogEntries(document, analyzeRequest, notFoundEntries, analyzeRequest.getAnalysisNumber()).stream()
+        AtomicReference<List<EntityLogEntry>> newEntityLogEntriesReference = new AtomicReference<>();
-                .filter(entry -> entry.getContainingNodeId().isEmpty() || sectionsToReanalyseIds.contains(entry.getContainingNodeId()
+
+        Observation.createNotStarted("EntityLogCreatorService", observationRegistry)
+                .contextualName("create-new-entity-log-entries")
+                .observe(() -> newEntityLogEntriesReference.set(createEntityLogEntries(document, analyzeRequest, notFoundEntries, analyzeRequest.getAnalysisNumber()).stream()
+                                                                        .filter(entry -> entry.getContainingNodeId().isEmpty()
+                                                                                         || sectionsToReanalyseIds.contains(entry.getContainingNodeId()
                                                                                                                                     .get(0)))
-                .collect(Collectors.toList());
+                                                                        .collect(Collectors.toList())));
 
         List<EntityLogEntry> previousEntriesFromReAnalyzedSections = redactionStorageService.findEntriesContainedBySectionsOrNotContained(analyzeRequest.getDossierId(),
                                                                                                                                           analyzeRequest.getFileId(),
                                                                                                                                           sectionsToReanalyseIds);
 
-        EntryChanges entryChanges = entityChangeLogService.computeChanges(previousEntriesFromReAnalyzedSections, newEntityLogEntries, analyzeRequest.getAnalysisNumber());
+        EntryChanges entryChanges = entityChangeLogService.computeChanges(previousEntriesFromReAnalyzedSections,
+                                                                          newEntityLogEntriesReference.get(),
+                                                                          analyzeRequest.getAnalysisNumber());
 
         return updateVersionsAndReturnChanges(entityLogWithoutEntries, dictionaryVersion, analyzeRequest, entryChanges.inserted(), entryChanges.updated());
     }
@@ -154,26 +176,47 @@ public class EntityLogCreatorService {
 
         List<EntityLogEntry> entries = new ArrayList<>();
 
-        document.getEntities()
+        List<TextEntity> textEntities = document.getEntities()
                 .stream()
                 .filter(entity -> !entity.getValue().isEmpty())
                 .filter(EntityLogCreatorService::notFalsePositiveOrFalseRecommendationOrRemoval)
                 .filter(entity -> !entity.removed())
-                .forEach(entityNode -> entries.addAll(toEntityLogEntries(entityNode, analysisNumber, analyzeRequest.getDossierId(), analyzeRequest.getFileId())));
+                .toList();
 
-        document.streamAllImages()
+        List<Image> images = document.streamAllImages()
                 .filter(entity -> !entity.removed())
-                .forEach(imageNode -> entries.add(createEntityLogEntry(imageNode, dossierTemplateId, analysisNumber, analyzeRequest.getDossierId(), analyzeRequest.getFileId())));
+                .toList();
 
-        notFoundPrecursorEntries.stream()
+        List<PrecursorEntity> notFoundPrecursorEntities = notFoundPrecursorEntries.stream()
                 .filter(entity -> !entity.removed())
-                .forEach(precursorEntity -> entries.add(createEntityLogEntry(precursorEntity, analysisNumber, analyzeRequest.getDossierId(), analyzeRequest.getFileId())));
+                .toList();
+
+        List<String> allIds = new ArrayList<>();
+        allIds.addAll(textEntities.stream()
+                              .flatMap(entity -> entity.getPositionsOnPagePerPage()
+                                      .stream()
+                                      .map(PositionOnPage::getId))
+                              .toList());
+        allIds.addAll(images.stream()
+                              .map(Image::getId)
+                              .toList());
+        allIds.addAll(notFoundPrecursorEntities.stream()
+                              .map(PrecursorEntity::getId)
+                              .toList());
+
+        Map<String, List<ManualChange>> manualChangesMap = getManualChangesByEntityLogIds(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), allIds);
+
+        textEntities.forEach(entityNode -> entries.addAll(toEntityLogEntries(entityNode, analysisNumber, manualChangesMap)));
+
+        images.forEach(imageNode -> entries.add(createEntityLogEntry(imageNode, dossierTemplateId, analysisNumber, manualChangesMap.getOrDefault(imageNode.getId(), new ArrayList<>()))));
+
+        notFoundPrecursorEntities.forEach(precursorEntity -> entries.add(createEntityLogEntry(precursorEntity, analysisNumber, manualChangesMap.getOrDefault(precursorEntity.getId(), new ArrayList<>()))));
 
         return entries;
     }
 
 
-    private List<EntityLogEntry> toEntityLogEntries(TextEntity textEntity, int analysisNumber, String dossierId, String fileId) {
+    private List<EntityLogEntry> toEntityLogEntries(TextEntity textEntity, int analysisNumber, Map<String, List<ManualChange>> existingManualChangesMap) {
 
         List<EntityLogEntry> entityLogEntries = new ArrayList<>();
 
@@ -185,7 +228,7 @@ public class EntityLogCreatorService {
                     .map(rectangle2D -> new Position(rectangle2D, positionOnPage.getPage().getNumber()))
                     .toList();
 
-            EntityLogEntry entityLogEntry = createEntityLogEntry(textEntity, analysisNumber, positionOnPage.getId(), dossierId, fileId);
+            EntityLogEntry entityLogEntry = createEntityLogEntry(textEntity, analysisNumber, existingManualChangesMap.getOrDefault(positionOnPage.getId(), new ArrayList<>()));
 
             // set the ID from the positions, since it might contain a "-" with the page number if the entity is split across multiple pages
             entityLogEntry.setId(positionOnPage.getId());
@@ -197,12 +240,11 @@ public class EntityLogCreatorService {
     }
 
 
-    private EntityLogEntry createEntityLogEntry(Image image, String dossierTemplateId, int analysisNumber, String dossierId, String fileId) {
+    private EntityLogEntry createEntityLogEntry(Image image, String dossierTemplateId, int analysisNumber, List<ManualChange> existingManualChanges) {
 
         String imageType = image.getImageType().equals(ImageType.OTHER) ? "image" : image.getImageType().toString().toLowerCase(Locale.ENGLISH);
         boolean isHint = dictionaryService.isHint(imageType, dossierTemplateId);
 
-        List<ManualChange> existingManualChanges = getManualChangesByEntityLogId(dossierId, fileId, image.getId());
         List<ManualChange> allManualChanges = ManualChangeFactory.toLocalManualChangeList(image.getManualOverwrite().getManualChangeLog(), true, analysisNumber);
 
         return EntityLogEntry.builder()
@@ -230,12 +272,11 @@ public class EntityLogCreatorService {
     }
 
 
-    private EntityLogEntry createEntityLogEntry(PrecursorEntity precursorEntity, int analysisNumber, String dossierId, String fileId) {
+    private EntityLogEntry createEntityLogEntry(PrecursorEntity precursorEntity, int analysisNumber, List<ManualChange> existingManualChanges) {
 
         String type = precursorEntity.getManualOverwrite().getType()
                 .orElse(precursorEntity.getType());
 
-        List<ManualChange> existingManualChanges = getManualChangesByEntityLogId(dossierId, fileId, precursorEntity.getId());
         List<ManualChange> allManualChanges = ManualChangeFactory.toLocalManualChangeList(precursorEntity.getManualOverwrite().getManualChangeLog(), true, analysisNumber);
 
         return EntityLogEntry.builder()
@@ -273,7 +314,7 @@ public class EntityLogCreatorService {
     }
 
 
-    private EntityLogEntry createEntityLogEntry(TextEntity entity, int analysisNumber, String id, String dossierId, String fileId) {
+    private EntityLogEntry createEntityLogEntry(TextEntity entity, int analysisNumber, List<ManualChange> existingManualChanges) {
 
         Set<String> referenceIds = new HashSet<>();
         entity.references()
@@ -284,7 +325,6 @@ public class EntityLogCreatorService {
 
         EntryType entryType = buildEntryType(entity);
 
-        List<ManualChange> existingManualChanges = getManualChangesByEntityLogId(dossierId, fileId, id);
         List<ManualChange> allManualChanges = ManualChangeFactory.toLocalManualChangeList(entity.getManualOverwrite().getManualChangeLog(), true, analysisNumber);
 
         return EntityLogEntry.builder()
@@ -421,16 +461,16 @@ public class EntityLogCreatorService {
     }
 
 
-    public List<ManualChange> getManualChangesByEntityLogId(String dossierId, String fileId, String id) {
+    public Map<String, List<ManualChange>> getManualChangesByEntityLogIds(String dossierId, String fileId, List<String> ids) {
 
-        List<ManualChange> manualChanges = new ArrayList<>();
+        Map<String, List<ManualChange>> manualChangesMap = new HashMap<>();
-        List<EntityLogEntry> entityLogEntries = entityLogMongoService.findEntityLogEntriesByIds(dossierId, fileId, List.of(id));
+        List<EntityLogEntry> entityLogEntries = entityLogMongoService.findEntityLogEntriesByIds(dossierId, fileId, ids);
 
         for (EntityLogEntry entry : entityLogEntries) {
-            manualChanges.addAll(entry.getManualChanges());
+            manualChangesMap.put(entry.getId(), entry.getManualChanges());
         }
 
-        return manualChanges;
+        return manualChangesMap;
     }
 
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/NotFoundImportedEntitiesService.java

@@ -9,8 +9,9 @@ import java.util.Map;
 import java.util.Set;
 import java.util.stream.Collectors;
 
+import org.springframework.stereotype.Service;
+
 import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Change;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ChangeType;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
@@ -20,11 +21,9 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ChangeFactory;
 import com.iqser.red.service.redaction.v1.server.model.PrecursorEntity;
 import com.iqser.red.service.redaction.v1.server.model.RectangleWithPage;
-import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
-
-import org.springframework.stereotype.Service;
 
 import io.micrometer.core.annotation.Timed;
+import io.micrometer.observation.annotation.Observed;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
 
@@ -37,6 +36,7 @@ public class NotFoundImportedEntitiesService {
 
 
     @Timed("redactmanager_processEntityLog")
+    @Observed(name = "NotFoundImportedEntitiesService", contextualName = "process-entity-log")
     public void processEntityLog(EntityLog entityLog, AnalyzeRequest analyzeRequest, List<PrecursorEntity> notFoundEntities) {
         // recreate imported redactions with manual changes
         if (notFoundEntities == null || notFoundEntities.isEmpty()) {
@@ -65,7 +65,7 @@ public class NotFoundImportedEntitiesService {
     }
 
 
-    public Map<Integer, List<PrecursorEntity>> mapImportedRedactionsOnPage(List<PrecursorEntity> importedEntities) {
+    private Map<Integer, List<PrecursorEntity>> mapImportedRedactionsOnPage(List<PrecursorEntity> importedEntities) {
 
         Map<Integer, List<PrecursorEntity>> importedRedactionsMap = new HashMap<>();
         Set<Integer> pageNumbers = importedEntities.stream()
@@ -82,7 +82,7 @@ public class NotFoundImportedEntitiesService {
     }
 
 
-    public void addIntersections(EntityLogEntry entityLogEntry, Map<Integer, List<PrecursorEntity>> importedEntitiesMap, int analysisNumber) {
+    private void addIntersections(EntityLogEntry entityLogEntry, Map<Integer, List<PrecursorEntity>> importedEntitiesMap, int analysisNumber) {
 
         for (Position rectangle : entityLogEntry.getPositions()) {
             if (importedEntitiesMap.containsKey(rectangle.getPageNumber())) {

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/DocumentGraphMapper.java

@@ -1,9 +1,10 @@
 package com.iqser.red.service.redaction.v1.server.service.document;
 
+import static com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPageProto.DocumentPage;
+import static com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPositionDataProto.AllDocumentPositionData;
+import static com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.EntryDataProto.EntryData;
 
 import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
 import java.util.HashSet;
 import java.util.LinkedList;
 import java.util.List;
@@ -27,10 +28,7 @@ import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.AtomicTextBlock;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlockCollector;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPage;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentTextDataProto.AllDocumentTextData;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPositionData;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentStructure;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentTextData;
 
 import lombok.experimental.UtilityClass;
 
@@ -43,27 +41,29 @@ public class DocumentGraphMapper {
         DocumentTree documentTree = new DocumentTree(document);
         Context context = new Context(documentData, documentTree);
 
-        context.pageData.addAll(Arrays.stream(documentData.getDocumentPages())
+        context.pageData.addAll(documentData.getDocumentPages().getDocumentPagesList()
+                                        .stream()
                                         .map(DocumentGraphMapper::buildPage)
                                         .toList());
 
-        context.documentTree.getRoot().getChildren().addAll(buildEntries(documentData.getDocumentStructure().getRoot().getChildren(), context));
+        context.documentTree.getRoot().getChildren().addAll(buildEntries(documentData.getDocumentStructure().getRoot().getChildrenList(), context));
 
         document.setDocumentTree(context.documentTree);
         document.setPages(new HashSet<>(context.pageData));
-        document.setNumberOfPages(documentData.getDocumentPages().length);
+        document.setNumberOfPages(documentData.getDocumentPages().getDocumentPagesCount());
 
         document.setTextBlock(document.getTextBlock());
         return document;
     }
 
 
-    private List<DocumentTree.Entry> buildEntries(List<DocumentStructure.EntryData> entries, Context context) {
+    private List<DocumentTree.Entry> buildEntries(List<EntryData> entries, Context context) {
 
         List<DocumentTree.Entry> newEntries = new ArrayList<>(entries.size());
-        for (DocumentStructure.EntryData entryData : entries) {
+        for (EntryData entryData : entries) {
 
-            List<Page> pages = Arrays.stream(entryData.getPageNumbers())
+            List<Page> pages = entryData.getPageNumbersList()
+                    .stream()
                     .map(pageNumber -> getPage(pageNumber, context))
                     .toList();
 
@@ -76,22 +76,17 @@ public class DocumentGraphMapper {
                 case FOOTER -> buildFooter(context);
                 case TABLE -> buildTable(context, entryData.getProperties());
                 case TABLE_CELL -> buildTableCell(context, entryData.getProperties());
-                case IMAGE -> buildImage(context, entryData.getProperties(), entryData.getPageNumbers());
+                case IMAGE -> buildImage(context, entryData.getProperties(), entryData.getPageNumbersList());
                 default -> throw new UnsupportedOperationException("Not yet implemented for type " + entryData.getType());
             };
 
-            if (entryData.getAtomicBlockIds().length > 0) {
+            if (entryData.getAtomicBlockIdsCount() > 0) {
-                TextBlock textBlock = toTextBlock(entryData.getAtomicBlockIds(), context, node);
+                TextBlock textBlock = toTextBlock(entryData.getAtomicBlockIdsList(), context, node);
                 node.setLeafTextBlock(textBlock);
             }
-            List<Integer> treeId = Arrays.stream(entryData.getTreeId()).boxed()
+            List<Integer> treeId = entryData.getTreeIdList();
-                    .toList();
+            entryData.getEnginesList()
-            if (entryData.getEngines() != null) {
-                entryData.getEngines()
                     .forEach(node::addEngine);
-            } else {
-                entryData.setEngines(Collections.emptySet());
-            }
             node.setTreeId(treeId);
 
             switch (entryData.getType()) {
@@ -100,9 +95,8 @@ public class DocumentGraphMapper {
                 default -> pages.forEach(page -> page.getMainBody().add(node));
             }
 
-            newEntries.add(DocumentTree.Entry.builder().treeId(treeId).children(buildEntries(entryData.getChildren(), context)).node(node).build());
+            newEntries.add(DocumentTree.Entry.builder().treeId(treeId).children(buildEntries(entryData.getChildrenList(), context)).node(node).build());
-        }
+        } return newEntries;
-        return newEntries;
     }
 
 
@@ -112,10 +106,10 @@ public class DocumentGraphMapper {
     }
 
 
-    private Image buildImage(Context context, Map<String, String> properties, Long[] pageNumbers) {
+    private Image buildImage(Context context, Map<String, String> properties, List<Long> pageNumbers) {
 
-        assert pageNumbers.length == 1;
+        assert pageNumbers.size() == 1;
-        Page page = getPage(pageNumbers[0], context);
+        Page page = getPage(pageNumbers.get(0), context);
         var builder = Image.builder();
         PropertiesMapper.parseImageProperties(properties, builder);
         return builder.documentTree(context.documentTree).page(page).build();
@@ -161,13 +155,14 @@ public class DocumentGraphMapper {
         return SuperSection.builder().documentTree(context.documentTree).build();
     }
 
+
     private Paragraph buildParagraph(Context context, Map<String, String> properties) {
 
         if (PropertiesMapper.isDuplicateParagraph(properties)) {
 
             DuplicatedParagraph duplicatedParagraph = DuplicatedParagraph.builder().documentTree(context.documentTree).build();
 
-            Long[] unsortedTextblockIds = PropertiesMapper.getUnsortedTextblockIds(properties);
+            var unsortedTextblockIds = PropertiesMapper.getUnsortedTextblockIds(properties);
             duplicatedParagraph.setUnsortedLeafTextBlock(toTextBlock(unsortedTextblockIds, context, duplicatedParagraph));
             return duplicatedParagraph;
         }
@@ -176,9 +171,9 @@ public class DocumentGraphMapper {
     }
 
 
-    private TextBlock toTextBlock(Long[] atomicTextBlockIds, Context context, SemanticNode parent) {
+    private TextBlock toTextBlock(List<Long> atomicTextBlockIds, Context context, SemanticNode parent) {
 
-        return Arrays.stream(atomicTextBlockIds)
+        return atomicTextBlockIds.stream()
                 .map(atomicTextBlockId -> getAtomicTextBlock(context, parent, atomicTextBlockId))
                 .collect(new TextBlockCollector());
     }
@@ -186,10 +181,10 @@ public class DocumentGraphMapper {
 
     private AtomicTextBlock getAtomicTextBlock(Context context, SemanticNode parent, Long atomicTextBlockId) {
 
-        return AtomicTextBlock.fromAtomicTextBlockData(context.documentTextData.get(Math.toIntExact(atomicTextBlockId)),
+        return AtomicTextBlock.fromAtomicTextBlockData(context.documentTextData.getDocumentTextData(Math.toIntExact(atomicTextBlockId)),
-                                                       context.documentPositionData.get(Math.toIntExact(atomicTextBlockId)),
+                                                       context.documentPositionData.getDocumentPositionData(Math.toIntExact(atomicTextBlockId)),
                                                        parent,
-                                                       getPage(context.documentTextData.get(Math.toIntExact(atomicTextBlockId)).getPage(), context));
+                                                       getPage(context.documentTextData.getDocumentTextData(Math.toIntExact(atomicTextBlockId)).getPage(), context));
     }
 
 
@@ -211,18 +206,16 @@ public class DocumentGraphMapper {
 
         private final DocumentTree documentTree;
         private final List<Page> pageData;
-        private final List<DocumentTextData> documentTextData;
+        private final AllDocumentTextData documentTextData;
-        private final List<DocumentPositionData> documentPositionData;
+        private final AllDocumentPositionData documentPositionData;
 
 
         Context(DocumentData documentData, DocumentTree documentTree) {
 
             this.documentTree = documentTree;
             this.pageData = new ArrayList<>();
-            this.documentTextData = Arrays.stream(documentData.getDocumentTextData())
+            this.documentTextData = documentData.getDocumentTextData();
-                    .toList();
+            this.documentPositionData = documentData.getDocumentPositionData();
-            this.documentPositionData = Arrays.stream(documentData.getDocumentPositionData())
-                    .toList();
 
         }
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/PropertiesMapper.java

@@ -9,7 +9,7 @@ import com.iqser.red.service.redaction.v1.server.model.document.nodes.Image;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.ImageType;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Table;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentStructure;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentStructureWrapper;
 
 import lombok.experimental.UtilityClass;
 
@@ -18,32 +18,32 @@ public class PropertiesMapper {
 
     public void parseImageProperties(Map<String, String> properties, Image.ImageBuilder builder) {
 
-        builder.imageType(ImageType.fromString(properties.get(DocumentStructure.ImageProperties.IMAGE_TYPE)));
+        builder.imageType(ImageType.fromString(properties.get(DocumentStructureWrapper.ImageProperties.IMAGE_TYPE)));
-        builder.transparent(Boolean.parseBoolean(properties.get(DocumentStructure.ImageProperties.TRANSPARENT)));
+        builder.transparent(Boolean.parseBoolean(properties.get(DocumentStructureWrapper.ImageProperties.TRANSPARENT)));
-        builder.position(parseRectangle2D(properties.get(DocumentStructure.ImageProperties.POSITION)));
+        builder.position(parseRectangle2D(properties.get(DocumentStructureWrapper.ImageProperties.POSITION)));
-        builder.id(properties.get(DocumentStructure.ImageProperties.ID));
+        builder.id(properties.get(DocumentStructureWrapper.ImageProperties.ID));
     }
 
 
     public void parseTableCellProperties(Map<String, String> properties, TableCell.TableCellBuilder builder) {
 
-        builder.row(Integer.parseInt(properties.get(DocumentStructure.TableCellProperties.ROW)));
+        builder.row(Integer.parseInt(properties.get(DocumentStructureWrapper.TableCellProperties.ROW)));
-        builder.col(Integer.parseInt(properties.get(DocumentStructure.TableCellProperties.COL)));
+        builder.col(Integer.parseInt(properties.get(DocumentStructureWrapper.TableCellProperties.COL)));
-        builder.header(Boolean.parseBoolean(properties.get(DocumentStructure.TableCellProperties.HEADER)));
+        builder.header(Boolean.parseBoolean(properties.get(DocumentStructureWrapper.TableCellProperties.HEADER)));
-        builder.bBox(parseRectangle2D(properties.get(DocumentStructure.TableCellProperties.B_BOX)));
+        builder.bBox(parseRectangle2D(properties.get(DocumentStructureWrapper.TableCellProperties.B_BOX)));
     }
 
 
     public void parseTableProperties(Map<String, String> properties, Table.TableBuilder builder) {
 
-        builder.numberOfRows(Integer.parseInt(properties.get(DocumentStructure.TableProperties.NUMBER_OF_ROWS)));
+        builder.numberOfRows(Integer.parseInt(properties.get(DocumentStructureWrapper.TableProperties.NUMBER_OF_ROWS)));
-        builder.numberOfCols(Integer.parseInt(properties.get(DocumentStructure.TableProperties.NUMBER_OF_COLS)));
+        builder.numberOfCols(Integer.parseInt(properties.get(DocumentStructureWrapper.TableProperties.NUMBER_OF_COLS)));
     }
 
 
     private Rectangle2D parseRectangle2D(String bBox) {
 
-        List<Float> floats = Arrays.stream(bBox.split(DocumentStructure.RECTANGLE_DELIMITER))
+        List<Float> floats = Arrays.stream(bBox.split(DocumentStructureWrapper.RECTANGLE_DELIMITER))
                 .map(Float::parseFloat)
                 .toList();
         return new Rectangle2D.Float(floats.get(0), floats.get(1), floats.get(2), floats.get(3));
@@ -52,19 +52,19 @@ public class PropertiesMapper {
 
     public static boolean isDuplicateParagraph(Map<String, String> properties) {
 
-        return properties.containsKey(DocumentStructure.DuplicateParagraphProperties.UNSORTED_TEXTBLOCK_ID);
+        return properties.containsKey(DocumentStructureWrapper.DuplicateParagraphProperties.UNSORTED_TEXTBLOCK_ID);
     }
 
 
-    public static Long[] getUnsortedTextblockIds(Map<String, String> properties) {
+    public static List<Long> getUnsortedTextblockIds(Map<String, String> properties) {
 
-        return toLongArray(properties.get(DocumentStructure.DuplicateParagraphProperties.UNSORTED_TEXTBLOCK_ID));
+        return toLongList(properties.get(DocumentStructureWrapper.DuplicateParagraphProperties.UNSORTED_TEXTBLOCK_ID));
     }
 
 
-    public static Long[] toLongArray(String ids) {
+    public static List<Long> toLongList(String ids) {
 
-        return Arrays.stream(ids.substring(1, ids.length() - 1).trim().split(",")).map(Long::valueOf).toArray(Long[]::new);
+        return Arrays.stream(ids.substring(1, ids.length() - 1).trim().split(",")).map(Long::valueOf).toList();
     }
 
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/RedactionStorageService.java

@@ -1,5 +1,7 @@
 package com.iqser.red.service.redaction.v1.server.storage;
 
+import static com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPositionDataProto.AllDocumentPositionData;
+
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.InputStream;
@@ -15,6 +17,7 @@ import org.springframework.core.task.TaskExecutor;
 import org.springframework.stereotype.Service;
 
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentLog;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedactions;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedactionsPerPage;
@@ -23,17 +26,17 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlo
 import com.iqser.red.service.persistence.service.v1.api.shared.mongo.service.EntityLogMongoService;
 import com.iqser.red.service.redaction.v1.server.client.model.NerEntitiesModel;
 import com.iqser.red.service.redaction.v1.server.model.document.DocumentData;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
 import com.iqser.red.service.redaction.v1.server.utils.exception.NotFoundException;
 import com.iqser.red.storage.commons.exception.StorageObjectDoesNotExist;
 import com.iqser.red.storage.commons.service.StorageService;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPage;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPageProto.AllDocumentPages;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPositionData;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentStructureProto.DocumentStructure;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentStructure;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentStructureWrapper;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentTextData;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentTextDataProto.AllDocumentTextData;
 import com.knecon.fforesight.tenantcommons.TenantContext;
 
 import io.micrometer.core.annotation.Timed;
+import io.micrometer.observation.annotation.Observed;
 import lombok.Getter;
 import lombok.RequiredArgsConstructor;
 import lombok.SneakyThrows;
@@ -228,32 +231,32 @@ public class RedactionStorageService {
     public DocumentData getDocumentData(String dossierId, String fileId) {
 
         try {
-            Supplier<DocumentStructure> documentStructureSupplier = () -> storageService.readJSONObject(TenantContext.getTenantId(),
+            Supplier<DocumentStructure> documentStructureSupplier = () -> storageService.readProtoObject(TenantContext.getTenantId(),
                                                                                                         StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_STRUCTURE),
-                                                                                                        DocumentStructure.class);
+                                                                                                        DocumentStructure.parser());
             CompletableFuture<DocumentStructure> documentStructureFuture = CompletableFuture.supplyAsync(documentStructureSupplier, taskExecutor);
 
-            Supplier<DocumentTextData[]> documentTextDataSupplier = () -> storageService.readJSONObject(TenantContext.getTenantId(),
+            Supplier<AllDocumentTextData> documentTextDataSupplier = () -> storageService.readProtoObject(TenantContext.getTenantId(),
                                                                                                          StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_TEXT),
-                                                                                                        DocumentTextData[].class);
+                                                                                                         AllDocumentTextData.parser());
-            CompletableFuture<DocumentTextData[]> documentTextDataFuture = CompletableFuture.supplyAsync(documentTextDataSupplier, taskExecutor);
+            CompletableFuture<AllDocumentTextData> documentTextDataFuture = CompletableFuture.supplyAsync(documentTextDataSupplier, taskExecutor);
 
-            Supplier<DocumentPositionData[]> documentPositionDataSupplier = () -> storageService.readJSONObject(TenantContext.getTenantId(),
+            Supplier<AllDocumentPositionData> documentPositionDataSupplier = () -> storageService.readProtoObject(TenantContext.getTenantId(),
                                                                                                                 StorageIdUtils.getStorageId(dossierId,
                                                                                                                                             fileId,
                                                                                                                                             FileType.DOCUMENT_POSITION),
-                                                                                                                DocumentPositionData[].class);
+                                                                                                                  AllDocumentPositionData.parser());
-            CompletableFuture<DocumentPositionData[]> documentPositionDataFuture = CompletableFuture.supplyAsync(documentPositionDataSupplier, taskExecutor);
+            CompletableFuture<AllDocumentPositionData> documentPositionDataFuture = CompletableFuture.supplyAsync(documentPositionDataSupplier, taskExecutor);
 
-            Supplier<DocumentPage[]> documentPageSupplier = () -> storageService.readJSONObject(TenantContext.getTenantId(),
+            Supplier<AllDocumentPages> documentPageSupplier = () -> storageService.readProtoObject(TenantContext.getTenantId(),
                                                                                                   StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_PAGES),
-                                                                                                DocumentPage[].class);
+                                                                                                  AllDocumentPages.parser());
-            CompletableFuture<DocumentPage[]> documentPagesFuture = CompletableFuture.supplyAsync(documentPageSupplier, taskExecutor);
+            CompletableFuture<AllDocumentPages> documentPagesFuture = CompletableFuture.supplyAsync(documentPageSupplier, taskExecutor);
 
             CompletableFuture.allOf(documentStructureFuture, documentTextDataFuture, documentPositionDataFuture, documentPagesFuture).join();
 
             return DocumentData.builder()
-                    .documentStructure(documentStructureFuture.get())
+                    .documentStructureWrapper(new DocumentStructureWrapper(documentStructureFuture.get()))
                     .documentTextData(documentTextDataFuture.get())
                     .documentPositionData(documentPositionDataFuture.get())
                     .documentPages(documentPagesFuture.get())
@@ -310,6 +313,10 @@ public class RedactionStorageService {
 
             return dossierId + "/" + fileId + "." + fileType.name() + fileType.getExtension();
         }
+        public static String getStorageId(String userId, String dossierId, String filename) {
+
+            return userId + "/" + dossierId + "/" + filename;
+        }
 
     }
 

redaction-service-v1/redaction-service-server-v1/src/main/resources/drools/all_rules_documine.drl

@@ -47,7 +47,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange;
 
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngineProto.LayoutEngine;
 
 global Document document
 global EntityCreationService entityCreationService

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AnalysisEnd2EndTest.java

@@ -9,9 +9,15 @@ import static org.mockito.Mockito.when;
 
 import java.io.File;
 import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.ObjectInputStream;
+import java.io.ObjectOutputStream;
 import java.nio.file.FileVisitOption;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.time.OffsetDateTime;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
@@ -27,7 +33,6 @@ import java.util.stream.Collectors;
 import java.util.zip.GZIPInputStream;
 
 import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.extension.ExtendWith;
 import org.springframework.amqp.rabbit.core.RabbitTemplate;
@@ -36,15 +41,23 @@ import org.springframework.boot.test.context.SpringBootTest;
 import org.springframework.boot.test.mock.mockito.MockBean;
 import org.springframework.context.annotation.Import;
 import org.springframework.core.io.ClassPathResource;
+import org.springframework.data.redis.listener.RedisMessageListenerContainer;
+import org.springframework.data.util.Pair;
 import org.springframework.test.context.junit.jupiter.SpringExtension;
 
+import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
 import com.google.common.collect.Sets;
 import com.iqser.red.commons.jackson.ObjectMapperFactory;
 import com.iqser.red.service.dictionarymerge.commons.DictionaryEntryModel;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
 import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
@@ -56,13 +69,26 @@ import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryMode
 import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryVersion;
 import com.iqser.red.service.redaction.v1.server.service.AnalyzeService;
 import com.iqser.red.service.redaction.v1.server.service.DictionaryService;
+import com.iqser.red.service.redaction.v1.server.service.websocket.RedisSyncedWebSocketService;
 import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
+import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService.StorageIdUtils;
 import com.iqser.red.service.redaction.v1.server.testcontainers.MongoDBTestContainer;
 import com.iqser.red.service.redaction.v1.server.utils.exception.NotFoundException;
+import com.iqser.red.storage.commons.exception.StorageObjectDoesNotExist;
 import com.iqser.red.storage.commons.service.StorageService;
 import com.knecon.fforesight.keycloakcommons.security.TenantAuthenticationManagerResolver;
 import com.knecon.fforesight.mongo.database.commons.liquibase.TenantMongoLiquibaseExecutor;
 import com.knecon.fforesight.mongo.database.commons.service.MongoConnectionProvider;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPage;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPageProto;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPositionData;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPositionDataProto;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentStructure;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentStructureProto;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentTextData;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentTextDataProto;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.EntryDataProto;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeTypeProto;
 import com.knecon.fforesight.tenantcommons.TenantContext;
 import com.knecon.fforesight.tenantcommons.TenantProvider;
 import com.knecon.fforesight.tenantcommons.model.MongoDBConnection;
@@ -74,7 +100,7 @@ import lombok.extern.slf4j.Slf4j;
 @ExtendWith(SpringExtension.class)
 @SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
 @Import(RedactionIntegrationTest.RedactionIntegrationTestConfiguration.class)
-@Disabled
+//@Disabled
 /*
  * This test is meant to be used directly with a download from blob storage (e.g. minio). You need to define the dossier template you want to use by supplying an absolute path.
  * The dossier template will then be parsed for dictionaries, colors, entities, and rules. This is defined for the all tests once.
@@ -83,8 +109,7 @@ import lombok.extern.slf4j.Slf4j;
  * This way you can recreate what is happening on the stack almost exactly.
  */ public class AnalysisEnd2EndTest {
 
-    Path dossierTemplateToUse = Path.of(
+    Path dossierTemplateToUse = Path.of("/Users/maverickstuder/Documents/syngenta/redactmanager/prod-cp-eu-reg/EFSA_sanitisation_pre_GFL_v1"); // Add your dossier-template here
-            "/Users/maverickstuder/Documents/syngenta/redactmanager/prod-cp-eu-reg/EFSA_sanitisation_pre_GFL_v1"); // Add your dossier-template here
     ObjectMapper mapper = ObjectMapperFactory.create();
     final String TENANT_ID = "tenant";
 
@@ -123,12 +148,122 @@ import lombok.extern.slf4j.Slf4j;
     @MockBean
     protected TenantAuthenticationManagerResolver tenantAuthenticationManagerResolver;
 
+    @MockBean
+    private RedisSyncedWebSocketService redisSyncedWebSocketService;
+
+    @MockBean
+    private RedisMessageListenerContainer redisPubsubContainer;
+
+    @Autowired
+    RedactionStorageService redactionStorageService;
+
+    private static EntryDataProto.EntryData convertEntryData(DocumentStructure.EntryData oldEntryData) {
+        EntryDataProto.EntryData.Builder builder = EntryDataProto.EntryData.newBuilder();
+
+        builder.setType(NodeTypeProto.NodeType.valueOf(oldEntryData.getType().name()));
+        builder.addAllTreeId(Arrays.stream(oldEntryData.getTreeId()).boxed().collect(Collectors.toList()));
+        builder.addAllAtomicBlockIds(Arrays.asList(oldEntryData.getAtomicBlockIds()));
+        builder.addAllPageNumbers(Arrays.asList(oldEntryData.getPageNumbers()));
+
+        builder.putAllProperties(oldEntryData.getProperties());
+
+        // Recursively converting children
+        if (oldEntryData.getChildren() != null) {
+            oldEntryData.getChildren().forEach(child -> builder.addChildren(convertEntryData(child)));
+        }
+
+        return builder.build();
+    }
+
+    private static DocumentStructureProto.DocumentStructure convertDocumentStructure(DocumentStructure oldStructure) {
+        DocumentStructureProto.DocumentStructure.Builder newBuilder = DocumentStructureProto.DocumentStructure.newBuilder();
+
+        if (oldStructure.getRoot() != null) {
+            newBuilder.setRoot(convertEntryData(oldStructure.getRoot()));
+        }
+
+        return newBuilder.build();
+    }
+
+    private static DocumentPageProto.DocumentPage convertDocumentPage(DocumentPage oldPage) {
+        return DocumentPageProto.DocumentPage.newBuilder()
+                .setNumber(oldPage.getNumber())
+                .setHeight(oldPage.getHeight())
+                .setWidth(oldPage.getWidth())
+                .setRotation(oldPage.getRotation())
+                .build();
+    }
+
+    private static DocumentPageProto.AllDocumentPages convertAllDocumentPages(DocumentPage[] oldPages) {
+        DocumentPageProto.AllDocumentPages.Builder allPagesBuilder = DocumentPageProto.AllDocumentPages.newBuilder();
+
+        for (DocumentPage oldPage : oldPages) {
+            DocumentPageProto.DocumentPage newPage = convertDocumentPage(oldPage);
+            allPagesBuilder.addDocumentPages(newPage);
+        }
+
+        return allPagesBuilder.build();
+    }
+
+    private static DocumentPositionDataProto.DocumentPositionData convertPositionData(DocumentPositionData oldData) {
+        DocumentPositionDataProto.DocumentPositionData.Builder builder = DocumentPositionDataProto.DocumentPositionData.newBuilder()
+                .setId(oldData.getId())
+                .addAllStringIdxToPositionIdx(Arrays.stream(oldData.getStringIdxToPositionIdx()).boxed().collect(Collectors.toList()));
+
+        // Convert positions array to Position messages
+        for (float[] pos : oldData.getPositions()) {
+            DocumentPositionDataProto.DocumentPositionData.Position position = DocumentPositionDataProto.DocumentPositionData.Position.newBuilder()
+                    .setX(pos[0])
+                    .setY(pos[1])
+                    .setWidth(pos[2])
+                    .setHeight(pos[3])
+                    .build();
+            builder.addPositions(position);
+        }
+
+        return builder.build();
+    }
+
+    private static DocumentPositionDataProto.AllDocumentPositionData convertAllPositionData(DocumentPositionData[] oldDataList) {
+        DocumentPositionDataProto.AllDocumentPositionData.Builder allDataBuilder = DocumentPositionDataProto.AllDocumentPositionData.newBuilder();
+
+        for (DocumentPositionData oldData : oldDataList) {
+            allDataBuilder.addDocumentPositionData(convertPositionData(oldData));
+        }
+
+        return allDataBuilder.build();
+    }
+
+    public static DocumentTextDataProto.DocumentTextData convertTextData(DocumentTextData oldData) {
+        DocumentTextDataProto.DocumentTextData.Builder builder = DocumentTextDataProto.DocumentTextData.newBuilder()
+                .setId(oldData.getId())
+                .setPage(oldData.getPage())
+                .setSearchText(oldData.getSearchText())
+                .setNumberOnPage(oldData.getNumberOnPage())
+                .setStart(oldData.getStart())
+                .setEnd(oldData.getEnd())
+                .addAllLineBreaks(Arrays.stream(oldData.getLineBreaks()).boxed().collect(Collectors.toList()));
+
+        return builder.build();
+    }
+
+    public static DocumentTextDataProto.AllDocumentTextData convertAllTextData(DocumentTextData[] oldDataList) {
+        DocumentTextDataProto.AllDocumentTextData.Builder allDataBuilder = DocumentTextDataProto.AllDocumentTextData.newBuilder();
+
+        for (DocumentTextData oldData : oldDataList) {
+            allDataBuilder.addDocumentTextData(convertTextData(oldData));
+        }
+
+        return allDataBuilder.build();
+    }
+
+
 
     @Test
     @SneakyThrows
     public void runAnalysisEnd2End() {
 
-        String folder = "/Users/maverickstuder/Documents/RedactManager/redaction-service/redaction-service-v1/redaction-service-server-v1/src/test/resources/files/files_end2end/file0"; // Should contain all files from minio directly, still zipped. Can contain multiple files.
+        String folder = "/Users/maverickstuder/Documents/RedactManager/redaction-service/redaction-service-v1/redaction-service-server-v1/src/test/resources/files/files_end2end/file_large"; // Should contain all files from minio directly, still zipped. Can contain multiple files.
 
         Path absoluteFolderPath;
         if (folder.startsWith("files")) { // if it starts with "files" it is most likely in the resources folder, else it should be an absolute path
@@ -145,16 +280,146 @@ import lombok.extern.slf4j.Slf4j;
             AnalyzeRequest analyzeRequest = analyzeRequests.get(i);
             log.info("{}/{}: Starting analysis for file {}", i + 1, analyzeRequests.size(), analyzeRequest.getFileId());
             var times = new LinkedList<Long>();
-            for (int j = 1; j <= 10; j++) {
+            for (int j = 1; j <= 1; j++) {
                 var start = System.currentTimeMillis();
-                analyzeService.analyze(analyzeRequest);
+                ObjectMapper objectMapper = new ObjectMapper();
+                objectMapper.registerModule(new JavaTimeModule());
+
+                String oldDocumentStructureStorageId = StorageIdUtils.getStorageId(analyzeRequest.getDossierId(),
+                                                                                   analyzeRequest.getFileId(),
+                                                                                   FileType.DOCUMENT_STRUCTURE.name() + ".json");
+                String newDocumentStructureStorageId = StorageIdUtils.getStorageId(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.DOCUMENT_STRUCTURE);
+                try (InputStream inputStream = redactionStorageService.getStoredObject(oldDocumentStructureStorageId)) {
+                    DocumentStructure oldDocumentStructure = objectMapper.readValue(inputStream, DocumentStructure.class);
+                    DocumentStructureProto.DocumentStructure newDocumentStructure = convertDocumentStructure(oldDocumentStructure);
+                    storageService.storeProtoObject(TenantContext.getTenantId(), newDocumentStructureStorageId, newDocumentStructure);
+                    storageService.deleteObject(TenantContext.getTenantId(), oldDocumentStructureStorageId);
+                } catch (StorageObjectDoesNotExist e) {
+                    log.info("No document structure found for {}, {}, ignoring....", analyzeRequest.getDossierId(), analyzeRequest.getFileId());
+                }
+
+                String oldDocumentPagesStorageId = StorageIdUtils.getStorageId(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.DOCUMENT_PAGES.name() + ".json");
+                String newDocumentPagesStorageId = StorageIdUtils.getStorageId(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.DOCUMENT_PAGES);
+                try (InputStream inputStream = redactionStorageService.getStoredObject(oldDocumentPagesStorageId)) {
+                    DocumentPage[] oldDocumentPages = objectMapper.readValue(inputStream, DocumentPage[].class);
+                    DocumentPageProto.AllDocumentPages newDocumentPages = convertAllDocumentPages(oldDocumentPages);
+                    storageService.storeProtoObject(TenantContext.getTenantId(), newDocumentPagesStorageId, newDocumentPages);
+                    storageService.deleteObject(TenantContext.getTenantId(), oldDocumentPagesStorageId);
+                } catch (StorageObjectDoesNotExist e) {
+                    log.info("No document pages found for {}, {}, ignoring....", analyzeRequest.getDossierId(), analyzeRequest.getFileId());
+                }
+
+                String oldDocumentPositionDataStorageId = StorageIdUtils.getStorageId(analyzeRequest.getDossierId(),
+                                                                                      analyzeRequest.getFileId(),
+                                                                                      FileType.DOCUMENT_POSITION.name() + ".json");
+                String newDocumentPositionDataStorageId = StorageIdUtils.getStorageId(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.DOCUMENT_POSITION);
+                try (InputStream inputStream = redactionStorageService.getStoredObject(oldDocumentPositionDataStorageId)) {
+                    DocumentPositionData[] oldDocumentPositionData = objectMapper.readValue(inputStream, DocumentPositionData[].class);
+                    DocumentPositionDataProto.AllDocumentPositionData newDocumentPositionData = convertAllPositionData(oldDocumentPositionData);
+                    storageService.storeProtoObject(TenantContext.getTenantId(), newDocumentPositionDataStorageId, newDocumentPositionData);
+                    storageService.deleteObject(TenantContext.getTenantId(), oldDocumentPositionDataStorageId);
+                } catch (StorageObjectDoesNotExist e) {
+                    log.info("No document position data found for {}, {}, ignoring....", analyzeRequest.getDossierId(), analyzeRequest.getFileId());
+                }
+
+                String oldDocumentTextDataStorageId = StorageIdUtils.getStorageId(analyzeRequest.getDossierId(),
+                                                                                  analyzeRequest.getFileId(),
+                                                                                  FileType.DOCUMENT_TEXT.name() + ".json");
+                String newDocumentTextDataStorageId = StorageIdUtils.getStorageId(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.DOCUMENT_TEXT);
+                try (InputStream inputStream = redactionStorageService.getStoredObject(oldDocumentTextDataStorageId)) {
+                    DocumentTextData[] oldDocumentTextData = objectMapper.readValue(inputStream, DocumentTextData[].class);
+                    DocumentTextDataProto.AllDocumentTextData newDocumentTextData = convertAllTextData(oldDocumentTextData);
+                    storageService.storeProtoObject(TenantContext.getTenantId(), newDocumentTextDataStorageId, newDocumentTextData);
+                    storageService.deleteObject(TenantContext.getTenantId(), oldDocumentTextDataStorageId);
+                } catch (StorageObjectDoesNotExist e) {
+                    log.info("No document text data found for {}, {}, ignoring....", analyzeRequest.getDossierId(), analyzeRequest.getFileId());
+                }
+
+                analyzeService.reanalyze(analyzeRequest);
+                storeEntityLogAsTempFile(analyzeRequest, objectMapper);
+
+                EntityLog entityLog = redactionStorageService.getEntityLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
                 times.add(System.currentTimeMillis() - start);
+
+                EntityLog otherEntityLog = getEntityLogFromTemp("/tmp/entityLog2.ser", objectMapper);
+
+                List<String> entryIds = entityLog.getEntityLogEntry()
+                        .stream()
+                        .map(EntityLogEntry::getId)
+                        .toList();
+                List<String> otherEntryIds = otherEntityLog.getEntityLogEntry()
+                        .stream()
+                        .map(EntityLogEntry::getId)
+                        .toList();
+                if(entryIds.containsAll(otherEntryIds) && otherEntryIds.containsAll(entryIds)) {
+                    System.out.println("We won");
+                } else {
+                    System.out.println("We lost");
+                }
+
             }
             System.out.println("times in ms for each analyze run: " + times);
         }
     }
 
 
+    private static void addManualRedactionEntryToFile(AnalyzeRequest analyzeRequest) {
+
+        ManualRedactionEntry manualRedactionEntry = new ManualRedactionEntry();
+        manualRedactionEntry.setAnnotationId("yourAnnotationId");
+        manualRedactionEntry.setFileId("fileId");
+        manualRedactionEntry.setType("CBI_author");
+        manualRedactionEntry.setValue("7232");
+        manualRedactionEntry.setReason(
+                "(Regulations (EU) 2016/679 and (EU) 2018/1725 shall apply to the processing of personal data carried out pursuant to this Regulation. Any personal data made public pursuant to Article 38 of this Regulation and this Article shall only be used to ensure the transparency of the risk assessment under this Regulation and shall not be further processed in a manner that is incompatible with these purposes, in accordance with point (b) of Article 5(1) of Regulation (EU) 2016/679 and point (b) of Article 4(1) of Regulation (EU) 2018/1725, as the case may be)");
+        manualRedactionEntry.setLegalBasis("Article 39(e)(3) of Regulation (EC) No 178/2002");
+        manualRedactionEntry.setProcessedDate(OffsetDateTime.now());
+        manualRedactionEntry.setRequestDate(OffsetDateTime.now());
+        manualRedactionEntry.setPositions(List.of(Rectangle.builder().topLeftX(332.134f).topLeftY(689.72f).width(26.688f).height(13.872f).page(1).build()));
+
+        analyzeRequest.setManualRedactions(ManualRedactions.builder().entriesToAdd(Set.of(manualRedactionEntry)).build());
+    }
+
+
+    private void restoreEntityLogFromTempFile(AnalyzeRequest analyzeRequest, ObjectMapper objectMapper) throws JsonProcessingException {
+        //Deserialize entityLog from a file
+        String entityLogString = null;
+        try (FileInputStream fileIn = new FileInputStream("/tmp/entityLog.ser"); ObjectInputStream in = new ObjectInputStream(fileIn)) {
+            entityLogString = (String) in.readObject();
+        } catch (IOException | ClassNotFoundException e) {
+            e.printStackTrace();
+        }
+        var entityLog = objectMapper.readValue(entityLogString, EntityLog.class);
+        redactionStorageService.saveEntityLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), entityLog);
+    }
+
+    private EntityLog getEntityLogFromTemp(String fileName, ObjectMapper objectMapper) throws JsonProcessingException {
+        //Deserialize entityLog from a file
+        String entityLogString = null;
+        try (FileInputStream fileIn = new FileInputStream(fileName); ObjectInputStream in = new ObjectInputStream(fileIn)) {
+            entityLogString = (String) in.readObject();
+        } catch (IOException | ClassNotFoundException e) {
+            e.printStackTrace();
+        }
+        return objectMapper.readValue(entityLogString, EntityLog.class);
+    }
+
+
+    private void storeEntityLogAsTempFile(AnalyzeRequest analyzeRequest, ObjectMapper objectMapper) throws JsonProcessingException {
+
+        EntityLog entityLog = redactionStorageService.getEntityLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
+
+        // Serialize
+        String jsonString = objectMapper.writeValueAsString(entityLog);
+        // Serialize entityLog to a file
+        try (FileOutputStream fileOut = new FileOutputStream("/tmp/entityLog.ser"); ObjectOutputStream out = new ObjectOutputStream(fileOut)) {
+            out.writeObject(jsonString);
+        } catch (IOException e) {
+            e.printStackTrace();
+        }
+    }
+
+
     @BeforeEach
     public void setup() {
 
@@ -198,6 +463,8 @@ import lombok.extern.slf4j.Slf4j;
     }
 
 
+
+
     @SneakyThrows
     private List<AnalyzeRequest> prepareStorageForFolder(Path folder) {
 
@@ -231,28 +498,27 @@ import lombok.extern.slf4j.Slf4j;
             request.setManualRedactions(new ManualRedactions());
         }
 
-        Set<FileType> endingsToUpload = Set.of("ORIGIN",
+        Set<FileType> endingsToUpload = Set.of("ORIGIN", "IMAGE_INFO", "NER_ENTITIES", "TABLES", "IMPORTED_REDACTIONS")
-                                               "DOCUMENT_PAGES",
-                                               "DOCUMENT_POSITION",
-                                               "DOCUMENT_STRUCTURE",
-                                               "DOCUMENT_TEXT",
-                                               "IMAGE_INFO",
-                                               "NER_ENTITIES",
-                                               "TABLES",
-                                               "IMPORTED_REDACTIONS")
                 .stream()
                 .map(FileType::valueOf)
                 .collect(Collectors.toSet());
 
+        Set<String> oldFileSuffixes = Set.of("DOCUMENT_PAGES", "DOCUMENT_POSITION", "DOCUMENT_STRUCTURE", "DOCUMENT_TEXT");
+
         Set<FileType> uploadedFileTypes = Files.walk(folder)
                 .filter(path -> path.toFile().isFile())
                 .filter(path -> parseFileTypeFromPath(path).map(endingsToUpload::contains)
                         .orElse(false))
-                .map(filePath -> uploadFile(filePath, request))
+                .map(filePath -> uploadFile(filePath, request, true))
                 .filter(Optional::isPresent)
                 .map(Optional::get)
                 .collect(Collectors.toUnmodifiableSet());
 
+        Files.walk(folder)
+                .filter(path -> path.toFile().isFile())
+                .filter(path -> oldFileSuffixes.contains(parseFileTypeAndExtensionFromPath(path).getFirst()))
+                .forEach(filePath -> uploadFile(filePath, request, false));
+
         Set<FileType> missingFileTypes = Sets.difference(endingsToUpload, uploadedFileTypes);
         if (!missingFileTypes.isEmpty()) {
             log.error("Folder {} is missing files of type {}",
@@ -277,16 +543,36 @@ import lombok.extern.slf4j.Slf4j;
     }
 
 
-    @SneakyThrows
+    private static Pair<String, String> parseFileTypeAndExtensionFromPath(Path path) {
-    private Optional<FileType> uploadFile(Path path, AnalyzeRequest request) {
 
-        Optional<FileType> fileType = parseFileTypeFromPath(path);
+        String[] parts = path.getFileName().toString().split("\\.");
-        if (fileType.isEmpty()) {
+        String fileType = parts[1];
-            return Optional.empty();
+        String extension = parts[2];
+        return Pair.of(fileType, extension);
     }
-        try (var fis = new FileInputStream(path.toFile()); var in = new GZIPInputStream(fis);) {
-            storageService.storeObject(TENANT_ID, RedactionStorageService.StorageIdUtils.getStorageId(request.getDossierId(), request.getFileId(), fileType.get()), in);
 
+
+    @SneakyThrows
+    private Optional<FileType> uploadFile(Path path, AnalyzeRequest request, boolean parseAsFileType) {
+
+        Optional<FileType> fileType = Optional.empty();
+        String storageId = null;
+        if (parseAsFileType) {
+
+            fileType = parseFileTypeFromPath(path);
+            if (fileType.isPresent()) {
+                storageId = StorageIdUtils.getStorageId(request.getDossierId(), request.getFileId(), fileType.get());
+            }
+        } else {
+            Pair<String, String> fileTypeAndExtension = parseFileTypeAndExtensionFromPath(path);
+            storageId = StorageIdUtils.getStorageId(request.getDossierId(), request.getFileId(), fileTypeAndExtension.getFirst() + "." + fileTypeAndExtension.getSecond());
+
+        }
+        if(storageId != null) {
+            try (var fis = new FileInputStream(path.toFile()); var in = new GZIPInputStream(fis)) {
+                storageService.storeObject(TENANT_ID, storageId, in);
+
+            }
         }
         return fileType;
     }

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java

@@ -7,7 +7,9 @@ import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.ArgumentMatchers.anyString;
 import static org.mockito.Mockito.when;
 
+import java.io.BufferedWriter;
 import java.io.FileOutputStream;
+import java.io.FileWriter;
 import java.io.IOException;
 import java.net.URI;
 import java.net.URL;
@@ -67,6 +69,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemp
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Point;
 import com.iqser.red.service.redaction.v1.server.annotate.AnnotateRequest;
 import com.iqser.red.service.redaction.v1.server.annotate.AnnotateResponse;
+import com.iqser.red.service.redaction.v1.server.model.document.DocumentData;
 import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Section;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.OsUtils;
@@ -78,6 +81,7 @@ import com.iqser.red.storage.commons.service.StorageService;
 import com.iqser.red.storage.commons.utils.FileSystemBackedStorageService;
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
 import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingServiceProcessorConfiguration;
+import com.knecon.fforesight.service.layoutparser.processor.utils.ProtobufUtil;
 import com.knecon.fforesight.tenantcommons.TenantContext;
 
 import lombok.SneakyThrows;
@@ -623,8 +627,8 @@ public class RedactionIntegrationTest extends RulesIntegrationTest {
 
         System.out.println("first analysis duration: " + (end - start));
 
-        try (FileOutputStream fileOutputStream = new FileOutputStream(OsUtils.getTemporaryDirectory() + "/Test.json")) {
+        try (BufferedWriter writer = new BufferedWriter(new FileWriter(OsUtils.getTemporaryDirectory() + "/Test.json"))) {
-            fileOutputStream.write(objectMapper.writeValueAsBytes(redactionStorageService.getDocumentData(TEST_DOSSIER_ID, TEST_FILE_ID)));
+            writer.write(getDocumentDataString(redactionStorageService.getDocumentData(TEST_DOSSIER_ID, TEST_FILE_ID)));
         }
 
         List<String> valuesInDocument = entityLog.getEntityLogEntry()
@@ -702,6 +706,16 @@ public class RedactionIntegrationTest extends RulesIntegrationTest {
         System.out.println("Output file:" + outputFileName);
     }
 
+    @SneakyThrows
+    private String getDocumentDataString(DocumentData documentData) {
+
+        String result = ProtobufUtil.toJson(documentData.getDocumentPages());
+        result += ",\n" + ProtobufUtil.toJson(documentData.getDocumentTextData());
+        result += ",\n" + ProtobufUtil.toJson(documentData.getDocumentPositionData());
+        result += ",\n" + ProtobufUtil.toJson(documentData.getDocumentStructure());
+        return result;
+    }
+
 
     @Test
     public void testRemovePublishedInformations() throws IOException {

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/acceptance_rules.drl

@@ -55,7 +55,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange;
 
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngineProto.LayoutEngine;
 
 global Document document
 global EntityCreationService entityCreationService

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/all_redact_manager_rules.drl

@@ -55,7 +55,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange;
 
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngineProto.LayoutEngine;
 
 global Document document
 global EntityCreationService entityCreationService

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora.drl

@@ -47,7 +47,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange;
 
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngineProto.LayoutEngine;
 
 global Document document
 global EntityCreationService entityCreationService

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/efsa_sanitisation.drl

@@ -55,7 +55,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange;
 
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngineProto.LayoutEngine;
 
 global Document document
 global EntityCreationService entityCreationService

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/manual_redaction_rules.drl

@@ -55,7 +55,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange;
 
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngineProto.LayoutEngine;
 
 global Document document
 global EntityCreationService entityCreationService

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl

@@ -55,7 +55,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange;
 
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngineProto.LayoutEngine;
 
 global Document document
 global EntityCreationService entityCreationService

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_v2.drl

@@ -55,7 +55,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange;
 
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngineProto.LayoutEngine;
 
 global Document document
 global EntityCreationService entityCreationService

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/table_demo.drl

@@ -55,7 +55,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange;
 
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngineProto.LayoutEngine;
 
 global Document document
 global EntityCreationService entityCreationService

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/test_rules.drl

@@ -55,7 +55,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange;
 
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngineProto.LayoutEngine;
 
 global Document document
 global EntityCreationService entityCreationService

redaction-service-v1/rules-management/src/main/resources/all_redact_manager_rules.drl

@@ -55,7 +55,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange;
 
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngineProto.LayoutEngine;
 
 global Document document
 global EntityCreationService entityCreationService

redaction-service-v1/rules-management/src/main/resources/all_rules_documine.drl

@@ -47,7 +47,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange;
 
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngineProto.LayoutEngine;
 
 global Document document
 global EntityCreationService entityCreationService