Merge branch 'hotfix' into 'master'

rule validation hotfix:

See merge request redactmanager/redaction-service!191

publish-custom-image.sh

@@ -9,7 +9,7 @@ branch=$(git rev-parse --abbrev-ref HEAD)
 commit_hash=$(git rev-parse --short=5 HEAD)
 
 # Combine branch and commit hash
-buildName="${branch}-${commit_hash}"
+buildName="${USER}-${branch}-${commit_hash}"
 
 gradle bootBuildImage --cleanCache --publishImage -PbuildbootDockerHostNetwork=true -Pversion=$buildName
 echo "nexus.knecon.com:5001/red/${dir}-server-v1:$buildName"

redaction-service-v1/redaction-service-api-v1/build.gradle.kts

@@ -1,13 +1,13 @@
 plugins {
     id("com.iqser.red.service.java-conventions")
-    id("io.freefair.lombok") version "8.2.2"
+    id("io.freefair.lombok") version "8.4"
 }
 
 description = "redaction-service-api-v1"
 
 dependencies {
-    implementation("org.springframework:spring-web:6.0.11")
-    implementation("com.iqser.red.service:persistence-service-internal-api-v1:2.163.0")
+    implementation("org.springframework:spring-web:6.0.12")
+    implementation("com.iqser.red.service:persistence-service-internal-api-v1:2.207.0")
 }
 
 publishing {

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/MigrationRequest.java

@@ -0,0 +1,17 @@
+package com.iqser.red.service.redaction.v1.model;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
+public class MigrationRequest {
+
+    String dossierId;
+    String fileId;
+
+}

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/MigrationResponse.java

@@ -0,0 +1,17 @@
+package com.iqser.red.service.redaction.v1.model;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
+public class MigrationResponse {
+
+    String dossierId;
+    String fileId;
+
+}

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/QueueNames.java

@@ -0,0 +1,14 @@
+package com.iqser.red.service.redaction.v1.model;
+
+public class QueueNames {
+
+    public static final String REDACTION_QUEUE = "redactionQueue";
+    public static final String REDACTION_DQL = "redactionDQL";
+
+    public static final String REDACTION_PRIORITY_QUEUE = "redactionPriorityQueue";
+
+    public static final String MIGRATION_QUEUE = "migrationQueue";
+    public static final String MIGRATION_DLQ = "migrationDLQ";
+    public static final String MIGRATION_RESPONSE_QUEUE = "migrationResponseQueue";
+
+}

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/UnprocessedManualEntity.java

@@ -0,0 +1,32 @@
+package com.iqser.red.service.redaction.v1.model;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.EqualsAndHashCode;
+import lombok.NoArgsConstructor;
+
+@Data
+@AllArgsConstructor
+@NoArgsConstructor
+@Builder
+@EqualsAndHashCode
+public class UnprocessedManualEntity {
+
+    private String annotationId;
+    private String textBefore;
+    private String textAfter;
+    private int startOffset;
+    private int endOffset;
+    private String closestHeadline;
+    private String section;
+    private float[] color;
+    private List<Integer> containingNodeId;
+    @Builder.Default
+    private List<Position> positions = new ArrayList<>();
+}

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/resources/UnprocessedManualEntityResource.java

@@ -0,0 +1,20 @@
+package com.iqser.red.service.redaction.v1.resources;
+
+import java.util.List;
+
+import org.springframework.http.MediaType;
+import org.springframework.web.bind.annotation.PostMapping;
+import org.springframework.web.bind.annotation.RequestBody;
+import org.springframework.web.bind.annotation.RequestParam;
+
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
+import com.iqser.red.service.redaction.v1.model.UnprocessedManualEntity;
+
+public interface UnprocessedManualEntityResource {
+
+    @PostMapping(value = "/mergeUnprocessedManualEntities", produces = MediaType.APPLICATION_JSON_VALUE)
+    List<UnprocessedManualEntity> mergeUnprocessedManualEntities(@RequestParam(value = "fileId") String fileId,
+                                                                 @RequestParam(value = "dossierId") String dossierId,
+                                                                 @RequestParam(value = "dossierTemplateId") String dosserTemplateId,
+                                                                 @RequestBody ManualRedactions manualRedactions);
+}

redaction-service-v1/redaction-service-server-v1/build.gradle.kts

@@ -3,20 +3,20 @@ import org.springframework.boot.gradle.tasks.bundling.BootBuildImage
 plugins {
     application
     id("com.iqser.red.service.java-conventions")
-    id("org.springframework.boot") version "3.1.2"
+    id("org.springframework.boot") version "3.1.4"
     id("io.spring.dependency-management") version "1.1.3"
-    id("org.sonarqube") version "4.3.0.3225"
-    id("io.freefair.lombok") version "8.2.2"
+    id("org.sonarqube") version "4.4.1.3373"
+    id("io.freefair.lombok") version "8.4"
 }
 
 description = "redaction-service-server-v1"
 
 
-val layoutParserVersion = "0.70.0"
+val layoutParserVersion = "0.74.0"
 val jacksonVersion = "2.15.2"
-val droolsVersion = "8.43.0.Final"
+val droolsVersion = "9.44.0.Final"
 val pdfBoxVersion = "3.0.0"
-val persistenceServiceVersion = "2.168.0"
+val persistenceServiceVersion = "2.231.0"
 
 configurations {
     all {
@@ -30,12 +30,12 @@ dependencies {
     implementation("com.iqser.red.service:persistence-service-internal-api-v1:${persistenceServiceVersion}") { exclude(group = "org.springframework.boot") }
     implementation("com.knecon.fforesight:layoutparser-service-internal-api:${layoutParserVersion}")
 
-    implementation("com.iqser.red.commons:spring-commons:2.7.0")
+    implementation("com.iqser.red.commons:spring-commons:6.2.0")
     implementation("com.iqser.red.commons:metric-commons:2.3.0")
 
     implementation("com.iqser.red.commons:dictionary-merge-commons:1.5.0")
-    implementation("com.iqser.red.commons:storage-commons:2.43.0")
-    implementation("com.knecon.fforesight:tenant-commons:0.13.0")
+    implementation("com.iqser.red.commons:storage-commons:2.45.0")
+    implementation("com.knecon.fforesight:tenant-commons:0.14.0")
 
     implementation("com.fasterxml.jackson.module:jackson-module-afterburner:${jacksonVersion}")
     implementation("com.fasterxml.jackson.datatype:jackson-datatype-jsr310:${jacksonVersion}")
@@ -49,12 +49,12 @@ dependencies {
     implementation("org.locationtech.jts:jts-core:1.19.0")
 
     implementation("org.springframework.cloud:spring-cloud-starter-openfeign:4.0.4")
-    implementation("org.springframework.boot:spring-boot-starter-amqp:3.1.2")
+    implementation("org.springframework.boot:spring-boot-starter-amqp:3.1.4")
 
     testImplementation("org.apache.pdfbox:pdfbox:${pdfBoxVersion}")
     testImplementation("org.apache.pdfbox:pdfbox-tools:${pdfBoxVersion}")
 
-    testImplementation("org.springframework.boot:spring-boot-starter-test:3.1.2")
+    testImplementation("org.springframework.boot:spring-boot-starter-test:3.1.4")
     testImplementation("com.knecon.fforesight:layoutparser-service-processor:${layoutParserVersion}") {
         exclude(
             group = "com.iqser.red.service",

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/UnprocessedManualEntityController.java

@@ -0,0 +1,29 @@
+package com.iqser.red.service.redaction.v1.server.controller;
+
+import java.util.List;
+
+import org.springframework.web.bind.annotation.RestController;
+
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
+import com.iqser.red.service.redaction.v1.model.UnprocessedManualEntity;
+import com.iqser.red.service.redaction.v1.resources.UnprocessedManualEntityResource;
+import com.iqser.red.service.redaction.v1.server.service.UnprocessedChangesService;
+
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+@RestController
+@RequiredArgsConstructor
+public class UnprocessedManualEntityController implements UnprocessedManualEntityResource {
+
+    private final UnprocessedChangesService unprocessedChangesService;
+
+
+    @Override
+    public List<UnprocessedManualEntity> mergeUnprocessedManualEntities(String fileId, String dossierId, String dossierTemplateId, ManualRedactions manualRedactions) {
+
+        return unprocessedChangesService.addInformationToUnprocessedManualEntities(fileId, dossierId, dossierTemplateId, manualRedactions);
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/ManualEntity.java

@@ -4,10 +4,9 @@ import java.util.List;
 import java.util.PriorityQueue;
 
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
 import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
-import com.iqser.red.service.redaction.v1.server.model.document.entity.IEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
+import com.iqser.red.service.redaction.v1.server.model.document.entity.IEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.ManualChangeOverwrite;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule;
 
@@ -44,29 +43,7 @@ public class ManualEntity implements IEntity {
     ManualChangeOverwrite manualOverwrite = new ManualChangeOverwrite();
 
 
-    public static ManualEntity fromRedactionLogEntry(RedactionLogEntry redactionLogEntry) {
-
-        String ruleIdentifier = redactionLogEntry.getType() + "." + redactionLogEntry.getMatchedRule() + ".0";
-        List<RectangleWithPage> rectangleWithPages = redactionLogEntry.getPositions().stream().map(RectangleWithPage::fromRedactionLogRectangle).toList();
-        return ManualEntity.builder()
-                .id(redactionLogEntry.getId())
-                .value(redactionLogEntry.getValue())
-                .entityPosition(rectangleWithPages)
-                .ruleIdentifier(ruleIdentifier)
-                .reason(redactionLogEntry.getReason())
-                .legalBasis(redactionLogEntry.getLegalBasis())
-                .type(redactionLogEntry.getType())
-                .section(redactionLogEntry.getSection())
-                .entityType(redactionLogEntry.isRecommendation() ? EntityType.RECOMMENDATION : EntityType.ENTITY)
-                .applied(redactionLogEntry.isRedacted())
-                .isDictionaryEntry(redactionLogEntry.isDictionaryEntry())
-                .isDossierDictionaryEntry(redactionLogEntry.isDossierDictionaryEntry())
-                .rectangle(redactionLogEntry.isRectangle())
-                .build();
-    }
-
-
-    public static ManualEntity fromManualRedactionEntry(ManualRedactionEntry manualRedactionEntry) {
+    public static ManualEntity fromManualRedactionEntry(ManualRedactionEntry manualRedactionEntry, boolean hint) {
 
         List<RectangleWithPage> rectangleWithPages = manualRedactionEntry.getPositions().stream().map(RectangleWithPage::fromAnnotationRectangle).toList();
         ManualChangeOverwrite manualChangeOverwrite = new ManualChangeOverwrite();
@@ -80,7 +57,7 @@ public class ManualEntity implements IEntity {
                 .legalBasis(manualRedactionEntry.getLegalBasis())
                 .type(manualRedactionEntry.getType())
                 .section(manualRedactionEntry.getSection())
-                .entityType(EntityType.ENTITY)
+                .entityType(hint ? EntityType.HINT : EntityType.ENTITY)
                 .applied(true)
                 .isDictionaryEntry(false)
                 .isDossierDictionaryEntry(false)

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/MigratedEntityLog.java

@@ -0,0 +1,20 @@
+package com.iqser.red.service.redaction.v1.server.model;
+
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.migration.MigratedIds;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
+public class MigratedEntityLog {
+
+    MigratedIds migratedIds;
+    EntityLog entityLog;
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/MigrationEntity.java

@@ -0,0 +1,305 @@
+package com.iqser.red.service.redaction.v1.server.model;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ChangeType;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryType;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Change;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ManualChange;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ManualRedactionType;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
+import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
+import com.iqser.red.service.redaction.v1.server.model.document.entity.IEntity;
+import com.iqser.red.service.redaction.v1.server.model.document.entity.PositionOnPage;
+import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.Image;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.ImageType;
+
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.RequiredArgsConstructor;
+
+@Data
+@AllArgsConstructor
+@RequiredArgsConstructor
+public final class MigrationEntity {
+
+    private final ManualEntity manualEntity;
+    private final RedactionLogEntry redactionLogEntry;
+    private IEntity migratedEntity;
+    private String oldId;
+    private String newId;
+
+
+    public static MigrationEntity fromRedactionLogEntry(RedactionLogEntry redactionLogEntry) {
+
+        return new MigrationEntity(createManualEntity(redactionLogEntry), redactionLogEntry);
+    }
+
+
+    public static ManualEntity createManualEntity(RedactionLogEntry redactionLogEntry) {
+
+        String ruleIdentifier = "OLD." + redactionLogEntry.getMatchedRule() + ".0";
+        List<RectangleWithPage> rectangleWithPages = redactionLogEntry.getPositions().stream().map(RectangleWithPage::fromRedactionLogRectangle).toList();
+
+        return ManualEntity.builder()
+                .id(redactionLogEntry.getId())
+                .value(redactionLogEntry.getValue())
+                .entityPosition(rectangleWithPages)
+                .ruleIdentifier(ruleIdentifier)
+                .reason(redactionLogEntry.getReason())
+                .legalBasis(redactionLogEntry.getLegalBasis())
+                .type(redactionLogEntry.getType())
+                .section(redactionLogEntry.getSection())
+                .entityType(getEntityType(redactionLogEntry))
+                .applied(redactionLogEntry.isRedacted())
+                .isDictionaryEntry(redactionLogEntry.isDictionaryEntry())
+                .isDossierDictionaryEntry(redactionLogEntry.isDossierDictionaryEntry())
+                .rectangle(redactionLogEntry.isRectangle())
+                .build();
+    }
+
+
+    private static EntityType getEntityType(RedactionLogEntry redactionLogEntry) {
+
+        if (redactionLogEntry.isRecommendation()) {
+            return EntityType.RECOMMENDATION;
+        }
+        if (redactionLogEntry.isHint()) {
+            return EntityType.HINT;
+        }
+        return EntityType.ENTITY;
+    }
+
+
+    private static com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Change toEntityLogChanges(Change change) {
+
+        return new com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Change(change.getAnalysisNumber(),
+                toEntityLogType(change.getType()),
+                change.getDateTime());
+    }
+
+
+    private static EntryType getEntryType(EntityType entityType) {
+
+        return switch (entityType) {
+            case ENTITY -> EntryType.ENTITY;
+            case HINT -> EntryType.HINT;
+            case FALSE_POSITIVE -> EntryType.FALSE_POSITIVE;
+            case RECOMMENDATION -> EntryType.RECOMMENDATION;
+            case FALSE_RECOMMENDATION -> EntryType.FALSE_RECOMMENDATION;
+        };
+    }
+
+
+    private static com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualChange toEntityLogManualChanges(ManualChange manualChange) {
+
+        return new com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualChange(toManualRedactionType(manualChange.getManualRedactionType()),
+                manualChange.getProcessedDate(),
+                manualChange.getRequestedDate(),
+                manualChange.getUserId(),
+                manualChange.getPropertyChanges());
+    }
+
+
+    private static ChangeType toEntityLogType(com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ChangeType type) {
+
+        return switch (type) {
+            case ADDED -> ChangeType.ADDED;
+            case REMOVED -> ChangeType.REMOVED;
+            case CHANGED -> ChangeType.CHANGED;
+        };
+    }
+
+
+    private static com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType toManualRedactionType(ManualRedactionType manualRedactionType) {
+
+        return switch (manualRedactionType) {
+            case ADD_LOCALLY -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.ADD_LOCALLY;
+            case ADD_TO_DICTIONARY -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.ADD_TO_DICTIONARY;
+            case REMOVE_LOCALLY -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.REMOVE_LOCALLY;
+            case REMOVE_FROM_DICTIONARY -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.REMOVE_FROM_DICTIONARY;
+            case FORCE_REDACT -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.FORCE_REDACT;
+            case FORCE_HINT -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.FORCE_HINT;
+            case RECATEGORIZE -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.RECATEGORIZE;
+            case LEGAL_BASIS_CHANGE -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.LEGAL_BASIS_CHANGE;
+            case RESIZE -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.RESIZE;
+        };
+    }
+
+
+    public EntityLogEntry toEntityLogEntry(Map<String, String> oldToNewIdMapping) {
+
+        EntityLogEntry entityLogEntry;
+        if (migratedEntity instanceof Image image) {
+            entityLogEntry = createEntityLogEntry(image);
+        } else if (migratedEntity instanceof TextEntity textEntity) {
+            entityLogEntry = createEntityLogEntry(textEntity);
+        } else if (migratedEntity instanceof ManualEntity entity) {
+            entityLogEntry = createEntityLogEntry(entity);
+        } else {
+            throw new UnsupportedOperationException("Unknown subclass " + migratedEntity.getClass());
+        }
+        entityLogEntry.setChanges(redactionLogEntry.getChanges().stream().map(MigrationEntity::toEntityLogChanges).toList());
+        entityLogEntry.setManualChanges(redactionLogEntry.getManualChanges()
+                .stream()
+                .filter(manualChange -> manualChange.getAnnotationStatus().equals(AnnotationStatus.APPROVED))
+                .map(MigrationEntity::toEntityLogManualChanges)
+                .toList());
+        entityLogEntry.setColor(redactionLogEntry.getColor());
+        entityLogEntry.setReference(migrateSetOfIds(redactionLogEntry.getReference(), oldToNewIdMapping));
+        entityLogEntry.setImportedRedactionIntersections(migrateSetOfIds(redactionLogEntry.getImportedRedactionIntersections(), oldToNewIdMapping));
+        entityLogEntry.setEngines(redactionLogEntry.getEngines().stream().map(this::toEntityLogEngine).collect(Collectors.toSet()));
+        if (redactionLogEntry.getLegalBasis() != null) {
+            entityLogEntry.setLegalBasis(redactionLogEntry.getLegalBasis());
+        }
+
+        return entityLogEntry;
+    }
+
+
+    private Set<String> migrateSetOfIds(Set<String> ids, Map<String, String> oldToNewIdMapping) {
+
+        if (ids == null) {
+            return Collections.emptySet();
+        }
+        return ids.stream().map(oldToNewIdMapping::get).collect(Collectors.toSet());
+    }
+
+
+    private com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine toEntityLogEngine(Engine engine) {
+
+        return switch (engine) {
+            case DICTIONARY -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine.DICTIONARY;
+            case NER -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine.NER;
+            case RULE -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine.RULE;
+        };
+    }
+
+
+    public EntityLogEntry createEntityLogEntry(Image image) {
+
+        String imageType = image.getImageType().equals(ImageType.OTHER) ? "image" : image.getImageType().toString().toLowerCase(Locale.ENGLISH);
+        return EntityLogEntry.builder()
+                .id(image.getId())
+                .value(image.value())
+                .type(imageType)
+                .reason(image.buildReasonWithManualChangeDescriptions())
+                .legalBasis(image.legalBasis())
+                .matchedRule(image.getMatchedRule().getRuleIdentifier().toString())
+                .dictionaryEntry(false)
+                .positions(List.of(new Position(image.getPosition(), image.getPage().getNumber())))
+                .containingNodeId(image.getTreeId())
+                .closestHeadline(image.getHeadline().getTextBlock().getSearchText())
+                .section(image.getManualOverwrite().getSection().orElse(image.getParent().toString()))
+                .imageHasTransparency(image.isTransparent())
+                .state(buildEntryState(image)).entryType(redactionLogEntry.isHint() ? EntryType.IMAGE_HINT : EntryType.IMAGE)
+                .build();
+
+    }
+
+
+    public EntityLogEntry createEntityLogEntry(ManualEntity manualEntity) {
+
+        String type = manualEntity.getManualOverwrite().getType().orElse(manualEntity.getType());
+        return EntityLogEntry.builder()
+                .id(manualEntity.getId())
+                .reason(manualEntity.buildReasonWithManualChangeDescriptions())
+                .legalBasis(manualEntity.legalBasis())
+                .value(manualEntity.value())
+                .type(type)
+                .state(buildEntryState(manualEntity))
+                .entryType(buildEntryType(manualEntity))
+                .section(manualEntity.getManualOverwrite().getSection().orElse(manualEntity.getSection()))
+                .containingNodeId(Collections.emptyList())
+                .closestHeadline("")
+                .matchedRule(manualEntity.getMatchedRule().getRuleIdentifier().toString())
+                .dictionaryEntry(manualEntity.isDictionaryEntry())
+                .dossierDictionaryEntry(manualEntity.isDossierDictionaryEntry())
+                .textAfter("")
+                .textBefore("")
+                .startOffset(-1)
+                .endOffset(-1)
+                .positions(manualEntity.getManualOverwrite()
+                        .getPositions()
+                        .orElse(manualEntity.getEntityPosition())
+                        .stream()
+                        .map(entityPosition -> new Position(entityPosition.rectangle2D(), entityPosition.pageNumber()))
+                        .toList())
+                .engines(Collections.emptySet())
+                .build();
+    }
+
+
+    public EntityLogEntry createEntityLogEntry(TextEntity entity) {
+
+        assert entity.getPositionsOnPagePerPage().size() == 1;
+        PositionOnPage positionOnPage = entity.getPositionsOnPagePerPage().get(0);
+        List<Position> rectanglesPerLine = positionOnPage.getRectanglePerLine()
+                .stream()
+                .map(rectangle2D -> new Position(rectangle2D, positionOnPage.getPage().getNumber()))
+                .toList();
+        return EntityLogEntry.builder()
+                .id(positionOnPage.getId())
+                .positions(rectanglesPerLine)
+                .reason(entity.buildReasonWithManualChangeDescriptions())
+                .legalBasis(entity.legalBasis())
+                .value(entity.getManualOverwrite().getValue().orElse(entity.getMatchedRule().isWriteValueWithLineBreaks() ? entity.getValueWithLineBreaks() : entity.getValue()))
+                .type(entity.getType())
+                .section(entity.getManualOverwrite().getSection().orElse(entity.getDeepestFullyContainingNode().toString()))
+                .containingNodeId(entity.getDeepestFullyContainingNode().getTreeId())
+                .closestHeadline(entity.getDeepestFullyContainingNode().getHeadline().getTextBlock().getSearchText())
+                .matchedRule(entity.getMatchedRule().getRuleIdentifier().toString())
+                .dictionaryEntry(entity.isDictionaryEntry())
+                .textAfter(entity.getTextAfter())
+                .textBefore(entity.getTextBefore())
+                .startOffset(entity.getTextRange().start())
+                .endOffset(entity.getTextRange().end())
+                .dossierDictionaryEntry(entity.isDossierDictionaryEntry())
+                .engines(entity.getEngines() != null ? entity.getEngines() : Collections.emptySet())
+                .state(buildEntryState(entity))
+                .entryType(buildEntryType(entity))
+                .build();
+    }
+
+
+    private EntryState buildEntryState(IEntity entity) {
+
+        if (entity.applied() && entity.active()) {
+            return EntryState.APPLIED;
+        } else if (entity.skipped() && entity.active()) {
+            return EntryState.SKIPPED;
+        } else if (entity.ignored()) {
+            return EntryState.IGNORED;
+        } else {
+            return EntryState.REMOVED;
+        }
+    }
+
+
+    private EntryType buildEntryType(IEntity entity) {
+
+        if (entity instanceof TextEntity textEntity) {
+            return getEntryType(textEntity.getEntityType());
+        } else if (entity instanceof ManualEntity manualEntity) {
+            if (manualEntity.isRectangle()) {
+                return EntryType.AREA;
+            }
+            return getEntryType(manualEntity.getEntityType());
+        } else if (entity instanceof Image) {
+            return EntryType.IMAGE;
+        }
+        throw new UnsupportedOperationException(String.format("Entity subclass %s is not implemented!", entity.getClass()));
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/RectangleWithPage.java

@@ -20,13 +20,13 @@ public record RectangleWithPage(int pageNumber, Rectangle2D rectangle2D) {
 
     private static Rectangle2D toRectangle2D(Rectangle rectangle) {
 
-        return new Rectangle2D.Float(rectangle.getTopLeft().getX(), rectangle.getTopLeft().getY() + rectangle.getHeight(), rectangle.getWidth(), -rectangle.getHeight());
+        return new Rectangle2D.Float(rectangle.getTopLeft().getX(), rectangle.getTopLeft().getY(), rectangle.getWidth(), rectangle.getHeight());
     }
 
 
     private static Rectangle2D toRectangle2D(com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle rectangle) {
 
-        return new Rectangle2D.Float(rectangle.getTopLeft().getX(), rectangle.getTopLeft().getY() + rectangle.getHeight(), rectangle.getWidth(), -rectangle.getHeight());
+        return new Rectangle2D.Float(rectangle.getTopLeft().getX(), rectangle.getTopLeft().getY(), rectangle.getWidth(), rectangle.getHeight());
     }
 
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/component/Component.java

@@ -20,9 +20,9 @@ import lombok.experimental.FieldDefaults;
 public class Component {
 
     RuleIdentifier matchedRule;
-    String category;
+    String name;
     String value;
-    String transformation;
+    String valueDescription;
 
     List<Entity> references;
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/component/Entity.java

@@ -10,6 +10,8 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryType;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualChange;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
 
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
@@ -34,11 +36,11 @@ public class Entity {
 
     boolean imported;
 
+    SemanticNode containingNode;
     String section;
     float[] color;
 
     List<Position> positions;
-    int sectionNumber;
 
     String textBefore;
     String textAfter;
@@ -65,7 +67,7 @@ public class Entity {
     Set<String> importedRedactionIntersections;
 
 
-    public static Entity fromEntityLogEntry(EntityLogEntry e) {
+    public static Entity fromEntityLogEntry(EntityLogEntry e, Document document) {
 
         return Entity.builder()
                 .id(e.getId())
@@ -78,9 +80,7 @@ public class Entity {
                 .legalBasis(e.getLegalBasis())
                 .imported(e.isImported())
                 .section(e.getSection())
-                .color(e.getColor())
-                .positions(e.getPositions())
-                .sectionNumber(e.getSectionNumber())
+                .color(e.getColor()).positions(e.getPositions()).containingNode(document.getDocumentTree().getEntryById(e.getContainingNodeId()).getNode())
                 .textBefore(e.getTextBefore())
                 .textAfter(e.getTextAfter())
                 .startOffset(e.getStartOffset())

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/EntityType.java

@@ -2,6 +2,7 @@ package com.iqser.red.service.redaction.v1.server.model.document.entity;
 
 public enum EntityType {
     ENTITY,
+    HINT,
     RECOMMENDATION,
     FALSE_POSITIVE,
     FALSE_RECOMMENDATION

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/IEntity.java

@@ -35,7 +35,7 @@ public interface IEntity {
 
     default String value() {
 
-        return getManualOverwrite().getValue().orElse(getValue());
+        return getManualOverwrite().getValue().orElse(getValue() == null ? "" : getValue());
     }
 
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/ManualChangeOverwrite.java

@@ -10,10 +10,11 @@ import java.util.Optional;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.BaseAnnotation;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
+import com.iqser.red.service.redaction.v1.server.model.RectangleWithPage;
 
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
@@ -42,6 +43,7 @@ public class ManualChangeOverwrite {
     String legalBasis;
     String section;
     String value;
+    List<RectangleWithPage> positions;
     Boolean applied;
     Boolean removed;
     Boolean ignored;
@@ -96,9 +98,12 @@ public class ManualChangeOverwrite {
                 value = manualLegalBasisChange.getValue();
             }
 
-            if (manualChange instanceof ManualResizeRedaction) {
-                // resizing logic happens in ManualChangesApplicationService.
+            if (manualChange instanceof ManualResizeRedaction manualResizeRedaction) {
+                // resizing logic for TextEntities and Images happens in ManualChangesApplicationService.
                 resized = true;
+                // This is only for not found Manual Entities.
+                value = manualResizeRedaction.getValue();
+                positions = manualResizeRedaction.getPositions().stream().map(RectangleWithPage::fromAnnotationRectangle).toList();
             }
 
             if (manualChange instanceof ManualRecategorization recategorization) {
@@ -197,4 +202,11 @@ public class ManualChangeOverwrite {
         return descriptions == null ? Collections.emptyList() : descriptions;
     }
 
+
+    public Optional<List<RectangleWithPage>> getPositions() {
+
+        calculateCurrentOverride();
+        return positions == null ? Optional.empty() : Optional.of(positions);
+    }
+
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/SemanticNode.java

@@ -249,7 +249,7 @@ public interface SemanticNode {
      * Ignores Entity with ignored == true or removed == true.
      *
      * @param types an array of strings representing the types of entities to check for
-     * @return true, if this SemanticNode has at least one Entity of any of the provided types
+     * @return true, if this SemanticNode has at least one Entity of the provided types
      */
     default boolean hasEntitiesOfAnyType(String... types) {
 
@@ -425,6 +425,71 @@ public interface SemanticNode {
     }
 
 
+    /**
+     * Checks whether this SemanticNode contains exactly the provided String as a word.
+     * @param word - String which the TextBlock might contain
+     * @return true, if this node's TextBlock contains string
+     */
+    default boolean containsWord(String word) {
+
+        return getTextBlock().getWords().stream().anyMatch(s -> s.equals(word));
+    }
+
+
+    /**
+     * Checks whether this SemanticNode contains exactly the provided String as a word ignoring case.
+     * @param word - String which the TextBlock might contain
+     * @return true, if this node's TextBlock contains string
+     */
+    default boolean containsWordIgnoreCase(String word) {
+
+        return getTextBlock().getWords().stream().map(String::toLowerCase).anyMatch(s -> s.equals(word.toLowerCase(Locale.ENGLISH)));
+    }
+
+
+    /**
+     * Checks whether this SemanticNode contains any of the provided Strings as a word.
+     * @param words - A List of Strings which the TextBlock might contain
+     * @return true, if this node's TextBlock contains any of the provided strings
+     */
+    default boolean containsAnyWord(String... words) {
+
+        return Arrays.stream(words).anyMatch(word -> getTextBlock().getWords().stream().anyMatch(word::equals));
+    }
+
+
+    /**
+     * Checks whether this SemanticNode contains any of the provided Strings as a word ignoring case.
+     * @param words - A List of Strings which the TextBlock might contain
+     * @return true, if this node's TextBlock contains any of the provided strings
+     */
+    default boolean containsAnyWordIgnoreCase(String... words) {
+
+        return Arrays.stream(words).map(String::toLowerCase).anyMatch(word -> getTextBlock().getWords().stream().map(String::toLowerCase).anyMatch(word::equals));
+    }
+
+
+    /**
+     * Checks whether this SemanticNode contains all the provided Strings as word.
+     * @param words - A List of Strings which the TextBlock might contain
+     * @return true, if this node's TextBlock contains all the provided strings
+     */
+    default boolean containsAllWords(String... words) {
+
+        return Arrays.stream(words).allMatch(word -> getTextBlock().getWords().stream().anyMatch(word::equals));
+    }
+
+
+    /**
+     * Checks whether this SemanticNode contains all the provided Strings as word ignoring case.
+     * @param words - A List of Strings which the TextBlock might contain
+     * @return true, if this node's TextBlock contains all the provided strings
+     */
+    default boolean containsAllWordsIgnoreCase(String... words) {
+
+        return Arrays.stream(words).map(String::toLowerCase).allMatch(word -> getTextBlock().getWords().stream().map(String::toLowerCase).anyMatch(word::equals));
+    }
+
     /**
      * Checks whether this SemanticNode matches the provided regex pattern.
      *
@@ -455,7 +520,7 @@ public interface SemanticNode {
      * @param y the lower left corner Y value
      * @param w width
      * @param h height
-     * @param pageNumber the pagenumber of the rectangle
+     * @param pageNumber the pageNumber of the rectangle
      * @return true if intersects, false otherwise
      */
     default boolean intersectsRectangle(int x, int y, int w, int h, int pageNumber) {
@@ -607,7 +672,7 @@ public interface SemanticNode {
     private Map<Page, Rectangle2D> getBBoxFromLeafTextBlock(Map<Page, Rectangle2D> bBoxPerPage) {
 
         Map<Page, List<AtomicTextBlock>> atomicTextBlockPerPage = getTextBlock().getAtomicTextBlocks().stream().collect(Collectors.groupingBy(AtomicTextBlock::getPage));
-        atomicTextBlockPerPage.forEach((page, atbs) -> bBoxPerPage.put(page, RectangleTransformations.atomicTextBlockBBox(atbs)));
+        atomicTextBlockPerPage.forEach((page, atomicTextBlocks) -> bBoxPerPage.put(page, RectangleTransformations.atomicTextBlockBBox(atomicTextBlocks)));
         return bBoxPerPage;
     }
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/textblock/AtomicTextBlock.java

@@ -3,11 +3,14 @@ package com.iqser.red.service.redaction.v1.server.model.document.textblock;
 import static java.lang.String.format;
 
 import java.awt.geom.Rectangle2D;
+import java.text.BreakIterator;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
+import java.util.Locale;
 import java.util.Map;
 import java.util.Set;
 import java.util.stream.Collectors;
@@ -40,6 +43,7 @@ public class AtomicTextBlock implements TextBlock {
     //string coordinates
     TextRange textRange;
     String searchText;
+    List<String> words;
     List<Integer> lineBreaks;
 
     //position coordinates
@@ -114,6 +118,20 @@ public class AtomicTextBlock implements TextBlock {
         return new TextRange(lineBreaks.get(lineNumber - 1) + textRange.start(), lineBreaks.get(lineNumber) + textRange.start());
     }
 
+    public List<String> getWords() {
+
+        if (words == null) {
+            words = new ArrayList<>();
+            BreakIterator iterator = BreakIterator.getWordInstance(Locale.ENGLISH);
+            iterator.setText(searchText);
+            int start = iterator.first();
+            for (int end = iterator.next(); end != BreakIterator.DONE; start = end, end = iterator.next()) {
+                words.add(searchText.substring(start, end));
+            }
+        }
+        return words;
+    }
+
 
     @Override
     public List<AtomicTextBlock> getAtomicTextBlocks() {

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/textblock/ConcatenatedTextBlock.java

@@ -7,6 +7,7 @@ import java.util.Collections;
 import java.util.HashMap;
 import java.util.LinkedList;
 import java.util.List;
+import java.util.Collection;
 import java.util.Map;
 import java.util.stream.Stream;
 
@@ -86,6 +87,13 @@ public class ConcatenatedTextBlock implements TextBlock {
     }
 
 
+    @Override
+    public List<String> getWords() {
+
+        return atomicTextBlocks.stream().map(AtomicTextBlock::getWords).flatMap(Collection::stream).toList();
+    }
+
+
     @Override
     public int numberOfLines() {
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/textblock/TextBlock.java

@@ -18,6 +18,7 @@ public interface TextBlock extends CharSequence {
 
     String getSearchText();
 
+    List<String> getWords();
 
     List<AtomicTextBlock> getAtomicTextBlocks();
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/drools/RuleFileBluePrint.java

@@ -1,10 +1,12 @@
 package com.iqser.red.service.redaction.v1.server.model.drools;
 
+import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
 import java.util.Objects;
 import java.util.Optional;
+import java.util.Set;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
@@ -39,6 +41,12 @@ public final class RuleFileBluePrint {
     }
 
 
+    public Set<String> getImportSplitByKeyword() {
+
+        return Arrays.stream(imports.replaceAll("\n", "").split("import")).map(String::trim).collect(Collectors.toSet());
+    }
+
+
     public List<BasicRule> findRulesByIdentifier(RuleIdentifier ruleIdentifier) {
 
         if (Objects.isNull(ruleIdentifier.unit())) {

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/drools/RuleIdentifier.java

@@ -1,9 +1,6 @@
 package com.iqser.red.service.redaction.v1.server.model.drools;
 
-import java.util.Arrays;
 import java.util.Objects;
-import java.util.Set;
-import java.util.stream.Collectors;
 
 import lombok.NonNull;
 
@@ -35,18 +32,6 @@ public record RuleIdentifier(@NonNull RuleType type, Integer unit, Integer id) {
     }
 
 
-    public static Set<RuleIdentifier> fromListOfIdentifiersString(String input) {
-
-        return Arrays.stream(input.split(",")).map(String::trim).map(RuleIdentifier::fromString).collect(Collectors.toSet());
-    }
-
-
-    private static Integer parseIntOrStar(String value) {
-
-        return !value.equals("*") ? Integer.parseInt(value) : null;
-    }
-
-
     /**
      * This is used to filter rules, if the field Integer unit or Integer id is null, the field will match any other RuleIdentifier.
      * Therefore, to compare RuleIdentifiers one should always use the function matches.
@@ -66,6 +51,10 @@ public record RuleIdentifier(@NonNull RuleType type, Integer unit, Integer id) {
     @Override
     public String toString() {
 
+        if (type.name().isBlank() || type.name().isEmpty()) {
+            return "";
+        }
+
         StringBuilder sb = new StringBuilder();
         sb.append(type().name());
         if (Objects.nonNull(unit())) {

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/queue/MessageReceiver.java

@@ -1,13 +1,11 @@
 package com.iqser.red.service.redaction.v1.server.queue;
 
-import static com.iqser.red.service.redaction.v1.server.queue.MessagingConfiguration.REDACTION_QUEUE;
+import static com.iqser.red.service.redaction.v1.model.QueueNames.REDACTION_QUEUE;
 
 import org.springframework.amqp.core.Message;
 import org.springframework.amqp.rabbit.annotation.RabbitHandler;
 import org.springframework.amqp.rabbit.annotation.RabbitListener;
 
-import com.fasterxml.jackson.core.JsonProcessingException;
-
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/queue/MessagingConfiguration.java

@@ -1,6 +1,10 @@
 package com.iqser.red.service.redaction.v1.server.queue;
 
-import lombok.RequiredArgsConstructor;
+import static com.iqser.red.service.redaction.v1.model.QueueNames.MIGRATION_DLQ;
+import static com.iqser.red.service.redaction.v1.model.QueueNames.MIGRATION_QUEUE;
+import static com.iqser.red.service.redaction.v1.model.QueueNames.REDACTION_DQL;
+import static com.iqser.red.service.redaction.v1.model.QueueNames.REDACTION_PRIORITY_QUEUE;
+import static com.iqser.red.service.redaction.v1.model.QueueNames.REDACTION_QUEUE;
 
 import org.springframework.amqp.core.Queue;
 import org.springframework.amqp.core.QueueBuilder;
@@ -8,15 +12,12 @@ import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;
 
+import lombok.RequiredArgsConstructor;
+
 @Configuration
 @RequiredArgsConstructor
 public class MessagingConfiguration {
 
-    public static final String REDACTION_QUEUE = "redactionQueue";
-    public static final String REDACTION_DQL = "redactionDQL";
-
-    public static final String REDACTION_PRIORITY_QUEUE = "redactionPriorityQueue";
-
     public static final String X_ERROR_INFO_HEADER = "x-error-message";
     public static final String X_ERROR_INFO_TIMESTAMP_HEADER = "x-error-message-timestamp";
 
@@ -37,6 +38,20 @@ public class MessagingConfiguration {
     }
 
 
+    @Bean
+    public Queue migrationQueue() {
+
+        return QueueBuilder.durable(MIGRATION_QUEUE).withArgument("x-dead-letter-exchange", "").withArgument("x-dead-letter-routing-key", MIGRATION_DLQ).maxPriority(2).build();
+    }
+
+
+    @Bean
+    public Queue migrationDLQ() {
+
+        return QueueBuilder.durable(MIGRATION_DLQ).build();
+    }
+
+
     @Bean
     public Queue redactionQueue() {
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/queue/MigrationMessageReceiver.java

@@ -0,0 +1,66 @@
+package com.iqser.red.service.redaction.v1.server.queue;
+
+import static com.iqser.red.service.redaction.v1.model.QueueNames.MIGRATION_QUEUE;
+import static com.iqser.red.service.redaction.v1.model.QueueNames.MIGRATION_RESPONSE_QUEUE;
+
+import org.springframework.amqp.core.Message;
+import org.springframework.amqp.rabbit.annotation.RabbitHandler;
+import org.springframework.amqp.rabbit.annotation.RabbitListener;
+import org.springframework.amqp.rabbit.core.RabbitTemplate;
+import org.springframework.stereotype.Service;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog;
+import com.iqser.red.service.redaction.v1.model.MigrationRequest;
+import com.iqser.red.service.redaction.v1.model.MigrationResponse;
+import com.iqser.red.service.redaction.v1.server.model.MigratedEntityLog;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
+import com.iqser.red.service.redaction.v1.server.service.RedactionLogToEntityLogMigrationService;
+import com.iqser.red.service.redaction.v1.server.service.document.DocumentGraphMapper;
+import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
+
+import lombok.AccessLevel;
+import lombok.RequiredArgsConstructor;
+import lombok.SneakyThrows;
+import lombok.experimental.FieldDefaults;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+@Service
+@RequiredArgsConstructor
+@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
+public class MigrationMessageReceiver {
+
+    ObjectMapper objectMapper;
+    RedactionLogToEntityLogMigrationService redactionLogToEntityLogMigrationService;
+    RedactionStorageService redactionStorageService;
+    RabbitTemplate rabbitTemplate;
+
+
+    @SneakyThrows
+    @RabbitHandler
+    @RabbitListener(queues = MIGRATION_QUEUE)
+    public void receiveMigrationRequest(Message message) {
+
+        MigrationRequest migrationRequest = objectMapper.readValue(message.getBody(), MigrationRequest.class);
+
+        RedactionLog redactionLog = redactionStorageService.getRedactionLog(migrationRequest.getDossierId(), migrationRequest.getFileId());
+        Document document = DocumentGraphMapper.toDocumentGraph(redactionStorageService.getDocumentData(migrationRequest.getDossierId(), migrationRequest.getFileId()));
+
+        MigratedEntityLog migratedEntityLog = redactionLogToEntityLogMigrationService.migrate(redactionLog, document);
+
+        redactionStorageService.storeObject(migrationRequest.getDossierId(), migrationRequest.getFileId(), FileType.ENTITY_LOG, migratedEntityLog.getEntityLog());
+        redactionStorageService.storeObject(migrationRequest.getDossierId(), migrationRequest.getFileId(), FileType.MIGRATED_IDS, migratedEntityLog.getMigratedIds());
+
+        sendFinished(MigrationResponse.builder().dossierId(migrationRequest.getDossierId()).fileId(migrationRequest.getFileId()).build());
+    }
+
+
+    @SneakyThrows
+    public void sendFinished(MigrationResponse migrationResponse) {
+
+        rabbitTemplate.convertAndSend(MIGRATION_RESPONSE_QUEUE, migrationResponse);
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/queue/PriorityMessageReceiver.java

@@ -1,13 +1,11 @@
 package com.iqser.red.service.redaction.v1.server.queue;
 
-import static com.iqser.red.service.redaction.v1.server.queue.MessagingConfiguration.REDACTION_PRIORITY_QUEUE;
+import static com.iqser.red.service.redaction.v1.model.QueueNames.REDACTION_PRIORITY_QUEUE;
 
 import org.springframework.amqp.core.Message;
 import org.springframework.amqp.rabbit.annotation.RabbitHandler;
 import org.springframework.amqp.rabbit.annotation.RabbitListener;
 
-import com.fasterxml.jackson.core.JsonProcessingException;
-
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/queue/RedactionMessageReceiver.java

@@ -1,8 +1,8 @@
 package com.iqser.red.service.redaction.v1.server.queue;
 
-import static com.iqser.red.service.redaction.v1.server.queue.MessagingConfiguration.REDACTION_DQL;
-import static com.iqser.red.service.redaction.v1.server.queue.MessagingConfiguration.REDACTION_PRIORITY_QUEUE;
-import static com.iqser.red.service.redaction.v1.server.queue.MessagingConfiguration.REDACTION_QUEUE;
+import static com.iqser.red.service.redaction.v1.model.QueueNames.REDACTION_DQL;
+import static com.iqser.red.service.redaction.v1.model.QueueNames.REDACTION_PRIORITY_QUEUE;
+import static com.iqser.red.service.redaction.v1.model.QueueNames.REDACTION_QUEUE;
 import static com.iqser.red.service.redaction.v1.server.queue.MessagingConfiguration.X_ERROR_INFO_HEADER;
 import static com.iqser.red.service.redaction.v1.server.queue.MessagingConfiguration.X_ERROR_INFO_TIMESTAMP_HEADER;
 import static java.lang.String.format;
@@ -23,8 +23,8 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeResu
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileErrorInfo;
 import com.iqser.red.service.redaction.v1.server.client.FileStatusProcessingUpdateClient;
 import com.iqser.red.service.redaction.v1.server.client.RulesClient;
-import com.iqser.red.service.redaction.v1.server.utils.exception.DroolsTimeoutException;
 import com.iqser.red.service.redaction.v1.server.service.AnalyzeService;
+import com.iqser.red.service.redaction.v1.server.utils.exception.DroolsTimeoutException;
 
 import lombok.RequiredArgsConstructor;
 import lombok.SneakyThrows;
@@ -128,7 +128,7 @@ public class RedactionMessageReceiver {
 
     @RabbitHandler
     @RabbitListener(queues = REDACTION_DQL)
-    public void receiveAnalyzeRequestDQL(Message in) throws IOException {
+    public void receiveAnalyzeRequestDLQ(Message in) throws IOException {
 
         var analyzeRequest = objectMapper.readValue(in.getBody(), AnalyzeRequest.class);
         log.info("Failed to process analyze request: {}", analyzeRequest);

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalyzeService.java

@@ -18,9 +18,6 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileTyp
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentLog;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogChanges;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogLegalBasis;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Comment;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.legalbasis.LegalBasis;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog;
@@ -33,10 +30,10 @@ import com.iqser.red.service.redaction.v1.server.client.model.NerEntitiesModel;
 import com.iqser.red.service.redaction.v1.server.model.KieWrapper;
 import com.iqser.red.service.redaction.v1.server.model.ManualEntity;
 import com.iqser.red.service.redaction.v1.server.model.NerEntities;
+import com.iqser.red.service.redaction.v1.server.model.component.Component;
 import com.iqser.red.service.redaction.v1.server.model.dictionary.Dictionary;
 import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryIncrement;
 import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryVersion;
-import com.iqser.red.service.redaction.v1.server.model.component.Component;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
 import com.iqser.red.service.redaction.v1.server.service.document.DocumentGraphMapper;
@@ -73,7 +70,6 @@ public class AnalyzeService {
     ComponentLogCreatorService componentLogCreatorService;
     RedactionStorageService redactionStorageService;
     RedactionChangeLogService redactionChangeLogService;
-    EntityChangeLogService entityChangeLogService;
     LegalBasisClient legalBasisClient;
     RedactionServiceSettings redactionServiceSettings;
     ImportedRedactionService importedRedactionService;
@@ -83,85 +79,6 @@ public class AnalyzeService {
     FunctionTimerValues redactmanagerAnalyzePagewiseValues;
 
 
-    @Timed("redactmanager_analyze")
-    public AnalyzeResult analyze(AnalyzeRequest analyzeRequest) {
-
-        long startTime = System.currentTimeMillis();
-
-        var kieWrapperEntityRules = kieContainerCreationService.getLatestKieContainer(analyzeRequest.getDossierTemplateId(), RuleFileType.ENTITY);
-        log.info("Updated Rules to Version {} for file {} in dossier {}", kieWrapperEntityRules.rulesVersion(), analyzeRequest.getFileId(), analyzeRequest.getDossierId());
-
-        var kieWrapperComponentRules = kieContainerCreationService.getLatestKieContainer(analyzeRequest.getDossierTemplateId(), RuleFileType.COMPONENT);
-        log.info("Updated Rules to Version {} for file {} in dossier {}", kieWrapperEntityRules.rulesVersion(), analyzeRequest.getFileId(), analyzeRequest.getDossierId());
-
-        Document document = DocumentGraphMapper.toDocumentGraph(redactionStorageService.getDocumentData(analyzeRequest.getDossierId(), analyzeRequest.getFileId()));
-        log.info("Loaded Document Graph for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
-
-        NerEntities nerEntities = getEntityRecognitionEntities(analyzeRequest, document);
-        log.info("Loaded Ner Entities for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
-
-        dictionaryService.updateDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId());
-        Dictionary dictionary = dictionaryService.getDeepCopyDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId());
-        log.info("Updated Dictionaries for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
-
-        List<ManualEntity> notFoundManualRedactionEntries = manualRedactionEntryService.addManualRedactionEntriesAndReturnNotFoundEntries(analyzeRequest, document);
-
-        dictionarySearchService.addDictionaryEntities(dictionary, document);
-        log.info("Finished Dictionary Search for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
-
-        List<FileAttribute> allFileAttributes = entityDroolsExecutionService.executeRules(kieWrapperEntityRules.container(),
-                document,
-                dictionary,
-                analyzeRequest.getFileAttributes(),
-                analyzeRequest.getManualRedactions(),
-                nerEntities);
-        log.info("Finished entity rule execution for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
-
-        RedactionLog redactionLog = createRedactionLog(analyzeRequest, document, notFoundManualRedactionEntries, dictionary, kieWrapperEntityRules);
-        EntityLog entityLog = createEntityLog(analyzeRequest, document, notFoundManualRedactionEntries, dictionary, kieWrapperEntityRules);
-
-        return finalizeAnalysis(analyzeRequest, startTime, kieWrapperComponentRules, entityLog,
-                redactionLog,
-                document.getNumberOfPages(), dictionary.getVersion(), false, new HashSet<>(allFileAttributes));
-    }
-
-
-    private EntityLog createEntityLog(AnalyzeRequest analyzeRequest,
-                                      Document document,
-                                      List<ManualEntity> notFoundManualRedactionEntries,
-                                      Dictionary dictionary,
-                                      KieWrapper wrapper) {
-
-        List<EntityLogEntry> entityLogEntries = entityLogCreatorService.createEntityLogEntries(document, analyzeRequest.getDossierTemplateId(), notFoundManualRedactionEntries);
-
-        List<LegalBasis> legalBasis = legalBasisClient.getLegalBasisMapping(analyzeRequest.getDossierTemplateId());
-
-        EntityLog entityLog = new EntityLog(redactionServiceSettings.getAnalysisVersion(),
-                analyzeRequest.getAnalysisNumber(),
-                entityLogEntries,
-                toEntityLogLegalBasis(legalBasis),
-                dictionary.getVersion().getDossierTemplateVersion(),
-                dictionary.getVersion().getDossierVersion(),
-                wrapper.rulesVersion(),
-                legalBasisClient.getVersion(analyzeRequest.getDossierTemplateId()));
-
-        List<EntityLogEntry> importedRedactionFilteredEntries = importedRedactionService.processImportedEntities(analyzeRequest.getDossierTemplateId(),
-                analyzeRequest.getDossierId(),
-                analyzeRequest.getFileId(),
-                entityLog.getEntityLogEntry(),
-                true);
-
-        entityLog.setEntityLogEntry(importedRedactionFilteredEntries);
-        return entityLog;
-    }
-
-
-    private List<EntityLogLegalBasis> toEntityLogLegalBasis(List<LegalBasis> legalBasis) {
-
-        return legalBasis.stream().map(l -> new EntityLogLegalBasis(l.getName(), l.getDescription(), l.getReason())).collect(Collectors.toList());
-    }
-
-
     @Timed("redactmanager_reanalyze")
     @SneakyThrows
     public AnalyzeResult reanalyze(@RequestBody AnalyzeRequest analyzeRequest) {
@@ -189,10 +106,14 @@ public class AnalyzeService {
         log.info("{} Sections to reanalyze found for file {} in dossier {}", sectionsToReanalyseIds.size(), analyzeRequest.getFileId(), analyzeRequest.getDossierId());
 
         if (sectionsToReAnalyse.isEmpty()) {
+
             return finalizeAnalysis(analyzeRequest,
                     startTime,
                     kieContainerCreationService.getLatestKieContainer(analyzeRequest.getDossierTemplateId(), RuleFileType.COMPONENT),
-                    previousEntityLog,
+                    entityLogCreatorService.updateVersionsAndReturnChanges(previousEntityLog,
+                            dictionaryIncrement.getDictionaryVersion(),
+                            analyzeRequest.getDossierTemplateId(),
+                            false), document,
                     previousRedactionLog,
                     document.getNumberOfPages(),
                     dictionaryIncrement.getDictionaryVersion(),
@@ -205,7 +126,9 @@ public class AnalyzeService {
         NerEntities nerEntities = getEntityRecognitionEntitiesFilteredBySectionIds(analyzeRequest, document, sectionsToReanalyseIds);
         log.info("Loaded Ner Entities for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
 
-        List<ManualEntity> notFoundManualRedactionEntries = manualRedactionEntryService.addManualRedactionEntriesAndReturnNotFoundEntries(analyzeRequest, document);
+        List<ManualEntity> notFoundManualRedactionEntries = manualRedactionEntryService.addManualRedactionEntriesAndReturnNotFoundEntries(analyzeRequest,
+                document,
+                analyzeRequest.getDossierTemplateId());
 
         Dictionary dictionary = dictionaryService.getDeepCopyDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId());
         log.info("Updated Dictionaries for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
@@ -223,12 +146,18 @@ public class AnalyzeService {
         log.info("Finished entity rule execution for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
 
         RedactionLog redactionLog = updatePreviousRedactionLog(analyzeRequest, document, notFoundManualRedactionEntries, previousRedactionLog, sectionsToReanalyseIds);
-        EntityLog entityLog = updatePreviousEntityLog(analyzeRequest, document, notFoundManualRedactionEntries, previousEntityLog, sectionsToReanalyseIds);
+        EntityLogChanges entityLogChanges = entityLogCreatorService.updatePreviousEntityLog(analyzeRequest,
+                document,
+                notFoundManualRedactionEntries,
+                previousEntityLog,
+                sectionsToReanalyseIds,
+                dictionary.getVersion());
 
         return finalizeAnalysis(analyzeRequest,
                 startTime,
                 kieContainerCreationService.getLatestKieContainer(analyzeRequest.getDossierTemplateId(), RuleFileType.COMPONENT),
-                entityLog,
+                entityLogChanges,
+                document,
                 redactionLog,
                 document.getNumberOfPages(),
                 dictionaryIncrement.getDictionaryVersion(),
@@ -237,6 +166,58 @@ public class AnalyzeService {
     }
 
 
+    @Timed("redactmanager_analyze")
+    public AnalyzeResult analyze(AnalyzeRequest analyzeRequest) {
+
+        long startTime = System.currentTimeMillis();
+
+        var kieWrapperEntityRules = kieContainerCreationService.getLatestKieContainer(analyzeRequest.getDossierTemplateId(), RuleFileType.ENTITY);
+        log.info("Updated Rules to Version {} for file {} in dossier {}", kieWrapperEntityRules.rulesVersion(), analyzeRequest.getFileId(), analyzeRequest.getDossierId());
+
+        var kieWrapperComponentRules = kieContainerCreationService.getLatestKieContainer(analyzeRequest.getDossierTemplateId(), RuleFileType.COMPONENT);
+        log.info("Updated Rules to Version {} for file {} in dossier {}", kieWrapperEntityRules.rulesVersion(), analyzeRequest.getFileId(), analyzeRequest.getDossierId());
+
+        Document document = DocumentGraphMapper.toDocumentGraph(redactionStorageService.getDocumentData(analyzeRequest.getDossierId(), analyzeRequest.getFileId()));
+        log.info("Loaded Document Graph for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
+
+        NerEntities nerEntities = getEntityRecognitionEntities(analyzeRequest, document);
+        log.info("Loaded Ner Entities for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
+
+        dictionaryService.updateDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId());
+        Dictionary dictionary = dictionaryService.getDeepCopyDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId());
+        log.info("Updated Dictionaries for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
+
+        List<ManualEntity> notFoundManualRedactionEntries = manualRedactionEntryService.addManualRedactionEntriesAndReturnNotFoundEntries(analyzeRequest,
+                document,
+                analyzeRequest.getDossierTemplateId());
+
+        dictionarySearchService.addDictionaryEntities(dictionary, document);
+        log.info("Finished Dictionary Search for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
+
+        List<FileAttribute> allFileAttributes = entityDroolsExecutionService.executeRules(kieWrapperEntityRules.container(),
+                document,
+                dictionary,
+                analyzeRequest.getFileAttributes(),
+                analyzeRequest.getManualRedactions(),
+                nerEntities);
+        log.info("Finished entity rule execution for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
+
+        RedactionLog redactionLog = createRedactionLog(analyzeRequest, document, notFoundManualRedactionEntries, dictionary, kieWrapperEntityRules);
+        EntityLog entityLog = entityLogCreatorService.createInitialEntityLog(analyzeRequest,
+                document,
+                notFoundManualRedactionEntries,
+                dictionary.getVersion(),
+                kieWrapperEntityRules.rulesVersion());
+
+        return finalizeAnalysis(analyzeRequest, startTime, kieWrapperComponentRules, new EntityLogChanges(entityLog, false), document,
+                redactionLog,
+                document.getNumberOfPages(),
+                dictionary.getVersion(),
+                false,
+                new HashSet<>(allFileAttributes));
+    }
+
+
     @Deprecated(forRemoval = true)
     private RedactionLog updatePreviousRedactionLog(AnalyzeRequest analyzeRequest,
                                                     Document document,
@@ -245,9 +226,7 @@ public class AnalyzeService {
                                                     Set<Integer> sectionsToReanalyseIds) {
 
         List<RedactionLogEntry> newRedactionLogEntries = redactionLogCreatorService.createRedactionLog(document,
-                analyzeRequest.getDossierTemplateId(),
-                notFoundManualRedactionEntries,
-                getComments(analyzeRequest));
+                analyzeRequest.getDossierTemplateId(), notFoundManualRedactionEntries);
 
         var importedRedactionFilteredEntries = importedRedactionService.processImportedRedactions(analyzeRequest.getDossierTemplateId(),
                 analyzeRequest.getDossierId(),
@@ -262,38 +241,21 @@ public class AnalyzeService {
     }
 
 
-    private EntityLog updatePreviousEntityLog(AnalyzeRequest analyzeRequest,
-                                              Document document,
-                                              List<ManualEntity> notFoundManualRedactionEntries,
-                                              EntityLog previousEntityLog,
-                                              Set<Integer> sectionsToReanalyseIds) {
-
-        List<EntityLogEntry> newEntityLogEntries = entityLogCreatorService.createEntityLogEntries(document, analyzeRequest.getDossierTemplateId(), notFoundManualRedactionEntries);
-
-        var importedRedactionFilteredEntries = importedRedactionService.processImportedEntities(analyzeRequest.getDossierTemplateId(),
-                analyzeRequest.getDossierId(),
-                analyzeRequest.getFileId(),
-                newEntityLogEntries,
-                false);
-
-        previousEntityLog.getEntityLogEntry()
-                .removeIf(entry -> sectionsToReanalyseIds.contains(entry.getSectionNumber()) && !entry.getType().equals(ImportedRedactionService.IMPORTED_REDACTION_TYPE));
-        previousEntityLog.getEntityLogEntry().addAll(importedRedactionFilteredEntries);
-        return previousEntityLog;
-    }
-
-
-    private AnalyzeResult finalizeAnalysis(AnalyzeRequest analyzeRequest, long startTime, KieWrapper kieWrapperComponentRules, EntityLog entityLog,
+    private AnalyzeResult finalizeAnalysis(AnalyzeRequest analyzeRequest, long startTime, KieWrapper kieWrapperComponentRules, EntityLogChanges entityLogChanges, Document document,
                                            RedactionLog redactionLog,
                                            int numberOfPages,
                                            DictionaryVersion dictionaryVersion,
                                            boolean isReanalysis,
                                            Set<FileAttribute> addedFileAttributes) {
 
-        EntityLogChanges entityLogChanges = finalizeEntityLog(analyzeRequest, entityLog, redactionLog, dictionaryVersion);
+        finalizeRedactionLog(analyzeRequest, redactionLog, dictionaryVersion);
 
-        if (entityLogChanges.isHasChanges()) {
-            computeComponentsWhenRulesArePresent(analyzeRequest, kieWrapperComponentRules, addedFileAttributes, entityLogChanges);
+        EntityLog entityLog = entityLogChanges.getEntityLog();
+        redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.ENTITY_LOG, entityLogChanges.getEntityLog());
+
+        log.info("Created entity log for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
+        if (entityLogChanges.isHasChanges() || !isReanalysis) {
+            computeComponentsWhenRulesArePresent(analyzeRequest, kieWrapperComponentRules, document, addedFileAttributes, entityLogChanges, dictionaryVersion);
         }
 
         log.info("Stored analysis logs for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
@@ -310,7 +272,8 @@ public class AnalyzeService {
                 .hasUpdates(entityLogChanges.isHasChanges())
                 .analysisVersion(redactionServiceSettings.getAnalysisVersion())
                 .analysisNumber(analyzeRequest.getAnalysisNumber())
-                .rulesVersion(entityLog.getRulesVersion()).componentRulesVersion(kieWrapperComponentRules.rulesVersion())
+                .rulesVersion(entityLog.getRulesVersion())
+                .componentRulesVersion(kieWrapperComponentRules.rulesVersion())
                 .dictionaryVersion(entityLog.getDictionaryVersion())
                 .legalBasisVersion(entityLog.getLegalBasisVersion())
                 .dossierDictionaryVersion(entityLog.getDossierDictionaryVersion())
@@ -321,53 +284,40 @@ public class AnalyzeService {
     }
 
 
-    private void computeComponentsWhenRulesArePresent(AnalyzeRequest analyzeRequest,
-                                                      KieWrapper kieWrapperComponentRules,
+    private void computeComponentsWhenRulesArePresent(AnalyzeRequest analyzeRequest, KieWrapper kieWrapperComponentRules, Document document,
                                                       Set<FileAttribute> addedFileAttributes,
-                                                      EntityLogChanges entityLogChanges) {
+                                                      EntityLogChanges entityLogChanges,
+                                                      DictionaryVersion dictionaryVersion) {
 
         if (!kieWrapperComponentRules.isPresent()) {
             return;
         }
 
-        List<Component> components = componentDroolsExecutionService.executeRules(kieWrapperComponentRules.container(),
-                entityLogChanges.getEntityLog(),
+        List<Component> components = componentDroolsExecutionService.executeRules(kieWrapperComponentRules.container(), entityLogChanges.getEntityLog(), document,
                 addedFileAttributes.stream().toList());
         log.info("Finished component rule execution for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
 
-        ComponentLog componentLog = componentLogCreatorService.buildComponentLog(analyzeRequest.getAnalysisNumber(), components);
+        ComponentLog componentLog = componentLogCreatorService.buildComponentLog(analyzeRequest.getAnalysisNumber(), components, kieWrapperComponentRules.rulesVersion());
 
         redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.COMPONENT_LOG, componentLog);
 
     }
 
 
-    private EntityLogChanges finalizeEntityLog(AnalyzeRequest analyzeRequest, EntityLog entityLog, RedactionLog redactionLog, DictionaryVersion dictionaryVersion) {
-
+    private RedactionLogChanges finalizeRedactionLog(AnalyzeRequest analyzeRequest, RedactionLog redactionLog, DictionaryVersion dictionaryVersion) {
         // TODO: remove redactionLog related stuff
+
+        RedactionLog previousRedactionLog = redactionStorageService.getRedactionLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
+
         redactionLog.setDictionaryVersion(dictionaryVersion.getDossierTemplateVersion());
         redactionLog.setDossierDictionaryVersion(dictionaryVersion.getDossierVersion());
 
         excludeExcludedPages(redactionLog, analyzeRequest.getExcludedPages());
-        RedactionLogChanges redactionLogChange = redactionChangeLogService.computeChanges(analyzeRequest.getDossierId(),
-                analyzeRequest.getFileId(),
-                redactionLog,
-                analyzeRequest.getAnalysisNumber());
-
-        entityLog.setDictionaryVersion(dictionaryVersion.getDossierTemplateVersion());
-        entityLog.setDossierDictionaryVersion(dictionaryVersion.getDossierVersion());
-
-        excludeExcludedPages(entityLog, analyzeRequest.getExcludedPages());
-
-        EntityLogChanges entityLogChanges = entityChangeLogService.computeChanges(analyzeRequest.getDossierId(),
-                analyzeRequest.getFileId(),
-                entityLog,
-                analyzeRequest.getAnalysisNumber());
-        log.info("Created entity log for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
+        RedactionLogChanges redactionLogChange = redactionChangeLogService.computeChanges(previousRedactionLog, redactionLog, analyzeRequest.getAnalysisNumber());
 
         redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.REDACTION_LOG, redactionLogChange.getRedactionLog());
-        redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.ENTITY_LOG, entityLogChanges.getEntityLog());
-        return entityLogChanges;
+
+        return redactionLogChange;
     }
 
 
@@ -427,9 +377,7 @@ public class AnalyzeService {
                                             KieWrapper wrapper) {
 
         List<RedactionLogEntry> redactionLogEntries = redactionLogCreatorService.createRedactionLog(document,
-                analyzeRequest.getDossierTemplateId(),
-                notFoundManualRedactionEntries,
-                getComments(analyzeRequest));
+                analyzeRequest.getDossierTemplateId(), notFoundManualRedactionEntries);
 
         List<LegalBasis> legalBasis = legalBasisClient.getLegalBasisMapping(analyzeRequest.getDossierTemplateId());
         RedactionLog redactionLog = new RedactionLog(redactionServiceSettings.getAnalysisVersion(),
@@ -451,18 +399,6 @@ public class AnalyzeService {
     }
 
 
-    private static Map<String, List<Comment>> getComments(AnalyzeRequest analyzeRequest) {
-
-        if (analyzeRequest.getManualRedactions() == null) {
-            return Collections.emptyMap();
-        }
-        if (analyzeRequest.getManualRedactions().getComments() == null) {
-            return Collections.emptyMap();
-        }
-        return analyzeRequest.getManualRedactions().getComments();
-    }
-
-
     public List<RedactionLogLegalBasis> toRedactionLogLegalBasis(List<LegalBasis> legalBasis) {
 
         return legalBasis.stream().map(l -> new RedactionLogLegalBasis(l.getName(), l.getDescription(), l.getReason())).collect(Collectors.toList());
@@ -480,17 +416,4 @@ public class AnalyzeService {
         }
     }
 
-
-    private void excludeExcludedPages(EntityLog entityLog, Set<Integer> excludedPages) {
-
-        if (excludedPages != null && !excludedPages.isEmpty()) {
-            entityLog.getEntityLogEntry().forEach(entry -> entry.getPositions().forEach(pos -> {
-                if (excludedPages.contains(pos.getPageNumber())) {
-                    entry.setExcluded(true);
-                }
-            }));
-        }
-
-    }
-
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/ComponentLogCreatorService.java

@@ -1,57 +1,61 @@
 package com.iqser.red.service.redaction.v1.server.service;
 
+import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.List;
-import java.util.stream.Collectors;
+import java.util.Map;
 
 import org.springframework.stereotype.Service;
 
-import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentEntityReference;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentLog;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentLogCategory;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentLogEntityReference;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentLogEntry;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentLogEntryValue;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position;
 import com.iqser.red.service.redaction.v1.server.model.component.Component;
 import com.iqser.red.service.redaction.v1.server.model.component.Entity;
+import com.iqser.red.service.redaction.v1.server.service.document.ComponentComparator;
 import com.iqser.red.service.redaction.v1.server.service.document.EntityComparators;
 
 @Service
 public class ComponentLogCreatorService {
 
-    public ComponentLog buildComponentLog(int analysisNumber, List<Component> components) {
+    public ComponentLog buildComponentLog(int analysisNumber, List<Component> components, long componentRulesVersion) {
 
-        List<ComponentLogCategory> componentLogCategories = components.stream()
-                .collect(Collectors.groupingBy(Component::getCategory, Collectors.mapping(this::buildComponentLogEntry, Collectors.toList())))
+        Map<String, List<ComponentLogEntryValue>> map = new HashMap<>();
+        components.stream().sorted(ComponentComparator.first()).forEach(component -> {
+            ComponentLogEntryValue componentLogEntryValue = buildComponentLogEntry(component);
+            map.computeIfAbsent(component.getName(), k -> new ArrayList<>()).add(componentLogEntryValue);
+        });
+        List<ComponentLogEntry> componentLogComponents = map
                 .entrySet()
-                .stream()
-                .map(entry -> new ComponentLogCategory(entry.getKey(), entry.getValue()))
+                .stream().map(entry -> new ComponentLogEntry(entry.getKey(), entry.getValue()))
                 .toList();
-        return new ComponentLog(analysisNumber, componentLogCategories, -1, -1, -1);
+        return new ComponentLog(analysisNumber, componentRulesVersion, componentLogComponents);
     }
 
 
-    private ComponentLogEntry buildComponentLogEntry(Component component) {
+    private ComponentLogEntryValue buildComponentLogEntry(Component component) {
 
-        return ComponentLogEntry.builder()
-                .value(component.getValue())
-                .transformation(component.getTransformation()).originalValue(component.getReferences().stream().sorted(EntityComparators.start()).map(Entity::getValue).toList())
-                .componentEntityReferences(toComponentEntityReferences(component.getReferences()))
+        return ComponentLogEntryValue.builder()
+                .value(component.getValue()).originalValue(component.getValue())
+                .componentRuleId(component.getMatchedRule().toString())
+                .valueDescription(component.getValueDescription())
+                .componentLogEntityReferences(toComponentEntityReferences(component.getReferences().stream().sorted(EntityComparators.first()).toList()))
                 .build();
     }
 
 
-    private List<ComponentEntityReference> toComponentEntityReferences(List<Entity> references) {
+    private List<ComponentLogEntityReference> toComponentEntityReferences(List<Entity> references) {
 
         return references.stream().map(this::toComponentEntityReference).toList();
     }
 
 
-    private ComponentEntityReference toComponentEntityReference(Entity entity) {
+    private ComponentLogEntityReference toComponentEntityReference(Entity entity) {
 
-        return ComponentEntityReference.builder()
-                .id(entity.getId())
-                .page(entity.getPositions().stream().findFirst().map(Position::getPageNumber).orElse(0))
-                .reason(entity.getReason())
-                .ruleIdentifier(entity.getMatchedRule())
+        return ComponentLogEntityReference.builder().id(entity.getId())
+                .page(entity.getPositions().stream().findFirst().map(Position::getPageNumber).orElse(0)).entityRuleId(entity.getMatchedRule())
                 .type(entity.getType())
                 .build();
     }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/DictionarySearchService.java

@@ -27,7 +27,7 @@ public class DictionarySearchService {
     public void addDictionaryEntities(Dictionary dictionary, SemanticNode node) {
 
         for (var model : dictionary.getDictionaryModels()) {
-            bySearchImplementationAsDictionary(model.getEntriesSearch(), model.getType(), EntityType.ENTITY, node, model.isDossierDictionary());
+            bySearchImplementationAsDictionary(model.getEntriesSearch(), model.getType(), model.isHint() ? EntityType.HINT : EntityType.ENTITY, node, model.isDossierDictionary());
             bySearchImplementationAsDictionary(model.getFalsePositiveSearch(), model.getType(), EntityType.FALSE_POSITIVE, node, model.isDossierDictionary());
             bySearchImplementationAsDictionary(model.getFalseRecommendationsSearch(), model.getType(), EntityType.FALSE_RECOMMENDATION, node, model.isDossierDictionary());
         }
@@ -43,7 +43,7 @@ public class DictionarySearchService {
         EntityCreationService entityCreationService = new EntityCreationService(entityEnrichmentService);
         searchImplementation.getBoundaries(node.getTextBlock(), node.getTextRange())
                 .stream().filter(boundary -> entityCreationService.isValidEntityTextRange(node.getTextBlock(), boundary))
-                .map(bounds -> entityCreationService.forceByBoundary(bounds, type, entityType, node))
+                .map(bounds -> entityCreationService.forceByTextRange(bounds, type, entityType, node))
                 .peek(entity -> entity.setDictionaryEntry(true))
                 .peek(entity -> entity.setDossierDictionaryEntry(isDossierDictionaryEntry))
                 .forEach(entity -> entity.addEngine(Engine.DICTIONARY));

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/DictionaryService.java

@@ -47,7 +47,7 @@ import lombok.extern.slf4j.Slf4j;
 @RequiredArgsConstructor
 public class DictionaryService {
 
-    private static final String DEFAULT_COLOR = "#cccccc";
+    public static final String DEFAULT_COLOR = "#cccccc";
     private final DictionaryClient dictionaryClient;
 
     private final RedactionServiceSettings settings;

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityChangeLogService.java

@@ -1,11 +1,8 @@
 package com.iqser.red.service.redaction.v1.server.service;
 
 import java.time.OffsetDateTime;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.HashSet;
 import java.util.List;
-import java.util.Map;
+import java.util.Optional;
 import java.util.Set;
 import java.util.stream.Collectors;
 
@@ -13,10 +10,8 @@ import org.springframework.stereotype.Service;
 
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Change;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ChangeType;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogChanges;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
-import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState;
 
 import io.micrometer.core.annotation.Timed;
 import lombok.RequiredArgsConstructor;
@@ -27,93 +22,72 @@ import lombok.extern.slf4j.Slf4j;
 @RequiredArgsConstructor
 public class EntityChangeLogService {
 
-    private final RedactionStorageService redactionStorageService;
-
 
     @Timed("redactmanager_computeChanges")
-    public EntityLogChanges computeChanges(String dossierId, String fileId, EntityLog currentEntityLog, int analysisNumber) {
+    public boolean computeChanges(List<EntityLogEntry> previousEntityLogEntries, List<EntityLogEntry> newEntityLogEntries, int analysisNumber) {
 
-        long start = System.currentTimeMillis();
-
-        EntityLog previousEntityLog = redactionStorageService.getEntityLog(dossierId, fileId);
-
-        if (previousEntityLog == null) {
-            currentEntityLog.getEntityLogEntry().forEach(entry -> {
-                entry.getChanges().add(new Change(analysisNumber, ChangeType.ADDED, OffsetDateTime.now()));
-            });
-            return new EntityLogChanges(currentEntityLog, false);
+        var now = OffsetDateTime.now();
+        if (previousEntityLogEntries.isEmpty()) {
+            newEntityLogEntries.forEach(entry -> entry.getChanges().add(new Change(analysisNumber, ChangeType.ADDED, now)));
+            return true;
         }
 
-        List<EntityLogEntry> previouslyExistingEntries = previousEntityLog.getEntityLogEntry().stream().filter(entry -> !lastChangeIsRemoved(entry)).toList();
+        boolean hasChanges = false;
 
-        Map<String, EntityLogEntry> addedEntryIds = getEntriesThatExistInCurrentButNotInPreviousEntityLog(currentEntityLog, previouslyExistingEntries);
-        Set<String> removedIds = getEntryIdsThatExistInPreviousButNotInCurrentEntityLog(currentEntityLog, previouslyExistingEntries);
-
-        List<EntityLogEntry> newEntityLogEntries = previousEntityLog.getEntityLogEntry();
-
-        List<EntityLogEntry> toRemove = new ArrayList<>();
-        newEntityLogEntries.forEach(entry -> {
-            if (removedIds.contains(entry.getId()) && addedEntryIds.containsKey(entry.getId())) {
-                List<Change> changes = entry.getChanges();
-                changes.add(new Change(analysisNumber, ChangeType.CHANGED, OffsetDateTime.now()));
-                var newEntry = addedEntryIds.get(entry.getId());
-                newEntry.setChanges(changes);
-                addedEntryIds.put(entry.getId(), newEntry);
-                toRemove.add(entry);
-            } else if (removedIds.contains(entry.getId())) {
-                entry.getChanges().add(new Change(analysisNumber, ChangeType.REMOVED, OffsetDateTime.now()));
-            } else if (addedEntryIds.containsKey(entry.getId())) {
-                List<Change> changes = entry.getChanges();
-                changes.add(new Change(analysisNumber, ChangeType.ADDED, OffsetDateTime.now()));
-                var newEntry = addedEntryIds.get(entry.getId());
-                newEntry.setChanges(changes);
-                addedEntryIds.put(entry.getId(), newEntry);
-                toRemove.add(entry);
+        for (EntityLogEntry entityLogEntry : newEntityLogEntries) {
+            Optional<EntityLogEntry> optionalPreviousEntity = previousEntityLogEntries.stream().filter(entry -> entry.getId().equals(entityLogEntry.getId())).findAny();
+            if (optionalPreviousEntity.isEmpty()) {
+                hasChanges = true;
+                entityLogEntry.getChanges().add(new Change(analysisNumber, ChangeType.ADDED, now));
+                continue;
             }
-        });
-
-        newEntityLogEntries.removeAll(toRemove);
-
-        addedEntryIds.forEach((k, v) -> {
-            if (v.getChanges().isEmpty()) {
-                v.getChanges().add(new Change(analysisNumber, ChangeType.ADDED, OffsetDateTime.now()));
+            var previousEntity = optionalPreviousEntity.get();
+            entityLogEntry.getChanges().addAll(previousEntity.getChanges());
+            if (!previousEntity.getState().equals(entityLogEntry.getState())) {
+                hasChanges = true;
+                ChangeType changeType = calculateChangeType(entityLogEntry.getState(), previousEntity.getState());
+                if (changeType != null) {
+                    entityLogEntry.getChanges().add(new Change(analysisNumber, changeType, now));
+                }
             }
-            newEntityLogEntries.add(v);
-        });
-
-        currentEntityLog.setEntityLogEntry(newEntityLogEntries);
-
-        log.debug("Change computation took: {}", System.currentTimeMillis() - start);
-        return new EntityLogChanges(currentEntityLog, !addedEntryIds.isEmpty() || !removedIds.isEmpty());
+        }
+        addRemovedEntriesAsRemoved(previousEntityLogEntries, newEntityLogEntries, analysisNumber, now);
+        return hasChanges;
     }
 
 
-    private static Set<String> getEntryIdsThatExistInPreviousButNotInCurrentEntityLog(EntityLog currentEntityLog, List<EntityLogEntry> previouslyExistingEntries) {
+    private static void addRemovedEntriesAsRemoved(List<EntityLogEntry> previousEntityLogEntries,
+                                                   List<EntityLogEntry> newEntityLogEntries,
+                                                   int analysisNumber,
+                                                   OffsetDateTime now) {
 
-        Set<EntityLogEntry> removed = new HashSet<>(previouslyExistingEntries);
-        currentEntityLog.getEntityLogEntry().forEach(removed::remove);
-
-        Set<String> removedIds = removed.stream().map(EntityLogEntry::getId).collect(Collectors.toSet());
-        return removedIds;
+        Set<String> existingIds = newEntityLogEntries.stream().map(EntityLogEntry::getId).collect(Collectors.toSet());
+        List<EntityLogEntry> removedEntries = previousEntityLogEntries.stream().filter(entry -> !existingIds.contains(entry.getId())).toList();
+        removedEntries.forEach(entry -> entry.getChanges().add(new Change(analysisNumber, ChangeType.REMOVED, now)));
+        removedEntries.forEach(entry -> entry.setState(EntryState.REMOVED));
+        newEntityLogEntries.addAll(removedEntries);
     }
 
 
-    private static Map<String, EntityLogEntry> getEntriesThatExistInCurrentButNotInPreviousEntityLog(EntityLog currentEntityLog, List<EntityLogEntry> previouslyExistingEntries) {
+    private ChangeType calculateChangeType(EntryState state, EntryState previousState) {
 
-        Set<EntityLogEntry> currentExistingEntries = currentEntityLog.getEntityLogEntry().stream().filter(entry -> !lastChangeIsRemoved(entry)).collect(Collectors.toSet());
-
-        previouslyExistingEntries.forEach(currentExistingEntries::remove);
-        Map<String, EntityLogEntry> addedIds = new HashMap<>();
-        currentExistingEntries.forEach(entry -> {
-            addedIds.put(entry.getId(), entry);
-        });
-        return addedIds;
+        if (state.equals(previousState)) {
+            throw new IllegalArgumentException("States are equal, can't calculate ChangeType.");
+        }
+        if (!isRemoved(previousState) && isRemoved(state)) {
+            return ChangeType.REMOVED;
+        }
+        if (isRemoved(previousState) && !isRemoved(state)) {
+            return ChangeType.ADDED;
+        }
+        return ChangeType.CHANGED;
     }
 
 
-    private static boolean lastChangeIsRemoved(EntityLogEntry entry) {
+    private static boolean isRemoved(EntryState state) {
 
-        return entry.getChanges().stream().reduce((a, b) -> b).map(change -> change.getType().equals(ChangeType.REMOVED)).orElse(false);
+        return (state.equals(EntryState.REMOVED) || state.equals(EntryState.IGNORED));
     }
 
+
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java

@@ -1,19 +1,30 @@
 package com.iqser.red.service.redaction.v1.server.service;
 
+import static com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryType.IMAGE;
+
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Locale;
 import java.util.Set;
+import java.util.stream.Collectors;
 
 import org.springframework.stereotype.Service;
 
+import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogChanges;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogLegalBasis;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryType;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.legalbasis.LegalBasis;
+import com.iqser.red.service.redaction.v1.server.RedactionServiceSettings;
+import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient;
 import com.iqser.red.service.redaction.v1.server.model.ManualEntity;
+import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryVersion;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.IEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.PositionOnPage;
@@ -21,42 +32,135 @@ import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntit
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Image;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.ImageType;
+import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
 
+import lombok.AccessLevel;
 import lombok.RequiredArgsConstructor;
+import lombok.experimental.FieldDefaults;
 import lombok.extern.slf4j.Slf4j;
 
 @Service
 @Slf4j
 @RequiredArgsConstructor
+@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
 public class EntityLogCreatorService {
 
-    private final DictionaryService dictionaryService;
-    private final ManualChangeFactory manualChangeFactory;
+    DictionaryService dictionaryService;
+    ManualChangeFactory manualChangeFactory;
+    ImportedRedactionService importedRedactionService;
+    RedactionServiceSettings redactionServiceSettings;
+    LegalBasisClient legalBasisClient;
+    EntityChangeLogService entityChangeLogService;
+    RedactionStorageService redactionStorageService;
 
 
-    public List<EntityLogEntry> createEntityLogEntries(Document document, String dossierTemplateId, List<ManualEntity> notFoundManualRedactionEntries) {
+    private static boolean notFalsePositiveOrFalseRecommendation(TextEntity textEntity) {
+
+        return !(textEntity.getEntityType().equals(EntityType.FALSE_POSITIVE) || textEntity.getEntityType().equals(EntityType.FALSE_RECOMMENDATION));
+    }
+
+
+    public EntityLog createInitialEntityLog(AnalyzeRequest analyzeRequest, Document document, List<ManualEntity> notFoundManualEntities,
+                                            DictionaryVersion dictionaryVersion,
+                                            long rulesVersion) {
+
+        List<EntityLogEntry> entityLogEntries = createEntityLogEntries(document, analyzeRequest.getDossierTemplateId(), notFoundManualEntities);
+
+        List<LegalBasis> legalBasis = legalBasisClient.getLegalBasisMapping(analyzeRequest.getDossierTemplateId());
+        EntityLog entityLog = new EntityLog(redactionServiceSettings.getAnalysisVersion(),
+                analyzeRequest.getAnalysisNumber(),
+                entityLogEntries,
+                toEntityLogLegalBasis(legalBasis),
+                dictionaryVersion.getDossierTemplateVersion(),
+                dictionaryVersion.getDossierVersion(),
+                rulesVersion,
+                legalBasisClient.getVersion(analyzeRequest.getDossierTemplateId()));
+
+        List<EntityLogEntry> importedRedactionFilteredEntries = importedRedactionService.processImportedEntities(analyzeRequest.getDossierTemplateId(),
+                analyzeRequest.getDossierId(),
+                analyzeRequest.getFileId(),
+                entityLog.getEntityLogEntry(),
+                true);
+
+        entityLog.setEntityLogEntry(importedRedactionFilteredEntries);
+        List<EntityLogEntry> previousExistingEntityLogEntries = getPreviousEntityLogEntries(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
+        entityChangeLogService.computeChanges(previousExistingEntityLogEntries, entityLogEntries, analyzeRequest.getAnalysisNumber());
+        excludeExcludedPages(entityLog, analyzeRequest.getExcludedPages());
+        return entityLog;
+    }
+
+
+    private List<EntityLogEntry> getPreviousEntityLogEntries(String dossierId, String fileId) {
+
+        if (redactionStorageService.entityLogExists(dossierId, fileId)) {
+            return redactionStorageService.getEntityLog(dossierId, fileId).getEntityLogEntry();
+        } else {
+            return Collections.emptyList();
+        }
+    }
+
+
+    public EntityLogChanges updateVersionsAndReturnChanges(EntityLog entityLog, DictionaryVersion dictionaryVersion, String dossierTemplateId, boolean hasChanges) {
+
+        List<LegalBasis> legalBasis = legalBasisClient.getLegalBasisMapping(dossierTemplateId);
+        entityLog.setLegalBasisVersion(legalBasisClient.getVersion(dossierTemplateId));
+        entityLog.setLegalBasis(toEntityLogLegalBasis(legalBasis));
+        entityLog.setDictionaryVersion(dictionaryVersion.getDossierTemplateVersion());
+        entityLog.setDossierDictionaryVersion(dictionaryVersion.getDossierVersion());
+
+        return new EntityLogChanges(entityLog, hasChanges);
+    }
+
+
+    public EntityLogChanges updatePreviousEntityLog(AnalyzeRequest analyzeRequest,
+                                                    Document document,
+                                                    List<ManualEntity> notFoundManualRedactionEntries,
+                                                    EntityLog previousEntityLog,
+                                                    Set<Integer> sectionsToReanalyseIds,
+                                                    DictionaryVersion dictionaryVersion) {
+
+        List<EntityLogEntry> newEntityLogEntries = createEntityLogEntries(document, analyzeRequest.getDossierTemplateId(), notFoundManualRedactionEntries);
+        Set<String> newEntityIds = newEntityLogEntries.stream().map(EntityLogEntry::getId).collect(Collectors.toSet());
+        List<EntityLogEntry> previousEntriesFromReAnalyzedSections = previousEntityLog.getEntityLogEntry()
+                .stream()
+                .filter(entry -> (newEntityIds.contains(entry.getId()) || sectionsToReanalyseIds.contains(entry.getContainingNodeId().get(0))) && !entry.getType()
+                        .equals(ImportedRedactionService.IMPORTED_REDACTION_TYPE))
+                .toList();
+        previousEntityLog.getEntityLogEntry().removeAll(previousEntriesFromReAnalyzedSections);
+
+        boolean hasChanges = entityChangeLogService.computeChanges(previousEntriesFromReAnalyzedSections, newEntityLogEntries, analyzeRequest.getAnalysisNumber());
+
+        var newEntityLogWithImportedEntities = importedRedactionService.processImportedEntities(analyzeRequest.getDossierTemplateId(),
+                analyzeRequest.getDossierId(),
+                analyzeRequest.getFileId(),
+                newEntityLogEntries,
+                false);
+
+        previousEntityLog.getEntityLogEntry().addAll(newEntityLogWithImportedEntities);
+
+        excludeExcludedPages(previousEntityLog, analyzeRequest.getExcludedPages());
+
+        return updateVersionsAndReturnChanges(previousEntityLog, dictionaryVersion, analyzeRequest.getDossierTemplateId(), hasChanges);
+    }
+
+
+    private List<EntityLogEntry> createEntityLogEntries(Document document, String dossierTemplateId, List<ManualEntity> notFoundManualRedactionEntries) {
 
         List<EntityLogEntry> entries = new ArrayList<>();
+        Set<String> processedIds = new HashSet<>();
         document.getEntities()
                 .stream()
-                .filter(EntityLogCreatorService::isEntityOrRecommendationType)
+                .filter(EntityLogCreatorService::notFalsePositiveOrFalseRecommendation)
                 .filter(entity -> !entity.removed())
-                .forEach(entityNode -> entries.addAll(toEntityLogEntries(entityNode, dossierTemplateId)));
-        document.streamAllImages().filter(image -> !image.removed()).forEach(imageNode -> entries.add(createEntityLogEntry(imageNode, dossierTemplateId)));
-        notFoundManualRedactionEntries.forEach(entityIdentifier -> entries.add(createEntityLogEntry(entityIdentifier, dossierTemplateId)));
+                .forEach(entityNode -> entries.addAll(toEntityLogEntries(entityNode, dossierTemplateId, processedIds)));
+        document.streamAllImages().filter(entity -> !entity.removed()).forEach(imageNode -> entries.add(createEntityLogEntry(imageNode, dossierTemplateId)));
+        notFoundManualRedactionEntries.stream().filter(entity -> !entity.removed()).forEach(manualEntity -> entries.add(createEntityLogEntry(manualEntity, dossierTemplateId)));
         return entries;
     }
 
 
-    private static boolean isEntityOrRecommendationType(TextEntity textEntity) {
+    private List<EntityLogEntry> toEntityLogEntries(TextEntity textEntity, String dossierTemplateId, Set<String> processedIds) {
 
-        return textEntity.getEntityType() == EntityType.ENTITY || textEntity.getEntityType() == EntityType.RECOMMENDATION;
-    }
-
-
-    private List<EntityLogEntry> toEntityLogEntries(TextEntity textEntity, String dossierTemplateId) {
-
-        Set<String> processedIds = new HashSet<>();
         List<EntityLogEntry> redactionLogEntities = new ArrayList<>();
 
         for (PositionOnPage positionOnPage : textEntity.getPositionsOnPagePerPage()) {
@@ -67,72 +171,75 @@ public class EntityLogCreatorService {
             }
             processedIds.add(positionOnPage.getId());
 
-            EntityLogEntry redactionLogEntry = createEntityLogEntry(textEntity, dossierTemplateId);
-            redactionLogEntry.setId(positionOnPage.getId());
+            EntityLogEntry entityLogEntries = createEntityLogEntry(textEntity, dossierTemplateId);
+            entityLogEntries.setId(positionOnPage.getId());
 
-            List<Position> rectanglesPerLine = positionOnPage.getRectanglePerLine()
-                    .stream().map(rectangle2D -> new Position(rectangle2D, positionOnPage.getPage().getNumber()))
+            List<Position> rectanglesPerLine = positionOnPage.getRectanglePerLine().stream().map(rectangle2D -> new Position(rectangle2D, positionOnPage.getPage().getNumber()))
                     .toList();
 
-            redactionLogEntry.setPositions(rectanglesPerLine);
-            redactionLogEntities.add(redactionLogEntry);
+            entityLogEntries.setPositions(rectanglesPerLine);
+            redactionLogEntities.add(entityLogEntries);
         }
 
         return redactionLogEntities;
     }
 
 
-    private EntityLogEntry createEntityLogEntry(TextEntity entity, String dossierTemplateId) {
+    private EntityLogEntry createEntityLogEntry(Image image, String dossierTemplateId) {
 
-        Set<String> referenceIds = new HashSet<>();
-        entity.references().stream().filter(TextEntity::active).forEach(ref -> ref.getPositionsOnPagePerPage().forEach(pos -> referenceIds.add(pos.getId())));
-        int sectionNumber = entity.getDeepestFullyContainingNode().getTreeId().isEmpty() ? 0 : entity.getDeepestFullyContainingNode().getTreeId().get(0);
-        boolean isHint = isHint(entity.getType(), dossierTemplateId);
+        String imageType = image.getImageType().equals(ImageType.OTHER) ? "image" : image.getImageType().toString().toLowerCase(Locale.ENGLISH);
+        boolean isHint = dictionaryService.isHint(imageType, dossierTemplateId);
         return EntityLogEntry.builder()
-                .color(getColor(entity.getType(), dossierTemplateId, entity.applied()))
-                .reason(entity.buildReasonWithManualChangeDescriptions())
-                .legalBasis(entity.legalBasis())
-                .value(entity.getManualOverwrite().getValue().orElse(entity.getMatchedRule().isWriteValueWithLineBreaks() ? entity.getValueWithLineBreaks() : entity.getValue()))
-                .type(entity.getType())
-                .section(entity.getManualOverwrite().getSection().orElse(entity.getDeepestFullyContainingNode().toString()))
-                .sectionNumber(sectionNumber)
-                .matchedRule(entity.getMatchedRule().getRuleIdentifier().toString()).dictionaryEntry(entity.isDictionaryEntry())
-                .textAfter(entity.getTextAfter())
-                .textBefore(entity.getTextBefore())
-                .startOffset(entity.getTextRange().start())
-                .endOffset(entity.getTextRange().end()).dossierDictionaryEntry(entity.isDossierDictionaryEntry())
-                .engines(entity.getEngines() != null ? entity.getEngines() : Collections.emptySet())
-                .reference(referenceIds)
-                .manualChanges(manualChangeFactory.toManualChangeList(entity.getManualOverwrite().getManualChangeLog(), isHint))
-                .state(buildEntryState(entity))
-                .entryType(buildEntryType(entity, isHint))
+                .id(image.getId())
+                .value(image.value())
+                .color(getColor(imageType, dossierTemplateId, image.applied(), isHint))
+                .value(image.value())
+                .type(imageType)
+                .reason(image.buildReasonWithManualChangeDescriptions())
+                .legalBasis(image.legalBasis())
+                .matchedRule(image.getMatchedRule().getRuleIdentifier().toString())
+                .dictionaryEntry(false)
+                .positions(List.of(new Position(image.getPosition(), image.getPage().getNumber())))
+                .containingNodeId(image.getTreeId())
+                .closestHeadline(image.getHeadline().getTextBlock().getSearchText())
+                .section(image.getManualOverwrite().getSection().orElse(image.getParent().toString()))
+                .imageHasTransparency(image.isTransparent())
+                .manualChanges(manualChangeFactory.toManualChangeList(image.getManualOverwrite().getManualChangeLog(), isHint))
+                .state(buildEntryState(image)).entryType(isHint ? EntryType.IMAGE_HINT : EntryType.IMAGE)
                 .build();
+
     }
 
 
-    public EntityLogEntry createEntityLogEntry(ManualEntity manualEntity, String dossierTemplateId) {
+    private EntityLogEntry createEntityLogEntry(ManualEntity manualEntity, String dossierTemplateId) {
 
         String type = manualEntity.getManualOverwrite().getType().orElse(manualEntity.getType());
-        boolean isHint = isHint(type, dossierTemplateId);
+        boolean isHint = isHint(manualEntity.getEntityType());
         return EntityLogEntry.builder()
                 .id(manualEntity.getId())
-                .color(getColor(type, dossierTemplateId, manualEntity.applied()))
+                .color(getColor(type, dossierTemplateId, manualEntity.applied(), isHint))
                 .reason(manualEntity.buildReasonWithManualChangeDescriptions())
                 .legalBasis(manualEntity.legalBasis())
-                .value(manualEntity.getManualOverwrite().getValue().orElse(manualEntity.getValue()))
+                .value(manualEntity.value())
                 .type(type)
                 .state(buildEntryState(manualEntity))
-                .entryType(buildEntryType(manualEntity, isHint))
+                .entryType(buildEntryType(manualEntity))
                 .section(manualEntity.getManualOverwrite().getSection().orElse(manualEntity.getSection()))
-                .sectionNumber(0)
-                .matchedRule("ManualRedaction")
+                .containingNodeId(Collections.emptyList())
+                .closestHeadline("")
+                .matchedRule(manualEntity.getMatchedRule().getRuleIdentifier().toString())
                 .dictionaryEntry(manualEntity.isDictionaryEntry())
                 .dossierDictionaryEntry(manualEntity.isDossierDictionaryEntry())
                 .textAfter("")
                 .textBefore("")
                 .startOffset(-1)
                 .endOffset(-1)
-                .positions(manualEntity.getEntityPosition().stream().map(entityPosition -> new Position(entityPosition.rectangle2D(), entityPosition.pageNumber())).toList())
+                .positions(manualEntity.getManualOverwrite()
+                        .getPositions()
+                        .orElse(manualEntity.getEntityPosition())
+                        .stream()
+                        .map(entityPosition -> new Position(entityPosition.rectangle2D(), entityPosition.pageNumber()))
+                        .toList())
                 .engines(Collections.emptySet())
                 .reference(Collections.emptySet())
                 .manualChanges(manualChangeFactory.toManualChangeList(manualEntity.getManualOverwrite().getManualChangeLog(), isHint))
@@ -140,49 +247,57 @@ public class EntityLogCreatorService {
     }
 
 
-    public EntityLogEntry createEntityLogEntry(Image image, String dossierTemplateId) {
+    private EntityLogEntry createEntityLogEntry(TextEntity entity, String dossierTemplateId) {
 
-        String imageType = image.getImageType().equals(ImageType.OTHER) ? "image" : image.getImageType().toString().toLowerCase(Locale.ENGLISH);
-        boolean isHint = dictionaryService.isHint(imageType, dossierTemplateId);
+        Set<String> referenceIds = new HashSet<>();
+        entity.references().stream().filter(TextEntity::active).forEach(ref -> ref.getPositionsOnPagePerPage().forEach(pos -> referenceIds.add(pos.getId())));
+        boolean isHint = isHint(entity.getEntityType());
         return EntityLogEntry.builder()
-                .id(image.getId())
-                .color(getColor(imageType, dossierTemplateId, image.applied()))
-                .type(imageType)
-                .reason(image.buildReasonWithManualChangeDescriptions())
-                .legalBasis(image.legalBasis())
-                .matchedRule(image.getMatchedRule().getRuleIdentifier().toString())
-                .dictionaryEntry(false).positions(List.of(new Position(image.getPosition(), image.getPage().getNumber())))
-                .sectionNumber(image.getTreeId().get(0))
-                .section(image.getManualOverwrite().getSection().orElse(image.getParent().toString()))
-                .imageHasTransparency(image.isTransparent())
-                .manualChanges(manualChangeFactory.toManualChangeList(image.getManualOverwrite().getManualChangeLog(), isHint))
-                .state(buildEntryState(image))
-                .entryType(buildEntryType(image, isHint))
+                .color(getColor(entity.getType(), dossierTemplateId, entity.applied(), isHint))
+                .reason(entity.buildReasonWithManualChangeDescriptions())
+                .legalBasis(entity.legalBasis())
+                .value(entity.getManualOverwrite().getValue().orElse(entity.getMatchedRule().isWriteValueWithLineBreaks() ? entity.getValueWithLineBreaks() : entity.getValue()))
+                .type(entity.getType())
+                .section(entity.getManualOverwrite().getSection().orElse(entity.getDeepestFullyContainingNode().toString()))
+                .containingNodeId(entity.getDeepestFullyContainingNode().getTreeId())
+                .closestHeadline(entity.getDeepestFullyContainingNode().getHeadline().getTextBlock().getSearchText())
+                .matchedRule(entity.getMatchedRule().getRuleIdentifier().toString())
+                .dictionaryEntry(entity.isDictionaryEntry())
+                .textAfter(entity.getTextAfter())
+                .textBefore(entity.getTextBefore())
+                .startOffset(entity.getTextRange().start())
+                .endOffset(entity.getTextRange().end())
+                .dossierDictionaryEntry(entity.isDossierDictionaryEntry())
+                .engines(entity.getEngines() != null ? entity.getEngines() : Collections.emptySet())
+                .reference(referenceIds)
+                .manualChanges(manualChangeFactory.toManualChangeList(entity.getManualOverwrite().getManualChangeLog(), isHint))
+                .state(buildEntryState(entity))
+                .entryType(buildEntryType(entity))
                 .build();
-
     }
 
 
-    private float[] getColor(String type, String dossierTemplateId, boolean isRedaction) {
+    private boolean isHint(EntityType entityType) {
 
-        if (!isRedaction && !isHint(type, dossierTemplateId)) {
+        return entityType.equals(EntityType.HINT);
+    }
+
+
+    private float[] getColor(String type, String dossierTemplateId, boolean isApplied, boolean isHint) {
+
+        if (!isApplied && !isHint) {
             return dictionaryService.getNotRedactedColor(dossierTemplateId);
         }
         return dictionaryService.getColor(type, dossierTemplateId);
-    }
 
-
-    private boolean isHint(String type, String dossierTemplateId) {
-
-        return dictionaryService.isHint(type, dossierTemplateId);
     }
 
 
     private EntryState buildEntryState(IEntity entity) {
 
-        if (entity.applied()) {
+        if (entity.applied() && entity.active()) {
             return EntryState.APPLIED;
-        } else if (entity.skipped()) {
+        } else if (entity.skipped() && entity.active()) {
             return EntryState.SKIPPED;
         } else if (entity.ignored()) {
             return EntryState.IGNORED;
@@ -192,30 +307,48 @@ public class EntityLogCreatorService {
     }
 
 
-    private EntryType buildEntryType(IEntity entity, boolean isHint) {
+    private EntryType buildEntryType(IEntity entity) {
 
         if (entity instanceof TextEntity textEntity) {
-            return getEntryType(isHint, textEntity.getEntityType());
+            return getEntryType(textEntity.getEntityType());
         } else if (entity instanceof ManualEntity manualEntity) {
-            if (((ManualEntity) entity).isRectangle()) {
+            if (manualEntity.isRectangle()) {
                 return EntryType.AREA;
             }
-            return getEntryType(isHint, manualEntity.getEntityType());
-        } else if (entity instanceof Image) {
-            return EntryType.IMAGE;
+            return getEntryType(manualEntity.getEntityType());
         }
-        throw new UnsupportedOperationException("Entity subclass %s not implemented!");
+        throw new UnsupportedOperationException(String.format("Entity subclass %s is not implemented!", entity.getClass()));
     }
 
 
-    private static EntryType getEntryType(boolean isHint, EntityType entityType) {
+    private static EntryType getEntryType(EntityType entityType) {
 
         return switch (entityType) {
-            case ENTITY -> isHint ? EntryType.HINT : EntryType.ENTITY;
+            case ENTITY -> EntryType.ENTITY;
+            case HINT -> EntryType.HINT;
             case FALSE_POSITIVE -> EntryType.FALSE_POSITIVE;
             case RECOMMENDATION -> EntryType.RECOMMENDATION;
             case FALSE_RECOMMENDATION -> EntryType.FALSE_RECOMMENDATION;
         };
     }
 
+
+    private List<EntityLogLegalBasis> toEntityLogLegalBasis(List<LegalBasis> legalBasis) {
+
+        return legalBasis.stream().map(l -> new EntityLogLegalBasis(l.getName(), l.getDescription(), l.getReason())).collect(Collectors.toList());
+    }
+
+
+    private void excludeExcludedPages(EntityLog entityLog, Set<Integer> excludedPages) {
+
+        if (excludedPages != null && !excludedPages.isEmpty()) {
+            entityLog.getEntityLogEntry().forEach(entry -> entry.getPositions().forEach(pos -> {
+                if (excludedPages.contains(pos.getPageNumber())) {
+                    entry.setExcluded(true);
+                }
+            }));
+        }
+
+    }
+
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/ImportedRedactionService.java

@@ -7,6 +7,7 @@ import org.springframework.stereotype.Service;
 
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryType;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Point;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
@@ -54,22 +55,21 @@ public class ImportedRedactionService {
     @Timed("redactmanager_processImportedRedactions")
     public List<EntityLogEntry> processImportedEntities(String dossierTemplateId,
                                                         String dossierId,
-                                                        String fileId,
-                                                        List<EntityLogEntry> redactionLogEntries,
+                                                        String fileId, List<EntityLogEntry> entityLogEntries,
                                                         boolean addImportedRedactions) {
 
-        var importedRedactions = redactionStorageService.getImportedRedactions(dossierId, fileId);
+        ImportedRedactions importedRedactions = redactionStorageService.getImportedRedactions(dossierId, fileId);
         if (importedRedactions == null) {
-            return redactionLogEntries;
+            return entityLogEntries;
         }
 
-        redactionLogEntries.forEach(redactionLogEntry -> addIntersections(redactionLogEntry, importedRedactions));
+        entityLogEntries.forEach(redactionLogEntry -> addIntersections(redactionLogEntry, importedRedactions));
 
         if (addImportedRedactions) {
-            return addImportedRedactionsEntityLogEntries(dossierTemplateId, redactionLogEntries, importedRedactions);
+            return addImportedRedactionsEntityLogEntries(dossierTemplateId, entityLogEntries, importedRedactions);
         }
 
-        return redactionLogEntries;
+        return entityLogEntries;
     }
 
 
@@ -103,7 +103,7 @@ public class ImportedRedactionService {
             for (var importedRedaction : importedRedactionsValues) {
                 EntityLogEntry redactionLogEntry = EntityLogEntry.builder()
                         .id(importedRedaction.getId())
-                        .type(IMPORTED_REDACTION_TYPE)
+                        .type(IMPORTED_REDACTION_TYPE).entryType(EntryType.ENTITY)
                         .imported(true)
                         .state(EntryState.APPLIED)
                         .positions(toPositions(importedRedaction.getPositions()))

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/ManualChangesApplicationService.java

@@ -16,13 +16,15 @@ import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNo
 import com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService;
 import com.iqser.red.service.redaction.v1.server.utils.RectangleTransformations;
 
+import lombok.AccessLevel;
 import lombok.RequiredArgsConstructor;
+import lombok.experimental.FieldDefaults;
 
 @RequiredArgsConstructor
+@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
 public class ManualChangesApplicationService {
 
-    private final EntityCreationService entityCreationService;
-
+    EntityCreationService entityCreationService;
 
     public void recategorize(IEntity IEntityToBeReCategorized, ManualRecategorization manualRecategorization) {
 
@@ -32,10 +34,10 @@ public class ManualChangesApplicationService {
         }
         // need to create a new entity and copy over all values, since type is part of the primary key for entities and should never be changed!
         if (IEntityToBeReCategorized instanceof TextEntity textEntity) {
-            TextEntity recategorizedEntity = entityCreationService.copyEntity(textEntity, manualRecategorization.getType(), textEntity.getEntityType(), textEntity.getDeepestFullyContainingNode());
+            TextEntity recategorizedEntity = entityCreationService.copyEntityWithoutRules(textEntity, manualRecategorization.getType(), textEntity.getEntityType(), textEntity.getDeepestFullyContainingNode());
             recategorizedEntity.setPositionsOnPagePerPage(textEntity.getPositionsOnPagePerPage());
             recategorizedEntity.getManualOverwrite().addChange(manualRecategorization);
-            textEntity.removeFromGraph();
+            textEntity.remove("FINAL.0.0", "removed by manual recategorization");
         }
     }
 
@@ -90,7 +92,7 @@ public class ManualChangesApplicationService {
 
     private static Rectangle2D toRectangle2D(com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle rect) {
 
-        return new Rectangle2D.Double(rect.getTopLeftX() - rect.getWidth(), rect.getTopLeftY() - rect.getHeight(), rect.getWidth(), rect.getHeight());
+        return new Rectangle2D.Double(rect.getTopLeftX(), rect.getTopLeftY(), rect.getWidth(), rect.getHeight());
     }
 
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RedactionChangeLogService.java

@@ -22,6 +22,7 @@ import io.micrometer.core.annotation.Timed;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
 
+@Deprecated(forRemoval = true)
 @Slf4j
 @Service
 @RequiredArgsConstructor
@@ -31,12 +32,10 @@ public class RedactionChangeLogService {
 
 
     @Timed("redactmanager_computeChanges")
-    public RedactionLogChanges computeChanges(String dossierId, String fileId, RedactionLog currentRedactionLog, int analysisNumber) {
+    public RedactionLogChanges computeChanges(RedactionLog previousRedactionLog, RedactionLog currentRedactionLog, int analysisNumber) {
 
         long start = System.currentTimeMillis();
 
-        RedactionLog previousRedactionLog = redactionStorageService.getRedactionLog(dossierId, fileId);
-
         if (previousRedactionLog == null) {
             currentRedactionLog.getRedactionLogEntry().forEach(entry -> {
                 entry.getChanges().add(new Change(analysisNumber, ChangeType.ADDED, OffsetDateTime.now()));

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RedactionLogCreatorService.java

@@ -6,7 +6,6 @@ import java.util.Collections;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Locale;
-import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
 import java.util.stream.Collectors;
@@ -14,25 +13,27 @@ import java.util.stream.Collectors;
 import org.springframework.beans.BeanUtils;
 import org.springframework.stereotype.Service;
 
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ManualChange;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Comment;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ManualChange;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ManualRedactionType;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Point;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogComment;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
+import com.iqser.red.service.redaction.v1.server.model.ManualEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
+import com.iqser.red.service.redaction.v1.server.model.document.entity.IEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.ManualChangeOverwrite;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.PositionOnPage;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Image;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.ImageType;
-import com.iqser.red.service.redaction.v1.server.model.ManualEntity;
 
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
 
+@Deprecated(forRemoval = true)
 @Service
 @Slf4j
 @RequiredArgsConstructor
@@ -42,32 +43,26 @@ public class RedactionLogCreatorService {
     private final ManualChangeFactory manualChangeFactory;
 
 
-    public List<RedactionLogEntry> createRedactionLog(Document document,
-                                                      String dossierTemplateId,
-                                                      List<ManualEntity> notFoundManualRedactionEntries,
-                                                      Map<String, List<Comment>> comments) {
+    public List<RedactionLogEntry> createRedactionLog(Document document, String dossierTemplateId, List<ManualEntity> notFoundManualRedactionEntries) {
 
         List<RedactionLogEntry> entries = new ArrayList<>();
-        document.getEntities()
-                .stream()
-                .filter(RedactionLogCreatorService::isEntityOrRecommendationType)
-                .filter(entity -> !entity.removed())
-                .forEach(entityNode -> entries.addAll(toRedactionLogEntries(entityNode, dossierTemplateId, comments)));
-        document.streamAllImages().filter(image -> !image.removed()).forEach(imageNode -> entries.add(createRedactionLogEntry(imageNode, dossierTemplateId, comments)));
-        notFoundManualRedactionEntries.forEach(entityIdentifier -> entries.add(createRedactionLogEntry(entityIdentifier, dossierTemplateId, comments)));
+        Set<String> processIds = new HashSet<>();
+        document.getEntities().stream().filter(RedactionLogCreatorService::notFalsePositiveOrFalseRecommendation)
+                .filter(IEntity::active).forEach(entityNode -> entries.addAll(toRedactionLogEntries(entityNode, dossierTemplateId, processIds)));
+        document.streamAllImages().filter(image -> !image.removed()).forEach(imageNode -> entries.add(createRedactionLogEntry(imageNode, dossierTemplateId)));
+        notFoundManualRedactionEntries.forEach(entityIdentifier -> entries.add(createRedactionLogEntry(entityIdentifier, dossierTemplateId)));
         return entries;
     }
 
 
-    private static boolean isEntityOrRecommendationType(TextEntity textEntity) {
+    private static boolean notFalsePositiveOrFalseRecommendation(TextEntity textEntity) {
 
-        return textEntity.getEntityType() == EntityType.ENTITY || textEntity.getEntityType() == EntityType.RECOMMENDATION;
+        return !(textEntity.getEntityType() == EntityType.FALSE_POSITIVE || textEntity.getEntityType() == EntityType.FALSE_RECOMMENDATION);
     }
 
 
-    private List<RedactionLogEntry> toRedactionLogEntries(TextEntity textEntity, String dossierTemplateId, Map<String, List<Comment>> comments) {
+    private List<RedactionLogEntry> toRedactionLogEntries(TextEntity textEntity, String dossierTemplateId, Set<String> processedIds) {
 
-        Set<String> processedIds = new HashSet<>();
         List<RedactionLogEntry> redactionLogEntities = new ArrayList<>();
 
         for (PositionOnPage positionOnPage : textEntity.getPositionsOnPagePerPage()) {
@@ -80,7 +75,6 @@ public class RedactionLogCreatorService {
 
             RedactionLogEntry redactionLogEntry = createRedactionLogEntry(textEntity, dossierTemplateId);
             redactionLogEntry.setId(positionOnPage.getId());
-            redactionLogEntry.setComments(buildRedactionLogComments(comments, positionOnPage.getId()));
 
             List<Rectangle> rectanglesPerLine = positionOnPage.getRectanglePerLine()
                     .stream()
@@ -95,45 +89,13 @@ public class RedactionLogCreatorService {
     }
 
 
-    private List<RedactionLogComment> buildRedactionLogComments(Map<String, List<Comment>> commentsPerId, String id) {
-
-        if (!commentsPerId.containsKey(id)) {
-            return Collections.emptyList();
-        }
-        List<Comment> comments = commentsPerId.get(id);
-        if (comments == null || comments.isEmpty()) {
-            return Collections.emptyList();
-        }
-        return toRedactionLogComments(comments);
-    }
-
-
-    private List<RedactionLogComment> toRedactionLogComments(List<Comment> comments) {
-
-        return comments.stream().map(this::toRedactionLogComment).toList();
-    }
-
-
-    private RedactionLogComment toRedactionLogComment(Comment comment) {
-
-        return new RedactionLogComment(comment.getId(),
-                comment.getUser(),
-                comment.getText(),
-                comment.getAnnotationId(),
-                comment.getFileId(),
-                comment.getDate(),
-                comment.getSoftDeletedTime());
-    }
-
-
     private RedactionLogEntry createRedactionLogEntry(TextEntity entity, String dossierTemplateId) {
 
         Set<String> referenceIds = new HashSet<>();
         entity.references().stream().filter(TextEntity::active).forEach(ref -> ref.getPositionsOnPagePerPage().forEach(pos -> referenceIds.add(pos.getId())));
         int sectionNumber = entity.getDeepestFullyContainingNode().getTreeId().isEmpty() ? 0 : entity.getDeepestFullyContainingNode().getTreeId().get(0);
-        boolean isHint = isHint(entity.getType(), dossierTemplateId);
-        return RedactionLogEntry.builder()
-                .color(getColor(entity.getType(), dossierTemplateId, entity.applied()))
+        boolean isHint = isHint(entity.getEntityType());
+        return RedactionLogEntry.builder().color(getColor(entity.getType(), dossierTemplateId, entity.applied(), isHint))
                 .reason(entity.buildReasonWithManualChangeDescriptions())
                 .legalBasis(entity.legalBasis())
                 .value(entity.getManualOverwrite().getValue().orElse(entity.getMatchedRule().isWriteValueWithLineBreaks() ? entity.getValueWithLineBreaks() : entity.getValue()))
@@ -175,40 +137,9 @@ public class RedactionLogCreatorService {
     }
 
 
-    public RedactionLogEntry createRedactionLogEntry(ManualEntity manualEntity, String dossierTemplateId, Map<String, List<Comment>> comments) {
+    private boolean isHint(EntityType entityType) {
 
-        String type = manualEntity.getManualOverwrite().getType().orElse(manualEntity.getType());
-        boolean isHint = isHint(type, dossierTemplateId);
-        return RedactionLogEntry.builder()
-                .id(manualEntity.getId())
-                .color(getColor(type, dossierTemplateId, manualEntity.applied()))
-                .reason(manualEntity.buildReasonWithManualChangeDescriptions())
-                .legalBasis(manualEntity.legalBasis())
-                .value(manualEntity.getManualOverwrite().getValue().orElse(manualEntity.getValue()))
-                .type(type)
-                .redacted(manualEntity.applied())
-                .isHint(isHint)
-                .isRecommendation(manualEntity.getEntityType().equals(EntityType.RECOMMENDATION))
-                .isFalsePositive(manualEntity.getEntityType().equals(EntityType.FALSE_POSITIVE) || manualEntity.getEntityType().equals(EntityType.FALSE_RECOMMENDATION))
-                .section(manualEntity.getManualOverwrite().getSection().orElse(manualEntity.getSection()))
-                .sectionNumber(0)
-                .matchedRule("ManualRedaction")
-                .rectangle(manualEntity.isRectangle())
-                .isDictionaryEntry(manualEntity.isDictionaryEntry())
-                .isDossierDictionaryEntry(manualEntity.isDossierDictionaryEntry())
-                .textAfter("")
-                .textBefore("")
-                .startOffset(-1)
-                .endOffset(-1)
-                .positions(manualEntity.getEntityPosition()
-                        .stream()
-                        .map(entityPosition -> toRedactionLogRectangle(entityPosition.rectangle2D(), entityPosition.pageNumber()))
-                        .collect(Collectors.toList()))
-                .engines(Collections.emptySet())
-                .reference(Collections.emptySet())
-                .manualChanges(mapManualChanges(manualEntity.getManualOverwrite(), isHint))
-                .comments(buildRedactionLogComments(comments, manualEntity.getId()))
-                .build();
+        return entityType.equals(EntityType.HINT);
     }
 
 
@@ -222,18 +153,41 @@ public class RedactionLogCreatorService {
 
         ManualChange manualChange1 = new ManualChange();
         BeanUtils.copyProperties(manualChange, manualChange1);
+        manualChange1.setManualRedactionType(switch (manualChange.getManualRedactionType()) {
+            case ADD_LOCALLY -> ManualRedactionType.ADD_LOCALLY;
+            case ADD_TO_DICTIONARY -> ManualRedactionType.ADD_TO_DICTIONARY;
+            case REMOVE_LOCALLY -> ManualRedactionType.REMOVE_LOCALLY;
+            case REMOVE_FROM_DICTIONARY -> ManualRedactionType.REMOVE_FROM_DICTIONARY;
+            case FORCE_REDACT -> ManualRedactionType.FORCE_REDACT;
+            case FORCE_HINT -> ManualRedactionType.FORCE_HINT;
+            case RECATEGORIZE -> ManualRedactionType.RECATEGORIZE;
+            case LEGAL_BASIS_CHANGE -> ManualRedactionType.LEGAL_BASIS_CHANGE;
+            case RESIZE -> ManualRedactionType.RESIZE;
+        });
+        manualChange1.setProcessedDate(manualChange.getProcessedDate());
+        manualChange1.setRequestedDate(manualChange.getRequestedDate());
+        manualChange1.setPropertyChanges(manualChange.getPropertyChanges());
+        manualChange1.setAnnotationStatus(AnnotationStatus.APPROVED);
         return manualChange1;
     }
 
 
-    public RedactionLogEntry createRedactionLogEntry(Image image, String dossierTemplateId, Map<String, List<Comment>> comments) {
+    private float[] getColor(String type, String dossierTemplateId, boolean isRedaction, boolean isHint) {
+
+        if (!isRedaction && isHint) {
+            return dictionaryService.getNotRedactedColor(dossierTemplateId);
+        }
+        return dictionaryService.getColor(type, dossierTemplateId);
+    }
+
+
+    public RedactionLogEntry createRedactionLogEntry(Image image, String dossierTemplateId) {
 
         String imageType = image.getImageType().equals(ImageType.OTHER) ? "image" : image.getImageType().toString().toLowerCase(Locale.ENGLISH);
         boolean isHint = dictionaryService.isHint(imageType, dossierTemplateId);
-        return RedactionLogEntry.builder()
-                .id(image.getId())
-                .color(getColor(imageType, dossierTemplateId, image.applied()))
+        return RedactionLogEntry.builder().id(image.getId()).color(getColor(imageType, dossierTemplateId, image.applied(), isHint))
                 .isImage(true)
+                .value(image.value())
                 .type(imageType)
                 .redacted(image.applied())
                 .reason(image.buildReasonWithManualChangeDescriptions())
@@ -247,33 +201,46 @@ public class RedactionLogCreatorService {
                 .section(image.getManualOverwrite().getSection().orElse(image.getParent().toString()))
                 .imageHasTransparency(image.isTransparent())
                 .manualChanges(mapManualChanges(image.getManualOverwrite(), isHint))
-                .comments(buildRedactionLogComments(comments, image.getId()))
                 .build();
 
     }
 
 
-    private float[] getColor(String type, String dossierTemplateId, boolean isRedaction) {
-
-        if (!isRedaction && !isHint(type, dossierTemplateId)) {
-            return dictionaryService.getNotRedactedColor(dossierTemplateId);
-        }
-        return dictionaryService.getColor(type, dossierTemplateId);
-    }
-
-
-    private boolean isHint(String type, String dossierTemplateId) {
-
-        return dictionaryService.isHint(type, dossierTemplateId);
-    }
-
-
     private Rectangle toRedactionLogRectangle(Rectangle2D rectangle2D, int pageNumber) {
 
-        return new Rectangle(new Point((float) rectangle2D.getMinX(), (float) (rectangle2D.getMinY() + rectangle2D.getHeight())),
-                (float) rectangle2D.getWidth(),
-                -(float) rectangle2D.getHeight(),
-                pageNumber);
+        return new Rectangle(new Point((float) rectangle2D.getMinX(), (float) rectangle2D.getMinY()), (float) rectangle2D.getWidth(), (float) rectangle2D.getHeight(), pageNumber);
+    }
+
+
+    public RedactionLogEntry createRedactionLogEntry(ManualEntity manualEntity, String dossierTemplateId) {
+
+        String type = manualEntity.getManualOverwrite().getType().orElse(manualEntity.getType());
+        boolean isHint = isHint(manualEntity.getEntityType());
+        return RedactionLogEntry.builder()
+                .id(manualEntity.getId())
+                .color(getColor(type, dossierTemplateId, manualEntity.applied(), isHint))
+                .reason(manualEntity.buildReasonWithManualChangeDescriptions())
+                .legalBasis(manualEntity.legalBasis())
+                .value(manualEntity.value())
+                .type(type)
+                .redacted(manualEntity.applied())
+                .isHint(isHint)
+                .isRecommendation(manualEntity.getEntityType().equals(EntityType.RECOMMENDATION))
+                .isFalsePositive(manualEntity.getEntityType().equals(EntityType.FALSE_POSITIVE) || manualEntity.getEntityType().equals(EntityType.FALSE_RECOMMENDATION))
+                .section(manualEntity.getManualOverwrite().getSection().orElse(manualEntity.getSection()))
+                .sectionNumber(0)
+                .matchedRule(manualEntity.getMatchedRule().getRuleIdentifier().toString())
+                .rectangle(manualEntity.isRectangle())
+                .isDictionaryEntry(manualEntity.isDictionaryEntry())
+                .isDossierDictionaryEntry(manualEntity.isDossierDictionaryEntry())
+                .textAfter("")
+                .textBefore("")
+                .startOffset(-1).endOffset(-1).positions(manualEntity.getManualOverwrite().getPositions().orElse(manualEntity.getEntityPosition())
+                        .stream().map(entityPosition -> toRedactionLogRectangle(entityPosition.rectangle2D(), entityPosition.pageNumber())).toList())
+                .engines(Collections.emptySet())
+                .reference(Collections.emptySet())
+                .manualChanges(mapManualChanges(manualEntity.getManualOverwrite(), isHint))
+                .build();
     }
 
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RedactionLogToEntityLogMigrationService.java

@@ -0,0 +1,187 @@
+package com.iqser.red.service.redaction.v1.server.service;
+
+import java.util.Collection;
+import java.util.Comparator;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import org.springframework.stereotype.Service;
+
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogLegalBasis;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.migration.MigratedIds;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogLegalBasis;
+import com.iqser.red.service.redaction.v1.server.model.ManualEntity;
+import com.iqser.red.service.redaction.v1.server.model.MigratedEntityLog;
+import com.iqser.red.service.redaction.v1.server.model.MigrationEntity;
+import com.iqser.red.service.redaction.v1.server.model.RectangleWithPage;
+import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
+import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.Image;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
+import com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService;
+import com.iqser.red.service.redaction.v1.server.service.document.EntityEnrichmentService;
+import com.iqser.red.service.redaction.v1.server.service.document.EntityFindingUtility;
+import com.iqser.red.service.redaction.v1.server.utils.MigratedIdsCollector;
+
+import lombok.AccessLevel;
+import lombok.RequiredArgsConstructor;
+import lombok.experimental.FieldDefaults;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+@Service
+@RequiredArgsConstructor
+@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
+//TODO: remove this, once the migration is done
+public class RedactionLogToEntityLogMigrationService {
+
+    private static final double MATCH_THRESHOLD = 10;
+    EntityFindingUtility entityFindingUtility;
+    EntityEnrichmentService entityEnrichmentService;
+
+
+    public MigratedEntityLog migrate(RedactionLog redactionLog, Document document) {
+
+        List<MigrationEntity> entitiesToMigrate = calculateMigrationEntitiesFromRedactionLog(redactionLog, document);
+        MigratedIds migratedIds = entitiesToMigrate.stream().collect(new MigratedIdsCollector());
+
+        EntityLog entityLog = new EntityLog();
+        entityLog.setAnalysisNumber(redactionLog.getAnalysisNumber());
+        entityLog.setRulesVersion(redactionLog.getRulesVersion());
+        entityLog.setDictionaryVersion(redactionLog.getDictionaryVersion());
+        entityLog.setDossierDictionaryVersion(redactionLog.getDossierDictionaryVersion());
+        entityLog.setLegalBasisVersion(redactionLog.getLegalBasisVersion());
+        entityLog.setAnalysisVersion(redactionLog.getAnalysisVersion());
+        entityLog.setLegalBasis(redactionLog.getLegalBasis().stream().map(RedactionLogToEntityLogMigrationService::toEntityLogLegalBasis).toList());
+
+        Map<String, String> oldToNewIDMapping = migratedIds.buildOldToNewMapping();
+        entityLog.setEntityLogEntry(entitiesToMigrate.stream().map(migrationEntity -> migrationEntity.toEntityLogEntry(oldToNewIDMapping)).toList());
+
+        if (redactionLog.getRedactionLogEntry().size() != entityLog.getEntityLogEntry().size()) {
+            String message = String.format("Not all entities have been found during the migration redactionLog has %d entries and new entityLog %d",
+                    redactionLog.getRedactionLogEntry().size(),
+                    entityLog.getEntityLogEntry().size());
+            log.error(message);
+            throw new AssertionError(message);
+        }
+
+        return new MigratedEntityLog(migratedIds, entityLog);
+    }
+
+
+    private List<MigrationEntity> calculateMigrationEntitiesFromRedactionLog(RedactionLog redactionLog, Document document) {
+
+        List<MigrationEntity> images = getImageBasedMigrationEntities(redactionLog, document);
+        List<MigrationEntity> textMigrationEntities = getTextBasedMigrationEntities(redactionLog, document);
+        return Stream.of(textMigrationEntities.stream(), images.stream()).flatMap(Function.identity()).toList();
+    }
+
+
+    private static EntityLogLegalBasis toEntityLogLegalBasis(RedactionLogLegalBasis redactionLogLegalBasis) {
+
+        return new EntityLogLegalBasis(redactionLogLegalBasis.getName(), redactionLogLegalBasis.getDescription(), redactionLogLegalBasis.getReason());
+    }
+
+
+    private List<MigrationEntity> getImageBasedMigrationEntities(RedactionLog redactionLog, Document document) {
+
+        List<Image> images = document.streamAllImages().collect(Collectors.toList());
+        List<RedactionLogEntry> redactionLogImages = redactionLog.getRedactionLogEntry().stream().filter(RedactionLogEntry::isImage).toList();
+        List<MigrationEntity> migrationEntities = new LinkedList<>();
+        for (RedactionLogEntry redactionLogImage : redactionLogImages) {
+            List<RectangleWithPage> imagePositions = redactionLogImage.getPositions().stream().map(RectangleWithPage::fromRedactionLogRectangle).toList();
+            assert imagePositions.size() == 1;
+            Image closestImage = images.stream()
+                    .filter(image -> image.onPage(redactionLogImage.getPositions().get(0).getPage()))
+                    .min(Comparator.comparingDouble(image -> entityFindingUtility.calculateDistance(image.getPosition(), imagePositions.get(0).rectangle2D())))
+                    .orElseThrow(() -> new RuntimeException("Image from redaction log not found: " + redactionLogImage));
+
+            double minDistance = entityFindingUtility.calculateDistance(closestImage.getPosition(), imagePositions.get(0).rectangle2D());
+            if (minDistance > MATCH_THRESHOLD) {
+                throw new RuntimeException(String.format("Closest image has a distance of %.2f which is higher than the allowed %.2f", minDistance, MATCH_THRESHOLD));
+            }
+
+            images.remove(closestImage);
+
+            String ruleIdentifier = "OLDIMG." + redactionLogImage.getMatchedRule() + ".0";
+            if (redactionLogImage.lastChangeIsRemoved()) {
+                closestImage.remove(ruleIdentifier, redactionLogImage.getReason());
+            } else if (redactionLogImage.isRedacted()) {
+                closestImage.apply(ruleIdentifier, redactionLogImage.getReason(), redactionLogImage.getLegalBasis());
+            } else {
+                closestImage.skip(ruleIdentifier, redactionLogImage.getReason());
+            }
+            migrationEntities.add(new MigrationEntity(null, redactionLogImage, closestImage, redactionLogImage.getId(), closestImage.getId()));
+        }
+        return migrationEntities;
+    }
+
+
+    private List<MigrationEntity> getTextBasedMigrationEntities(RedactionLog redactionLog, Document document) {
+
+        List<MigrationEntity> entitiesToMigrate = redactionLog.getRedactionLogEntry()
+                .stream()
+                .filter(redactionLogEntry -> !redactionLogEntry.isImage())
+                .map(MigrationEntity::fromRedactionLogEntry)
+                .peek(migrationEntity -> {
+                    if (migrationEntity.getRedactionLogEntry().lastChangeIsRemoved()) {
+                        migrationEntity.getManualEntity().remove(migrationEntity.getManualEntity().getRuleIdentifier(), migrationEntity.getRedactionLogEntry().getReason());
+                    } else if (migrationEntity.getManualEntity().isApplied()) {
+                        migrationEntity.getManualEntity()
+                                .apply(migrationEntity.getManualEntity().getRuleIdentifier(),
+                                        migrationEntity.getManualEntity().getReason(),
+                                        migrationEntity.getManualEntity().getLegalBasis());
+                    } else {
+                        migrationEntity.getManualEntity().skip(migrationEntity.getManualEntity().getRuleIdentifier(), migrationEntity.getManualEntity().getReason());
+                    }
+                })
+                .toList();
+
+        Map<String, List<TextEntity>> tempEntitiesByValue = entityFindingUtility.findAllPossibleEntitiesAndGroupByValue(document,
+                entitiesToMigrate.stream().map(MigrationEntity::getManualEntity).toList());
+
+        for (MigrationEntity migrationEntity : entitiesToMigrate) {
+            Optional<TextEntity> optionalTextEntity = entityFindingUtility.findClosestEntityAndReturnEmptyIfNotFound(migrationEntity.getManualEntity(),
+                    tempEntitiesByValue,
+                    MATCH_THRESHOLD);
+
+            if (optionalTextEntity.isEmpty()) {
+                migrationEntity.setMigratedEntity(migrationEntity.getManualEntity());
+                migrationEntity.setOldId(migrationEntity.getManualEntity().getId());
+                migrationEntity.setNewId(migrationEntity.getManualEntity().getId());
+                continue;
+            }
+
+            TextEntity entity = createCorrectEntity(migrationEntity.getManualEntity(), document, optionalTextEntity.get().getTextRange());
+            migrationEntity.setMigratedEntity(entity);
+            migrationEntity.setOldId(migrationEntity.getManualEntity().getId());
+            migrationEntity.setNewId(entity.getPositionsOnPagePerPage().get(0).getId()); // Can only be on one page, since redactionLogEntries can only be on one page
+        }
+
+        tempEntitiesByValue.values().stream().flatMap(Collection::stream).forEach(TextEntity::removeFromGraph);
+        return entitiesToMigrate;
+    }
+
+
+    private TextEntity createCorrectEntity(ManualEntity manualEntity, SemanticNode node, TextRange closestTextRange) {
+
+        EntityCreationService entityCreationService = new EntityCreationService(entityEnrichmentService);
+        TextEntity correctEntity = entityCreationService.forceByTextRange(closestTextRange, manualEntity.getType(), manualEntity.getEntityType(), node);
+
+        correctEntity.addMatchedRules(manualEntity.getMatchedRuleList());
+        correctEntity.setDictionaryEntry(manualEntity.isDictionaryEntry());
+        correctEntity.setDossierDictionaryEntry(manualEntity.isDossierDictionaryEntry());
+        correctEntity.getManualOverwrite().addChanges(manualEntity.getManualOverwrite().getManualChangeLog());
+        return correctEntity;
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/UnprocessedChangesService.java

@@ -0,0 +1,101 @@
+package com.iqser.red.service.redaction.v1.server.service;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import org.springframework.stereotype.Service;
+
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
+import com.iqser.red.service.redaction.v1.model.UnprocessedManualEntity;
+import com.iqser.red.service.redaction.v1.server.model.ManualEntity;
+import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
+import com.iqser.red.service.redaction.v1.server.service.document.DocumentGraphMapper;
+import com.iqser.red.service.redaction.v1.server.service.document.ManualEntityCreationService;
+import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
+
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+@Service
+@RequiredArgsConstructor
+public class UnprocessedChangesService {
+
+    private final RedactionStorageService redactionStorageService;
+    private final DictionaryService dictionaryService;
+    private final ManualEntityCreationService manualEntityCreationService;
+
+
+    public List<UnprocessedManualEntity> addInformationToUnprocessedManualEntities(String fileId, String dossierId, String dossierTemplateId, ManualRedactions manualRedactions) {
+
+        Document document = DocumentGraphMapper.toDocumentGraph(redactionStorageService.getDocumentData(dossierId, fileId));
+        List<UnprocessedManualEntity> unprocessedManualEntities = new ArrayList<>();
+        Set<String> annotationIds = manualRedactions.getEntriesToAdd().stream().map(ManualRedactionEntry::getAnnotationId).collect(Collectors.toSet());
+
+        List<ManualEntity> notFoundManualEntities = manualEntityCreationService.toTextEntity(manualEntitiesConverter(manualRedactions), document);
+
+        document.getEntities().forEach(textEntity -> {
+            Set<String> processedIds = new HashSet<>();
+            for (var positionsOnPerPage : textEntity.getPositionsOnPagePerPage()) {
+                if (processedIds.contains(positionsOnPerPage.getId())) {
+                    continue;
+                }
+                processedIds.add(positionsOnPerPage.getId());
+                List<Position> positions = positionsOnPerPage.getRectanglePerLine().stream().map(rectangle2D -> new Position(rectangle2D, positionsOnPerPage.getPage().getNumber()))
+                        .toList();
+                unprocessedManualEntities.add(UnprocessedManualEntity.builder()
+                        .annotationId(annotationIds.stream().filter(textEntity::matchesAnnotationId).findFirst().orElse(""))
+                        .closestHeadline(textEntity.getDeepestFullyContainingNode().getHeadline().getTextBlock().getSearchText())
+                        .color(getColor(textEntity.getType(), dossierTemplateId, textEntity.applied(), textEntity.getEntityType().equals(EntityType.HINT)))
+                        .startOffset(textEntity.getTextRange().start())
+                        .endOffset(textEntity.getTextRange().end())
+                        .containingNodeId(textEntity.getDeepestFullyContainingNode().getTreeId())
+                        .textBefore(textEntity.getTextBefore())
+                        .textAfter(textEntity.getTextAfter())
+                        .section(textEntity.getManualOverwrite().getSection().orElse(textEntity.getDeepestFullyContainingNode().toString()))
+                        .positions(positions)
+                        .build());
+            }
+
+        });
+
+        notFoundManualEntities.forEach(manualEntity -> unprocessedManualEntities.add(UnprocessedManualEntity.builder()
+                .annotationId(manualEntity.getId())
+                .color(getColor(manualEntity.getType(), dossierTemplateId, manualEntity.applied(), manualEntity.getEntityType().equals(EntityType.HINT)))
+                .closestHeadline("")
+                .startOffset(-1)
+                .endOffset(-1)
+                .containingNodeId(Collections.emptyList())
+                .textAfter("")
+                .textBefore("")
+                .section("")
+                .build()));
+
+        return unprocessedManualEntities;
+    }
+
+
+    private float[] getColor(String type, String dossierTemplateId, boolean isApplied, boolean isHint) {
+
+        if (!isApplied && !isHint) {
+            return dictionaryService.getNotRedactedColor(dossierTemplateId);
+        }
+        return dictionaryService.getColor(type, dossierTemplateId);
+    }
+
+
+    private List<ManualEntity> manualEntitiesConverter(ManualRedactions manualRedactions) {
+
+        return manualRedactions.getEntriesToAdd().stream()
+                .map(manualRedactionEntry -> ManualEntity.fromManualRedactionEntry(manualRedactionEntry,
+                        manualRedactionEntry.getType() != null && manualRedactionEntry.getType().equals("hint_only"))).toList();
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/ComponentComparator.java

@@ -0,0 +1,33 @@
+package com.iqser.red.service.redaction.v1.server.service.document;
+
+import java.util.Comparator;
+
+import com.iqser.red.service.redaction.v1.server.model.component.Component;
+
+public class ComponentComparator implements Comparator<Component> {
+
+    public static ComponentComparator first() {
+
+        return new ComponentComparator();
+    }
+
+
+    @Override
+    public int compare(Component component1, Component component2) {
+
+        var firstEntity1 = component1.getReferences().stream().min(EntityComparators.first());
+        var firstEntity2 = component2.getReferences().stream().min(EntityComparators.first());
+        if (firstEntity1.isEmpty() && firstEntity2.isEmpty()) {
+            return 0;
+        }
+        if (firstEntity1.isEmpty() && firstEntity2.isPresent()) {
+            return -1;
+        }
+        if (firstEntity1.isPresent() && firstEntity2.isEmpty()) {
+            return 1;
+        }
+
+        return new EntityComparators.FirstEntity().compare(firstEntity1.get(), firstEntity2.get());
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/ComponentCreationService.java

@@ -1,17 +1,27 @@
 package com.iqser.red.service.redaction.v1.server.service.document;
 
+import java.text.BreakIterator;
 import java.util.Collection;
 import java.util.Collections;
+import java.util.Comparator;
+import java.util.Date;
 import java.util.HashSet;
 import java.util.LinkedList;
 import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Optional;
 import java.util.Set;
 import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
 import org.kie.api.runtime.KieSession;
 
 import com.iqser.red.service.redaction.v1.server.model.component.Component;
 import com.iqser.red.service.redaction.v1.server.model.component.Entity;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.Table;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell;
 import com.iqser.red.service.redaction.v1.server.model.drools.RuleIdentifier;
 import com.iqser.red.service.redaction.v1.server.utils.DateConverter;
 
@@ -27,103 +37,446 @@ public class ComponentCreationService {
     Set<Entity> referencedEntities = new HashSet<>();
 
 
-    public void firstOrElse(String ruleIdentifier, String category, Collection<Entity> entities, String fallback) {
+    private static List<Entity> findEntitiesFromLongestSection(Collection<Entity> entities) {
 
-        String transformation = String.format("First found value or else '%s'", fallback);
-        String value = entities.stream().min(EntityComparators.start()).map(Entity::getValue).orElse(fallback);
-        create(ruleIdentifier, category, value, transformation, entities);
+        var entitiesBySection = entities.stream().collect(Collectors.groupingBy(entity -> entity.getContainingNode().getHighestParent()));
+        Optional<SemanticNode> longestSection = entitiesBySection.entrySet()
+                .stream()
+                .sorted(Comparator.comparingInt(ComponentCreationService::getTotalLengthOfEntities).reversed())
+                .map(Map.Entry::getKey)
+                .findFirst();
+
+        if (longestSection.isEmpty()) {
+            return Collections.emptyList();
+        }
+
+        return entitiesBySection.get(longestSection.get());
     }
 
 
-    public void joining(String ruleIdentifier, String category, Collection<Entity> entities) {
+    /**
+     * Joins entity values, and creates a component from the result.
+     *
+     * @param ruleIdentifier The identifier of the rule.
+     * @param name           The name of the entity.
+     * @param entities       The collection of entities to process.
+     */
+    public void joining(String ruleIdentifier, String name, Collection<Entity> entities) {
 
-        joining(ruleIdentifier, category, entities, ", ");
+        joining(ruleIdentifier, name, entities, ", ");
     }
 
 
-    public void joining(String ruleIdentifier, String category, Collection<Entity> entities, String delimiter) {
+    /**
+     * Finds the first value from the collection of entities and creates a component from it. If no value is found, the fallback value is used instead.
+     *
+     * @param ruleIdentifier the identifier for the rule
+     * @param name           the name of the operation
+     * @param entities       the collection of entities to search for the first value
+     * @param fallback       the value to be returned if no value is found in the collection
+     */
+    public void firstOrElse(String ruleIdentifier, String name, Collection<Entity> entities, String fallback) {
 
-        String transformation = String.format("Joining all values with '%s'", delimiter);
-        String value = entities.stream().sorted(EntityComparators.start()).map(Entity::getValue).collect(Collectors.joining(delimiter));
-        create(ruleIdentifier, category, value, transformation, entities);
+        String valueDescription = String.format("First found value of type %s or else '%s'", joinTypes(entities), fallback);
+        String value = entities.stream().min(EntityComparators.first()).map(Entity::getValue).orElse(fallback);
+        create(ruleIdentifier, name, value, valueDescription, entities);
     }
 
 
-    public void joiningUnique(String ruleIdentifier, String category, Collection<Entity> entities) {
+    private static String joinTypes(Collection<Entity> entities) {
 
-        joining(ruleIdentifier, category, entities, ", ");
+        return entities.stream().map(Entity::getType).distinct().collect(Collectors.joining(", "));
     }
 
 
-    public void joiningUnique(String ruleIdentifier, String category, Collection<Entity> entities, String delimiter) {
-
-        String transformation = String.format("Joining all values with '%s'", delimiter);
-        String value = entities.stream().sorted(EntityComparators.start()).map(Entity::getValue).distinct().collect(Collectors.joining(delimiter));
-        create(ruleIdentifier, category, value, transformation, entities);
-    }
-
-
-    public void convertDates(String ruleIdentifier, String category, Collection<Entity> entities) {
-
-        convertDates(ruleIdentifier, category, entities, "dd/MM/yyyy");
-    }
-
-
-    public void convertDates(String ruleIdentifier, String category, Collection<Entity> entities, String resultFormat) {
-
-        String transformation = "Convert values of type to dd/MM/yyyy joined with ', '";
-        String date = entities.stream().map(Entity::getValue).map(value -> DateConverter.convertDate(value, resultFormat)).collect(Collectors.joining(", "));
-        create(ruleIdentifier, category, date, transformation, entities);
-    }
-
-
-    public void createComponentsForUnMappedEntities(String ruleIdentifier, Collection<Entity> entities) {
-
-        entities.forEach(entity -> create(ruleIdentifier, entity.getType(), entity.getValue(), "Unmapped Entity", List.of(entity)));
-    }
-
-
-    public void create(String ruleIdentifier, String category, String value, String transformation, Collection<Entity> references) {
+    /**
+     * Creates a new component with the given parameters and inserts it into the kieSession.
+     *
+     * @param ruleIdentifier   The rule identifier for the component.
+     * @param name             The name of the component.
+     * @param value            The value of the component.
+     * @param valueDescription The description of the value.
+     * @param references       A collection of Entity objects that the component references.
+     */
+    public void create(String ruleIdentifier, String name, String value, String valueDescription, Collection<Entity> references) {
 
         referencedEntities.addAll(references);
 
-        kieSession.insert(Component.builder()
-                .matchedRule(RuleIdentifier.fromString(ruleIdentifier))
-                .category(category)
-                .value(value)
-                .transformation(transformation)
-                .references(new LinkedList<>(references)));
+        kieSession.insert(Component.builder().matchedRule(RuleIdentifier.fromString(ruleIdentifier)).name(name).value(value).valueDescription(valueDescription)
+                .references(new LinkedList<>(references))
+                .build());
     }
 
 
-    public void create(String ruleIdentifier, String category, String value, String transformation, Entity reference) {
+    /**
+     * Joins entity values from the first section entities appear in, and creates a component from the result.
+     *
+     * @param ruleIdentifier The identifier of the rule.
+     * @param name           The name of the entity.
+     * @param entities       The collection of entities to process.
+     */
+    public void joiningFromFirstSectionOnly(String ruleIdentifier, String name, Collection<Entity> entities) {
+
+        joiningFromFirstSectionOnly(ruleIdentifier, name, entities, ", ");
+    }
+
+
+    /**
+     * Joins unique entity values from the first section entities appear in, and creates a component from the result.
+     *
+     * @param ruleIdentifier The identifier of the rule.
+     * @param name           The name of the entity.
+     * @param entities       The collection of entities to process.
+     */
+    public void joiningFromFirstSectionOnly(String ruleIdentifier, String name, Collection<Entity> entities, String delimiter) {
+
+        List<Entity> entitiesFromFirstSection = findEntitiesFromFirstSection(entities);
+        joining(ruleIdentifier, name, entitiesFromFirstSection, delimiter);
+    }
+
+
+    private static List<Entity> findEntitiesFromFirstSection(Collection<Entity> entities) {
+
+        var entitiesBySection = entities.stream().collect(Collectors.groupingBy(entity -> entity.getContainingNode().getHighestParent()));
+        Optional<SemanticNode> firstSection = entitiesBySection.keySet().stream().min(SemanticNodeComparators.first());
+        if (firstSection.isEmpty()) {
+            return Collections.emptyList();
+        }
+        return entitiesBySection.get(firstSection.get());
+    }
+
+
+    /**
+     * Joins unique entity values from the first section entities appear in, and creates a component from the result.
+     *
+     * @param ruleIdentifier The identifier of the rule.
+     * @param name           The name of the entity.
+     * @param entities       The collection of entities to process.
+     */
+    public void joiningUniqueFromFirstSectionOnly(String ruleIdentifier, String name, Collection<Entity> entities) {
+
+        joiningUniqueFromFirstSectionOnly(ruleIdentifier, name, entities, ", ");
+    }
+
+
+    /**
+     * Joins entity values from the first section entities appear in, and creates a component from the result.
+     *
+     * @param ruleIdentifier The identifier of the rule.
+     * @param name           The name of the entity.
+     * @param entities       The collection of entities to process.
+     */
+    public void joiningUniqueFromFirstSectionOnly(String ruleIdentifier, String name, Collection<Entity> entities, String delimiter) {
+
+        List<Entity> entitiesFromFirstSection = findEntitiesFromFirstSection(entities);
+        joiningUnique(ruleIdentifier, name, entitiesFromFirstSection, delimiter);
+    }
+
+
+    /**
+     * Joins entity values, and creates a component from the result.
+     *
+     * @param ruleIdentifier The identifier of the rule.
+     * @param name           The name of the entity.
+     * @param entities       The collection of entities to process.
+     */
+    public void joining(String ruleIdentifier, String name, Collection<Entity> entities, String delimiter) {
+
+        String valueDescription = String.format("Joining all values of type %s with '%s'", joinTypes(entities), delimiter);
+        String value = entities.stream().sorted(EntityComparators.first()).map(Entity::getValue).collect(Collectors.joining(delimiter));
+        create(ruleIdentifier, name, value, valueDescription, entities);
+    }
+
+
+    /**
+     * Joins entity values with delimiter ', ' from the section with the longest combined entity values only, and creates a component from the result.
+     *
+     * @param ruleIdentifier The identifier of the rule.
+     * @param name           The name of the entity.
+     * @param entities       The collection of entities to process.
+     */
+    public void joiningFromLongestSectionOnly(String ruleIdentifier, String name, Collection<Entity> entities) {
+
+        joiningFromLongestSectionOnly(ruleIdentifier, name, entities, ", ");
+    }
+
+
+    /**
+     * Joins entity values from the section with the longest combined entity values only, and creates a component from the result.
+     *
+     * @param ruleIdentifier The identifier of the rule.
+     * @param name           The name of the entity.
+     * @param entities       The collection of entities to process.
+     */
+    public void joiningFromLongestSectionOnly(String ruleIdentifier, String name, Collection<Entity> entities, String delimiter) {
+
+        List<Entity> entitiesFromLongestSection = findEntitiesFromLongestSection(entities);
+        joining(ruleIdentifier, name, entitiesFromLongestSection, delimiter);
+    }
+
+
+    /**
+     * Joins all unique values from a collection of entities into a single string using a specified delimiter and creates a component from the result.
+     *
+     * @param ruleIdentifier the identifier of the rule
+     * @param name           the name of the joining operation
+     * @param entities       the collection of entities
+     * @param delimiter      the delimiter to use for joining the values
+     */
+    public void joiningUnique(String ruleIdentifier, String name, Collection<Entity> entities, String delimiter) {
+
+        String valueDescription = String.format("Joining all unique values of type %s with '%s'", joinTypes(entities), delimiter);
+        String value = entities.stream().sorted(EntityComparators.first()).map(Entity::getValue).distinct().collect(Collectors.joining(delimiter));
+        create(ruleIdentifier, name, value, valueDescription, entities);
+    }
+
+
+    private static int getTotalLengthOfEntities(Map.Entry<SemanticNode, List<Entity>> entry) {
+
+        return entry.getValue().stream().mapToInt(Entity::getLength).sum();
+    }
+
+
+    /**
+     * Joins unique entity values with delimiter ', ' from the section with the longest combined entity values only, and creates a component from the result.
+     *
+     * @param ruleIdentifier The identifier of the rule.
+     * @param name           The name of the entity.
+     * @param entities       The collection of entities to process.
+     */
+    public void joiningUniqueFromLongestSectionOnly(String ruleIdentifier, String name, Collection<Entity> entities) {
+
+        joiningUniqueFromLongestSectionOnly(ruleIdentifier, name, entities, ", ");
+    }
+
+
+    /**
+     * Joins unique entity values from the section with the longest combined entity values only, and creates a component from the result.
+     *
+     * @param ruleIdentifier The identifier of the rule.
+     * @param name           The name of the entity.
+     * @param entities       The collection of entities to process.
+     * @param delimiter      The string delimiter to join the entities.
+     */
+    public void joiningUniqueFromLongestSectionOnly(String ruleIdentifier, String name, Collection<Entity> entities, String delimiter) {
+
+        List<Entity> entitiesFromLongestSection = findEntitiesFromLongestSection(entities);
+        joiningUnique(ruleIdentifier, name, entitiesFromLongestSection, delimiter);
+    }
+
+
+    /**
+     * Joins the names of the entities in the provided collection, separated by a comma,
+     * and creates a component from the result.
+     *
+     * @param ruleIdentifier The unique identifier of the rule being applied.
+     * @param name           The name to which the joined result will be set.
+     * @param entities       The collection of entities whose names will be joined.
+     */
+    public void joiningUnique(String ruleIdentifier, String name, Collection<Entity> entities) {
+
+        joiningUnique(ruleIdentifier, name, entities, ", ");
+    }
+
+
+    /**
+     * Computes the number of unique values in the collection of entities and creates a component with the result.
+     *
+     * @param ruleIdentifier the identifier of the rule
+     * @param name           the name of the record
+     * @param entities       the collection of entities to compute unique values from
+     */
+    public void uniqueValueCount(String ruleIdentifier, String name, Collection<Entity> entities) {
+
+        long count = entities.stream().map(Entity::getValue).distinct().count();
+        create(ruleIdentifier, name, String.valueOf(count), "Number of unique values in the entity references", entities);
+    }
+
+
+    /**
+     * Creates a component for each sentence in the collection of entities.
+     *
+     * @param ruleIdentifier The identifier of the rule.
+     * @param name           The name of the entity.
+     * @param entities       A collection of Entity objects.
+     */
+    public void asSentences(String ruleIdentifier, String name, Collection<Entity> entities) {
+
+        if (entities.isEmpty()) {
+            return;
+        }
+        entities.stream().sorted(EntityComparators.first()).forEach(entity -> {
+            BreakIterator iterator = BreakIterator.getSentenceInstance(Locale.ENGLISH);
+            iterator.setText(entity.getValue());
+            int start = iterator.first();
+            for (int end = iterator.next(); end != BreakIterator.DONE; start = end, end = iterator.next()) {
+                create(ruleIdentifier,
+                        name,
+                        entity.getValue().substring(start, end).replaceAll("\\n", "").trim(),
+                        String.format("Values of type '%s' as sentences", entity.getType()),
+                        entity);
+            }
+        });
+    }
+
+
+    /**
+     * Creates a new component with the given rule identifier, name, value, value description, and reference.
+     * The component is built using the provided parameters and inserted into the knowledge session.
+     * The reference is added to the referencedEntities list.
+     *
+     * @param ruleIdentifier   The identifier of the rule for the component.
+     * @param name             The name of the component.
+     * @param value            The value of the component.
+     * @param valueDescription The description of the value.
+     * @param reference        The reference entity for the component.
+     */
+    public void create(String ruleIdentifier, String name, String value, String valueDescription, Entity reference) {
 
         referencedEntities.add(reference);
         List<Entity> referenceList = new LinkedList<>();
         referenceList.add(reference);
         kieSession.insert(Component.builder()
                 .matchedRule(RuleIdentifier.fromString(ruleIdentifier))
-                .category(category)
+                .name(name)
                 .value(value)
-                .transformation(transformation)
+                .valueDescription(valueDescription)
                 .references(referenceList)
                 .build());
     }
 
 
-    public void create(String ruleIdentifier, String category, String value, String transformation) {
+    /**
+     * Creates components for unmapped entities.
+     *
+     * @param ruleIdentifier The identifier of the rule being applied.
+     * @param entities       The collection of entities to create components for.
+     */
+    public void createComponentsForUnMappedEntities(String ruleIdentifier, Collection<Entity> entities) {
 
-        create(ruleIdentifier, category, value, transformation, Collections.emptyList());
+        entities.stream().filter(entity -> !referencedEntities.contains(entity)).sorted(EntityComparators.first())
+                .forEach(entity -> create(ruleIdentifier, entity.getType(), entity.getValue(), "Unmapped Entity", List.of(entity)));
     }
 
 
-    public void create(String ruleIdentifier, String category, String value) {
+    /**
+     * Converts entity values to the 'dd/MM/yyyy' format and joins them with ', '. If the value could not be parsed as a date, it will be created as is.
+     *
+     * @param ruleIdentifier the identifier of the rule
+     * @param name           the name of the entity
+     * @param entities       the collection of entities
+     */
+    public void convertDates(String ruleIdentifier, String name, Collection<Entity> entities) {
 
-        kieSession.insert(Component.builder()
-                .matchedRule(RuleIdentifier.fromString(ruleIdentifier))
-                .category(category)
+        convertDates(ruleIdentifier, name, entities, "dd/MM/yyyy");
+    }
+
+
+    /**
+     * Converts entity values to provided format and joins them with ', '. If the value could not be parsed as a date, it will be created as is.
+     *
+     * @param ruleIdentifier the identifier of the rule
+     * @param name           the name of the entity
+     * @param entities       the collection of entities
+     * @param resultFormat   the desired format for the converted dates
+     */
+    public void convertDates(String ruleIdentifier, String name, Collection<Entity> entities, String resultFormat) {
+
+        String valueDescription = String.format("Convert values of type '%s' to %s joined with ', '", joinTypes(entities), resultFormat);
+        List<String> unparsedDates = new LinkedList<>();
+        List<Date> dates = new LinkedList<>();
+
+        for (Entity entity : entities) {
+            String value = entity.getValue();
+            Optional<Date> optionalDate = DateConverter.parseDate(value);
+            if (optionalDate.isPresent()) {
+                dates.add(optionalDate.get());
+            } else {
+                unparsedDates.add(value);
+            }
+        }
+
+        String formattedDateStrings = Stream.concat(//
+                        dates.stream().sorted().map(date -> DateConverter.convertDate(date, resultFormat)), //
+                        unparsedDates.stream())//
+                .collect(Collectors.joining(", "));
+
+        create(ruleIdentifier, name, formattedDateStrings, valueDescription, entities);
+    }
+
+
+    /**
+     * Joins values from entities that are in the same table row. If entities are not in a table cell they are added as a single component.
+     *
+     * @param ruleIdentifier the identifier of the rule
+     * @param name           the name of the entity
+     * @param entities       the collection of entities
+     */
+    public void joiningFromSameTableRow(String ruleIdentifier, String name, Collection<Entity> entities) {
+
+        String types = entities.stream().map(Entity::getType).sorted(Comparator.reverseOrder()).distinct().collect(Collectors.joining(", "));
+        String valueDescription = String.format("Combine values of %s that are in same table row", types);
+        entities.stream().collect(Collectors.groupingBy(this::getFirstTable)).forEach((optionalTable, groupedEntities) -> {
+            if (optionalTable.isEmpty()) {
+                groupedEntities.forEach(entity -> create(ruleIdentifier, name, entity.getValue(), valueDescription, entity));
+            }
+
+            groupedEntities.stream()
+                    .filter(entity -> entity.getContainingNode() instanceof TableCell)
+                    .collect(Collectors.groupingBy(entity -> ((TableCell) entity.getContainingNode()).getRow()))
+                    .entrySet()
+                    .stream()
+                    .sorted(Comparator.comparingInt(Map.Entry::getKey))
+                    .map(Map.Entry::getValue)
+                    .forEach(entitiesInSameRow -> create(ruleIdentifier,
+                            name, entitiesInSameRow.stream().sorted(EntityComparators.first()).map(Entity::getValue).collect(Collectors.joining(", ")),
+                            valueDescription,
+                            entitiesInSameRow));
+        });
+    }
+
+
+    private Optional<Table> getFirstTable(Entity entity) {
+
+        SemanticNode node = entity.getContainingNode();
+        while (!(node instanceof Table)) {
+            if (!node.hasParent()) {
+                return Optional.empty();
+            }
+            node = node.getParent();
+        }
+
+        return Optional.of((Table) node);
+    }
+
+
+    /**
+     * Creates a new component with the given rule identifier, name, value, and value description.
+     * If the component is part of a table, it also takes a list of entities that belong to the same table row.
+     *
+     * @param ruleIdentifier   the identifier of the rule
+     * @param name             the name of the entity
+     * @param value            the value of the component
+     * @param valueDescription the description of the value
+     */
+    public void create(String ruleIdentifier, String name, String value, String valueDescription) {
+
+        create(ruleIdentifier, name, value, valueDescription, Collections.emptyList());
+    }
+
+
+    /**
+     * Creates a new component with the given rule identifier, name, and value.
+     *
+     * @param ruleIdentifier the identifier of the rule
+     * @param name           the name of the entity
+     * @param value          the value of the component
+     */
+    public void create(String ruleIdentifier, String name, String value) {
+
+        kieSession.insert(Component.builder().matchedRule(RuleIdentifier.fromString(ruleIdentifier)).name(name)
                 .value(value)
-                .transformation("")
+                .valueDescription("")
                 .references(Collections.emptyList())
                 .build());
     }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityComparators.java

@@ -6,7 +6,12 @@ import com.iqser.red.service.redaction.v1.server.model.component.Entity;
 
 public abstract class EntityComparators implements Comparator<Entity> {
 
-    private static class LongestEntity implements Comparator<Entity> {
+    public static Comparator<Entity> first() {
+
+        return new FirstEntity();
+    }
+
+    public static class LongestEntity implements Comparator<Entity> {
 
         @Override
         public int compare(Entity Entity, Entity otherEntity) {
@@ -16,26 +21,20 @@ public abstract class EntityComparators implements Comparator<Entity> {
 
     }
 
-    private static class FirstEntity implements Comparator<Entity> {
-
-        @Override
-        public int compare(Entity Entity, Entity otherEntity) {
-
-            return Integer.compare(Entity.getStartOffset(), otherEntity.getStartOffset());
-        }
-
-    }
-
 
     public static Comparator<Entity> length() {
 
         return new LongestEntity();
     }
 
+    public static class FirstEntity implements Comparator<Entity> {
 
-    public static Comparator<Entity> start() {
+        @Override
+        public int compare(Entity Entity, Entity otherEntity) {
+
+            return Integer.compare(Entity.getStartOffset(), otherEntity.getStartOffset());
+        }
 
-        return new FirstEntity();
     }
 
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityCreationService.java

@@ -36,6 +36,7 @@ import com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility;
 import com.iqser.red.service.redaction.v1.server.model.NerEntities;
 import com.iqser.red.service.redaction.v1.server.model.dictionary.SearchImplementation;
 import com.iqser.red.service.redaction.v1.server.utils.IdBuilder;
+import com.iqser.red.service.redaction.v1.server.utils.exception.NotFoundException;
 
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
@@ -57,6 +58,8 @@ public class EntityCreationService {
 
     public Stream<TextEntity> betweenStrings(String start, String stop, String type, EntityType entityType, SemanticNode node) {
 
+        checkIfBothStartAndEndAreEmpty(start, stop);
+
         List<TextRange> startTextRanges = RedactionSearchUtility.findTextRangesByString(start, node.getTextBlock());
         List<TextRange> stopTextRanges = RedactionSearchUtility.findTextRangesByString(stop, node.getTextBlock());
 
@@ -66,6 +69,8 @@ public class EntityCreationService {
 
     public Stream<TextEntity> betweenStringsIgnoreCase(String start, String stop, String type, EntityType entityType, SemanticNode node) {
 
+        checkIfBothStartAndEndAreEmpty(start, stop);
+
         List<TextRange> startBoundaries = RedactionSearchUtility.findTextRangesByStringIgnoreCase(start, node.getTextBlock());
         List<TextRange> stopBoundaries = RedactionSearchUtility.findTextRangesByStringIgnoreCase(stop, node.getTextBlock());
 
@@ -75,6 +80,8 @@ public class EntityCreationService {
 
     public Stream<TextEntity> betweenStringsIncludeStart(String start, String stop, String type, EntityType entityType, SemanticNode node) {
 
+        checkIfBothStartAndEndAreEmpty(start, stop);
+
         List<TextRange> startBoundaries = RedactionSearchUtility.findTextRangesByString(start, node.getTextBlock());
         List<TextRange> stopBoundaries = RedactionSearchUtility.findTextRangesByString(stop, node.getTextBlock());
 
@@ -89,6 +96,8 @@ public class EntityCreationService {
 
     public Stream<TextEntity> betweenStringsIncludeStartIgnoreCase(String start, String stop, String type, EntityType entityType, SemanticNode node) {
 
+        checkIfBothStartAndEndAreEmpty(start, stop);
+
         List<TextRange> startBoundaries = RedactionSearchUtility.findTextRangesByStringIgnoreCase(start, node.getTextBlock());
         List<TextRange> stopBoundaries = RedactionSearchUtility.findTextRangesByStringIgnoreCase(stop, node.getTextBlock());
 
@@ -103,6 +112,8 @@ public class EntityCreationService {
 
     public Stream<TextEntity> betweenStringsIncludeEnd(String start, String stop, String type, EntityType entityType, SemanticNode node) {
 
+        checkIfBothStartAndEndAreEmpty(start, stop);
+
         List<TextRange> startBoundaries = RedactionSearchUtility.findTextRangesByString(start, node.getTextBlock());
         List<TextRange> stopBoundaries = RedactionSearchUtility.findTextRangesByString(stop, node.getTextBlock());
 
@@ -117,6 +128,8 @@ public class EntityCreationService {
 
     public Stream<TextEntity> betweenStringsIncludeEndIgnoreCase(String start, String stop, String type, EntityType entityType, SemanticNode node) {
 
+        checkIfBothStartAndEndAreEmpty(start, stop);
+
         List<TextRange> startBoundaries = RedactionSearchUtility.findTextRangesByStringIgnoreCase(start, node.getTextBlock());
         List<TextRange> stopBoundaries = RedactionSearchUtility.findTextRangesByStringIgnoreCase(stop, node.getTextBlock());
 
@@ -131,6 +144,8 @@ public class EntityCreationService {
 
     public Stream<TextEntity> betweenStringsIncludeStartAndEnd(String start, String stop, String type, EntityType entityType, SemanticNode node) {
 
+        checkIfBothStartAndEndAreEmpty(start, stop);
+
         List<TextRange> startBoundaries = RedactionSearchUtility.findTextRangesByString(start, node.getTextBlock());
         List<TextRange> stopBoundaries = RedactionSearchUtility.findTextRangesByString(stop, node.getTextBlock());
 
@@ -149,6 +164,8 @@ public class EntityCreationService {
 
     public Stream<TextEntity> betweenStringsIncludeStartAndEndIgnoreCase(String start, String stop, String type, EntityType entityType, SemanticNode node) {
 
+        checkIfBothStartAndEndAreEmpty(start, stop);
+
         List<TextRange> startBoundaries = RedactionSearchUtility.findTextRangesByStringIgnoreCase(start, node.getTextBlock());
         List<TextRange> stopBoundaries = RedactionSearchUtility.findTextRangesByStringIgnoreCase(stop, node.getTextBlock());
 
@@ -165,6 +182,38 @@ public class EntityCreationService {
     }
 
 
+    public Stream<TextEntity> shortestBetweenAnyString(List<String> starts, List<String> stops, String type, EntityType entityType, SemanticNode node) {
+
+        checkIfBothStartAndEndAreEmpty(starts, stops);
+
+        List<TextRange> startTextRanges = RedactionSearchUtility.findTextRangesByList(starts, node.getTextBlock());
+        List<TextRange> stopTextRanges = RedactionSearchUtility.findTextRangesByList(stops, node.getTextBlock());
+
+        return betweenTextRanges(startTextRanges, stopTextRanges, type, entityType, node);
+    }
+
+
+    public Stream<TextEntity> shortestBetweenAnyStringIgnoreCase(List<String> starts, List<String> stops, String type, EntityType entityType, SemanticNode node) {
+
+        checkIfBothStartAndEndAreEmpty(starts, stops);
+
+        List<TextRange> startTextRanges = RedactionSearchUtility.findTextRangesByListIgnoreCase(starts, node.getTextBlock());
+        List<TextRange> stopTextRanges = RedactionSearchUtility.findTextRangesByListIgnoreCase(stops, node.getTextBlock());
+
+        return betweenTextRanges(startTextRanges, stopTextRanges, type, entityType, node);
+    }
+
+    public Stream<TextEntity> shortestBetweenAnyStringIgnoreCase(List<String> starts, List<String> stops, String type, EntityType entityType, SemanticNode node, int limit) {
+
+        checkIfBothStartAndEndAreEmpty(starts, stops);
+
+        List<TextRange> startTextRanges = RedactionSearchUtility.findTextRangesByListIgnoreCase(starts, node.getTextBlock());
+        List<TextRange> stopTextRanges = RedactionSearchUtility.findTextRangesByListIgnoreCase(stops, node.getTextBlock());
+
+        return betweenTextRanges(startTextRanges, stopTextRanges, type, entityType, node, limit);
+    }
+
+
     public Stream<TextEntity> betweenRegexes(String regexStart, String regexStop, String type, EntityType entityType, SemanticNode node) {
 
         TextBlock textBlock = node.getTextBlock();
@@ -184,14 +233,17 @@ public class EntityCreationService {
         return betweenTextRanges(startBoundaries, stopBoundaries, type, entityType, node);
     }
 
-
     public Stream<TextEntity> betweenTextRanges(List<TextRange> startBoundaries, List<TextRange> stopBoundaries, String type, EntityType entityType, SemanticNode node) {
 
-        if (startBoundaries.isEmpty() || stopBoundaries.isEmpty()) {
-            return Stream.empty();
-        }
+        return betweenTextRanges(startBoundaries, stopBoundaries, type, entityType, node,0);
+    }
+
+
+    public Stream<TextEntity> betweenTextRanges(List<TextRange> startBoundaries, List<TextRange> stopBoundaries, String type, EntityType entityType, SemanticNode node, int limit) {
+
         List<TextRange> entityBoundaries = findNonOverlappingBoundariesBetweenBoundariesWithMinimalDistances(startBoundaries, stopBoundaries);
         return entityBoundaries.stream()
+                .filter(range -> (limit == 0 || range.length() <= limit))
                 .map(textRange -> textRange.trim(node.getTextBlock()))
                 .filter(textRange -> isValidEntityTextRange(node.getTextBlock(), textRange))
                 .map(textRange -> byTextRange(textRange, type, entityType, node))
@@ -208,7 +260,7 @@ public class EntityCreationService {
                     .filter(stopBoundary -> stopBoundary.start() > startTextRange.end())
                     .min(Comparator.comparingInt(TextRange::start));
             if (optionalStopTextRangeWithMinimalDistance.isEmpty()) {
-                break;
+                continue;
             }
             entityBoundaries.add(new TextRange(startTextRange.end(), optionalStopTextRangeWithMinimalDistance.get().start()));
         }
@@ -223,12 +275,25 @@ public class EntityCreationService {
         For Example consider the text: "a this is some text. a here is more text b". If "a" is the start string and "b" is the stop string, there are two possibilities.
         "this is some text. a here is more text" and "here is more text". We only want to keep the latter.
          */
-        return entityTextRanges.stream().filter(boundary -> entityTextRanges.stream().noneMatch(innerBoundary -> innerBoundary != boundary && innerBoundary.containedBy(boundary)))
+        return entityTextRanges.stream()
+                .filter(boundary -> entityTextRanges.stream().noneMatch(innerBoundary -> !innerBoundary.equals(boundary) && innerBoundary.containedBy(boundary)))
                 .toList();
 
     }
 
 
+    private void checkIfBothStartAndEndAreEmpty(String start, String end) {
+        checkIfBothStartAndEndAreEmpty(List.of(start), List.of(end));
+    }
+
+
+    private <T> void checkIfBothStartAndEndAreEmpty(List<T> start, List<T> end) {
+        if ((start == null || start.isEmpty()) && (end == null || end.isEmpty())) {
+            throw new IllegalArgumentException("Start and end values are empty!");
+        }
+    }
+
+
     public Stream<TextEntity> bySearchImplementation(SearchImplementation searchImplementation, String type, EntityType entityType, SemanticNode node) {
 
         return searchImplementation.getBoundaries(node.getTextBlock(), node.getTextRange())
@@ -500,7 +565,6 @@ public class EntityCreationService {
         return expandedEnd;
     }
 
-
     /**
      * Creates a redaction entity based on the given boundary, type, entity type, and semantic node.
      * If the document already contains an equal redaction entity, then the original Entity is returned.
@@ -514,22 +578,39 @@ public class EntityCreationService {
      */
     public Optional<TextEntity> byTextRange(TextRange textRange, String type, EntityType entityType, SemanticNode node) {
 
+        return byTextRangeWithEngine(textRange, type, entityType, node, Set.of(Engine.RULE));
+    }
+
+    /**
+     * Creates a redaction entity based on the given boundary, type, entity type, and semantic node.
+     * If the document already contains an equal redaction entity, then the original Entity is returned.
+     * Also inserts the Entity into the kieSession.
+     *
+     * @param textRange  The boundary of the redaction entity.
+     * @param type       The type of the redaction entity.
+     * @param entityType The entity type of the redaction entity.
+     * @param node       The semantic node to associate with the redaction entity.
+     * @param engines    Set of engines to be used.
+     * @return An Optional containing the redaction entity, or the previous entity if the entity already exists.
+     */
+    public Optional<TextEntity> byTextRangeWithEngine(TextRange textRange, String type, EntityType entityType, SemanticNode node, Set<Engine> engines) {
+
         if (!node.getTextRange().contains(textRange)) {
             throw new IllegalArgumentException(String.format("%s is not in the %s of the provided semantic node %s", textRange, node.getTextRange(), node));
         }
         TextRange trimmedTextRange = textRange.trim(node.getTextBlock());
         TextEntity entity = TextEntity.initialEntityNode(trimmedTextRange, type, entityType);
         if (node.getEntities().contains(entity)) {
-            return node.getEntities().stream().filter(entity::equals).peek(e -> e.addEngine(Engine.RULE)).findAny();
+            return node.getEntities().stream().filter(entity::equals).peek(e -> e.addEngines(engines)).findAny();
         }
         addEntityToGraph(entity, node);
-        entity.addEngine(Engine.RULE);
+        entity.addEngines(engines);
         insertToKieSession(entity);
         return Optional.of(entity);
     }
 
 
-    public TextEntity forceByBoundary(TextRange textRange, String type, EntityType entityType, SemanticNode node) {
+    public TextEntity forceByTextRange(TextRange textRange, String type, EntityType entityType, SemanticNode node) {
 
         TextRange trimmedTextRange = textRange.trim(node.getTextBlock());
         TextEntity entity = TextEntity.initialEntityNode(trimmedTextRange, type, entityType);
@@ -576,14 +657,18 @@ public class EntityCreationService {
 
     public TextEntity copyEntity(TextEntity entity, String type, EntityType entityType, SemanticNode node) {
 
-        TextEntity newEntity = TextEntity.initialEntityNode(entity.getTextRange(), type, entityType);
-        newEntity.addEngines(entity.getEngines());
+        var newEntity = copyEntityWithoutRules(entity, type, entityType, node);
         newEntity.addMatchedRules(entity.getMatchedRuleList());
+        return newEntity;
+    }
+
+    public TextEntity copyEntityWithoutRules(TextEntity entity, String type, EntityType entityType, SemanticNode node) {
+
+        TextEntity newEntity = byTextRangeWithEngine(entity.getTextRange(), type, entityType, node, entity.getEngines()).orElseThrow(() -> new NotFoundException(
+                "No entity present!"));
         newEntity.getManualOverwrite().addChanges(entity.getManualOverwrite().getManualChangeLog());
         newEntity.setDictionaryEntry(entity.isDictionaryEntry());
         newEntity.setDossierDictionaryEntry(entity.isDossierDictionaryEntry());
-        addEntityToGraph(newEntity, node);
-        insertToKieSession(newEntity);
         return newEntity;
     }
 
@@ -598,28 +683,22 @@ public class EntityCreationService {
 
     public TextEntity byNerEntity(NerEntities.NerEntity nerEntity, EntityType entityType, SemanticNode semanticNode) {
 
-        var entity = forceByBoundary(nerEntity.textRange(), nerEntity.type(), entityType, semanticNode);
-        entity.addEngine(Engine.NER);
-        insertToKieSession(entity);
-        return entity;
+        return byTextRangeWithEngine(nerEntity.textRange(), nerEntity.type(), entityType, semanticNode, Set.of(Engine.NER)).orElseThrow(() -> new NotFoundException("No entity present!"));
     }
 
 
     public TextEntity byNerEntity(NerEntities.NerEntity nerEntity, String type, EntityType entityType, SemanticNode semanticNode) {
 
-        var entity = forceByBoundary(nerEntity.textRange(), type, entityType, semanticNode);
-        entity.addEngine(Engine.NER);
-        insertToKieSession(entity);
-        return entity;
+        return byTextRangeWithEngine(nerEntity.textRange(), type, entityType, semanticNode, Set.of(Engine.NER)).orElseThrow(() -> new NotFoundException("No entity present!"));
     }
 
 
     public Stream<TextEntity> combineNerEntitiesToCbiAddressDefaults(NerEntities nerEntities, String type, EntityType entityType, SemanticNode semanticNode) {
 
         return NerEntitiesAdapter.combineNerEntitiesToCbiAddressDefaults(nerEntities)
-                .map(boundary -> forceByBoundary(boundary, type, entityType, semanticNode))
-                .peek(entity -> entity.addEngine(Engine.NER))
-                .peek(this::insertToKieSession);
+                .map(boundary -> byTextRangeWithEngine(boundary, type, entityType, semanticNode, Set.of(Engine.NER)))
+                .filter(Optional::isPresent)
+                .map(Optional::get);
     }
 
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityFindingUtility.java

@@ -0,0 +1,164 @@
+package com.iqser.red.service.redaction.v1.server.service.document;
+
+import static java.lang.String.format;
+import static java.util.stream.Collectors.groupingBy;
+
+import java.awt.geom.Rectangle2D;
+import java.util.Collection;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+
+import com.iqser.red.service.redaction.v1.server.model.ManualEntity;
+import com.iqser.red.service.redaction.v1.server.model.RectangleWithPage;
+import com.iqser.red.service.redaction.v1.server.model.dictionary.SearchImplementation;
+import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
+import com.iqser.red.service.redaction.v1.server.model.document.entity.PositionOnPage;
+import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.Page;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
+import com.iqser.red.service.redaction.v1.server.service.DictionaryService;
+
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+@Service
+public class EntityFindingUtility {
+
+    EntityCreationService entityCreationService;
+    DictionaryService dictionaryService;
+
+
+    @Autowired
+    public EntityFindingUtility(EntityEnrichmentService entityEnrichmentService, DictionaryService dictionaryService) {
+
+        entityCreationService = new EntityCreationService(entityEnrichmentService);
+        this.dictionaryService = dictionaryService;
+    }
+
+
+    public Optional<TextEntity> findClosestEntityAndReturnEmptyIfNotFound(ManualEntity identifier, Map<String, List<TextEntity>> entitiesWithSameValue, double matchThreshold) {
+
+        List<TextEntity> possibleEntities = entitiesWithSameValue.get(identifier.getValue().toLowerCase(Locale.ENGLISH));
+
+        if (entityIdentifierValueNotFound(possibleEntities)) {
+            log.warn("Entity could not be created with identifier: {}, due to the value {} not being found anywhere.", identifier, identifier.getValue());
+            return Optional.empty();
+        }
+
+        Optional<TextEntity> optionalClosestEntity = possibleEntities.stream()
+                .filter(entity -> pagesMatch(entity, identifier.getEntityPosition()))
+                .min(Comparator.comparingDouble(entity -> calculateMinDistance(identifier.getEntityPosition(), entity)));
+
+        if (optionalClosestEntity.isEmpty()) {
+            log.warn("No Entity with value {} found on page {}", identifier.getValue(), identifier.getEntityPosition());
+            return Optional.empty();
+        }
+
+        TextEntity closestEntity = optionalClosestEntity.get();
+        double distance = calculateMinDistance(identifier.getEntityPosition(), closestEntity);
+        if (distance > matchThreshold) {
+            log.warn("For entity {} on page {} with positions {} distance to closest found entity is {} and therefore higher than the threshold of {}",
+                    identifier.getValue(),
+                    identifier.getEntityPosition().get(0).pageNumber(),
+                    identifier.getEntityPosition().stream().map(RectangleWithPage::rectangle2D).toList(),
+                    distance,
+                    matchThreshold);
+            return Optional.empty();
+        }
+
+        return Optional.of(closestEntity);
+    }
+
+
+    private static boolean entityIdentifierValueNotFound(List<TextEntity> possibleEntities) {
+
+        return possibleEntities == null || possibleEntities.isEmpty();
+    }
+
+
+    private static boolean pagesMatch(TextEntity entity, List<RectangleWithPage> originalPositions) {
+
+        Set<Integer> entityPageNumbers = entity.getPositionsOnPagePerPage().stream().map(PositionOnPage::getPage).map(Page::getNumber).collect(Collectors.toSet());
+        Set<Integer> originalPageNumbers = originalPositions.stream().map(RectangleWithPage::pageNumber).collect(Collectors.toSet());
+        return entityPageNumbers.containsAll(originalPageNumbers);
+    }
+
+
+    private double calculateMinDistance(List<RectangleWithPage> originalPositions, TextEntity entity) {
+
+        if (originalPositions.size() != countRectangles(entity)) {
+            return Double.MAX_VALUE;
+        }
+        return originalPositions.stream()
+                .mapToDouble(rectangleWithPage -> calculateMinDistancePerRectangle(entity, rectangleWithPage.pageNumber(), rectangleWithPage.rectangle2D()))
+                .sum();
+    }
+
+
+    private static long countRectangles(TextEntity entity) {
+
+        return entity.getPositionsOnPagePerPage().stream().mapToLong(redactionPosition -> redactionPosition.getRectanglePerLine().size()).sum();
+    }
+
+
+    private double calculateMinDistancePerRectangle(TextEntity entity, int pageNumber, Rectangle2D originalRectangle) {
+
+        return entity.getPositionsOnPagePerPage()
+                .stream()
+                .filter(redactionPosition -> redactionPosition.getPage().getNumber() == pageNumber)
+                .map(PositionOnPage::getRectanglePerLine)
+                .flatMap(Collection::stream)
+                .mapToDouble(rectangle -> calculateDistance(rectangle, originalRectangle))
+                .min()
+                .orElse(Double.MAX_VALUE);
+    }
+
+
+    public double calculateDistance(Rectangle2D rectangle1, Rectangle2D rectangle2) {
+
+        // mirrored coordinates safe comparison
+        double minX1 = Math.min(rectangle1.getMinX(), rectangle1.getMaxX());
+        double minY1 = Math.min(rectangle1.getMinY(), rectangle1.getMaxY());
+        double maxX1 = Math.max(rectangle1.getMinX(), rectangle1.getMaxX());
+        double maxY1 = Math.max(rectangle1.getMinY(), rectangle1.getMaxY());
+
+        double minX2 = Math.min(rectangle2.getMinX(), rectangle2.getMaxX());
+        double minY2 = Math.min(rectangle2.getMinY(), rectangle2.getMaxY());
+        double maxX2 = Math.max(rectangle2.getMinX(), rectangle2.getMaxX());
+        double maxY2 = Math.max(rectangle2.getMinY(), rectangle2.getMaxY());
+
+        return Math.abs(minX1 - minX2) //
+                + Math.abs(minY1 - minY2) //
+                + Math.abs(maxX1 - maxX2) //
+                + Math.abs(maxY1 - maxY2);
+    }
+
+
+    public Map<String, List<TextEntity>> findAllPossibleEntitiesAndGroupByValue(SemanticNode node, List<ManualEntity> manualEntities) {
+
+        Set<Integer> pageNumbers = manualEntities.stream().flatMap(entry -> entry.getEntityPosition().stream().map(RectangleWithPage::pageNumber)).collect(Collectors.toSet());
+        Set<String> entryValues = manualEntities.stream().map(ManualEntity::getValue).map(String::toLowerCase).collect(Collectors.toSet());
+
+        if (!pageNumbers.stream().allMatch(node::onPage)) {
+            throw new IllegalArgumentException(format("SemanticNode \"%s\" does not contain these pages %s, it has pages: %s",
+                    node,
+                    pageNumbers.stream().filter(pageNumber -> !node.onPage(pageNumber)).toList(),
+                    node.getPages()));
+        }
+        SearchImplementation searchImplementation = new SearchImplementation(entryValues, true);
+
+        return searchImplementation.getBoundaries(node.getTextBlock(), node.getTextRange())
+                .stream()
+                .map(boundary -> entityCreationService.forceByTextRange(boundary, "temp", EntityType.ENTITY, node))
+                .collect(groupingBy(entity -> entity.getValue().toLowerCase(Locale.ROOT)));
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/ManualEntityCreationService.java

@@ -1,80 +1,76 @@
 package com.iqser.red.service.redaction.v1.server.service.document;
 
-import static java.lang.String.format;
-import static java.util.stream.Collectors.groupingBy;
-
-import java.awt.geom.Rectangle2D;
 import java.util.ArrayList;
 import java.util.Collection;
-import java.util.Comparator;
 import java.util.LinkedList;
 import java.util.List;
-import java.util.Locale;
 import java.util.Map;
 import java.util.Optional;
 import java.util.Set;
-import java.util.stream.Collectors;
 
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
 
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog;
 import com.iqser.red.service.redaction.v1.server.model.ManualEntity;
-import com.iqser.red.service.redaction.v1.server.model.RectangleWithPage;
-import com.iqser.red.service.redaction.v1.server.model.dictionary.SearchImplementation;
 import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.PositionOnPage;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
-import com.iqser.red.service.redaction.v1.server.model.document.nodes.Page;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
+import com.iqser.red.service.redaction.v1.server.service.DictionaryService;
 
+import lombok.AccessLevel;
+import lombok.experimental.FieldDefaults;
 import lombok.extern.slf4j.Slf4j;
 
 @Slf4j
 @Service
+@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
 public class ManualEntityCreationService {
 
-    private static final double MATCH_THRESHOLD = 5; // Is compared to the sum of distances in pdf coordinates for each corner of the bounding box of the entities
-    private final EntityCreationService entityCreationService;
+    static double MATCH_THRESHOLD = 5; // Is compared to the sum of distances in pdf coordinates for each corner of the bounding box of the entities
+    EntityFindingUtility entityFindingUtility;
+    EntityCreationService entityCreationService;
+    DictionaryService dictionaryService;
 
 
     @Autowired
-    public ManualEntityCreationService(EntityEnrichmentService entityEnrichmentService) {
+    public ManualEntityCreationService(EntityEnrichmentService entityEnrichmentService, DictionaryService dictionaryService, EntityFindingUtility entityFindingUtility) {
 
+        this.entityFindingUtility = entityFindingUtility;
         entityCreationService = new EntityCreationService(entityEnrichmentService);
+        this.dictionaryService = dictionaryService;
     }
 
 
-    public List<ManualEntity> toRedactionEntity(RedactionLog redactionLog, SemanticNode node) {
-
-        List<ManualEntity> manualEntities = redactionLog.getRedactionLogEntry().stream().map(ManualEntity::fromRedactionLogEntry).toList();
-        return toRedactionEntity(manualEntities, node);
-    }
-
-
-    public List<ManualEntity> createRedactionEntitiesIfFoundAndReturnNotFoundEntries(Set<ManualRedactionEntry> manualRedactionEntries, SemanticNode node) {
+    public List<ManualEntity> createRedactionEntitiesIfFoundAndReturnNotFoundEntries(Set<ManualRedactionEntry> manualRedactionEntries,
+                                                                                     SemanticNode node,
+                                                                                     String dossierTemplateId) {
 
         List<ManualEntity> manualEntities = manualRedactionEntries.stream()
                 .filter(manualRedactionEntry -> !(manualRedactionEntry.isAddToDictionary() || manualRedactionEntry.isAddToDossierDictionary()))
-                .map(ManualEntity::fromManualRedactionEntry)
+                .map(manualRedactionEntry -> ManualEntity.fromManualRedactionEntry(manualRedactionEntry,
+                        dictionaryService.isHint(manualRedactionEntry.getType(), dossierTemplateId))).peek(manualEntity -> {
+                    if (manualEntity.getEntityType().equals(EntityType.HINT)) {
+                        manualEntity.skip("MAN.5.1", "manual hint is skipped by default");
+                    } else {
+                        manualEntity.apply("MAN.5.0", "manual entries are applied by default", manualEntity.getLegalBasis());
+                    }
+                })
                 .toList();
 
-        return toRedactionEntity(manualEntities, node);
+        return toTextEntity(manualEntities, node);
     }
 
 
-    private List<ManualEntity> toRedactionEntity(List<ManualEntity> manualEntities, SemanticNode node) {
+    public List<ManualEntity> toTextEntity(List<ManualEntity> manualEntities, SemanticNode node) {
 
-        Set<Integer> pageNumbers = manualEntities.stream().flatMap(entry -> entry.getEntityPosition().stream().map(RectangleWithPage::pageNumber)).collect(Collectors.toSet());
-        Set<String> entryValues = manualEntities.stream().map(ManualEntity::getValue).map(String::toLowerCase).collect(Collectors.toSet());
-
-        Map<String, List<TextEntity>> tempEntitiesByValue = findAllPossibleEntitiesAndGroupByValue(node, pageNumbers, entryValues);
+        Map<String, List<TextEntity>> tempEntitiesByValue = entityFindingUtility.findAllPossibleEntitiesAndGroupByValue(node, manualEntities);
 
         List<ManualEntity> notFoundManualEntities = new LinkedList<>();
         for (ManualEntity manualEntity : manualEntities) {
-            Optional<TextEntity> optionalRedactionEntity = findClosestEntityAndReturnEmptyIfNotFound(manualEntity, tempEntitiesByValue);
+            Optional<TextEntity> optionalRedactionEntity = entityFindingUtility.findClosestEntityAndReturnEmptyIfNotFound(manualEntity, tempEntitiesByValue, MATCH_THRESHOLD);
             if (optionalRedactionEntity.isEmpty()) {
                 notFoundManualEntities.add(manualEntity);
                 continue;
@@ -89,24 +85,19 @@ public class ManualEntityCreationService {
     /**
      * Deletes the temp Entity and creates a RedactionEntity with correct values, based on the given parameters.
      *
-     * @param manualEntity The entity identifier for the RedactionEntity.
+     * @param manualEntity     The entity identifier for the RedactionEntity.
      * @param node             The SemanticNode associated with the RedactionEntity.
-     * @param closestTextRange  The closest Boundary to the RedactionEntity.
+     * @param closestTextRange The closest Boundary to the RedactionEntity.
      */
     private void createCorrectEntity(ManualEntity manualEntity, SemanticNode node, TextRange closestTextRange) {
 
-        TextEntity correctEntity = entityCreationService.forceByBoundary(closestTextRange, manualEntity.getType(), manualEntity.getEntityType(), node);
+        TextEntity correctEntity = entityCreationService.forceByTextRange(closestTextRange, manualEntity.getType(), manualEntity.getEntityType(), node);
 
-        if (manualEntity.isApplied()) {
-            correctEntity.apply(manualEntity.getRuleIdentifier(), manualEntity.getReason(), manualEntity.getLegalBasis());
-        } else {
-            correctEntity.skip(manualEntity.getRuleIdentifier(), manualEntity.getReason());
-        }
+        correctEntity.addMatchedRules(manualEntity.getMatchedRuleList());
         correctEntity.setDictionaryEntry(manualEntity.isDictionaryEntry());
         correctEntity.setDossierDictionaryEntry(manualEntity.isDossierDictionaryEntry());
         correctEntity.getManualOverwrite().addChanges(manualEntity.getManualOverwrite().getManualChangeLog());
 
-
         // AnnotationIds must match the IDs in the add requests, or comments break. Maybe think about migrating IDs on the fly!
         List<PositionOnPage> redactionPositionsWithIdOfManualOnPage = new ArrayList<>(correctEntity.getPositionsOnPagePerPage().size());
         for (PositionOnPage positionOnPage : correctEntity.getPositionsOnPagePerPage()) {
@@ -115,124 +106,4 @@ public class ManualEntityCreationService {
         correctEntity.setPositionsOnPagePerPage(redactionPositionsWithIdOfManualOnPage);
     }
 
-
-    private Optional<TextEntity> findClosestEntityAndReturnEmptyIfNotFound(ManualEntity identifier, Map<String, List<TextEntity>> entitiesWithSameValue) {
-
-        List<TextEntity> possibleEntities = entitiesWithSameValue.get(identifier.getValue().toLowerCase(Locale.ROOT));
-
-        if (entityIdentifierValueNotFound(possibleEntities)) {
-            log.warn("Entity could not be created with identifier: {}, due to the value {} not being found anywhere.", identifier, identifier.getValue());
-            return Optional.empty();
-        }
-
-        Optional<TextEntity> optionalClosestEntity = possibleEntities.stream()
-                .filter(entity -> pagesMatch(entity, identifier.getEntityPosition()))
-                .min(Comparator.comparingDouble(entity -> calculateMinDistance(identifier.getEntityPosition(), entity)));
-
-        if (optionalClosestEntity.isEmpty()) {
-            log.warn("No Entity with value {} found on page {}", identifier.getValue(), identifier.getEntityPosition());
-            return Optional.empty();
-        }
-
-        TextEntity closestEntity = optionalClosestEntity.get();
-        double distance = calculateMinDistance(identifier.getEntityPosition(), closestEntity);
-        if (distance > MATCH_THRESHOLD) {
-            log.warn(format("Distance to closest found entity is %.2f and therefore higher than the threshold of %.2f for \n%s \n%s",
-                    distance,
-                    MATCH_THRESHOLD,
-                    identifier.getEntityPosition(),
-                    closestEntity.getPositionsOnPagePerPage()));
-            return Optional.empty();
-        }
-
-        return Optional.of(closestEntity);
-    }
-
-
-    private static boolean entityIdentifierValueNotFound(List<TextEntity> possibleEntities) {
-
-        return possibleEntities == null || possibleEntities.isEmpty();
-    }
-
-
-    private Map<String, List<TextEntity>> findAllPossibleEntitiesAndGroupByValue(SemanticNode node, Set<Integer> pageNumbers, Set<String> entryValues) {
-
-        if (!pageNumbers.stream().allMatch(node::onPage)) {
-            throw new IllegalArgumentException(format("SemanticNode \"%s\" does not contain these pages %s, it has pages: %s",
-                    node,
-                    pageNumbers.stream().filter(pageNumber -> !node.onPage(pageNumber)).toList(),
-                    node.getPages()));
-        }
-        SearchImplementation searchImplementation = new SearchImplementation(entryValues, true);
-
-        return searchImplementation.getBoundaries(node.getTextBlock(), node.getTextRange())
-                .stream()
-                .map(boundary -> entityCreationService.forceByBoundary(boundary, "temp", EntityType.ENTITY, node))
-                .collect(groupingBy(entity -> entity.getValue().toLowerCase(Locale.ROOT)));
-    }
-
-
-    private static boolean allValuesFound(Map<String, List<TextEntity>> entitiesByValue, Set<String> entryValues) {
-
-        return entitiesByValue.keySet().equals(entryValues);
-    }
-
-
-    private static boolean pagesMatch(TextEntity entity, List<RectangleWithPage> originalPositions) {
-
-        Set<Integer> entityPageNumbers = entity.getPositionsOnPagePerPage().stream().map(PositionOnPage::getPage).map(Page::getNumber).collect(Collectors.toSet());
-        Set<Integer> originalPageNumbers = originalPositions.stream().map(RectangleWithPage::pageNumber).collect(Collectors.toSet());
-        return entityPageNumbers.containsAll(originalPageNumbers);
-    }
-
-
-    private double calculateMinDistance(List<RectangleWithPage> originalPositions, TextEntity entity) {
-
-        if (originalPositions.size() != countRectangles(entity)) {
-            return Double.MAX_VALUE;
-        }
-        return originalPositions.stream()
-                .mapToDouble(rectangleWithPage -> calculateMinDistancePerRectangle(entity, rectangleWithPage.pageNumber(), rectangleWithPage.rectangle2D()))
-                .sum();
-    }
-
-
-    private static long countRectangles(TextEntity entity) {
-
-        return entity.getPositionsOnPagePerPage().stream().mapToLong(redactionPosition -> redactionPosition.getRectanglePerLine().size()).sum();
-    }
-
-
-    private double calculateMinDistancePerRectangle(TextEntity entity, int pageNumber, Rectangle2D originalRectangle) {
-
-        return entity.getPositionsOnPagePerPage()
-                .stream()
-                .filter(redactionPosition -> redactionPosition.getPage().getNumber() == pageNumber)
-                .map(PositionOnPage::getRectanglePerLine)
-                .flatMap(Collection::stream)
-                .mapToDouble(rectangle -> calculateDistance(rectangle, originalRectangle))
-                .min()
-                .orElse(Double.MAX_VALUE);
-    }
-
-
-    private double calculateDistance(Rectangle2D rectangle, Rectangle2D rectangle2D) {
-
-        // mirrored coordinates safe comparison
-        double minX1 = Math.min(rectangle.getMinX(), rectangle.getMaxX());
-        double minY1 = Math.min(rectangle.getMinY(), rectangle.getMaxY());
-        double maxX1 = Math.max(rectangle.getMinX(), rectangle.getMaxX());
-        double maxY1 = Math.max(rectangle.getMinY(), rectangle.getMaxY());
-
-        double minX2 = Math.min(rectangle2D.getMinX(), rectangle2D.getMaxX());
-        double minY2 = Math.min(rectangle2D.getMinY(), rectangle2D.getMaxY());
-        double maxX2 = Math.max(rectangle2D.getMinX(), rectangle2D.getMaxX());
-        double maxY2 = Math.max(rectangle2D.getMinY(), rectangle2D.getMaxY());
-
-        return Math.abs(minX1 - minX2) //
-                + Math.abs(minY1 - minY2) //
-                + Math.abs(maxX1 - maxX2) //
-                + Math.abs(maxY1 - maxY2);
-    }
-
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/ManualRedactionEntryService.java

@@ -24,12 +24,12 @@ public class ManualRedactionEntryService {
     private final ManualEntityCreationService manualEntityCreationService;
 
 
-    public List<ManualEntity> addManualRedactionEntriesAndReturnNotFoundEntries(AnalyzeRequest analyzeRequest, Document document) {
+    public List<ManualEntity> addManualRedactionEntriesAndReturnNotFoundEntries(AnalyzeRequest analyzeRequest, Document document, String dossierTemplateId) {
 
         List<ManualEntity> notFoundManualRedactionEntries = Collections.emptyList();
         if (analyzeRequest.getManualRedactions() != null) {
             notFoundManualRedactionEntries = manualEntityCreationService.createRedactionEntitiesIfFoundAndReturnNotFoundEntries(analyzeRequest.getManualRedactions()
-                    .getEntriesToAdd(), document);
+                    .getEntriesToAdd(), document, dossierTemplateId);
             log.info("Added Manual redaction entries for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
         }
         if (notFoundManualRedactionEntries.isEmpty()) {

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/SectionFinderService.java

@@ -47,7 +47,10 @@ public class SectionFinderService {
         Set<Integer> sectionsToReanalyse = new HashSet<>();
         for (EntityLogEntry entry : entityLog.getEntityLogEntry()) {
             if (relevantManuallyModifiedAnnotationIds.contains(entry.getId())) {
-                sectionsToReanalyse.add(entry.getSectionNumber());
+                if (entry.getContainingNodeId().isEmpty()) {
+                    continue; // Empty list means either Entity has not been found or it is between main sections. Thus, this might lead to wrong reanalysis.
+                }
+                sectionsToReanalyse.add(entry.getContainingNodeId().get(0));
             }
         }
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/SemanticNodeComparators.java

@@ -0,0 +1,46 @@
+package com.iqser.red.service.redaction.v1.server.service.document;
+
+import java.util.Comparator;
+import java.util.List;
+
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
+
+public abstract class SemanticNodeComparators implements Comparator<SemanticNode> {
+
+    public static SemanticNodeComparators first() {
+
+        return new FirstSemanticNode();
+    }
+
+    public static class FirstSemanticNode extends SemanticNodeComparators {
+
+        @Override
+        public int compare(SemanticNode semanticNode, SemanticNode otherSemanticNode) {
+
+            List<Integer> treeId = semanticNode.getTreeId();
+            List<Integer> otherTreeId = otherSemanticNode.getTreeId();
+
+            int prefixLength = Math.min(treeId.size(), otherTreeId.size());
+            // Compare ids one by one
+            for (int i = 0; i < prefixLength; i++) {
+                int id1 = treeId.get(i);
+                int id2 = otherTreeId.get(i);
+
+                // If ids are different, return the comparison result
+                if (id1 != id2) {
+                    return Integer.compare(id1, id2);
+                }
+            }
+
+            // If all prefix ids are equal, shorter treeId is first
+            if (treeId.size() != otherTreeId.size()) {
+                return Integer.compare(treeId.size(), otherTreeId.size());
+            }
+
+            // Lists are equal
+            return 0;
+        }
+
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/ComponentDroolsExecutionService.java

@@ -16,9 +16,12 @@ import org.springframework.stereotype.Service;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState;
 import com.iqser.red.service.redaction.v1.server.RedactionServiceSettings;
 import com.iqser.red.service.redaction.v1.server.model.component.Component;
 import com.iqser.red.service.redaction.v1.server.model.component.Entity;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
+import com.iqser.red.service.redaction.v1.server.service.document.ComponentComparator;
 import com.iqser.red.service.redaction.v1.server.service.document.ComponentCreationService;
 import com.iqser.red.service.redaction.v1.server.utils.exception.DroolsTimeoutException;
 
@@ -36,20 +39,24 @@ public class ComponentDroolsExecutionService {
     RedactionServiceSettings settings;
 
 
-    public List<Component> executeRules(KieContainer kieContainer, EntityLog entityLog, List<FileAttribute> fileAttributes) {
+    public List<Component> executeRules(KieContainer kieContainer, EntityLog entityLog, Document document, List<FileAttribute> fileAttributes) {
 
         KieSession kieSession = kieContainer.newKieSession();
         ComponentCreationService componentCreationService = new ComponentCreationService(kieSession);
 
         kieSession.setGlobal("componentCreationService", componentCreationService);
-        entityLog.getEntityLogEntry().stream().map(Entity::fromEntityLogEntry).forEach(kieSession::insert);
+        entityLog.getEntityLogEntry()
+                .stream()
+                .filter(entityLogEntry -> entityLogEntry.getState().equals(EntryState.APPLIED))
+                .map(entry -> Entity.fromEntityLogEntry(entry, document))
+                .forEach(kieSession::insert);
         fileAttributes.stream().filter(f -> f.getValue() != null).forEach(kieSession::insert);
 
         CompletableFuture<Void> completableFuture = CompletableFuture.supplyAsync(() -> {
             kieSession.fireAllRules();
             return null;
         });
-        kieSession.halt();
+
         try {
             completableFuture.orTimeout(settings.getDroolsExecutionTimeoutSecs(), TimeUnit.SECONDS).get();
         } catch (ExecutionException e) {
@@ -59,11 +66,12 @@ public class ComponentDroolsExecutionService {
             }
             throw new RuntimeException(e);
         } catch (InterruptedException e) {
+            kieSession.dispose();
             throw new RuntimeException(e);
         }
 
         List<FileAttribute> resultingFileAttributes = getFileAttributes(kieSession);
-        List<Component> components = getComponents(kieSession);
+        List<Component> components = getComponents(kieSession).stream().sorted(ComponentComparator.first()).toList();
         kieSession.dispose();
         return components;
     }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/DroolsSyntaxValidationService.java

@@ -1,12 +1,15 @@
 package com.iqser.red.service.redaction.v1.server.service.drools;
 
 import java.util.List;
+import java.util.Locale;
+import java.util.Set;
 import java.util.stream.Collectors;
 
 import org.kie.api.builder.KieBuilder;
 import org.kie.api.builder.Message;
 import org.springframework.stereotype.Service;
 
+import com.google.common.collect.Sets;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType;
 import com.iqser.red.service.redaction.v1.model.DroolsSyntaxErrorMessage;
 import com.iqser.red.service.redaction.v1.model.DroolsSyntaxValidation;
@@ -46,7 +49,7 @@ public class DroolsSyntaxValidationService {
             case COMPONENT -> RuleFileParser.buildBluePrintFromRulesString(RuleManagementResources.getBaseComponentRuleFileString());
         };
 
-        if (!ruleFileBluePrint.getImports().equals(baseRuleFileBluePrint.getImports())) {
+        if (!importsAreValid(baseRuleFileBluePrint, ruleFileBluePrint)) {
             customSyntaxValidation.getDroolsSyntaxErrorMessages()
                     .add(DroolsSyntaxErrorMessage.builder()
                             .line(ruleFileBluePrint.getImportLine())
@@ -56,8 +59,10 @@ public class DroolsSyntaxValidationService {
         }
         if (!ruleFileBluePrint.getGlobals().equals(baseRuleFileBluePrint.getGlobals())) {
             customSyntaxValidation.getDroolsSyntaxErrorMessages()
-                    .add(DroolsSyntaxErrorMessage.builder().line(ruleFileBluePrint.getGlobalsLine())
-                            .column(0).message(String.format("Changing the globals is not allowed! Must be: %n%s", baseRuleFileBluePrint.getGlobals()))
+                    .add(DroolsSyntaxErrorMessage.builder()
+                            .line(ruleFileBluePrint.getGlobalsLine())
+                            .column(0)
+                            .message(String.format("Changing the globals is not allowed! Must be: %n%s", baseRuleFileBluePrint.getGlobals()))
                             .build());
         }
         baseRuleFileBluePrint.getQueries().forEach(basicQuery -> {
@@ -72,9 +77,8 @@ public class DroolsSyntaxValidationService {
         });
         baseRuleFileBluePrint.streamAllRules().forEach(basicRule -> {
             if (!validateRuleIsPresent(basicRule, ruleFileBluePrint)) {
-                customSyntaxValidation.getDroolsSyntaxErrorMessages()
-                        .add(DroolsSyntaxErrorMessage.builder()
-                                .line(basicRule.getLine())
+                int line = ruleFileBluePrint.findRulesByIdentifier(basicRule.getIdentifier()).stream().findFirst().map(BasicRule::getLine).orElse(basicRule.getLine());
+                customSyntaxValidation.getDroolsSyntaxErrorMessages().add(DroolsSyntaxErrorMessage.builder().line(line)
                                 .column(0)
                                 .message(String.format("Changing or removing the rule %s is not allowed! Must be: %n%s", basicRule.getName(), basicRule.getCode()))
                                 .build());
@@ -84,9 +88,26 @@ public class DroolsSyntaxValidationService {
     }
 
 
+    private boolean importsAreValid(RuleFileBluePrint baseRuleFileBluePrint, RuleFileBluePrint ruleFileBluePrint) {
+
+        // imports may shrink, but not add anything new!
+        Set<String> baseImports = baseRuleFileBluePrint.getImportSplitByKeyword();
+        Set<String> imports = ruleFileBluePrint.getImportSplitByKeyword();
+        return Sets.difference(imports, baseImports).isEmpty();
+    }
+
+
     private static boolean validateRuleIsPresent(BasicRule basicRule, RuleFileBluePrint ruleFileBluePrint) {
 
-        return ruleFileBluePrint.findRulesByIdentifier(basicRule.getIdentifier()).stream().anyMatch(otherRule -> otherRule.getCode().equals(basicRule.getCode()));
+        return ruleFileBluePrint.findRulesByIdentifier(basicRule.getIdentifier()).stream().anyMatch(otherRule -> rulesMatch(basicRule, otherRule));
+    }
+
+
+    private static boolean rulesMatch(BasicRule basicRule, BasicRule otherRule) {
+
+        // Name only needs to match with to lower case, everything else must match exactly
+        return otherRule.getName().toLowerCase(Locale.ENGLISH).equals(basicRule.getName().toLowerCase(Locale.ROOT)) && //
+                otherRule.getCode().replace(otherRule.getName(), "").equals(basicRule.getCode().replace(basicRule.getName(), ""));
     }
 
 
@@ -101,7 +122,9 @@ public class DroolsSyntaxValidationService {
         var versionId = System.currentTimeMillis();
         var testRules = "test-rules";
         KieBuilder kieBuilder = kieContainerCreationService.registerNewKieContainerVersion(testRules,
-                versionId, rules.getRulesString(), RuleFileType.valueOf(rules.getRuleFileType()));
+                versionId,
+                rules.getRulesString(),
+                RuleFileType.valueOf(rules.getRuleFileType()));
         return buildDroolsCompilerSyntaxValidation(kieBuilder);
     }
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/EntityDroolsExecutionService.java

@@ -2,7 +2,8 @@ package com.iqser.red.service.redaction.v1.server.service.drools;
 
 import java.util.LinkedList;
 import java.util.List;
-import java.util.concurrent.Executors;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ExecutionException;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
 
@@ -12,7 +13,6 @@ import org.kie.api.runtime.rule.QueryResults;
 import org.kie.api.runtime.rule.QueryResultsRow;
 import org.springframework.stereotype.Service;
 
-import com.google.common.util.concurrent.SimpleTimeLimiter;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
@@ -42,6 +42,7 @@ public class EntityDroolsExecutionService {
 
     RedactionServiceSettings settings;
 
+
     @Timed("redactmanager_executeRules")
     public List<FileAttribute> executeRules(KieContainer kieContainer,
                                             Document document,
@@ -80,7 +81,7 @@ public class EntityDroolsExecutionService {
         fileAttributes.stream().filter(f -> f.getValue() != null).forEach(kieSession::insert);
 
         if (manualRedactions != null) {
-            manualRedactions.getResizeRedactions().forEach(kieSession::insert);
+            manualRedactions.getResizeRedactions().stream().filter(manualResizeRedaction -> !manualResizeRedaction.getUpdateDictionary()).forEach(kieSession::insert);
             manualRedactions.getRecategorizations().forEach(kieSession::insert);
             manualRedactions.getEntriesToAdd().forEach(kieSession::insert);
             manualRedactions.getForceRedactions().forEach(kieSession::insert);
@@ -93,20 +94,24 @@ public class EntityDroolsExecutionService {
         kieSession.getAgenda().getAgendaGroup("LOCAL_DICTIONARY_ADDS").setFocus();
 
 
-        SimpleTimeLimiter limiter = SimpleTimeLimiter.create(Executors.newSingleThreadExecutor());
+        CompletableFuture<Void> completableFuture = CompletableFuture.supplyAsync(() -> {
+            kieSession.fireAllRules();
+            return null;
+        });
 
         try {
-            limiter.runWithTimeout(kieSession::fireAllRules, settings.getDroolsExecutionTimeoutSecs(), TimeUnit.SECONDS);
-
-        } catch (TimeoutException e) {
+            completableFuture.orTimeout(settings.getDroolsExecutionTimeoutSecs(), TimeUnit.SECONDS).get();
+        } catch (ExecutionException e) {
             kieSession.dispose();
-            throw new DroolsTimeoutException(e, false, RuleFileType.ENTITY);
+            if(e.getCause() instanceof TimeoutException){
+                throw new DroolsTimeoutException(e, false, RuleFileType.ENTITY);
+            }
+            throw new RuntimeException(e);
         } catch (InterruptedException e) {
             kieSession.dispose();
             throw new RuntimeException(e);
         }
 
-
         List<FileAttribute> resultingFileAttributes = getFileAttributes(kieSession);
         kieSession.dispose();
         return resultingFileAttributes;

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/KieContainerCreationService.java

@@ -31,23 +31,21 @@ public class KieContainerCreationService {
 
     public KieWrapper getLatestKieContainer(String dossierTemplateId, RuleFileType ruleFileType) {
 
-        long version = -1;
         try {
-            version = rulesClient.getVersion(dossierTemplateId, ruleFileType);
+            long version = rulesClient.getVersion(dossierTemplateId, ruleFileType);
+            if (version == -1) {
+                return KieWrapper.empty();
+            }
+            return new KieWrapper(getKieContainer(dossierTemplateId, version, ruleFileType), version);
         } catch (FeignException fe) {
             if (fe.status() == HttpStatus.UNPROCESSABLE_ENTITY.value()) {
                 throw new DroolsTimeoutException(fe.getCause(), true, ruleFileType);
             }
             if (fe.status() == HttpStatus.NOT_FOUND.value()) {
-                return new KieWrapper(null, -1);
+                return KieWrapper.empty();
             }
+            throw new RuntimeException(fe);
         }
-        if (version == -1) {
-            return new KieWrapper(null, version);
-        }
-
-        return new KieWrapper(getKieContainer(dossierTemplateId, version, ruleFileType), version);
-
     }
 
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/RedactionStorageService.java

@@ -11,10 +11,9 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.imported.ImportedRedactions;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.SectionGrid;
 import com.iqser.red.service.redaction.v1.server.client.model.NerEntitiesModel;
-import com.iqser.red.service.redaction.v1.server.utils.exception.NotFoundException;
 import com.iqser.red.service.redaction.v1.server.model.document.DocumentData;
+import com.iqser.red.service.redaction.v1.server.utils.exception.NotFoundException;
 import com.iqser.red.storage.commons.exception.StorageObjectDoesNotExist;
 import com.iqser.red.storage.commons.service.StorageService;
 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPage;
@@ -86,6 +85,7 @@ public class RedactionStorageService {
     }
 
 
+    @Deprecated(forRemoval = true)
     @Timed("redactmanager_getRedactionLog")
     public RedactionLog getRedactionLog(String dossierId, String fileId) {
 
@@ -139,17 +139,6 @@ public class RedactionStorageService {
     }
 
 
-    @Timed("redactmanager_getSectionGrid")
-    public SectionGrid getSectionGrid(String dossierId, String fileId) {
-
-        try {
-            return storageService.readJSONObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.SECTION_GRID), SectionGrid.class);
-        } catch (StorageObjectDoesNotExist e) {
-            throw new NotFoundException("Section Grid is not available.");
-        }
-    }
-
-
     public ComponentLog getComponentLog(String dossierId, String fileId) {
 
         try {
@@ -160,6 +149,12 @@ public class RedactionStorageService {
     }
 
 
+    public boolean entityLogExists(String dossierId, String fileId) {
+
+        return storageService.objectExists(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.ENTITY_LOG));
+    }
+
+
     @RequiredArgsConstructor
     public enum StorageType {
         PARSED_DOCUMENT(".json");

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/utils/DateConverter.java

@@ -5,6 +5,7 @@ import java.text.SimpleDateFormat;
 import java.util.Date;
 import java.util.List;
 import java.util.Locale;
+import java.util.Optional;
 
 import lombok.AccessLevel;
 import lombok.experimental.FieldDefaults;
@@ -24,9 +25,8 @@ public class DateConverter {
             new SimpleDateFormat("dd-MMM-yyyy", Locale.ENGLISH));
 
 
-    public String convertDate(String dateAsString, String resultFormat) {
+    public Optional<Date> parseDate(String dateAsString) {
 
-        DateFormat resultDateFormat = new SimpleDateFormat(resultFormat, Locale.ENGLISH);
         Date date = null;
         for (SimpleDateFormat format : formats) {
 
@@ -38,10 +38,16 @@ public class DateConverter {
                 // ignore, try next...
             }
         }
-
         if (date == null) {
-            return dateAsString;
+            return Optional.empty();
         }
+        return Optional.of(date);
+    }
+
+
+    public String convertDate(Date date, String resultFormat) {
+
+        DateFormat resultDateFormat = new SimpleDateFormat(resultFormat, Locale.ENGLISH);
 
         return resultDateFormat.format(date);
     }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/utils/MigratedIdsCollector.java

@@ -0,0 +1,54 @@
+package com.iqser.red.service.redaction.v1.server.utils;
+
+import java.util.LinkedList;
+import java.util.Set;
+import java.util.function.BiConsumer;
+import java.util.function.BinaryOperator;
+import java.util.function.Function;
+import java.util.function.Supplier;
+import java.util.stream.Collector;
+
+import com.google.common.base.Functions;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.migration.MigratedIds;
+import com.iqser.red.service.redaction.v1.server.model.MigrationEntity;
+
+public class MigratedIdsCollector implements Collector<MigrationEntity, MigratedIds, MigratedIds> {
+
+    @Override
+    public Supplier<MigratedIds> supplier() {
+
+        return () -> new MigratedIds(new LinkedList<>());
+    }
+
+
+    @Override
+    public BiConsumer<MigratedIds, MigrationEntity> accumulator() {
+
+        return (migratedIds, migrationEntity) -> migratedIds.addMapping(migrationEntity.getOldId(), migrationEntity.getNewId());
+    }
+
+
+    @Override
+    public BinaryOperator<MigratedIds> combiner() {
+
+        return (migratedIds, migratedIds2) -> {
+            migratedIds.getMappings().addAll(migratedIds2.getMappings());
+            return migratedIds;
+        };
+    }
+
+
+    @Override
+    public Function<MigratedIds, MigratedIds> finisher() {
+
+        return Functions.identity();
+    }
+
+
+    @Override
+    public Set<Characteristics> characteristics() {
+
+        return Set.of(Characteristics.IDENTITY_FINISH, Characteristics.CONCURRENT, Characteristics.UNORDERED);
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/utils/RectangleTransformations.java

@@ -12,7 +12,7 @@ import java.util.function.Function;
 import java.util.function.Supplier;
 import java.util.stream.Collector;
 
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.AtomicTextBlock;
 
 import lombok.AllArgsConstructor;
@@ -26,18 +26,9 @@ public class RectangleTransformations {
     }
 
 
-    public static Rectangle2D rectangleBBox(List<Rectangle> rectangles) {
+    public static Rectangle2D rectangleBBox(List<Position> positions) {
 
-        return rectangles.stream().map(RectangleTransformations::toRectangle2D).collect(new Rectangle2DBBoxCollector());
-    }
-
-
-    public static Rectangle2D toRectangle2D(Rectangle redactionLogRectangle) {
-
-        return new Rectangle2D.Double(redactionLogRectangle.getTopLeft().getX(),
-                redactionLogRectangle.getTopLeft().getY() + redactionLogRectangle.getHeight(),
-                redactionLogRectangle.getWidth(),
-                -redactionLogRectangle.getHeight());
+        return positions.stream().map(Position::toRectangle2D).collect(new Rectangle2DBBoxCollector());
     }
 
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/utils/RedactionSearchUtility.java

@@ -132,6 +132,12 @@ public class RedactionSearchUtility {
         return getTextRangesByPatternWithLineBreaks(textBlock, group, pattern);
     }
 
+    public static List<TextRange> findTextRangesByRegexIgnoreCase(String regexPattern, TextBlock textBlock) {
+
+        Pattern pattern = Patterns.getCompiledPattern(regexPattern, true);
+        return getTextRangesByPattern(textBlock, 0, pattern);
+    }
+
 
     public static List<TextRange> findTextRangesByRegexIgnoreCase(String regexPattern, int group, TextBlock textBlock) {
 
@@ -179,4 +185,22 @@ public class RedactionSearchUtility {
         return getTextRangesByPattern(textBlock, 0, pattern);
     }
 
+    public static List<TextRange> findTextRangesByList(List<String> searchList, TextBlock textBlock) {
+
+        List<TextRange> boundaries = new LinkedList<>();
+        for (var searchString: searchList) {
+            boundaries.addAll(findTextRangesByString(searchString, textBlock));
+        }
+        return boundaries;
+    }
+
+    public static List<TextRange> findTextRangesByListIgnoreCase(List<String> searchList, TextBlock textBlock) {
+
+        List<TextRange> boundaries = new LinkedList<>();
+        for (var searchString: searchList) {
+            boundaries.addAll(findTextRangesByStringIgnoreCase(searchString, textBlock));
+        }
+        return boundaries;
+    }
+
 }

redaction-service-v1/redaction-service-server-v1/src/main/resources/drools/base_rules.drl

@@ -22,6 +22,7 @@ import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRu
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.*;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Section;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Table;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Paragraph;
@@ -73,7 +74,7 @@ query "getFileAttributes"
 //------------------------------------ Local dictionary search rules ------------------------------------
 
 // Rule unit: LocalDictionarySearch.0
-rule "LDS.0.0: run local dictionary search"
+rule "LDS.0.0: Run local dictionary search"
     agenda-group "LOCAL_DICTIONARY_ADDS"
     salience -999
     when

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AbstractRedactionIntegrationTest.java

@@ -1,5 +1,28 @@
 package com.iqser.red.service.redaction.v1.server;
 
+import static org.mockito.Mockito.when;
+
+import java.io.File;
+import java.io.InputStream;
+import java.net.URL;
+import java.nio.file.Files;
+import java.time.OffsetDateTime;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import org.junit.jupiter.api.AfterEach;
+import org.mockito.stubbing.Answer;
+import org.springframework.amqp.rabbit.core.RabbitTemplate;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.test.mock.mockito.MockBean;
+import org.springframework.core.io.ClassPathResource;
+
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.iqser.red.service.dictionarymerge.commons.DictionaryEntry;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
@@ -12,33 +35,19 @@ import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient;
 import com.iqser.red.service.redaction.v1.server.client.RulesClient;
 import com.iqser.red.service.redaction.v1.server.controller.RedactionController;
 import com.iqser.red.service.redaction.v1.server.service.AnalyzeService;
-import com.iqser.red.service.redaction.v1.server.utils.ResourceLoader;
-import com.iqser.red.service.redaction.v1.server.utils.TextNormalizationUtilities;
 import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
 import com.iqser.red.service.redaction.v1.server.utils.LayoutParsingRequestProvider;
+import com.iqser.red.service.redaction.v1.server.utils.ResourceLoader;
+import com.iqser.red.service.redaction.v1.server.utils.TextNormalizationUtilities;
 import com.iqser.red.storage.commons.service.StorageService;
+import com.iqser.red.storage.commons.utils.FileSystemBackedStorageService;
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingFinishedEvent;
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
 import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingPipeline;
 import com.knecon.fforesight.tenantcommons.TenantContext;
 import com.knecon.fforesight.tenantcommons.TenantsClient;
+
 import lombok.SneakyThrows;
-import org.junit.jupiter.api.AfterEach;
-import org.mockito.stubbing.Answer;
-import org.springframework.amqp.rabbit.core.RabbitTemplate;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.boot.test.mock.mockito.MockBean;
-import org.springframework.core.io.ClassPathResource;
-
-import java.io.File;
-import java.io.InputStream;
-import java.net.URL;
-import java.nio.file.Files;
-import java.time.OffsetDateTime;
-import java.util.*;
-import java.util.stream.Collectors;
-
-import static org.mockito.Mockito.when;
 
 public abstract class AbstractRedactionIntegrationTest {
 
@@ -431,7 +440,6 @@ public abstract class AbstractRedactionIntegrationTest {
     @SneakyThrows
     protected LayoutParsingFinishedEvent analyzeDocumentStructure(LayoutParsingType layoutParsingType, AnalyzeRequest request) {
 
-
         return layoutParsingPipeline.parseLayoutAndSaveFilesToStorage(LayoutParsingRequestProvider.build(layoutParsingType, request));
     }
 

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AnalysisTest.java

@@ -0,0 +1,125 @@
+package com.iqser.red.service.redaction.v1.server;
+
+import static org.mockito.Mockito.when;
+
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.util.List;
+
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Disabled;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
+import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration;
+import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.ComponentScan;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.context.annotation.FilterType;
+import org.springframework.context.annotation.Import;
+import org.springframework.context.annotation.Primary;
+import org.springframework.test.context.junit.jupiter.SpringExtension;
+
+import com.iqser.red.commons.jackson.ObjectMapperFactory;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeResult;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type;
+import com.iqser.red.service.redaction.v1.server.annotate.AnnotateRequest;
+import com.iqser.red.service.redaction.v1.server.annotate.AnnotateResponse;
+import com.iqser.red.service.redaction.v1.server.redaction.utils.OsUtils;
+import com.iqser.red.storage.commons.StorageAutoConfiguration;
+import com.iqser.red.storage.commons.service.StorageService;
+import com.iqser.red.storage.commons.utils.FileSystemBackedStorageService;
+import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
+import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingServiceProcessorConfiguration;
+import com.knecon.fforesight.tenantcommons.TenantContext;
+
+@ExtendWith(SpringExtension.class)
+@SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT, properties = {"application.type=DocuMine"})
+@Import(AnalysisTest.RedactionIntegrationTestConfiguration.class)
+public class AnalysisTest extends AbstractRedactionIntegrationTest {
+
+    private static final String RULES = loadFromClassPath("drools/table_demo.drl");
+    private static final String COMPONENT_RULES = loadFromClassPath("drools/table_demo_components.drl");
+
+
+    @Test
+    @Disabled
+    public void analyzeTableDemoFile() {
+
+        AnalyzeRequest request = uploadFileToStorage("files/TableDemo/Table_examples.pdf");
+
+        System.out.println("Start Full integration test");
+        analyzeDocumentStructure(LayoutParsingType.DOCUMINE, request);
+        System.out.println("Finished structure analysis");
+        AnalyzeResult result = analyzeService.analyze(request);
+        System.out.println("Finished analysis");
+//        var redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID);
+//        var componentLog = redactionStorageService.getComponentLog(TEST_DOSSIER_ID, TEST_FILE_ID);
+        AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder().dossierId(TEST_DOSSIER_ID).fileId(TEST_FILE_ID).build());
+
+        String outputFileName = OsUtils.getTemporaryDirectory() + "/Documine.pdf";
+
+        try (FileOutputStream fileOutputStream = new FileOutputStream(outputFileName)) {
+            fileOutputStream.write(annotateResponse.getDocument());
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
+
+    }
+
+
+    @Configuration
+    @EnableAutoConfiguration(exclude = {RabbitAutoConfiguration.class})
+    @Import(LayoutParsingServiceProcessorConfiguration.class)
+    @ComponentScan(excludeFilters = {@ComponentScan.Filter(type = FilterType.ASSIGNABLE_TYPE, value = StorageAutoConfiguration.class)})
+    static class RedactionIntegrationTestConfiguration {
+
+        @Bean
+        @Primary
+        public StorageService inmemoryStorage() {
+
+            return new FileSystemBackedStorageService(ObjectMapperFactory.create());
+        }
+
+    }
+
+
+    @BeforeEach
+    public void stubClients() {
+
+        TenantContext.setTenantId("documine");
+
+        when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.ENTITY)).thenReturn(System.currentTimeMillis());
+        when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.ENTITY)).thenReturn(JSONPrimitive.of(RULES));
+        when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.COMPONENT)).thenReturn(System.currentTimeMillis());
+        when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.COMPONENT)).thenReturn(JSONPrimitive.of(COMPONENT_RULES));
+
+        loadDictionaryForTest();
+        loadTypeForTest();
+        loadNerForTest();
+        when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L);
+        when(dictionaryClient.getAllTypesForDossierTemplate(TEST_DOSSIER_TEMPLATE_ID, false)).thenReturn(getTypeResponse());
+
+        when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L);
+        when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, false)).thenReturn(List.of(Type.builder()
+                .id(DOSSIER_REDACTIONS_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID)
+                .type(DOSSIER_REDACTIONS_INDICATOR)
+                .dossierTemplateId(TEST_DOSSIER_ID)
+                .hexColor("#ffe187")
+                .isHint(hintTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
+                .isCaseInsensitive(caseInSensitiveMap.get(DOSSIER_REDACTIONS_INDICATOR))
+                .isRecommendation(recommendationTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
+                .rank(rankTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
+                .build()));
+
+        mockDictionaryCalls(null);
+
+        when(dictionaryClient.getColors(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(colors);
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DictionaryServiceTest.java

@@ -28,6 +28,7 @@ import org.springframework.context.annotation.Primary;
 import org.springframework.test.context.junit.jupiter.SpringExtension;
 
 import com.google.common.collect.Sets;
+import com.iqser.red.commons.jackson.ObjectMapperFactory;
 import com.iqser.red.service.dictionarymerge.commons.DictionaryEntry;
 import com.iqser.red.service.dictionarymerge.commons.DictionaryEntryModel;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.configuration.Colors;
@@ -36,6 +37,7 @@ import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
 import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryVersion;
 import com.iqser.red.service.redaction.v1.server.service.DictionaryService;
 import com.iqser.red.storage.commons.service.StorageService;
+import com.iqser.red.storage.commons.utils.FileSystemBackedStorageService;
 import com.knecon.fforesight.tenantcommons.TenantContext;
 import com.knecon.fforesight.tenantcommons.TenantsClient;
 
@@ -67,7 +69,7 @@ public class DictionaryServiceTest {
         @Primary
         public StorageService inmemoryStorage() {
 
-            return new FileSystemBackedStorageService();
+            return new FileSystemBackedStorageService(ObjectMapperFactory.create());
         }
 
     }

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DocumineFloraTest.java

@@ -1,6 +1,5 @@
 package com.iqser.red.service.redaction.v1.server;
 
-import static org.mockito.Mockito.doThrow;
 import static org.mockito.Mockito.when;
 
 import java.io.FileOutputStream;
@@ -22,6 +21,7 @@ import org.springframework.context.annotation.Import;
 import org.springframework.context.annotation.Primary;
 import org.springframework.test.context.junit.jupiter.SpringExtension;
 
+import com.iqser.red.commons.jackson.ObjectMapperFactory;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeResult;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType;
@@ -30,9 +30,9 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemp
 import com.iqser.red.service.redaction.v1.server.annotate.AnnotateRequest;
 import com.iqser.red.service.redaction.v1.server.annotate.AnnotateResponse;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.OsUtils;
-import com.iqser.red.service.redaction.v1.server.utils.ExceptionProvider;
 import com.iqser.red.storage.commons.StorageAutoConfiguration;
 import com.iqser.red.storage.commons.service.StorageService;
+import com.iqser.red.storage.commons.utils.FileSystemBackedStorageService;
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
 import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingServiceProcessorConfiguration;
 import com.knecon.fforesight.tenantcommons.TenantContext;
@@ -47,10 +47,10 @@ public class DocumineFloraTest extends AbstractRedactionIntegrationTest {
 
 
     @Test
-    @Disabled
+//    @Disabled
     public void titleExtraction() throws IOException {
 
-        AnalyzeRequest request = uploadFileToStorage("files/Documine/Flora/ProblemDocs/8.SYN524464 FS (A16148F) - Teste de Ames (1).pdf");
+        AnalyzeRequest request = uploadFileToStorage("files/Documine/Flora/402Study-ocred.pdf");
 
 //        AnalyzeRequest request = prepareStorage("files/Documine/Flora/ProblemDocs/SOLICITA_VICTRATO-GOLD-II_Item 21_Mutacao_Genica (1).pdf",
 //                "files/Documine/Flora/ProblemDocs/SOLICITA_VICTRATO-GOLD-II_Item 21_Mutacao_Genica (1).TABLES.json");
@@ -60,8 +60,8 @@ public class DocumineFloraTest extends AbstractRedactionIntegrationTest {
         System.out.println("Finished structure analysis");
         AnalyzeResult result = analyzeService.analyze(request);
         System.out.println("Finished analysis");
-        var redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID);
-
+        var entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
+        var componentLog = redactionStorageService.getComponentLog(TEST_DOSSIER_ID, TEST_FILE_ID);
         AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder().dossierId(TEST_DOSSIER_ID).fileId(TEST_FILE_ID).build());
 
         String outputFileName = OsUtils.getTemporaryDirectory() + "/Documine.pdf";
@@ -86,7 +86,7 @@ public class DocumineFloraTest extends AbstractRedactionIntegrationTest {
         System.out.println("Finished structure analysis");
         AnalyzeResult result = analyzeService.analyze(request);
         System.out.println("Finished analysis");
-        var redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID);
+        var entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
 
         AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder().dossierId(TEST_DOSSIER_ID).fileId(TEST_FILE_ID).build());
 
@@ -100,20 +100,20 @@ public class DocumineFloraTest extends AbstractRedactionIntegrationTest {
 
 
     @Test
-    @Disabled
+//    @Disabled
     public void testTopOfPage13InNotHeader() throws IOException {
 
         // Fix In BodyTextFrameService destroys header detection in files/new/SYNGENTA_EFSA_sanitisation_GFL_v1_moreSections.pdf
         // TODO unify logic
 
-        AnalyzeRequest request = uploadFileToStorage("files/Documine/Flora/ProblemDocs/SOLICITA_VICTRATO-GOLD-II_Item 15_Toxicidade Oral Aguda.pdf");
+        AnalyzeRequest request = uploadFileToStorage("files/Documine/Flora/402Study-ocred.pdf");
 
         System.out.println("Start Full integration test");
         analyzeDocumentStructure(LayoutParsingType.DOCUMINE, request);
         System.out.println("Finished structure analysis");
         AnalyzeResult result = analyzeService.analyze(request);
         System.out.println("Finished analysis");
-        var redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID);
+        var entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
 
         AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder().dossierId(TEST_DOSSIER_ID).fileId(TEST_FILE_ID).build());
 
@@ -136,7 +136,7 @@ public class DocumineFloraTest extends AbstractRedactionIntegrationTest {
         @Primary
         public StorageService inmemoryStorage() {
 
-            return new FileSystemBackedStorageService();
+            return new FileSystemBackedStorageService(ObjectMapperFactory.create());
         }
 
     }

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/FileSystemBackedStorageService.java

@@ -1,127 +0,0 @@
-package com.iqser.red.service.redaction.v1.server;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.iqser.red.commons.jackson.ObjectMapperFactory;
-import com.iqser.red.storage.commons.exception.StorageObjectDoesNotExist;
-import com.iqser.red.storage.commons.service.StorageService;
-import lombok.SneakyThrows;
-import org.apache.commons.io.IOUtils;
-import org.springframework.core.io.InputStreamResource;
-
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileOutputStream;
-import java.io.InputStream;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.stream.Collectors;
-
-import static java.io.File.createTempFile;
-
-public class FileSystemBackedStorageService implements StorageService {
-
-    private final Map<String, File> dataMap = new HashMap<>();
-
-
-    public FileSystemBackedStorageService() {
-
-    }
-
-
-    @SneakyThrows
-    @Override
-    public void downloadTo(String tenantId, String objectId, File destinationFile) {
-
-        var res = dataMap.get(objectId);
-        if (res == null) {
-            throw new StorageObjectDoesNotExist(new RuntimeException());
-        }
-
-        IOUtils.copy(new FileInputStream(res), new FileOutputStream(destinationFile));
-    }
-
-    @Override
-    @SneakyThrows
-    public InputStreamResource getObject(String tenantId, String objectId) {
-        return new InputStreamResource(new FileInputStream(dataMap.get(objectId)));
-    }
-
-
-    @Override
-    public void deleteObject(String tenantId, String objectId) {
-
-        dataMap.remove(objectId);
-    }
-
-
-    @Override
-    public boolean objectExists(String tenantId, String objectId) {
-
-        return dataMap.containsKey(objectId);
-    }
-
-
-    @Override
-    @SneakyThrows
-    public <T> void storeJSONObject(String tenantId, String objectId, T any) {
-
-        File tempFile = createTempFile("test", ".tmp");
-        getMapper().writeValue(new FileOutputStream(tempFile), any);
-        dataMap.put(objectId, tempFile);
-    }
-
-
-    private ObjectMapper getMapper() {
-
-        return ObjectMapperFactory.create();
-    }
-
-
-    @Override
-    @SneakyThrows
-    public <T> T readJSONObject(String tenantId, String objectId, Class<T> clazz) {
-
-        if (dataMap.get(objectId) == null || !dataMap.get(objectId).exists()) {
-            throw new StorageObjectDoesNotExist("Stored object not found");
-        }
-        return getMapper().readValue(new FileInputStream(dataMap.get(objectId)), clazz);
-    }
-
-
-    public List<String> listPaths() {
-
-        return new ArrayList<>(dataMap.keySet());
-    }
-
-
-    public List<String> listFilePaths() {
-
-        return dataMap.values().stream().map(File::getAbsolutePath).collect(Collectors.toList());
-    }
-
-
-    @Override
-    @SneakyThrows
-    public void storeObject(String tenantId, String objectId, InputStream stream) {
-
-        File tempFile = createTempFile("test", ".tmp");
-
-        try (var fileOutputStream = new FileOutputStream(tempFile)) {
-            IOUtils.copy(stream, fileOutputStream);
-        }
-
-        dataMap.put(objectId, tempFile);
-    }
-
-
-    public void clearStorage() {
-
-        this.dataMap.forEach((k, v) -> {
-            v.delete();
-        });
-        this.dataMap.clear();
-    }
-
-}

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/MigrationIntegrationTest.java

@@ -0,0 +1,168 @@
+package com.iqser.red.service.redaction.v1.server;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.nio.file.Path;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.context.annotation.Import;
+import org.springframework.core.io.ClassPathResource;
+import org.springframework.test.context.junit.jupiter.SpringExtension;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
+import com.iqser.red.service.redaction.v1.server.annotate.AnnotateRequest;
+import com.iqser.red.service.redaction.v1.server.annotate.AnnotateResponse;
+import com.iqser.red.service.redaction.v1.server.document.graph.BuildDocumentIntegrationTest;
+import com.iqser.red.service.redaction.v1.server.model.MigratedEntityLog;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
+import com.iqser.red.service.redaction.v1.server.redaction.utils.OsUtils;
+import com.iqser.red.service.redaction.v1.server.service.RedactionLogToEntityLogMigrationService;
+
+import lombok.SneakyThrows;
+
+@ExtendWith(SpringExtension.class)
+@SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
+@Import(MigrationIntegrationTest.TestConfiguration.class)
+public class MigrationIntegrationTest extends BuildDocumentIntegrationTest {
+
+    @Autowired
+    RedactionLogToEntityLogMigrationService redactionLogToEntityLogMigrationService;
+
+    @Autowired
+    ObjectMapper mapper;
+
+
+    @Test
+    @SneakyThrows
+    public void testMigration() {
+
+        String fileName = "files/migration/def8f960580f088b975ba806dfae1f87.ORIGIN.pdf";
+        String imageFileName = "files/migration/def8f960580f088b975ba806dfae1f87.IMAGE_INFO.json";
+        String tableFileName = "files/migration/def8f960580f088b975ba806dfae1f87.TABLES.json";
+
+        Document document = buildGraph(fileName, imageFileName, tableFileName);
+        RedactionLog redactionLog;
+        try (var in = new ClassPathResource("files/migration/def8f960580f088b975ba806dfae1f87.REDACTION_LOG.json").getInputStream()) {
+            redactionLog = mapper.readValue(in, RedactionLog.class);
+        }
+        MigratedEntityLog migratedEntityLog = redactionLogToEntityLogMigrationService.migrate(redactionLog, document);
+
+        redactionStorageService.storeObject(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.ENTITY_LOG, migratedEntityLog.getEntityLog());
+        assertEquals(redactionLog.getRedactionLogEntry().size(), migratedEntityLog.getEntityLog().getEntityLogEntry().size());
+        assertEquals(redactionLog.getRedactionLogEntry().size(), migratedEntityLog.getMigratedIds().getMappings().size());
+        EntityLog entityLog = migratedEntityLog.getEntityLog();
+        assertEquals(redactionLog.getAnalysisNumber(), entityLog.getAnalysisNumber());
+        assertEquals(redactionLog.getAnalysisVersion(), entityLog.getAnalysisVersion());
+        assertEquals(redactionLog.getDictionaryVersion(), entityLog.getDictionaryVersion());
+        assertEquals(redactionLog.getDossierDictionaryVersion(), entityLog.getDossierDictionaryVersion());
+        assertEquals(redactionLog.getLegalBasisVersion(), entityLog.getLegalBasisVersion());
+        assertEquals(redactionLog.getRulesVersion(), entityLog.getRulesVersion());
+        assertEquals(redactionLog.getLegalBasis().size(), entityLog.getLegalBasis().size());
+
+        Map<String, String> migratedIds = migratedEntityLog.getMigratedIds().buildOldToNewMapping();
+        migratedIds.forEach((oldId, newId) -> assertEntryIsEqual(oldId, newId, redactionLog, entityLog, migratedIds));
+
+        AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder().dossierId(TEST_DOSSIER_ID).fileId(TEST_FILE_ID).build());
+        File outputFile = Path.of(OsUtils.getTemporaryDirectory()).resolve(Path.of(fileName.replaceAll(".pdf", "_MIGRATED.pdf")).getFileName()).toFile();
+
+        try (FileOutputStream fileOutputStream = new FileOutputStream(outputFile)) {
+            fileOutputStream.write(annotateResponse.getDocument());
+        }
+    }
+
+
+    private void assertEntryIsEqual(String oldId, String newId, RedactionLog redactionLog, EntityLog entityLog, Map<String, String> oldToNewMapping) {
+
+        RedactionLogEntry redactionLogEntry = redactionLog.getRedactionLogEntry().stream().filter(entry -> entry.getId().equals(oldId)).findAny().orElseThrow();
+        EntityLogEntry entityLogEntry = entityLog.getEntityLogEntry().stream().filter(entry -> entry.getId().equals(newId)).findAny().orElseThrow();
+
+        if (!redactionLogEntry.isImage()) {
+            assertEquals(redactionLogEntry.getValue().toLowerCase(Locale.ENGLISH), entityLogEntry.getValue().toLowerCase(Locale.ENGLISH));
+        }
+        assertEquals(redactionLogEntry.getChanges().size(), entityLogEntry.getChanges().size());
+        assertEquals(redactionLogEntry.getManualChanges().size(), entityLogEntry.getManualChanges().size());
+        assertEquals(redactionLogEntry.getPositions().size(), entityLogEntry.getPositions().size());
+        assertTrue(positionsAlmostEqual(redactionLogEntry.getPositions(), entityLogEntry.getPositions()));
+        assertEquals(redactionLogEntry.getColor(), entityLogEntry.getColor());
+        assertEqualsNullSafe(redactionLogEntry.getLegalBasis(), entityLogEntry.getLegalBasis());
+        assertEqualsNullSafe(redactionLogEntry.getReason(), entityLogEntry.getReason());
+        assertReferencesEqual(redactionLogEntry.getReference(), entityLogEntry.getReference(), oldToNewMapping);
+        assertEquals(redactionLogEntry.isDictionaryEntry(), entityLogEntry.isDictionaryEntry());
+        assertEquals(redactionLogEntry.isDossierDictionaryEntry(), entityLogEntry.isDossierDictionaryEntry());
+        assertEquals(redactionLogEntry.getEngines().stream().map(Enum::name).collect(Collectors.toSet()),
+                entityLogEntry.getEngines().stream().map(Enum::name).collect(Collectors.toSet()));
+    }
+
+
+    private boolean positionsAlmostEqual(List<Rectangle> positions1, List<Position> positions2) {
+
+        double tolerance = 2;
+        for (int i = 0; i < positions1.size(); i++) {
+            Rectangle p1 = positions1.get(0);
+            Position p2 = positions2.get(0);
+            if (p1.getPage() != p2.getPageNumber()) {
+                return false;
+            }
+            if (Math.abs(p1.getHeight() - p2.getRectangle()[3]) > tolerance) {
+                return false;
+            }
+            if (Math.abs(p1.getWidth() - p2.getRectangle()[2]) > tolerance) {
+                return false;
+            }
+            if (Math.abs(p1.getTopLeft().getX() - p2.getRectangle()[0]) > tolerance) {
+                return false;
+            }
+            if (Math.abs(p1.getTopLeft().getY() - p2.getRectangle()[1]) > tolerance) {
+                return false;
+            }
+        }
+        return true;
+    }
+
+
+    private void assertEqualsNullSafe(String string1, String string2) {
+
+        if (Objects.isNull(string1) && Objects.isNull(string2)) {
+            assertEquals(string1, string2);
+        } else if (Objects.isNull(string1)) {
+            assertEquals("", string2);
+        } else if (Objects.isNull(string2)) {
+            assertEquals("", string1);
+        } else {
+            assertEquals(string1, string2);
+        }
+
+    }
+
+
+    private void assertReferencesEqual(Set<String> reference, Set<String> reference1, Map<String, String> oldToNewMapping) {
+
+        if (reference == null) {
+            assertTrue(reference1.isEmpty());
+            return;
+        }
+
+        assertEquals(reference.stream().map(oldToNewMapping::get).collect(Collectors.toSet()), reference1);
+
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionAcceptanceTest.java

@@ -1,8 +1,7 @@
 package com.iqser.red.service.redaction.v1.server;
 
-import static org.mockito.Mockito.doThrow;
+import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.mockito.Mockito.when;
-import static org.wildfly.common.Assert.assertTrue;
 
 import java.io.FileOutputStream;
 import java.io.IOException;
@@ -25,22 +24,25 @@ import org.springframework.context.annotation.Import;
 import org.springframework.context.annotation.Primary;
 import org.springframework.test.context.junit.jupiter.SpringExtension;
 
+import com.iqser.red.commons.jackson.ObjectMapperFactory;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeResult;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryType;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
 import com.iqser.red.service.redaction.v1.server.annotate.AnnotateRequest;
 import com.iqser.red.service.redaction.v1.server.annotate.AnnotateResponse;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.OsUtils;
-import com.iqser.red.service.redaction.v1.server.utils.ExceptionProvider;
 import com.iqser.red.storage.commons.StorageAutoConfiguration;
 import com.iqser.red.storage.commons.service.StorageService;
+import com.iqser.red.storage.commons.utils.FileSystemBackedStorageService;
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
 import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingServiceProcessorConfiguration;
 import com.knecon.fforesight.tenantcommons.TenantContext;
@@ -62,7 +64,7 @@ public class RedactionAcceptanceTest extends AbstractRedactionIntegrationTest {
         @Primary
         public StorageService inmemoryStorage() {
 
-            return new FileSystemBackedStorageService();
+            return new FileSystemBackedStorageService(ObjectMapperFactory.create());
         }
 
     }
@@ -75,7 +77,7 @@ public class RedactionAcceptanceTest extends AbstractRedactionIntegrationTest {
 
         when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.ENTITY)).thenReturn(System.currentTimeMillis());
         when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.ENTITY)).thenReturn(JSONPrimitive.of(RULES));
-        doThrow(ExceptionProvider.getMockNotFoundFeignException()).when(rulesClient).getVersion(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.COMPONENT);
+        when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.COMPONENT)).thenReturn(-1L);
 
         loadDictionaryForTest();
         loadTypeForTest();
@@ -110,13 +112,12 @@ public class RedactionAcceptanceTest extends AbstractRedactionIntegrationTest {
         System.out.println("Finished structure analysis");
         AnalyzeResult result = analyzeService.analyze(request);
         System.out.println("Finished analysis");
-        var redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID);
+        EntityLog entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
 
-        var publishedInformationEntry1 = findEntityByTypeAndValue(redactionLog, "published_information", "Oxford University Press").findFirst().orElseThrow();
-        var asyaLyon1 = findEntityByTypeAndValueAndSectionNumber(redactionLog, "CBI_author", "Asya Lyon", publishedInformationEntry1.getSectionNumber()).findFirst().orElseThrow();
+        var publishedInformationEntry1 = findEntityByTypeAndValue(entityLog, "published_information", "Oxford University Press").findFirst().orElseThrow();
+        var asyaLyon1 = findEntityByTypeAndValueAndSectionNumber(entityLog, "CBI_author", "Asya Lyon", publishedInformationEntry1.getContainingNodeId()).findFirst().orElseThrow();
 
-        // works in intellij, but not mvn install, but it works in UI so idk...
-        // assertFalse(asyaLyon1.isRedacted());
+        assertEquals(EntryState.SKIPPED, asyaLyon1.getState());
 
         var idRemoval = buildIdRemoval(publishedInformationEntry1.getId());
 
@@ -124,12 +125,12 @@ public class RedactionAcceptanceTest extends AbstractRedactionIntegrationTest {
         request.setManualRedactions(manualRedactions);
         analyzeService.reanalyze(request);
 
-        redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID);
+        entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
 
-        var publishedInformationEntry2 = findEntityByTypeAndValue(redactionLog, "published_information", "Oxford University Press").findFirst().orElseThrow();
-        var asyaLyon2 = findEntityByTypeAndValueAndSectionNumber(redactionLog, "CBI_author", "Asya Lyon", publishedInformationEntry2.getSectionNumber()).findFirst().orElseThrow();
+        var publishedInformationEntry2 = findEntityByTypeAndValue(entityLog, "published_information", "Oxford University Press").findFirst().orElseThrow();
+        var asyaLyon2 = findEntityByTypeAndValueAndSectionNumber(entityLog, "CBI_author", "Asya Lyon", publishedInformationEntry2.getContainingNodeId()).findFirst().orElseThrow();
 
-        assertTrue(asyaLyon2.isRedacted());
+        assertEquals(EntryState.APPLIED, asyaLyon2.getState());
 
         AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder().dossierId(TEST_DOSSIER_ID).fileId(TEST_FILE_ID).build());
 
@@ -142,19 +143,18 @@ public class RedactionAcceptanceTest extends AbstractRedactionIntegrationTest {
     }
 
 
-    private Stream<RedactionLogEntry> findEntityByTypeAndValueAndSectionNumber(RedactionLog redactionLog, String type, String value, int sectionNumber) {
+    private Stream<EntityLogEntry> findEntityByTypeAndValueAndSectionNumber(EntityLog redactionLog, String type, String value, List<Integer> sectionNumber) {
 
-        return redactionLog.getRedactionLogEntry()
+        return redactionLog.getEntityLogEntry()
                 .stream()
                 .filter(entry -> entry.getType().equals(type))
-                .filter(entry -> entry.getValue().equals(value))
-                .filter(entry -> entry.getSectionNumber() == sectionNumber);
+                .filter(entry -> entry.getValue().equals(value)).filter(entry -> entry.getContainingNodeId().get(0).equals(sectionNumber.get(0)));
     }
 
 
-    private static Stream<RedactionLogEntry> findEntityByTypeAndValue(RedactionLog redactionLog, String type, String value) {
+    private static Stream<EntityLogEntry> findEntityByTypeAndValue(EntityLog redactionLog, String type, String value) {
 
-        return redactionLog.getRedactionLogEntry().stream().filter(entry -> entry.getType().equals(type)).filter(entry -> entry.getValue().equals(value));
+        return redactionLog.getEntityLogEntry().stream().filter(entry -> entry.getType().equals(type)).filter(entry -> entry.getValue().equals(value));
     }
 
 
@@ -167,9 +167,9 @@ public class RedactionAcceptanceTest extends AbstractRedactionIntegrationTest {
         System.out.println("Finished structure analysis");
         AnalyzeResult result = analyzeService.analyze(request);
         System.out.println("Finished analysis");
-        var redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID);
+        var entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
 
-        RedactionLogEntry desireeEtAl = findEntityByTypeAndValue(redactionLog, "CBI_author", "Desiree").filter(e -> !e.isRecommendation())
+        var desireeEtAl = findEntityByTypeAndValue(entityLog, "CBI_author", "Desiree").filter(e -> e.getEntryType().equals(EntryType.ENTITY))
                 .filter(e -> e.getMatchedRule().startsWith("CBI.16"))
                 .findAny()
                 .orElseThrow();
@@ -179,9 +179,9 @@ public class RedactionAcceptanceTest extends AbstractRedactionIntegrationTest {
         analyzeService.reanalyze(request);
         System.out.println("Finished reanalysis");
 
-        var redactionLog2 = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID);
-        assertTrue(findEntityByTypeAndValue(redactionLog2, "CBI_author", "Desiree").filter(entry -> !entry.isRecommendation()).noneMatch(RedactionLogEntry::isRedacted));
-
+        var redactionLog2 = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
+        assertEquals(EntryState.IGNORED,
+                findEntityByTypeAndValue(redactionLog2, "CBI_author", "Desiree").filter(entry -> entry.getEntryType().equals(EntryType.ENTITY)).findFirst().get().getState());
     }
 
 

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java

@@ -4,7 +4,6 @@ import static org.assertj.core.api.Assertions.assertThat;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.mockito.ArgumentMatchers.anyLong;
 import static org.mockito.ArgumentMatchers.anyString;
-import static org.mockito.Mockito.doThrow;
 import static org.mockito.Mockito.when;
 
 import java.io.FileOutputStream;
@@ -20,6 +19,7 @@ import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 import java.util.Set;
 import java.util.UUID;
 import java.util.stream.Collectors;
@@ -41,12 +41,15 @@ import org.springframework.core.io.ClassPathResource;
 import org.springframework.test.context.junit.jupiter.SpringExtension;
 
 import com.fasterxml.jackson.core.type.TypeReference;
+import com.iqser.red.commons.jackson.ObjectMapperFactory;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeResult;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryType;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Comment;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;
@@ -60,7 +63,6 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemp
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Point;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
 import com.iqser.red.service.redaction.v1.server.annotate.AnnotateRequest;
 import com.iqser.red.service.redaction.v1.server.annotate.AnnotateResponse;
 import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
@@ -68,9 +70,9 @@ import com.iqser.red.service.redaction.v1.server.model.document.nodes.Section;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.OsUtils;
 import com.iqser.red.service.redaction.v1.server.service.document.DocumentGraphMapper;
 import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
-import com.iqser.red.service.redaction.v1.server.utils.ExceptionProvider;
 import com.iqser.red.storage.commons.StorageAutoConfiguration;
 import com.iqser.red.storage.commons.service.StorageService;
+import com.iqser.red.storage.commons.utils.FileSystemBackedStorageService;
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
 import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingServiceProcessorConfiguration;
 import com.knecon.fforesight.tenantcommons.TenantContext;
@@ -94,7 +96,7 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
         @Primary
         public StorageService inmemoryStorage() {
 
-            return new FileSystemBackedStorageService();
+            return new FileSystemBackedStorageService(ObjectMapperFactory.create());
         }
 
     }
@@ -107,7 +109,7 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
 
         when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.ENTITY)).thenReturn(System.currentTimeMillis());
         when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.ENTITY)).thenReturn(JSONPrimitive.of(RULES));
-        doThrow(ExceptionProvider.getMockNotFoundFeignException()).when(rulesClient).getVersion(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.COMPONENT);
+        when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.COMPONENT)).thenReturn(-1L);
 
         loadDictionaryForTest();
         loadTypeForTest();
@@ -163,11 +165,11 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
         analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request);
         AnalyzeResult result = analyzeService.analyze(request);
 
-        Map<String, List<RedactionLogEntry>> duplicates = new HashMap<>();
+        Map<String, List<EntityLogEntry>> duplicates = new HashMap<>();
 
-        var redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID);
+        var entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
 
-        redactionLog.getRedactionLogEntry().forEach(entry -> {
+        entityLog.getEntityLogEntry().forEach(entry -> {
             duplicates.computeIfAbsent(entry.getId(), v -> new ArrayList<>()).add(entry);
         });
 
@@ -194,6 +196,55 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
     }
 
 
+    @Test
+    public void manualResizeTest() throws IOException {
+
+//        c6be5277f5ee60dc3d83527798b7fe02
+//        Rectangle(topLeft=Point(x=236.8, y=182.90005), width=70.89604, height=12.642, page=7)
+
+//        {
+//            "page": 7,
+//                "topLeft": {
+//            "x": 200.8,
+//                    "y": 332.364
+//        },
+//            "height": 15.408000000000015,
+//                "width": 40.584
+//        }
+
+        AnalyzeRequest request = uploadFileToStorage("files/new/crafted document.pdf");
+
+        ManualRedactions manualRedactions = ManualRedactions.builder()
+                .resizeRedactions(Set.of(ManualResizeRedaction.builder()
+                        .annotationId("c6be5277f5ee60dc3d83527798b7fe02")
+                        .value("Dr. Alan")
+                        .positions(List.of(new Rectangle(236.8f, 182.90005f, 40.584f, 12.642f, 7)))
+                        .status(AnnotationStatus.APPROVED)
+                        .requestDate(OffsetDateTime.now())
+                        .updateDictionary(false)
+                        .build()))
+                .build();
+
+        request.setManualRedactions(manualRedactions);
+
+        System.out.println("Start Full integration test");
+        analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request);
+        System.out.println("Finished structure analysis");
+        AnalyzeResult result = analyzeService.analyze(request);
+        System.out.println("Finished analysis");
+        var redactionLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
+
+        AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder().dossierId(TEST_DOSSIER_ID).fileId(TEST_FILE_ID).build());
+
+        String outputFileName = OsUtils.getTemporaryDirectory() + "/Annotated.pdf";
+
+        try (FileOutputStream fileOutputStream = new FileOutputStream(outputFileName)) {
+            fileOutputStream.write(annotateResponse.getDocument());
+        }
+
+    }
+
+
     @Test
     public void redactionExpansionOverlap() {
 
@@ -204,9 +255,9 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
         analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request);
         AnalyzeResult result = analyzeService.analyze(request);
 
-        var redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID);
+        var redactionLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
 
-        var values = redactionLog.getRedactionLogEntry().stream().map(RedactionLogEntry::getValue).collect(Collectors.toList());
+        var values = redactionLog.getEntityLogEntry().stream().map(EntityLogEntry::getValue).collect(Collectors.toList());
 
         assertThat(values).containsExactlyInAnyOrder("Lastname M.", "Doe", "Doe J.", "M. Mustermann", "Mustermann M.", "F. Lastname");
     }
@@ -215,13 +266,13 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
     @Test
     public void titleExtraction() throws IOException {
 
-        AnalyzeRequest request = uploadFileToStorage("files/Metolachlor/S-Metolachlor_RAR_01_Volume_1_2018-09-06.pdf");
+        AnalyzeRequest request = uploadFileToStorage("files/migration/def8f960580f088b975ba806dfae1f87.ORIGIN.pdf");
         System.out.println("Start Full integration test");
         analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request);
         System.out.println("Finished structure analysis");
         AnalyzeResult result = analyzeService.analyze(request);
         System.out.println("Finished analysis");
-        var redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID);
+        var entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
 
         AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder().dossierId(TEST_DOSSIER_ID).fileId(TEST_FILE_ID).build());
 
@@ -259,7 +310,7 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
         analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request);
         analyzeService.analyze(request);
 
-        var redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID);
+        var entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
 
         var toRemove = IdRemoval.builder()
                 .annotationId("c630599611e6e3db314518374bcf70f7")
@@ -274,13 +325,13 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
         request.setManualRedactions(manualRedactions);
         analyzeService.reanalyze(request);
 
-        var mergedRedactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID);
+        var mergedEntityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
 
-        var cbiAddressBeforeHintRemoval = redactionLog.getRedactionLogEntry().stream().filter(re -> re.getType().equalsIgnoreCase("CBI_Address")).findAny().get();
-        assertThat(cbiAddressBeforeHintRemoval.isRedacted()).isFalse();
+        var cbiAddressBeforeHintRemoval = entityLog.getEntityLogEntry().stream().filter(re -> re.getType().equalsIgnoreCase("CBI_Address")).findAny().get();
+        assertThat(cbiAddressBeforeHintRemoval.getState().equals(EntryState.APPLIED)).isFalse();
 
-        var cbiAddressAfterHintRemoval = mergedRedactionLog.getRedactionLogEntry().stream().filter(re -> re.getType().equalsIgnoreCase("CBI_Address")).findAny().get();
-        assertThat(cbiAddressAfterHintRemoval.isRedacted()).isTrue();
+        var cbiAddressAfterHintRemoval = mergedEntityLog.getEntityLogEntry().stream().filter(re -> re.getType().equalsIgnoreCase("CBI_Address")).findAny().get();
+        assertThat(cbiAddressAfterHintRemoval.getState().equals(EntryState.APPLIED)).isTrue();
     }
 
 
@@ -305,11 +356,11 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
                     AnalyzeResult result = analyzeService.analyze(request);
                     System.out.println("analysis analysis duration: " + (System.currentTimeMillis() - fstart));
 
-                    Map<String, List<RedactionLogEntry>> duplicates = new HashMap<>();
+                    Map<String, List<EntityLogEntry>> duplicates = new HashMap<>();
 
-                    var redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID);
+                    var entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
 
-                    redactionLog.getRedactionLogEntry().forEach(entry -> {
+                    entityLog.getEntityLogEntry().forEach(entry -> {
                         duplicates.computeIfAbsent(entry.getId(), v -> new ArrayList<>()).add(entry);
                     });
 
@@ -354,7 +405,7 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
         analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request);
         AnalyzeResult result = analyzeService.analyze(request);
 
-        var redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID);
+        var entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
         var documentGraph = DocumentGraphMapper.toDocumentGraph(redactionStorageService.getDocumentData(TEST_DOSSIER_ID, TEST_FILE_ID));
 
         long end = System.currentTimeMillis();
@@ -367,15 +418,15 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
 
         int correctFound = 0;
         loop:
-        for (RedactionLogEntry redactionLogEntry : redactionLog.getRedactionLogEntry()) {
+        for (EntityLogEntry entityLogEntry : entityLog.getEntityLogEntry()) {
             for (Section section : documentGraph.getMainSections()) {
-                if (redactionLogEntry.isImage()) {
+                if (entityLogEntry.getEntryType().equals(EntryType.IMAGE)) {
                     correctFound++;
                     continue loop;
                 }
-                if (redactionLogEntry.getSectionNumber() == section.getTreeId().get(0)) {
-                    String value = section.getTextBlock().subSequence(new TextRange(redactionLogEntry.getStartOffset(), redactionLogEntry.getEndOffset())).toString();
-                    if (redactionLogEntry.getValue().equalsIgnoreCase(value)) {
+                if (Objects.equals(entityLogEntry.getContainingNodeId().get(0), section.getTreeId().get(0))) {
+                    String value = section.getTextBlock().subSequence(new TextRange(entityLogEntry.getStartOffset(), entityLogEntry.getEndOffset())).toString();
+                    if (entityLogEntry.getValue().equalsIgnoreCase(value)) {
                         correctFound++;
                     } else {
                         throw new RuntimeException("WTF");
@@ -416,7 +467,7 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
 
         AnalyzeResult reanalyzeResult = analyzeService.reanalyze(request);
 
-        redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID);
+        entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
 
         end = System.currentTimeMillis();
         System.out.println("reanalysis analysis duration: " + (end - start));
@@ -436,7 +487,7 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
 
         analyzeService.reanalyze(request);
 
-        redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID);
+        entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
 
         System.out.println("Output file:" + outputFileName);
     }
@@ -470,9 +521,9 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
         mockDictionaryCalls(3L);
         analyzeService.reanalyze(request);
 
-        var redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID);
+        var entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
 
-        var changes = redactionLog.getRedactionLogEntry().stream().filter(entry -> entry.getValue() != null && entry.getValue().equals("report")).findFirst().get().getChanges();
+        var changes = entityLog.getEntityLogEntry().stream().filter(entry -> entry.getValue() != null && entry.getValue().equals("report")).findFirst().get().getChanges();
 
         assertThat(changes.size()).isEqualTo(2);
 
@@ -509,7 +560,7 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
         analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request);
         AnalyzeResult result = analyzeService.analyze(request);
 
-        var redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID);
+        var entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
         var documentGraph = DocumentGraphMapper.toDocumentGraph(redactionStorageService.getDocumentData(TEST_DOSSIER_ID, TEST_FILE_ID));
 
         long end = System.currentTimeMillis();
@@ -520,13 +571,13 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
             fileOutputStream.write(objectMapper.writeValueAsBytes(redactionStorageService.getDocumentData(TEST_DOSSIER_ID, TEST_FILE_ID)));
         }
 
-        List<String> valuesInDocument = redactionLog.getRedactionLogEntry()
+        List<String> valuesInDocument = entityLog.getEntityLogEntry()
                 .stream()
-                .filter(e -> !e.isImage())
+                .filter(e -> !e.getEntryType().equals(EntryType.IMAGE))
                 .map(redactionLogEntry -> new TextRange(redactionLogEntry.getStartOffset(), redactionLogEntry.getEndOffset()))
                 .map(boundary -> documentGraph.getTextBlock().subSequence(boundary).toString())
                 .toList();
-        List<String> valuesInRedactionLog = redactionLog.getRedactionLogEntry().stream().filter(e -> !e.isImage()).map(RedactionLogEntry::getValue).toList();
+        List<String> valuesInRedactionLog = entityLog.getEntityLogEntry().stream().filter(e -> !e.getEntryType().equals(EntryType.IMAGE)).map(EntityLogEntry::getValue).toList();
 
         assertEquals(valuesInRedactionLog, valuesInDocument);
 
@@ -564,7 +615,7 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
 
         AnalyzeResult reanalyzeResult = analyzeService.reanalyze(request);
 
-        redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID);
+        entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
 
         end = System.currentTimeMillis();
         System.out.println("reanalysis analysis duration: " + (end - start));
@@ -585,7 +636,7 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
 
         analyzeService.reanalyze(request);
 
-        redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID);
+        entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
 
         System.out.println("Output file:" + outputFileName);
     }
@@ -812,8 +863,8 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
             fileOutputStream.write(annotateResponse.getDocument());
         }
 
-        var redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID);
-        assertThat(redactionLog.getRedactionLogEntry().size()).isEqualTo(5);
+        var entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
+        assertThat(entityLog.getEntityLogEntry().size()).isEqualTo(5);
 
         long end = System.currentTimeMillis();
 
@@ -840,26 +891,26 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
         AnalyzeRequest request = uploadFileToStorage("files/new/S157.pdf");
 
         ManualRedactions manualRedactions = new ManualRedactions();
-
-        manualRedactions.getResizeRedactions()
-                .add(ManualResizeRedaction.builder()
-                        .annotationId("ca2b437e2480a4b5966cb8386020d454")
-                        .fileId(TEST_FILE_ID)
-                        .status(AnnotationStatus.APPROVED)
-                        .requestDate(OffsetDateTime.of(2022, 05, 23, 8, 30, 07, 475479, ZoneOffset.UTC))
-                        .processedDate(OffsetDateTime.of(2022, 05, 23, 8, 30, 07, 483651, ZoneOffset.UTC))
-                        .value("Bera P")
-                        .positions(List.of(new Rectangle(382.12485f, 235.94246f, 8.768621f, -21.632504f, 1)))
-                        .textBefore("")
-                        .textAfter("")
-                        .build());
+        ManualResizeRedaction manualResizeRedaction = ManualResizeRedaction.builder()
+                .annotationId("ca2b437e2480a4b5966cb8386020d454")
+                .fileId(TEST_FILE_ID)
+                .status(AnnotationStatus.APPROVED)
+                .requestDate(OffsetDateTime.of(2022, 05, 23, 8, 30, 07, 475479, ZoneOffset.UTC))
+                .processedDate(OffsetDateTime.of(2022, 05, 23, 8, 30, 07, 483651, ZoneOffset.UTC))
+                .value("Bera P")
+                .positions(List.of(new Rectangle(382.12485f, 235.94246f, 8.768621f, -21.632504f, 1)))
+                .textBefore("")
+                .updateDictionary(false)
+                .textAfter("")
+                .build();
+        manualRedactions.getResizeRedactions().add(manualResizeRedaction);
 
         request.setManualRedactions(manualRedactions);
 
         analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request);
         AnalyzeResult result = analyzeService.analyze(request);
 
-        var redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID);
+        var entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
 
         AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder()
                 .manualRedactions(manualRedactions)
@@ -887,10 +938,10 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
         analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request);
         AnalyzeResult result = analyzeService.analyze(request);
 
-        var redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID);
+        var redactionLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
 
-        redactionLog.getRedactionLogEntry().forEach(entry -> {
-            if (!entry.isHint()) {
+        redactionLog.getEntityLogEntry().forEach(entry -> {
+            if (!entry.getEntryType().equals(EntryType.HINT)) {
                 assertThat(entry.getReason()).isEqualTo("Not redacted because it's row does not belong to a vertebrate study");
             }
         });
@@ -929,7 +980,6 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
 
         String manualAddId = UUID.randomUUID().toString();
 
-        Comment comment = Comment.builder().date(OffsetDateTime.now()).user("TEST_USER").text("This is a comment test").build();
         manualRedactions.setIdsToRemove(Set.of(IdRemoval.builder()
                 .annotationId("5b940b2cb401ed9f5be6fc24f6e77bcf")
                 .fileId("fileId")
@@ -946,10 +996,6 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
                 .processedDate(OffsetDateTime.now())
                 .build()));
 
-        manualRedactions.getComments().put("e5be0f1d941bbb92a068e198648d06c4", List.of(comment));
-        manualRedactions.getComments().put("0836727c3508a0b2ea271da69c04cc2f", List.of(comment));
-        manualRedactions.getComments().put(manualAddId, List.of(comment));
-
         ManualRedactionEntry manualRedactionEntry = new ManualRedactionEntry();
         manualRedactionEntry.setAnnotationId(manualAddId);
         manualRedactionEntry.setFileId("fileId");
@@ -996,6 +1042,7 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
                         Rectangle.builder().topLeftX(307.89517f).topLeftY(327.567f).width(7.648041f).height(61.670967f).page(1).build(),
                         Rectangle.builder().topLeftX(316.99985f).topLeftY(327.567f).width(7.648041f).height(38.104286f).page(1).build()))
                 .status(AnnotationStatus.APPROVED)
+                .updateDictionary(false)
                 .build()));
 
         analyzeService.reanalyze(request);
@@ -1050,7 +1097,7 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
         analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request);
         AnalyzeResult result = analyzeService.analyze(request);
 
-        var redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID);
+        var entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
 
         AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder().dossierId(TEST_DOSSIER_ID).fileId(TEST_FILE_ID).build());
 
@@ -1058,7 +1105,7 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
             fileOutputStream.write(annotateResponse.getDocument());
         }
 
-        redactionLog.getRedactionLogEntry().forEach(entry -> {
+        entityLog.getEntityLogEntry().forEach(entry -> {
             if (entry.getValue() == null) {
                 return;
             }
@@ -1096,8 +1143,8 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
             fileOutputStream.write(annotateResponse.getDocument());
         }
 
-        var redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID);
-        var values = redactionLog.getRedactionLogEntry().stream().map(RedactionLogEntry::getValue).collect(Collectors.toList());
+        var entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
+        var values = entityLog.getEntityLogEntry().stream().map(EntityLogEntry::getValue).collect(Collectors.toList());
 
         assertThat(values).contains("Mrs. Robinson");
         assertThat(values).contains("Mr. Bojangles");

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationV2Test.java

@@ -1,7 +1,6 @@
 package com.iqser.red.service.redaction.v1.server;
 
 import static org.assertj.core.api.Assertions.assertThat;
-import static org.mockito.Mockito.doThrow;
 import static org.mockito.Mockito.when;
 
 import java.util.List;
@@ -21,14 +20,17 @@ import org.springframework.context.annotation.Import;
 import org.springframework.context.annotation.Primary;
 import org.springframework.test.context.junit.jupiter.SpringExtension;
 
+import com.iqser.red.commons.jackson.ObjectMapperFactory;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
-import com.iqser.red.service.redaction.v1.server.utils.ExceptionProvider;
 import com.iqser.red.storage.commons.StorageAutoConfiguration;
 import com.iqser.red.storage.commons.service.StorageService;
+import com.iqser.red.storage.commons.utils.FileSystemBackedStorageService;
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
 import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingServiceProcessorConfiguration;
 import com.knecon.fforesight.tenantcommons.TenantContext;
@@ -53,7 +55,7 @@ public class RedactionIntegrationV2Test extends AbstractRedactionIntegrationTest
         @Primary
         public StorageService inmemoryStorage() {
 
-            return new FileSystemBackedStorageService();
+            return new FileSystemBackedStorageService(ObjectMapperFactory.create());
         }
 
     }
@@ -66,7 +68,7 @@ public class RedactionIntegrationV2Test extends AbstractRedactionIntegrationTest
 
         when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.ENTITY)).thenReturn(System.currentTimeMillis());
         when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.ENTITY)).thenReturn(JSONPrimitive.of(RULES));
-        doThrow(ExceptionProvider.getMockNotFoundFeignException()).when(rulesClient).getVersion(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.COMPONENT);
+        when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.COMPONENT)).thenReturn(-1L);
 
         loadDictionaryForTest();
         loadTypeForTest();
@@ -122,22 +124,20 @@ public class RedactionIntegrationV2Test extends AbstractRedactionIntegrationTest
         analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request);
         analyzeService.analyze(request);
 
-        var redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID);
+        var entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
 
-        assertThat(redactionLog.getRedactionLogEntry().size()).isEqualTo(1);
+        assertThat(entityLog.getEntityLogEntry().size()).isEqualTo(1);
 
-        RedactionLogEntry redactionLogEntry = redactionLog.getRedactionLogEntry().get(0);
+        EntityLogEntry redactionLogEntry = entityLog.getEntityLogEntry().get(0);
 
         assertThat(redactionLogEntry.getType()).isEqualTo(DICTIONARY_AUTHOR);
         assertThat(redactionLogEntry.getValue()).isEqualTo(entryAuthorAndPIIDictionary);
-        assertThat(redactionLogEntry.isRedacted()).isEqualTo(true);
-        assertThat(redactionLogEntry.isRecommendation()).isEqualTo(false);
-        assertThat(redactionLogEntry.isFalsePositive()).isEqualTo(false);
+        assertThat(redactionLogEntry.getState()).isEqualTo(EntryState.APPLIED);
         assertThat(redactionLogEntry.isExcluded()).isEqualTo(false);
         assertThat(redactionLogEntry.isDictionaryEntry()).isEqualTo(true);
 
         assertThat(redactionLogEntry.getEngines().size()).isEqualTo(1);
-        assertThat(redactionLogEntry.getEngines().contains(com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine.DICTIONARY)).isEqualTo(true);
+        assertThat(redactionLogEntry.getEngines().contains(Engine.DICTIONARY)).isEqualTo(true);
     }
 
 
@@ -160,22 +160,20 @@ public class RedactionIntegrationV2Test extends AbstractRedactionIntegrationTest
         analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request);
         analyzeService.analyze(request);
 
-        var redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID);
+        var redactionLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
 
-        assertThat(redactionLog.getRedactionLogEntry().size()).isEqualTo(1);
+        assertThat(redactionLog.getEntityLogEntry().size()).isEqualTo(1);
 
-        RedactionLogEntry redactionLogEntry = redactionLog.getRedactionLogEntry().get(0);
+        EntityLogEntry redactionLogEntry = redactionLog.getEntityLogEntry().get(0);
 
         assertThat(redactionLogEntry.getType()).isEqualTo(DICTIONARY_AUTHOR);
         assertThat(redactionLogEntry.getValue()).isEqualTo(entryAuthorDictionary);
-        assertThat(redactionLogEntry.isRedacted()).isEqualTo(true);
-        assertThat(redactionLogEntry.isRecommendation()).isEqualTo(false);
-        assertThat(redactionLogEntry.isFalsePositive()).isEqualTo(false);
+        assertThat(redactionLogEntry.getState()).isEqualTo(EntryState.APPLIED);
         assertThat(redactionLogEntry.isExcluded()).isEqualTo(false);
         assertThat(redactionLogEntry.isDictionaryEntry()).isEqualTo(true);
 
         assertThat(redactionLogEntry.getEngines().size()).isEqualTo(1);
-        assertThat(redactionLogEntry.getEngines().contains(com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine.DICTIONARY)).isEqualTo(true);
+        assertThat(redactionLogEntry.getEngines().contains(Engine.DICTIONARY)).isEqualTo(true);
 
     }
 

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RulesTest.java

@@ -2,7 +2,6 @@ package com.iqser.red.service.redaction.v1.server;
 
 import static java.util.Map.entry;
 import static org.assertj.core.api.Assertions.assertThat;
-import static org.mockito.Mockito.doThrow;
 import static org.mockito.Mockito.when;
 
 import java.io.BufferedReader;
@@ -58,6 +57,7 @@ import org.springframework.test.context.junit.jupiter.SpringExtension;
 import com.fasterxml.jackson.databind.DeserializationFeature;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
+import com.iqser.red.commons.jackson.ObjectMapperFactory;
 import com.iqser.red.service.dictionarymerge.commons.DictionaryEntry;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType;
@@ -81,12 +81,12 @@ import com.iqser.red.service.redaction.v1.server.client.RulesClient;
 import com.iqser.red.service.redaction.v1.server.controller.RedactionController;
 import com.iqser.red.service.redaction.v1.server.service.AnalyzeService;
 import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
-import com.iqser.red.service.redaction.v1.server.utils.ExceptionProvider;
 import com.iqser.red.service.redaction.v1.server.utils.LayoutParsingRequestProvider;
 import com.iqser.red.service.redaction.v1.server.utils.ResourceLoader;
 import com.iqser.red.service.redaction.v1.server.utils.TextNormalizationUtilities;
 import com.iqser.red.storage.commons.StorageAutoConfiguration;
 import com.iqser.red.storage.commons.service.StorageService;
+import com.iqser.red.storage.commons.utils.FileSystemBackedStorageService;
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingFinishedEvent;
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
 import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingPipeline;
@@ -264,7 +264,7 @@ public class RulesTest {
 
         when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.ENTITY)).thenReturn(System.currentTimeMillis());
         when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.ENTITY)).thenReturn(JSONPrimitive.of(RULES));
-        doThrow(ExceptionProvider.getMockNotFoundFeignException()).when(rulesClient).getVersion(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.COMPONENT);
+        when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.COMPONENT)).thenReturn(-1L);
 
         loadDictionaryForTest();
         loadTypeForTest();
@@ -473,7 +473,7 @@ public class RulesTest {
             assertThat(savedRedactionLogEntry.get().isFalsePositive()).isEqualTo(redactionLogEntry.getEntryType().equals(EntryType.FALSE_POSITIVE));
             assertThat(savedRedactionLogEntry.get().getSection()).isEqualTo(redactionLogEntry.getSection());
             assertThat(savedRedactionLogEntry.get().getColor()).isEqualTo(redactionLogEntry.getColor());
-            assertThat(savedRedactionLogEntry.get().getSectionNumber()).isEqualTo(redactionLogEntry.getSectionNumber());
+            assertThat(savedRedactionLogEntry.get().getSectionNumber()).isEqualTo(redactionLogEntry.getContainingNodeId().get(0));
             assertThat(savedRedactionLogEntry.get().getTextBefore()).isEqualTo(redactionLogEntry.getTextBefore());
             assertThat(savedRedactionLogEntry.get().getTextAfter()).isEqualTo(redactionLogEntry.getTextAfter());
             assertThat(savedRedactionLogEntry.get().getStartOffset()).isEqualTo(redactionLogEntry.getStartOffset());
@@ -510,7 +510,6 @@ public class RulesTest {
                 var savedManualChange = savedRedactionLogEntry.get()
                         .getManualChanges()
                         .stream()
-                        .filter(m -> m.getAnnotationStatus() == manualChange.getAnnotationStatus())
                         .filter(m -> m.getManualRedactionType().name().equals(manualChange.getManualRedactionType().name()))
                         .filter(m -> m.getUserId().equalsIgnoreCase(manualChange.getUserId()))
                         .filter(m -> m.getPropertyChanges() == manualChange.getPropertyChanges())
@@ -840,7 +839,7 @@ public class RulesTest {
         @Primary
         public StorageService inmemoryStorage() {
 
-            return new FileSystemBackedStorageService();
+            return new FileSystemBackedStorageService(ObjectMapperFactory.create());
         }
 
     }

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/annotate/AnnotationService.java

@@ -1,11 +1,9 @@
 package com.iqser.red.service.redaction.v1.server.annotate;
 
-import java.awt.Color;
 import java.awt.geom.Rectangle2D;
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.util.ArrayList;
-import java.util.GregorianCalendar;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -14,44 +12,38 @@ import java.util.stream.Stream;
 
 import org.apache.pdfbox.Loader;
 import org.apache.pdfbox.pdmodel.PDDocument;
+import org.apache.pdfbox.pdmodel.PDDocumentCatalog;
 import org.apache.pdfbox.pdmodel.PDPage;
-import org.apache.pdfbox.pdmodel.PDPageContentStream;
 import org.apache.pdfbox.pdmodel.common.PDRectangle;
-import org.apache.pdfbox.pdmodel.font.PDType1Font;
-import org.apache.pdfbox.pdmodel.font.Standard14Fonts;
 import org.apache.pdfbox.pdmodel.graphics.color.PDColor;
 import org.apache.pdfbox.pdmodel.graphics.color.PDDeviceRGB;
+import org.apache.pdfbox.pdmodel.graphics.optionalcontent.PDOptionalContentProperties;
 import org.apache.pdfbox.pdmodel.interactive.annotation.PDAnnotation;
 import org.apache.pdfbox.pdmodel.interactive.annotation.PDAnnotationHighlight;
-import org.apache.pdfbox.pdmodel.interactive.annotation.PDAnnotationText;
 import org.springframework.stereotype.Service;
 
 import com.google.common.primitives.Floats;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryType;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogComment;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.CellRectangle;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.SectionGrid;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.SectionRectangle;
-import com.iqser.red.service.redaction.v1.server.service.DictionaryService;
 import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
 import com.iqser.red.service.redaction.v1.server.utils.RectangleTransformations;
-import com.iqser.red.service.redaction.v1.server.utils.exception.RedactionException;
 
 import lombok.RequiredArgsConstructor;
+import lombok.SneakyThrows;
 
 @Service
 @RequiredArgsConstructor
 public class AnnotationService {
 
     private final RedactionStorageService redactionStorageService;
-    private final DictionaryService dictionaryService;
 
 
     /**
-     * This method draws a PDF Text Markup Annotation for each RedactionLogEntry and the section grid.
+     * This method draws a PDF Text Markup Annotation for each EntityLogEntry and the Viewer document.
      * A Text Markup Annotation has two main functionalities.
      * First, it highlights text with a color and second, when hovered over in a PDF Viewer it displays a popup message.
      * Where the Popup message appears is defined by the Rectangle provided to the Annotation.
@@ -64,46 +56,41 @@ public class AnnotationService {
      * @param annotateRequest containing the SectionGrid, RedactionLog, ManualRedactions and the colors for the highlights
      * @return the annotated PDFDoc
      */
+    @SneakyThrows
     public AnnotateResponse annotate(AnnotateRequest annotateRequest) {
 
         var storedObjectFile = redactionStorageService.getStoredObjectFile(RedactionStorageService.StorageIdUtils.getStorageId(annotateRequest.getDossierId(),
                 annotateRequest.getFileId(),
-                FileType.ORIGIN));
+                FileType.VIEWER_DOCUMENT));
 
-        var redactionLog = redactionStorageService.getRedactionLog(annotateRequest.getDossierId(), annotateRequest.getFileId());
-        var sectionsGrid = redactionStorageService.getSectionGrid(annotateRequest.getDossierId(), annotateRequest.getFileId());
+        var entityLog = redactionStorageService.getEntityLog(annotateRequest.getDossierId(), annotateRequest.getFileId());
 
         try (PDDocument pdDocument = Loader.loadPDF(storedObjectFile)) {
             pdDocument.setAllSecurityToBeRemoved(true);
 
-            dictionaryService.updateDictionary(annotateRequest.getDossierTemplateId(), annotateRequest.getDossierId());
-            annotate(pdDocument, redactionLog, sectionsGrid);
+            annotate(pdDocument, entityLog);
+
+            PDDocumentCatalog catalog = pdDocument.getDocumentCatalog();
+            PDOptionalContentProperties ocprops = catalog.getOCProperties();
+            ocprops.setGroupEnabled("Layout grid", true);
 
             try (ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream()) {
                 pdDocument.save(byteArrayOutputStream);
                 return AnnotateResponse.builder().document(byteArrayOutputStream.toByteArray()).build();
             }
-
-        } catch (Exception e) {
-            throw new RedactionException(e);
         }
     }
 
 
-    private void annotate(PDDocument document, RedactionLog redactionLog, SectionGrid sectionGrid) throws IOException {
+    private void annotate(PDDocument document, EntityLog entityLog) throws IOException {
 
-        Map<Integer, List<RedactionLogEntry>> redactionLogPerPage = convertRedactionLog(redactionLog);
+        Map<Integer, List<EntityLogEntry>> redactionLogPerPage = groupingByPageNumber(entityLog);
 
         for (int page = 1; page <= document.getNumberOfPages(); page++) {
 
             PDPage pdPage = document.getPage(page - 1);
 
-            List<SectionRectangle> sectionRectangles = sectionGrid.getRectanglesPerPage().get(page);
-            if (sectionRectangles != null && !sectionRectangles.isEmpty()) {
-                drawSectionGrid(document, pdPage, sectionRectangles);
-            }
-
-            List<RedactionLogEntry> logEntries = redactionLogPerPage.get(page);
+            List<EntityLogEntry> logEntries = redactionLogPerPage.get(page);
             if (logEntries != null && !logEntries.isEmpty()) {
                 addAnnotations(logEntries, pdPage, page);
             }
@@ -111,19 +98,67 @@ public class AnnotationService {
     }
 
 
-    private void addAnnotations(List<RedactionLogEntry> logEntries, PDPage pdPage, int page) throws IOException {
+    private Map<Integer, List<EntityLogEntry>> groupingByPageNumber(EntityLog redactionLog) {
+
+        Map<Integer, List<EntityLogEntry>> redactionLogPerPage = new HashMap<>();
+        if (redactionLog == null) {
+            return redactionLogPerPage;
+        }
+        for (EntityLogEntry entry : redactionLog.getEntityLogEntry()) {
+            int page = 0;
+            for (Position position : entry.getPositions()) {
+                if (position.getPageNumber() != page) {
+                    redactionLogPerPage.computeIfAbsent(position.getPageNumber(), x -> new ArrayList<>()).add(entry);
+                    page = position.getPageNumber();
+                }
+            }
+        }
+        return redactionLogPerPage;
+    }
+
+
+    private void addAnnotations(List<EntityLogEntry> logEntries, PDPage pdPage, int page) throws IOException {
 
         List<PDAnnotation> annotations = pdPage.getAnnotations();
 
-        for (RedactionLogEntry entry : logEntries) {
-            if (entry.lastChangeIsRemoved()) {
+        for (EntityLogEntry entry : logEntries) {
+            if (entry.getState().equals(EntryState.REMOVED)) {
                 continue;
             }
             annotations.addAll(createAnnotation(entry, page, pdPage.getRotation(), pdPage.getCropBox()));
         }
     }
 
-    public static PDRectangle toPDRectangleBBox(List<Rectangle> rectangles) {
+
+    private List<PDAnnotation> createAnnotation(EntityLogEntry redactionLogEntry, int page, int rotation, PDRectangle cropBox) {
+
+        List<PDAnnotation> annotations = new ArrayList<>();
+
+        List<Position> rectangles = redactionLogEntry.getPositions().stream().filter(pos -> pos.getPageNumber() == page).collect(Collectors.toList());
+
+        if (rectangles.isEmpty()) {
+            return annotations;
+        }
+
+        PDAnnotationHighlight annotation = new PDAnnotationHighlight();
+        annotation.constructAppearances();
+        PDRectangle pdRectangle = toPDRectangleBBox(rectangles);
+        annotation.setRectangle(pdRectangle);
+        annotation.setQuadPoints(Floats.toArray(toQuadPoints(rectangles)));
+        if (!(redactionLogEntry.getEntryType().equals(EntryType.HINT) || redactionLogEntry.getState().equals(EntryState.IGNORED))) {
+            annotation.setContents(redactionLogEntry.getValue() + " " + createAnnotationContent(redactionLogEntry));
+        }
+        annotation.setTitlePopup(redactionLogEntry.getId());
+        annotation.setAnnotationName(redactionLogEntry.getId());
+        annotation.setColor(new PDColor(redactionLogEntry.getColor(), PDDeviceRGB.INSTANCE));
+        annotation.setNoRotate(false);
+        annotations.add(annotation);
+
+        return annotations;
+    }
+
+
+    public static PDRectangle toPDRectangleBBox(List<Position> rectangles) {
 
         Rectangle2D rectangle2D = RectangleTransformations.rectangleBBox(rectangles);
 
@@ -135,66 +170,10 @@ public class AnnotationService {
         return annotationPosition;
     }
 
-    private List<PDAnnotation> createAnnotation(RedactionLogEntry redactionLogEntry, int page, int rotation, PDRectangle cropBox) {
 
-        List<PDAnnotation> annotations = new ArrayList<>();
+    public static List<Double> toQuadPoints(List<Position> rectangles) {
 
-        List<Rectangle> rectangles = redactionLogEntry.getPositions().stream().filter(pos -> pos.getPage() == page).collect(Collectors.toList());
-
-        if (rectangles.isEmpty()) {
-            return annotations;
-        }
-
-        PDAnnotationHighlight annotation = new PDAnnotationHighlight();
-        annotation.constructAppearances();
-        PDRectangle pdRectangle = toPDRectangleBBox(rectangles);
-        annotation.setRectangle(pdRectangle);
-        annotation.setQuadPoints(Floats.toArray(toQuadPoints(rectangles)));
-        if (!redactionLogEntry.isHint()) {
-            annotation.setContents(redactionLogEntry.getValue() + " " + createAnnotationContent(redactionLogEntry));
-        }
-        annotation.setTitlePopup(redactionLogEntry.getId());
-        annotation.setAnnotationName(redactionLogEntry.getId());
-        annotation.setColor(new PDColor(redactionLogEntry.getColor(), PDDeviceRGB.INSTANCE));
-        annotation.setNoRotate(false);
-        annotations.add(annotation);
-
-        if (redactionLogEntry.getComments() != null) {
-            for (RedactionLogComment comment : redactionLogEntry.getComments()) {
-                PDAnnotationText txtAnnot = new PDAnnotationText();
-                txtAnnot.setAnnotationName(String.valueOf(comment.getId()));
-                txtAnnot.setInReplyTo(annotation); // Reference to highlight annotation
-                txtAnnot.setName(PDAnnotationText.NAME_COMMENT);
-                txtAnnot.setCreationDate(GregorianCalendar.from(comment.getDate().toZonedDateTime()));
-                txtAnnot.setTitlePopup(comment.getUser());
-                txtAnnot.setContents(comment.getText());
-                txtAnnot.setRectangle(pdRectangle);
-                annotations.add(txtAnnot);
-            }
-        }
-
-        return annotations;
-    }
-
-
-    private String createAnnotationContent(RedactionLogEntry redactionLogEntry) {
-
-        return redactionLogEntry.getType() + " \nRule " + redactionLogEntry.getMatchedRule() + " matched\n\n" + redactionLogEntry.getReason() + "\n\nLegal basis:" + redactionLogEntry.getLegalBasis() + "\n\nIn section: \"" + redactionLogEntry.getSection() + "\"";
-    }
-
-
-    public static List<Double> toQuadPoints(List<Rectangle> rectangles) {
-
-        return rectangles.stream().map(AnnotationService::toRectangle2D).flatMap(AnnotationService::toQuadPoints).toList();
-    }
-
-
-    private static Rectangle2D toRectangle2D(Rectangle redactionLogRectangle) {
-
-        return new Rectangle2D.Double(redactionLogRectangle.getTopLeft().getX(),
-                redactionLogRectangle.getTopLeft().getY(),
-                redactionLogRectangle.getWidth(),
-                redactionLogRectangle.getHeight());
+        return rectangles.stream().map(Position::toRectangle2D).flatMap(AnnotationService::toQuadPoints).toList();
     }
 
 
@@ -215,71 +194,9 @@ public class AnnotationService {
     }
 
 
-    private void drawSectionGrid(PDDocument document, PDPage pdPage, List<SectionRectangle> sectionRectangles) throws IOException {
+    private String createAnnotationContent(EntityLogEntry redactionLogEntry) {
 
-        PDPageContentStream contentStream = new PDPageContentStream(document, pdPage, PDPageContentStream.AppendMode.APPEND, true);
-        for (SectionRectangle sectionRectangle : sectionRectangles) {
-            drawSectionRectangle(contentStream, sectionRectangle);
-            drawSectionPartNumberText(contentStream, sectionRectangle);
-            drawTableCells(contentStream, sectionRectangle);
-        }
-        contentStream.close();
-    }
-
-
-    private void drawSectionRectangle(PDPageContentStream contentStream, SectionRectangle sectionRectangle) throws IOException {
-
-        contentStream.setStrokingColor(Color.LIGHT_GRAY);
-        contentStream.setLineWidth(0.5f);
-        contentStream.addRect(sectionRectangle.getTopLeft().getX(), sectionRectangle.getTopLeft().getY(), sectionRectangle.getWidth(), sectionRectangle.getHeight());
-        contentStream.stroke();
-    }
-
-
-    private void drawSectionPartNumberText(PDPageContentStream contentStream, SectionRectangle sectionRectangle) throws IOException {
-
-        contentStream.beginText();
-        contentStream.setNonStrokingColor(Color.DARK_GRAY);
-        contentStream.setFont(new PDType1Font(Standard14Fonts.FontName.TIMES_ROMAN), 8f);
-        if (sectionRectangle.getTableCells() == null) {
-            contentStream.newLineAtOffset(sectionRectangle.getTopLeft().getX(), sectionRectangle.getTopLeft().getY() + sectionRectangle.getHeight());
-        } else {
-            contentStream.newLineAtOffset(sectionRectangle.getTopLeft().getX(), sectionRectangle.getTopLeft().getY());
-        }
-        contentStream.showText(sectionRectangle.getPart() + "/" + sectionRectangle.getNumberOfParts());
-        contentStream.endText();
-    }
-
-
-    private void drawTableCells(PDPageContentStream contentStream, SectionRectangle sectionRectangle) throws IOException {
-
-        if (sectionRectangle.getTableCells() != null) {
-            for (CellRectangle cell : sectionRectangle.getTableCells()) {
-                contentStream.setLineWidth(0.5f);
-                contentStream.setStrokingColor(Color.CYAN);
-                contentStream.addRect(cell.getTopLeft().getX(), cell.getTopLeft().getY(), cell.getWidth(), cell.getHeight());
-                contentStream.stroke();
-            }
-        }
-    }
-
-
-    private Map<Integer, List<RedactionLogEntry>> convertRedactionLog(RedactionLog redactionLog) {
-
-        Map<Integer, List<RedactionLogEntry>> redactionLogPerPage = new HashMap<>();
-        if (redactionLog == null) {
-            return redactionLogPerPage;
-        }
-        for (RedactionLogEntry entry : redactionLog.getRedactionLogEntry()) {
-            int page = 0;
-            for (Rectangle position : entry.getPositions()) {
-                if (position.getPage() != page) {
-                    redactionLogPerPage.computeIfAbsent(position.getPage(), x -> new ArrayList<>()).add(entry);
-                    page = position.getPage();
-                }
-            }
-        }
-        return redactionLogPerPage;
+        return redactionLogEntry.getType() + " \nRule " + redactionLogEntry.getMatchedRule() + " matched\n\n" + redactionLogEntry.getReason() + "\n\nLegal basis:" + redactionLogEntry.getLegalBasis() + "\n\nIn section: \"" + redactionLogEntry.getSection() + "\"";
     }
 
 }

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/BuildDocumentIntegrationTest.java

@@ -12,14 +12,15 @@ import org.springframework.context.annotation.Import;
 import org.springframework.context.annotation.Primary;
 import org.springframework.test.context.junit.jupiter.SpringExtension;
 
+import com.iqser.red.commons.jackson.ObjectMapperFactory;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
 import com.iqser.red.service.redaction.v1.server.AbstractRedactionIntegrationTest;
 import com.iqser.red.service.redaction.v1.server.Application;
-import com.iqser.red.service.redaction.v1.server.FileSystemBackedStorageService;
 import com.iqser.red.service.redaction.v1.server.service.document.DocumentGraphMapper;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
 import com.iqser.red.storage.commons.StorageAutoConfiguration;
 import com.iqser.red.storage.commons.service.StorageService;
+import com.iqser.red.storage.commons.utils.FileSystemBackedStorageService;
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
 import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingServiceProcessorConfiguration;
 
@@ -40,11 +41,12 @@ public class BuildDocumentIntegrationTest extends AbstractRedactionIntegrationTe
         @Primary
         public StorageService inmemoryStorage() {
 
-            return new FileSystemBackedStorageService();
+            return new FileSystemBackedStorageService(ObjectMapperFactory.create());
         }
 
     }
 
+
     @SneakyThrows
     protected Document buildGraph(String filename) {
 
@@ -64,6 +66,20 @@ public class BuildDocumentIntegrationTest extends AbstractRedactionIntegrationTe
     }
 
 
+    @SneakyThrows
+    protected Document buildGraph(String filename, String imageFileName, String tableFileName) {
+
+        String fileNameWithPdf = filename;
+        if (!filename.endsWith(".pdf")) {
+            fileNameWithPdf = filename + ".pdf";
+        }
+        AnalyzeRequest request = prepareStorage(fileNameWithPdf, tableFileName, imageFileName);
+
+        analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request);
+        return DocumentGraphMapper.toDocumentGraph(redactionStorageService.getDocumentData(request.getDossierId(), request.getFileId()));
+    }
+
+
     @SneakyThrows
     protected Document buildGraphNoImages(String filename) {
 

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentPerformanceIntegrationTest.java

@@ -103,7 +103,7 @@ public class DocumentPerformanceIntegrationTest extends BuildDocumentIntegration
 
         when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.ENTITY)).thenReturn(System.currentTimeMillis());
         when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.ENTITY)).thenReturn(JSONPrimitive.of(RULES));
-        doThrow(ExceptionProvider.getMockNotFoundFeignException()).when(rulesClient).getVersion(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.COMPONENT);
+        when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.COMPONENT)).thenReturn(-1L);
 
         loadDictionaryForTest();
         loadTypeForTest();

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/MigrationPocTest.java

@@ -1,164 +0,0 @@
-package com.iqser.red.service.redaction.v1.server.document.graph;
-
-import static java.util.stream.Collectors.toMap;
-import static org.junit.jupiter.api.Assertions.assertTrue;
-import static org.mockito.Mockito.doThrow;
-import static org.mockito.Mockito.when;
-
-import java.io.IOException;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-import java.util.stream.Stream;
-
-import org.drools.io.ClassPathResource;
-import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.Disabled;
-import org.junit.jupiter.api.Test;
-import org.springframework.beans.factory.annotation.Autowired;
-
-import com.google.common.base.Functions;
-import com.google.common.collect.Sets;
-import com.iqser.red.commons.jackson.ObjectMapperFactory;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeResult;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
-import com.iqser.red.service.redaction.v1.server.service.document.DocumentGraphMapper;
-import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
-import com.iqser.red.service.redaction.v1.server.service.document.ManualEntityCreationService;
-import com.iqser.red.service.redaction.v1.server.model.ManualEntity;
-import com.iqser.red.service.redaction.v1.server.service.RedactionLogCreatorService;
-import com.iqser.red.service.redaction.v1.server.utils.ExceptionProvider;
-import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
-import com.knecon.fforesight.tenantcommons.TenantContext;
-
-import lombok.SneakyThrows;
-
-public class MigrationPocTest extends BuildDocumentIntegrationTest {
-
-    private static final String RULES = loadFromClassPath("drools/rules.drl");
-
-    @Autowired
-    private ManualEntityCreationService redactionLogAdapter;
-    @Autowired
-    private RedactionLogCreatorService redactionLogCreatorService;
-
-
-    @BeforeEach
-    public void stubClients() {
-
-        TenantContext.setTenantId("redaction");
-        when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.ENTITY)).thenReturn(System.currentTimeMillis());
-        when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.ENTITY)).thenReturn(JSONPrimitive.of(RULES));
-        doThrow(ExceptionProvider.getMockNotFoundFeignException()).when(rulesClient).getVersion(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.COMPONENT);
-
-        loadDictionaryForTest();
-        loadTypeForTest();
-        loadNerForTest();
-        when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L);
-        when(dictionaryClient.getAllTypesForDossierTemplate(TEST_DOSSIER_TEMPLATE_ID, false)).thenReturn(getTypeResponse());
-
-        when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L);
-        when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, false)).thenReturn(List.of(Type.builder()
-                .id(DOSSIER_REDACTIONS_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID)
-                .type(DOSSIER_REDACTIONS_INDICATOR)
-                .dossierTemplateId(TEST_DOSSIER_ID)
-                .hexColor("#ffe187")
-                .isHint(hintTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
-                .isCaseInsensitive(caseInSensitiveMap.get(DOSSIER_REDACTIONS_INDICATOR))
-                .isRecommendation(recommendationTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
-                .rank(rankTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
-                .build()));
-
-        mockDictionaryCalls(null);
-
-        when(dictionaryClient.getColors(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(colors);
-    }
-
-
-    @Test
-    @Disabled // Enable if you fix the TODO in EntityCreationService
-    @SneakyThrows
-    public void testMigration() {
-
-        AnalyzeRequest request = uploadFileToStorage("files/new/crafted document.pdf");
-        analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request);
-        AnalyzeResult result = analyzeService.analyze(request);
-        var newRedactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID);
-
-        var originalRedactionLog = getOriginalRedactionLog();
-
-        // IMPORTANT: always use the graph which is mapped from the DocumentData, since rounding errors occur during storage.
-        Document document = DocumentGraphMapper.toDocumentGraph(redactionStorageService.getDocumentData(request.getDossierId(), request.getFileId()));
-        List<ManualEntity> notFoundManualRedactionEntries = redactionLogAdapter.toRedactionEntity(originalRedactionLog, document);
-
-        var migratedRedactionLogEntries = redactionLogCreatorService.createRedactionLog(document, TEST_DOSSIER_TEMPLATE_ID, notFoundManualRedactionEntries, Collections.emptyMap());
-
-        Map<String, RedactionLogEntry> migratedIds = migratedRedactionLogEntries.stream().collect(toMap(RedactionLogEntry::getId, Functions.identity()));
-        Map<String, RedactionLogEntry> newIds = newRedactionLog.getRedactionLogEntry().stream().collect(toMap(RedactionLogEntry::getId, Functions.identity()));
-
-        logPrecision(migratedIds, newIds);
-        logRecall(migratedIds, newIds);
-    }
-
-
-    private static void logPrecision(Map<String, RedactionLogEntry> migratedIds, Map<String, RedactionLogEntry> newIds) {
-
-        var precision = computePrecision(migratedIds, newIds);
-        System.out.printf("precision %.2f\n", precision);
-        System.out.println("New Entries");
-        getAddedEntries(migratedIds, newIds).forEach(System.out::println);
-        assertTrue(precision >= 0.85);
-        System.out.println();
-    }
-
-
-    private static void logRecall(Map<String, RedactionLogEntry> migratedIds, Map<String, RedactionLogEntry> newIds) {
-
-        var recall = computeRecall(migratedIds, newIds);
-        System.out.printf("recall %.2f\n", recall);
-        System.out.println("Missing entries");
-        getMissingEntries(migratedIds, newIds).forEach(System.out::println);
-        assertTrue(recall >= 0.85);
-        System.out.println();
-    }
-
-
-    private static Stream<RedactionLogEntry> getMissingEntries(Map<String, RedactionLogEntry> migratedIds, Map<String, RedactionLogEntry> newIds) {
-
-        return migratedIds.entrySet().stream().filter(entry -> !newIds.containsKey(entry.getKey())).map(Map.Entry::getValue);
-    }
-
-
-    private static Stream<RedactionLogEntry> getAddedEntries(Map<String, RedactionLogEntry> migratedIds, Map<String, RedactionLogEntry> newIds) {
-
-        return newIds.entrySet().stream().filter(entry -> !migratedIds.containsKey(entry.getKey())).map(Map.Entry::getValue);
-    }
-
-
-    private static double computePrecision(Map<String, RedactionLogEntry> migratedIds, Map<String, RedactionLogEntry> newIds) {
-
-        return (double) Sets.intersection(newIds.keySet(), migratedIds.keySet()).size() / (double) newIds.size();
-    }
-
-
-    private static double computeRecall(Map<String, RedactionLogEntry> migratedIds, Map<String, RedactionLogEntry> newIds) {
-
-        return (double) Sets.intersection(newIds.keySet(), migratedIds.keySet()).size() / (double) migratedIds.size();
-    }
-
-
-    private static RedactionLog getOriginalRedactionLog() throws IOException {
-
-        RedactionLog originalRedactionLog;
-        try (var inputStream = new ClassPathResource("files/migration/legacy_redactionlog.json").getInputStream()) {
-            originalRedactionLog = ObjectMapperFactory.create().readValue(inputStream, RedactionLog.class);
-        }
-        return originalRedactionLog;
-    }
-
-}

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/drools/files/management/services/DroolsSyntaxValidationServiceTest.java

@@ -9,6 +9,7 @@ import java.nio.charset.StandardCharsets;
 import java.util.List;
 
 import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.Test;
 import org.mockito.MockitoAnnotations;
 import org.springframework.boot.test.mock.mockito.MockBean;
@@ -57,12 +58,28 @@ class DroolsSyntaxValidationServiceTest {
     }
 
 
+    @Test
+    @SneakyThrows
+    void testRulesWithRemovedImportsAndChangedName() {
+
+        DroolsSyntaxValidationService droolsSyntaxValidationService = new DroolsSyntaxValidationService(new KieContainerCreationService(rulesClient));
+        var rulesFile = new ClassPathResource("drools/rules.drl");
+
+        String rulesString = new String(rulesFile.getInputStream().readAllBytes());
+        rulesString = rulesString.replaceAll("import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell;", "");
+        rulesString = rulesString.replaceAll("rule \"LDS.0.0: Run local dictionary search\"", "rule \"LDS.0.0: run local dictionary search\"");
+        DroolsSyntaxValidation droolsSyntaxValidation = droolsSyntaxValidationService.testRules(new RuleValidationModel(RuleFileType.ENTITY.name(), rulesString));
+        droolsSyntaxValidation.getDroolsSyntaxErrorMessages().forEach(System.out::println);
+        assertTrue(droolsSyntaxValidation.isCompiled());
+    }
+
+
     @Test
     @SneakyThrows
     void testAllRules() {
 
         DroolsSyntaxValidationService droolsSyntaxValidationService = new DroolsSyntaxValidationService(new KieContainerCreationService(rulesClient));
-        var rulesFile = new ClassPathResource("drools/all_rules.drl");
+        var rulesFile = new ClassPathResource("drools/all_redact_manager_rules.drl");
 
         String rulesString = new String(rulesFile.getInputStream().readAllBytes());
 
@@ -155,7 +172,8 @@ class DroolsSyntaxValidationServiceTest {
         var rulesFile = new ClassPathResource("drools/rules.drl");
 
         String rulesString = new String(rulesFile.getInputStream().readAllBytes());
-        String corruptedRules = rulesString.replaceAll("import com.iqser.red.service.redaction.v1.server.model.document.nodes.ImageType;", "");
+        String corruptedRules = rulesString.replaceAll("import com.iqser.red.service.redaction.v1.server.model.document.nodes.ImageType;",
+                "import com.iqser.red.service.redaction.v1.server.model.document.nodes.ImageType;\nimport com.iqser.red.service.redaction.v1.server.model.document.nodes.SomethingElse;");
 
         DroolsSyntaxValidation droolsSyntaxValidation = droolsSyntaxValidationService.testRules(new RuleValidationModel(RuleFileType.ENTITY.name(), corruptedRules));
         assertFalse(droolsSyntaxValidation.isCompiled());
@@ -210,12 +228,12 @@ class DroolsSyntaxValidationServiceTest {
 
     @Test
     @SneakyThrows
+    @Disabled
     void attemptImportsFixToAllRuleFiles() {
 
         DroolsSyntaxValidationService droolsSyntaxValidationService = new DroolsSyntaxValidationService(new KieContainerCreationService(rulesClient));
 
-        List<String> ruleFiles = List.of("drools/rules.drl",
-                "drools/all_rules.drl",
+        List<String> ruleFiles = List.of("drools/rules.drl", "drools/all_redact_manager_rules.drl",
                 "drools/documine_flora.drl",
                 "drools/manual_redaction_rules.drl",
                 "drools/acceptance_rules.drl",
@@ -229,7 +247,7 @@ class DroolsSyntaxValidationServiceTest {
                 continue;
             }
             RuleFileBluePrint ruleFileBluePrint = RuleFileParser.buildBluePrintFromRulesString(rulesString);
-            RuleFileBluePrint baseRuleFileBluePrint = RuleFileParser.buildBluePrintFromRulesString(RuleManagementResources.getBaseComponentRuleFileString());
+            RuleFileBluePrint baseRuleFileBluePrint = RuleFileParser.buildBluePrintFromRulesString(RuleManagementResources.getBaseRuleFileString());
 
             rulesString = rulesString.replace(ruleFileBluePrint.getImports(), baseRuleFileBluePrint.getImports());
             rulesString = rulesString.replace(ruleFileBluePrint.getGlobals(), baseRuleFileBluePrint.getGlobals());

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesEnd2EndTest.java

@@ -2,9 +2,8 @@ package com.iqser.red.service.redaction.v1.server.manualchanges;
 
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertFalse;
-import static org.mockito.Mockito.doThrow;
+import static org.junit.jupiter.api.Assertions.assertNotEquals;
 import static org.mockito.Mockito.when;
-import static org.wildfly.common.Assert.assertTrue;
 
 import java.awt.geom.Rectangle2D;
 import java.io.File;
@@ -32,39 +31,40 @@ import org.springframework.context.annotation.Import;
 import org.springframework.context.annotation.Primary;
 import org.springframework.test.context.junit.jupiter.SpringExtension;
 
+import com.iqser.red.commons.jackson.ObjectMapperFactory;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeResult;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Comment;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ChangeType;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Point;
 import com.iqser.red.service.redaction.v1.server.AbstractRedactionIntegrationTest;
 import com.iqser.red.service.redaction.v1.server.Application;
-import com.iqser.red.service.redaction.v1.server.FileSystemBackedStorageService;
 import com.iqser.red.service.redaction.v1.server.annotate.AnnotateRequest;
 import com.iqser.red.service.redaction.v1.server.annotate.AnnotateResponse;
-import com.iqser.red.service.redaction.v1.server.service.document.DocumentGraphMapper;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
+import com.iqser.red.service.redaction.v1.server.redaction.utils.OsUtils;
+import com.iqser.red.service.redaction.v1.server.service.document.DocumentGraphMapper;
 import com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService;
 import com.iqser.red.service.redaction.v1.server.service.document.EntityEnrichmentService;
-import com.iqser.red.service.redaction.v1.server.redaction.utils.OsUtils;
-import com.iqser.red.service.redaction.v1.server.utils.ExceptionProvider;
 import com.iqser.red.storage.commons.StorageAutoConfiguration;
 import com.iqser.red.storage.commons.service.StorageService;
+import com.iqser.red.storage.commons.utils.FileSystemBackedStorageService;
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
 import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingServiceProcessorConfiguration;
 import com.knecon.fforesight.tenantcommons.TenantContext;
@@ -76,7 +76,8 @@ import lombok.SneakyThrows;
 @Import(ManualChangesEnd2EndTest.TestConfiguration.class)
 public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest {
 
-    private static final String RULES = loadFromClassPath("drools/rules.drl");
+    private static final String RULES = loadFromClassPath("drools/acceptance_rules.drl");
+    private static final String DM_RULES = loadFromClassPath("drools/documine_flora.drl");
     @Autowired
     private EntityEnrichmentService entityEnrichmentService;
 
@@ -92,7 +93,7 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest {
         @Primary
         public StorageService inmemoryStorage() {
 
-            return new FileSystemBackedStorageService();
+            return new FileSystemBackedStorageService(ObjectMapperFactory.create());
         }
 
     }
@@ -112,8 +113,7 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest {
 
         when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.ENTITY)).thenReturn(System.currentTimeMillis());
         when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.ENTITY)).thenReturn(JSONPrimitive.of(RULES));
-        doThrow(ExceptionProvider.getMockNotFoundFeignException()).when(rulesClient).getVersion(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.COMPONENT);
-
+        when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.COMPONENT)).thenReturn(-1L);
 
         loadDictionaryForTest();
         loadTypeForTest();
@@ -150,18 +150,18 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest {
 
         String testEntityValue1 = "Desiree";
         String testEntityValue2 = "Melanie";
-        RedactionLog redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID);
-        assertEquals(2, redactionLog.getRedactionLogEntry().stream().filter(entry -> entry.getValue().equals(testEntityValue1)).count());
-        assertEquals(2, redactionLog.getRedactionLogEntry().stream().filter(entry -> entry.getValue().equals(testEntityValue2)).count());
+        EntityLog redactionLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
+        assertEquals(2, redactionLog.getEntityLogEntry().stream().filter(entry -> entry.getValue().equals(testEntityValue1)).count());
+        assertEquals(2, redactionLog.getEntityLogEntry().stream().filter(entry -> entry.getValue().equals(testEntityValue2)).count());
 
         Document document = DocumentGraphMapper.toDocumentGraph(redactionStorageService.getDocumentData(TEST_DOSSIER_ID, TEST_FILE_ID));
         String expandedEntityKeyword = "Lorem ipsum dolor sit amet, consectetur adipiscing elit Desiree et al sed do eiusmod tempor incididunt ut labore et dolore magna aliqua Melanie et al. Reference No 12345 Lorem ipsum.";
         TextEntity expandedEntity = entityCreationService.byString(expandedEntityKeyword, "PII", EntityType.ENTITY, document).findFirst().get();
 
-        String idToResize = redactionLog.getRedactionLogEntry()
+        String idToResize = redactionLog.getEntityLogEntry()
                 .stream()
                 .filter(entry -> entry.getValue().equals(testEntityValue1))
-                .max(Comparator.comparingInt(RedactionLogEntry::getStartOffset))
+                .max(Comparator.comparingInt(EntityLogEntry::getStartOffset))
                 .get()
                 .getId();
         List<Rectangle> resizedPositions = expandedEntity.getPositionsOnPagePerPage()
@@ -176,25 +176,26 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest {
                 .positions(resizedPositions)
                 .requestDate(OffsetDateTime.now())
                 .status(AnnotationStatus.APPROVED)
+                .updateDictionary(false)
                 .build();
-
+        manualResizeRedaction.setUpdateDictionary(false);
         ManualRedactions manualRedactions = new ManualRedactions();
         manualRedactions.getResizeRedactions().add(manualResizeRedaction);
         request.setManualRedactions(manualRedactions);
         AnalyzeResult reanalyzeResult = analyzeService.reanalyze(request);
 
-        redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID);
+        redactionLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
         String annotatedFileName = Paths.get(filePath).getFileName().toString().replace(".pdf", "_annotated2.pdf");
         File tmpFile = Paths.get(OsUtils.getTemporaryDirectory(), annotatedFileName).toFile();
         AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder().dossierId(TEST_DOSSIER_ID).fileId(TEST_FILE_ID).build());
         try (FileOutputStream fileOutputStream = new FileOutputStream(tmpFile)) {
             fileOutputStream.write(annotateResponse.getDocument());
         }
-        RedactionLogEntry resizedEntry = redactionLog.getRedactionLogEntry().stream().filter(entry -> entry.getValue().equals(expandedEntityKeyword)).findFirst().get();
-        assertTrue(resizedEntry.getChanges().get(resizedEntry.getChanges().size() - 1).getType().equals(ChangeType.CHANGED));
+        EntityLogEntry resizedEntry = redactionLog.getEntityLogEntry().stream().filter(entry -> entry.getValue().equals(expandedEntityKeyword)).findFirst().get();
         assertEquals(idToResize, resizedEntry.getId());
-        assertEquals(1, redactionLog.getRedactionLogEntry().stream().filter(entry -> entry.getValue().equals(testEntityValue1)).count());
-        assertEquals(1, redactionLog.getRedactionLogEntry().stream().filter(entry -> entry.getValue().equals(testEntityValue2) && !entry.lastChangeIsRemoved()).count());
+        assertEquals(1, redactionLog.getEntityLogEntry().stream().filter(entry -> entry.getValue().equals(testEntityValue1)).count());
+        assertEquals(1,
+                redactionLog.getEntityLogEntry().stream().filter(entry -> entry.getValue().equals(testEntityValue2) && !entry.getState().equals(EntryState.REMOVED)).count());
     }
 
 
@@ -219,7 +220,6 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest {
 
         String manualAddId = UUID.randomUUID().toString();
 
-        Comment comment = Comment.builder().date(OffsetDateTime.now()).user("TEST_USER").text("This is a comment test").build();
         manualRedactions.setIdsToRemove(Set.of(IdRemoval.builder().annotationId("5b940b2cb401ed9f5be6fc24f6e77bcf").fileId("fileId").status(AnnotationStatus.DECLINED).build()));
         manualRedactions.setForceRedactions(Set.of(ManualForceRedaction.builder()
                 .annotationId("675eba69b0c2917de55462c817adaa05")
@@ -228,10 +228,6 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest {
                 .status(AnnotationStatus.APPROVED)
                 .build()));
 
-        manualRedactions.getComments().put("e5be0f1d941bbb92a068e198648d06c4", List.of(comment));
-        manualRedactions.getComments().put("0836727c3508a0b2ea271da69c04cc2f", List.of(comment));
-        manualRedactions.getComments().put(manualAddId, List.of(comment));
-
         ManualRedactionEntry manualRedactionEntry = new ManualRedactionEntry();
         manualRedactionEntry.setAnnotationId(manualAddId);
         manualRedactionEntry.setFileId("fileId");
@@ -259,7 +255,7 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest {
 
         analyzeService.reanalyze(request);
 
-        var redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID);
+        var redactionLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
 
         AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder().dossierId(TEST_DOSSIER_ID).fileId(TEST_FILE_ID).build());
 
@@ -280,24 +276,25 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest {
         AnalyzeRequest request = uploadFileToStorage(filePath);
         analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request);
         AnalyzeResult result = analyzeService.analyze(request);
-        RedactionLog redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID);
+        EntityLog redactionLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
 
-        var oxfordUniversityPress = redactionLog.getRedactionLogEntry()
+        var oxfordUniversityPress = redactionLog.getEntityLogEntry()
                 .stream()
                 .filter(entry -> entry.getType().equals("published_information"))
                 .filter(entry -> entry.getValue().equals("Oxford University Press"))
                 .findFirst()
                 .get();
 
-        var asyaLyon = redactionLog.getRedactionLogEntry()
+        var asyaLyon = redactionLog.getEntityLogEntry()
                 .stream()
                 .filter(entry -> entry.getType().equals("CBI_author"))
                 .filter(entry -> entry.getValue().equals("Asya Lyon"))
                 .findFirst()
                 .get();
 
-        assertEquals("CBI.3.2", asyaLyon.getMatchedRule());
-        assertEquals("No vertebrate found", asyaLyon.getReason());
+        assertEquals("CBI.7.0", asyaLyon.getMatchedRule());
+        assertEquals("Published Information found in section", asyaLyon.getReason());
+        assertNotEquals(asyaLyon.getState(), EntryState.APPLIED);
 
         ManualRecategorization recategorization = ManualRecategorization.builder()
                 .requestDate(OffsetDateTime.now())
@@ -311,20 +308,71 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest {
         request.getManualRedactions().setRecategorizations(Set.of(recategorization));
 
         analyzeService.reanalyze(request);
-        RedactionLog redactionLog2 = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID);
-        assertFalse(redactionLog2.getRedactionLogEntry()
+
+        EntityLog redactionLog2 = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
+        assertFalse(redactionLog2.getEntityLogEntry()
                 .stream()
                 .filter(entry -> entry.getType().equals("published_information"))
                 .anyMatch(entry -> entry.getValue().equals("Oxford University Press")));
 
-        var oxfordUniversityPressRecategorized = redactionLog2.getRedactionLogEntry()
+        var oxfordUniversityPressRecategorized = redactionLog2.getEntityLogEntry()
                 .stream()
                 .filter(entry -> entry.getType().equals("vertebrate"))
                 .filter(entry -> entry.getValue().equals("Oxford University Press"))
                 .findFirst()
                 .get();
 
+        var asyaLyon2 = redactionLog2.getEntityLogEntry()
+                .stream()
+                .filter(entry -> entry.getType().equals("CBI_author"))
+                .filter(entry -> entry.getValue().equals("Asya Lyon"))
+                .findFirst()
+                .get();
+
+        assertEquals(asyaLyon2.getState(), EntryState.APPLIED);
         assertEquals(1, oxfordUniversityPressRecategorized.getManualChanges().size());
     }
 
+
+    @Test
+    @SneakyThrows
+    public void testShrinkingResize() {
+
+        String filePath = "files/new/crafted document.pdf";
+        AnalyzeRequest request = uploadFileToStorage(filePath);
+        analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request);
+        AnalyzeResult result = analyzeService.analyze(request);
+        ManualRedactions manualRedactions = new ManualRedactions();
+        String annotationId = "testAnnotationId";
+
+        manualRedactions.setEntriesToAdd(Set.of(ManualRedactionEntry.builder()
+                .annotationId(annotationId)
+                .requestDate(OffsetDateTime.now())
+                .value("Expand to Hint Clarissa’s Donut ← not added to Dict, should be not annotated Simpson's Tower ← added to Authors-Dict, should be annotated")
+                .positions(List.of(//
+                        new Rectangle(new Point(56.8f, 496.27f), 61.25f, 12.83f, 2), //
+                        new Rectangle(new Point(56.8f, 482.26f), 303.804f, 15.408f, 2), //
+                        new Rectangle(new Point(56.8f, 468.464f), 314.496f, 15.408f, 2))) //
+                .status(AnnotationStatus.APPROVED)
+                .build()));
+        ManualResizeRedaction manualResizeRedaction = ManualResizeRedaction.builder()
+                .annotationId(annotationId)
+                .requestDate(OffsetDateTime.now())
+                .value("Expand to Hint")
+                .positions(List.of(new Rectangle(new Point(56.8f, 496.27f), 61.25f, 12.83f, 2)))
+                .status(AnnotationStatus.APPROVED)
+                .updateDictionary(false)
+                .build();
+        manualRedactions.setResizeRedactions(Set.of(manualResizeRedaction));
+        request.setManualRedactions(manualRedactions);
+
+        analyzeService.reanalyze(request);
+
+        EntityLog entityLog = redactionStorageService.getEntityLog(request.getDossierId(), request.getFileId());
+        EntityLogEntry entityLogEntry = entityLog.getEntityLogEntry().stream().filter(entry -> entry.getId().equals(annotationId)).findFirst().orElseThrow();
+        assertEquals("Expand to Hint", entityLogEntry.getValue());
+        assertEquals(1, entityLogEntry.getPositions().size());
+        assertEquals(ManualRedactionType.RESIZE, entityLogEntry.getManualChanges().get(entityLogEntry.getManualChanges().size() - 1).getManualRedactionType());
+    }
+
 }

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesIntegrationTest.java

@@ -7,7 +7,6 @@ import static org.wildfly.common.Assert.assertFalse;
 import java.awt.geom.Rectangle2D;
 import java.time.OffsetDateTime;
 import java.util.Collection;
-import java.util.Comparator;
 import java.util.List;
 import java.util.Set;
 import java.util.stream.Collectors;
@@ -31,18 +30,16 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
 import com.iqser.red.service.redaction.v1.server.document.graph.BuildDocumentIntegrationTest;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
-import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.PositionOnPage;
+import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Paragraph;
+import com.iqser.red.service.redaction.v1.server.service.ManualChangesApplicationService;
 import com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService;
 import com.iqser.red.service.redaction.v1.server.service.document.EntityEnrichmentService;
-import com.iqser.red.service.redaction.v1.server.service.ManualChangesApplicationService;
-import com.iqser.red.service.redaction.v1.server.utils.exception.NotFoundException;
 
 @Import(ManualChangesIntegrationTest.TestConfiguration.class)
 public class ManualChangesIntegrationTest extends BuildDocumentIntegrationTest {
@@ -108,9 +105,9 @@ public class ManualChangesIntegrationTest extends BuildDocumentIntegrationTest {
                 .positions(toAnnotationRectangles(biggerEntity.getPositionsOnPagePerPage().get(0)))
                 .status(AnnotationStatus.APPROVED)
                 .requestDate(OffsetDateTime.now())
+                .updateDictionary(false)
                 .build();
 
-        
         kieSession.insert(document);
         document.streamAllSubNodes().forEach(kieSession::insert);
         kieSession.insert(manualResizeRedaction);
@@ -144,7 +141,6 @@ public class ManualChangesIntegrationTest extends BuildDocumentIntegrationTest {
                 .requestDate(OffsetDateTime.now())
                 .build();
 
-        
         kieSession.insert(manualForceRedaction);
         kieSession.insert(document);
         document.streamAllSubNodes().forEach(kieSession::insert);
@@ -174,7 +170,6 @@ public class ManualChangesIntegrationTest extends BuildDocumentIntegrationTest {
         String initialId = entity.getPositionsOnPagePerPage().get(0).getId();
         IdRemoval idRemoval = IdRemoval.builder().annotationId(initialId).status(AnnotationStatus.APPROVED).requestDate(OffsetDateTime.now()).build();
 
-        
         kieSession.insert(document);
         document.streamAllSubNodes().forEach(kieSession::insert);
         kieSession.insert(idRemoval);
@@ -204,7 +199,6 @@ public class ManualChangesIntegrationTest extends BuildDocumentIntegrationTest {
                 .requestDate(OffsetDateTime.now())
                 .build();
 
-        
         kieSession.insert(document);
         document.streamAllSubNodes().forEach(kieSession::insert);
         kieSession.insert(idRemoval);
@@ -233,7 +227,6 @@ public class ManualChangesIntegrationTest extends BuildDocumentIntegrationTest {
         String initialId = entity.getPositionsOnPagePerPage().get(0).getId();
         IdRemoval idRemoval = IdRemoval.builder().annotationId(initialId).status(AnnotationStatus.REQUESTED).build();
 
-        
         kieSession.insert(idRemoval);
         kieSession.insert(document);
         document.streamAllSubNodes().forEach(kieSession::insert);
@@ -250,51 +243,6 @@ public class ManualChangesIntegrationTest extends BuildDocumentIntegrationTest {
     }
 
 
-    @Test
-    public void testManualRecategorizeRoundTrip() {
-
-        Document document = buildGraph("files/new/crafted document");
-        Set<TextEntity> entities = entityCreationService.byString("David Ksenia", "CBI_author", EntityType.ENTITY, document).collect(Collectors.toUnmodifiableSet());
-
-        TextEntity entity = entities.stream()
-                .min(Comparator.comparingInt(textEntity -> textEntity.getTextRange().start()))
-                .orElseThrow(() -> new NotFoundException("no entity found!"));
-        String originalType = entity.getType();
-        String initialId = entity.getPositionsOnPagePerPage().get(0).getId();
-
-        OffsetDateTime start = OffsetDateTime.now();
-        ManualRecategorization recategorization = ManualRecategorization.builder()
-                .type("any other type")
-                .annotationId(initialId)
-                .status(AnnotationStatus.APPROVED)
-                .requestDate(start)
-                .build();
-        ManualRecategorization recategorization2 = ManualRecategorization.builder()
-                .type(originalType)
-                .annotationId(initialId)
-                .status(AnnotationStatus.APPROVED)
-                .requestDate(start.plusSeconds(1))
-                .build();
-
-        
-        kieSession.insert(recategorization);
-        kieSession.insert(recategorization2);
-        kieSession.insert(document);
-        document.streamAllSubNodes().forEach(kieSession::insert);
-        kieSession.fireAllRules();
-        kieSession.dispose();
-
-        var recategorizedEntity = document.getEntities()
-                .stream()
-                .filter(entity2 -> entity2.matchesAnnotationId(initialId))
-                .findFirst()
-                .orElseThrow(() -> new NotFoundException("recategorized entity not found!"));
-
-        assertEquals(originalType, recategorizedEntity.getManualOverwrite().getType().orElse(recategorizedEntity.getType()));
-
-    }
-
-
     private void assertRectanglesAlmostEqual(Collection<Rectangle2D> rects1, Collection<Rectangle2D> rects2) {
 
         if (rects1.stream().allMatch(rect1 -> rects2.stream().anyMatch(rect2 -> rectanglesAlmostEqual(rect1, rect2)))) {
@@ -323,7 +271,7 @@ public class ManualChangesIntegrationTest extends BuildDocumentIntegrationTest {
 
     private static Rectangle toAnnotationRectangle(Rectangle2D rectangle2D, Integer number) {
 
-        return new Rectangle((float) rectangle2D.getMaxX(), (float) rectangle2D.getMaxY(), (float) rectangle2D.getWidth(), (float) rectangle2D.getHeight(), number);
+        return new Rectangle((float) rectangle2D.getMinX(), (float) rectangle2D.getMinY(), (float) rectangle2D.getWidth(), (float) rectangle2D.getHeight(), number);
     }
 
 }

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualEntityCreationServiceTest.java

@@ -1,12 +1,12 @@
 package com.iqser.red.service.redaction.v1.server.manualchanges;
 
 import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.Mockito.when;
 import static org.wildfly.common.Assert.assertFalse;
 import static org.wildfly.common.Assert.assertTrue;
 
 import java.awt.geom.Rectangle2D;
-import java.util.Collections;
 import java.util.List;
 import java.util.Set;
 
@@ -16,20 +16,22 @@ import org.mockito.MockitoAnnotations;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.boot.test.mock.mockito.MockBean;
 
+import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Point;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
 import com.iqser.red.service.redaction.v1.server.document.graph.BuildDocumentIntegrationTest;
+import com.iqser.red.service.redaction.v1.server.model.ManualEntity;
+import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryVersion;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
+import com.iqser.red.service.redaction.v1.server.service.DictionaryService;
+import com.iqser.red.service.redaction.v1.server.service.EntityLogCreatorService;
 import com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService;
 import com.iqser.red.service.redaction.v1.server.service.document.EntityEnrichmentService;
 import com.iqser.red.service.redaction.v1.server.service.document.ManualEntityCreationService;
-import com.iqser.red.service.redaction.v1.server.model.ManualEntity;
-import com.iqser.red.service.redaction.v1.server.service.DictionaryService;
-import com.iqser.red.service.redaction.v1.server.service.RedactionLogCreatorService;
 
 import lombok.SneakyThrows;
 
@@ -42,7 +44,7 @@ public class ManualEntityCreationServiceTest extends BuildDocumentIntegrationTes
     private ManualEntityCreationService manualEntityCreationService;
 
     @Autowired
-    private RedactionLogCreatorService redactionLogCreatorService;
+    private EntityLogCreatorService entityLogCreatorService;
 
     @MockBean
     private DictionaryService dictionaryService;
@@ -53,6 +55,7 @@ public class ManualEntityCreationServiceTest extends BuildDocumentIntegrationTes
 
         MockitoAnnotations.openMocks(this);
         when(dictionaryService.getColor(DICTIONARY_AUTHOR, TEST_DOSSIER_TEMPLATE_ID)).thenReturn(new float[]{0f, 0f, 0f});
+        when(dictionaryService.isHint(any(), any())).thenReturn(false);
     }
 
 
@@ -88,7 +91,9 @@ public class ManualEntityCreationServiceTest extends BuildDocumentIntegrationTes
         tempEntity.removeFromGraph();
         assertTrue(document.getEntities().isEmpty());
 
-        List<ManualEntity> notFoundManualEntities = manualEntityCreationService.createRedactionEntitiesIfFoundAndReturnNotFoundEntries(Set.of(manualRedactionEntry), document);
+        List<ManualEntity> notFoundManualEntities = manualEntityCreationService.createRedactionEntitiesIfFoundAndReturnNotFoundEntries(Set.of(manualRedactionEntry),
+                document,
+                TEST_DOSSIER_TEMPLATE_ID);
         assertTrue(notFoundManualEntities.isEmpty());
         assertEquals(1, document.getEntities().size());
     }
@@ -116,14 +121,17 @@ public class ManualEntityCreationServiceTest extends BuildDocumentIntegrationTes
 
         assertTrue(document.getEntities().isEmpty());
 
-        List<ManualEntity> notFoundManualEntities = manualEntityCreationService.createRedactionEntitiesIfFoundAndReturnNotFoundEntries(Set.of(manualRedactionEntry), document);
+        List<ManualEntity> notFoundManualEntities = manualEntityCreationService.createRedactionEntitiesIfFoundAndReturnNotFoundEntries(Set.of(manualRedactionEntry),
+                document,
+                TEST_DOSSIER_TEMPLATE_ID);
         assertEquals(1, notFoundManualEntities.size());
         assertTrue(document.getEntities().isEmpty());
 
-        List<RedactionLogEntry> redactionLogEntries = redactionLogCreatorService.createRedactionLog(document,
-                TEST_DOSSIER_TEMPLATE_ID,
+        List<EntityLogEntry> redactionLogEntries = entityLogCreatorService.createInitialEntityLog(new AnalyzeRequest(),
+                document,
                 notFoundManualEntities,
-                Collections.emptyMap());
+                new DictionaryVersion(),
+                0L).getEntityLogEntry();
 
         assertEquals(1, redactionLogEntries.size());
         assertEquals(value, redactionLogEntries.get(0).getValue());

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/realdata/LiveDataIntegrationTest.java

@@ -37,13 +37,13 @@ import org.springframework.test.context.junit.jupiter.SpringExtension;
 
 import com.fasterxml.jackson.core.type.TypeReference;
 import com.fasterxml.jackson.databind.ObjectMapper;
+import com.iqser.red.commons.jackson.ObjectMapperFactory;
 import com.iqser.red.service.dictionarymerge.commons.DictionaryEntry;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.configuration.Colors;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type;
 import com.iqser.red.service.redaction.v1.server.Application;
-import com.iqser.red.service.redaction.v1.server.FileSystemBackedStorageService;
 import com.iqser.red.service.redaction.v1.server.RedactionServiceSettings;
 import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
 import com.iqser.red.service.redaction.v1.server.client.FileStatusProcessingUpdateClient;
@@ -55,6 +55,7 @@ import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService
 import com.iqser.red.service.redaction.v1.server.utils.ExceptionProvider;
 import com.iqser.red.storage.commons.StorageAutoConfiguration;
 import com.iqser.red.storage.commons.service.StorageService;
+import com.iqser.red.storage.commons.utils.FileSystemBackedStorageService;
 import com.knecon.fforesight.tenantcommons.TenantContext;
 import com.knecon.fforesight.tenantcommons.TenantsClient;
 
@@ -105,9 +106,6 @@ public class LiveDataIntegrationTest {
     @Autowired
     protected RedactionMessageReceiver redactionMessageReceiver;
 
-    @Autowired
-    protected FileSystemBackedStorageService fileSystemBackedStorageService;
-
     @Autowired
     protected RedactionServiceSettings redactionServiceSettings;
 
@@ -125,7 +123,7 @@ public class LiveDataIntegrationTest {
         @Primary
         public StorageService inmemoryStorage() {
 
-            return new FileSystemBackedStorageService();
+            return new FileSystemBackedStorageService(ObjectMapperFactory.create());
         }
 
     }

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/service/document/EntityComparatorsTest.java

@@ -0,0 +1,23 @@
+package com.iqser.red.service.redaction.v1.server.service.document;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+import java.util.stream.Stream;
+
+import org.junit.jupiter.api.Test;
+
+import com.iqser.red.service.redaction.v1.server.model.component.Entity;
+
+class EntityComparatorsTest {
+
+    @Test
+    public void testFirstEntity() {
+
+        Entity entity1 = Entity.builder().startOffset(0).build();
+        Entity entity2 = Entity.builder().startOffset(2).build();
+        Entity entity3 = Entity.builder().startOffset(20).build();
+        assertEquals(entity1, Stream.of(entity2, entity3, entity1).min(EntityComparators.first()).orElseThrow());
+
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/service/document/SemanticNodeComparatorsTest.java

@@ -0,0 +1,67 @@
+package com.iqser.red.service.redaction.v1.server.service.document;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import org.junit.jupiter.api.Test;
+
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.Section;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
+
+class SemanticNodeComparatorsTest {
+
+    @Test
+    public void testFirstSemanticNode() {
+
+        var node = new Section(List.of(0, 1), null, null, null);
+        var otherNode = new Section(List.of(0, 2), null, null, null);
+        List<SemanticNode> list = new ArrayList<>();
+        list.add(otherNode);
+        list.add(node);
+        list.sort(SemanticNodeComparators.first());
+        assertEquals(node, list.get(0));
+    }
+
+
+    @Test
+    public void testFirstSemanticNode2() {
+
+        var node = new Section(Collections.emptyList(), null, null, null);
+        var otherNode = new Section(List.of(0, 2), null, null, null);
+        List<SemanticNode> list = new ArrayList<>();
+        list.add(otherNode);
+        list.add(node);
+        list.sort(SemanticNodeComparators.first());
+        assertEquals(node, list.get(0));
+    }
+
+
+    @Test
+    public void testFirstSemanticNode3() {
+
+        var node = new Section(List.of(1, 5, 8), null, null, null);
+        var otherNode = new Section(List.of(0, 2), null, null, null);
+        List<SemanticNode> list = new ArrayList<>();
+        list.add(otherNode);
+        list.add(node);
+        list.sort(SemanticNodeComparators.first());
+        assertEquals(otherNode, list.get(0));
+    }
+
+
+    @Test
+    public void testFirstSemanticNode4() {
+
+        var node = new Section(List.of(1, 5, 8), null, null, null);
+        var otherNode = new Section(List.of(1, 5, 9), null, null, null);
+        List<SemanticNode> list = new ArrayList<>();
+        list.add(otherNode);
+        list.add(node);
+        list.sort(SemanticNodeComparators.first());
+        assertEquals(node, list.get(0));
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/service/document/UnprocessedChangesServiceTest.java

@@ -0,0 +1,172 @@
+package com.iqser.red.service.redaction.v1.server.service.document;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.Mockito.when;
+
+import java.util.List;
+import java.util.Optional;
+import java.util.UUID;
+
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
+import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration;
+import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.boot.test.mock.mockito.MockBean;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.ComponentScan;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.context.annotation.FilterType;
+import org.springframework.context.annotation.Import;
+import org.springframework.context.annotation.Primary;
+import org.springframework.test.context.junit.jupiter.SpringExtension;
+
+import com.iqser.red.commons.jackson.ObjectMapperFactory;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
+import com.iqser.red.service.redaction.v1.model.UnprocessedManualEntity;
+import com.iqser.red.service.redaction.v1.server.AbstractRedactionIntegrationTest;
+import com.iqser.red.service.redaction.v1.server.Application;
+import com.iqser.red.service.redaction.v1.server.RedactionIntegrationTest;
+import com.iqser.red.service.redaction.v1.server.service.DictionaryService;
+import com.iqser.red.service.redaction.v1.server.service.UnprocessedChangesService;
+import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
+import com.iqser.red.storage.commons.StorageAutoConfiguration;
+import com.iqser.red.storage.commons.service.StorageService;
+import com.iqser.red.storage.commons.utils.FileSystemBackedStorageService;
+import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
+import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingServiceProcessorConfiguration;
+
+@ExtendWith(SpringExtension.class)
+@SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
+@Import(RedactionIntegrationTest.RedactionIntegrationTestConfiguration.class)
+public class UnprocessedChangesServiceTest extends AbstractRedactionIntegrationTest {
+
+    @Configuration
+    @EnableAutoConfiguration(exclude = {RabbitAutoConfiguration.class})
+    @Import({LayoutParsingServiceProcessorConfiguration.class})
+    @ComponentScan(excludeFilters = {@ComponentScan.Filter(type = FilterType.ASSIGNABLE_TYPE, value = StorageAutoConfiguration.class)})
+    public static class RedactionIntegrationTestConfiguration {
+
+        @Bean
+        @Primary
+        public StorageService inmemoryStorage() {
+
+            return new FileSystemBackedStorageService(ObjectMapperFactory.create());
+        }
+
+    }
+
+    @Autowired
+    RedactionStorageService redactionStorageService;
+
+    @MockBean
+    DictionaryService dictionaryService;
+
+    @Autowired
+    UnprocessedChangesService unprocessedChangesService;
+
+    @Test
+    public void testManualSurroundingText() {
+
+        String pdfFile = "files/new/S4.pdf";
+        when(dictionaryService.getColor(DICTIONARY_AUTHOR, TEST_DOSSIER_TEMPLATE_ID)).thenReturn(new float[]{0f, 0f, 0f});
+        when(dictionaryService.getNotRedactedColor(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(new float[]{0f, 0f, 0f});
+
+        ManualRedactions manualRedactions = new ManualRedactions();
+
+        ManualRedactionEntry manualRedactionEntry = new ManualRedactionEntry();
+        manualRedactionEntry.setAnnotationId(UUID.randomUUID().toString());
+        manualRedactionEntry.setFileId("fileId");
+        manualRedactionEntry.setStatus(AnnotationStatus.APPROVED);
+        manualRedactionEntry.setType("CBI_author");
+        manualRedactionEntry.setValue("rabbits");
+        manualRedactionEntry.setReason("Manual Redaction");
+        manualRedactionEntry.setPositions(List.of(Rectangle.builder().topLeftX(70.944f).topLeftY(670.1595f).width(30.07296f).height(10.048125f).page(1).build()));
+
+        ManualRedactionEntry manualRedactionEntry2 = new ManualRedactionEntry();
+        manualRedactionEntry2.setAnnotationId(UUID.randomUUID().toString());
+        manualRedactionEntry2.setFileId("fileId");
+        manualRedactionEntry2.setStatus(AnnotationStatus.APPROVED);
+        manualRedactionEntry2.setType("CBI_author");
+        manualRedactionEntry2.setValue("rabbits");
+        manualRedactionEntry2.setReason("Manual Redaction");
+        manualRedactionEntry2.setPositions(List.of(Rectangle.builder().topLeftX(470.5204f).topLeftY(746.1195f).width(29.96256f).height(10.048125f).page(1).build()));
+
+        var aoelId = UUID.randomUUID().toString();
+        ManualRedactionEntry manualRedactionEntry3 = new ManualRedactionEntry();
+        manualRedactionEntry3.setAnnotationId(aoelId);
+        manualRedactionEntry3.setFileId("fileId");
+        manualRedactionEntry3.setStatus(AnnotationStatus.APPROVED);
+        manualRedactionEntry3.setType("CBI_author");
+        manualRedactionEntry3.setValue("AOEL");
+        manualRedactionEntry3.setReason("Manual Redaction");
+        manualRedactionEntry3.setPositions(List.of(Rectangle.builder().topLeftX(355.53775f).topLeftY(266.1895f).width(29.32224f).height(10.048125f).page(1).build()));
+
+        var notFoundId = UUID.randomUUID().toString();
+        ManualRedactionEntry manualRedactionEntry4 = new ManualRedactionEntry();
+        manualRedactionEntry4.setAnnotationId(notFoundId);
+        manualRedactionEntry4.setFileId("fileId");
+        manualRedactionEntry4.setStatus(AnnotationStatus.APPROVED);
+        manualRedactionEntry4.setType("CBI_author");
+        manualRedactionEntry4.setValue("Random");
+        manualRedactionEntry4.setReason("Manual Redaction");
+        manualRedactionEntry4.setPositions(List.of(Rectangle.builder().topLeftX(1f).topLeftY(1f).width(1f).height(1f).page(1).build()));
+
+        manualRedactions.getEntriesToAdd().add(manualRedactionEntry);
+        manualRedactions.getEntriesToAdd().add(manualRedactionEntry2);
+        manualRedactions.getEntriesToAdd().add(manualRedactionEntry3);
+        manualRedactions.getEntriesToAdd().add(manualRedactionEntry4);
+
+        AnalyzeRequest request = uploadFileToStorage(pdfFile);
+        request.setManualRedactions(manualRedactions);
+        analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request);
+
+        List<UnprocessedManualEntity> unprocessedManualEntities = unprocessedChangesService.addInformationToUnprocessedManualEntities(TEST_FILE_ID, TEST_DOSSIER_ID, TEST_DOSSIER_TEMPLATE_ID, manualRedactions);
+
+        assertFalse(unprocessedManualEntities.isEmpty());
+        assertEquals(unprocessedManualEntities.size(), 4);
+
+        Optional<UnprocessedManualEntity> optionalUnprocessedManualEntity = unprocessedManualEntities.stream().filter(manualEntity -> manualEntity.getAnnotationId().equals(aoelId)).findFirst();
+        assertTrue(optionalUnprocessedManualEntity.isPresent());
+
+        UnprocessedManualEntity unprocessedManualEntity = optionalUnprocessedManualEntity.get();
+        assertEquals(unprocessedManualEntity.getTextBefore(), "was above the ");
+        assertEquals(unprocessedManualEntity.getTextAfter(), " without PPE (34%");
+        assertEquals(unprocessedManualEntity.getStartOffset(), 2766);
+        assertEquals(unprocessedManualEntity.getEndOffset(), 2770);
+        assertEquals(unprocessedManualEntity.getClosestHeadline(), "2.6.14 Summary of product exposure and risk assessment ");
+        assertEquals(unprocessedManualEntity.getSection(), "[1, 1]: Paragraph: A9396G containing 960 g/L");
+        assertEquals(unprocessedManualEntity.getContainingNodeId().get(0), 1);
+        assertEquals(unprocessedManualEntity.getContainingNodeId().get(1), 1);
+        assertEquals(unprocessedManualEntity.getColor()[0], 0f);
+        assertEquals(unprocessedManualEntity.getColor()[1], 0f);
+        assertEquals(unprocessedManualEntity.getColor()[2], 0f);
+        assertEquals(unprocessedManualEntity.getPositions().get(0).x(), 355.53775f);
+        assertEquals(unprocessedManualEntity.getPositions().get(0).y(), 266.49002f);
+        assertEquals(unprocessedManualEntity.getPositions().get(0).w(), 29.322266f);
+        assertEquals(unprocessedManualEntity.getPositions().get(0).h(), 11.017679f);
+
+        Optional<UnprocessedManualEntity> optionalNotFoundUnprocessedManualEntity = unprocessedManualEntities.stream().filter(manualEntity -> manualEntity.getAnnotationId().equals(notFoundId)).findFirst();
+        assertTrue(optionalNotFoundUnprocessedManualEntity.isPresent());
+
+        UnprocessedManualEntity unprocessedNotFoundManualEntity = optionalNotFoundUnprocessedManualEntity.get();
+        assertEquals(unprocessedNotFoundManualEntity.getTextBefore(), "");
+        assertEquals(unprocessedNotFoundManualEntity.getTextAfter(), "");
+        assertEquals(unprocessedNotFoundManualEntity.getStartOffset(), -1);
+        assertEquals(unprocessedNotFoundManualEntity.getEndOffset(), -1);
+        assertEquals(unprocessedNotFoundManualEntity.getClosestHeadline(), "");
+        assertEquals(unprocessedNotFoundManualEntity.getSection(), "");
+        assertTrue(unprocessedNotFoundManualEntity.getContainingNodeId().isEmpty());
+        assertEquals(unprocessedNotFoundManualEntity.getColor()[0], 0f);
+        assertEquals(unprocessedNotFoundManualEntity.getColor()[1], 0f);
+        assertEquals(unprocessedNotFoundManualEntity.getColor()[2], 0f);
+        assertTrue(unprocessedNotFoundManualEntity.getPositions().isEmpty());
+    }
+}

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/utils/LayoutParsingRequestProvider.java

@@ -31,7 +31,6 @@ public class LayoutParsingRequestProvider {
                 .textBlockFileStorageId(textBlockFileStorageId)
                 .positionBlockFileStorageId(positionBlockFileStorageId)
                 .pageFileStorageId(pageFileStorageId)
-                .sectionGridStorageId(sectionGridStorageId)
                 .simplifiedTextStorageId(simplifiedTextStorageId).viewerDocumentStorageId(viewerDocumentStorageId)
                 .build();
     }

redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/PII.txt

@@ -29,4 +29,4 @@ J.B. RASCLE
 青森植
 サンプル量
 供試試料 （無処理 区）
-材料
+材料

redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/PII_false_positive.txt

@@ -1 +1 @@
-C. J. Alfred Xinyi
+C. J. Alfred XinyiP

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/acceptance_rules.drl

@@ -22,6 +22,7 @@ import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRu
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.*;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Section;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Table;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Paragraph;
@@ -145,17 +146,13 @@ rule "CBI.7.0: Do not redact Names and Addresses if published information found
 
 rule "CBI.7.1: Do not redact Names and Addresses if published information found in same table row"
     when
-        $table: Table(hasEntitiesOfType("published_information"),
-                     (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
+        $table: Table(hasEntitiesOfType("published_information"), hasEntitiesOfType("CBI_author"))
+        $tableCell: TableCell() from $table.streamTableCells().toList()
+        $sameRowCell: TableCell(row == $tableCell.row) from $table.streamTableCells().toList()
+        TextEntity(type == "published_information") from $tableCell.getEntities()
+        $author: TextEntity(type == "CBI_author") from $tableCell.getEntities()
     then
-        $table.streamEntitiesWhereRowContainsEntitiesOfType(List.of("CBI_author", "CBI_address"))
-            .forEach(redactionEntity -> {
-                redactionEntity.skipWithReferences(
-                        "CBI.7.1",
-                        "Published Information found in row",
-                        $table.getEntitiesOfTypeInSameRow("published_information", redactionEntity)
-                );
-            });
+        $author.skipWithReferences("CBI.7.1", "Published Information found in row", $table.getEntitiesOfTypeInSameRow("published_information", $author));
     end
 
 
@@ -423,7 +420,7 @@ rule "ETC.0.0: Purity Hint"
     when
         $section: Section(containsStringIgnoreCase("purity"))
     then
-        entityCreationService.byRegexIgnoreCase("(purity ?( of|\\(.{1,20}\\))?( ?:)?) .{0,5}[\\d\\.]+( .{0,4}\\.)? ?%", "hint_only", EntityType.ENTITY, 1, $section)
+        entityCreationService.byRegexIgnoreCase("(purity ?( of|\\(.{1,20}\\))?( ?:)?) .{0,5}[\\d\\.]+( .{0,4}\\.)? ?%", "hint_only", EntityType.HINT, 1, $section)
             .forEach(hint -> hint.skip("ETC.0.0", "hint only"));
     end
 
@@ -639,7 +636,7 @@ rule "X.0.0: remove Entity contained by Entity of same type"
     salience 65
     when
         $larger: TextEntity($type: type, $entityType: entityType, active())
-        $contained: TextEntity(containedBy($larger), type == $type, entityType == $entityType, this != $larger, !resized(), active())
+        $contained: TextEntity(containedBy($larger), type == $type, entityType == $entityType, this != $larger, !hasManualChanges(), active())
     then
         $contained.remove("X.0.0", "remove Entity contained by Entity of same type");
         retract($contained);
@@ -651,7 +648,7 @@ rule "X.1.0: merge intersecting Entities of same type"
     salience 64
     when
         $first: TextEntity($type: type, $entityType: entityType, !resized(), active())
-        $second: TextEntity(intersects($first), type == $type, entityType == $entityType, this != $first, !resized(), active())
+        $second: TextEntity(intersects($first), type == $type, entityType == $entityType, this != $first, !hasManualChanges(), active())
     then
         TextEntity mergedEntity = entityCreationService.mergeEntitiesOfSameType(List.of($first, $second), $type, $entityType, document);
         $first.remove("X.1.0", "merge intersecting Entities of same type");
@@ -667,7 +664,7 @@ rule "X.2.0: remove Entity of type ENTITY when contained by FALSE_POSITIVE"
     salience 64
     when
         $falsePositive: TextEntity($type: type, entityType == EntityType.FALSE_POSITIVE, active())
-        $entity: TextEntity(containedBy($falsePositive), type == $type, entityType == EntityType.ENTITY, !resized(), active())
+        $entity: TextEntity(containedBy($falsePositive), type == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges(), active())
     then
         $entity.getIntersectingNodes().forEach(node -> update(node));
         $entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE");
@@ -680,7 +677,7 @@ rule "X.3.0: remove Entity of type RECOMMENDATION when contained by FALSE_RECOMM
     salience 64
     when
         $falseRecommendation: TextEntity($type: type, entityType == EntityType.FALSE_RECOMMENDATION, active())
-        $recommendation: TextEntity(containedBy($falseRecommendation), type == $type, entityType == EntityType.RECOMMENDATION, !resized(), active())
+        $recommendation: TextEntity(containedBy($falseRecommendation), type == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
     then
         $recommendation.remove("X.3.0", "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION");
         retract($recommendation);
@@ -691,8 +688,8 @@ rule "X.3.0: remove Entity of type RECOMMENDATION when contained by FALSE_RECOMM
 rule "X.4.0: remove Entity of type RECOMMENDATION when intersected by ENTITY with same type"
     salience 256
     when
-        $entity: TextEntity($type: type, entityType == EntityType.ENTITY, active())
-        $recommendation: TextEntity(intersects($entity), type == $type, entityType == EntityType.RECOMMENDATION, !resized(), active())
+        $entity: TextEntity($type: type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
+        $recommendation: TextEntity(intersects($entity), type == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
     then
         $entity.addEngines($recommendation.getEngines());
         $recommendation.remove("X.4.0", "remove Entity of type RECOMMENDATION when intersected by ENTITY with same type");
@@ -704,8 +701,8 @@ rule "X.4.0: remove Entity of type RECOMMENDATION when intersected by ENTITY wit
 rule "X.5.0: remove Entity of type RECOMMENDATION when contained by ENTITY"
     salience 256
     when
-        $entity: TextEntity(entityType == EntityType.ENTITY, active())
-        $recommendation: TextEntity(containedBy($entity), entityType == EntityType.RECOMMENDATION, !resized(), active())
+        $entity: TextEntity((entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
+        $recommendation: TextEntity(containedBy($entity), entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
     then
         $recommendation.remove("X.5.0", "remove Entity of type RECOMMENDATION when contained by ENTITY");
         retract($recommendation);
@@ -713,17 +710,28 @@ rule "X.5.0: remove Entity of type RECOMMENDATION when contained by ENTITY"
 
 
 // Rule unit: X.6
-rule "X.6.0: remove Entity of lower rank, when intersected by entity of type ENTITY"
+rule "X.6.0: remove Entity of lower rank, when contained by by entity of type ENTITY"
     salience 32
     when
-        $higherRank: TextEntity($type: type, entityType == EntityType.ENTITY, active())
-        $lowerRank: TextEntity(intersects($higherRank), type != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !resized(), active())
+        $higherRank: TextEntity($type: type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
+        $lowerRank: TextEntity(containedBy($higherRank), type != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active())
     then
         $lowerRank.getIntersectingNodes().forEach(node -> update(node));
-        $lowerRank.remove("X.6.0", "remove Entity of lower rank, when intersected by entity of type ENTITY");
+        $lowerRank.remove("X.6.0", "remove Entity of lower rank, when contained by entity of type ENTITY");
         retract($lowerRank);
     end
 
+rule "X.6.1: remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity"
+    salience 32
+    when
+        $higherRank: TextEntity($type: type, $value: value, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
+        $lowerRank: TextEntity(intersects($higherRank), type != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active(), $lowerRank.getValue().length() > $value.length())
+    then
+        $higherRank.getIntersectingNodes().forEach(node -> update(node));
+        $higherRank.remove("X.6.1", "remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity");
+        retract($higherRank);
+    end
+
 
 //------------------------------------ File attributes rules ------------------------------------
 
@@ -741,7 +749,7 @@ rule "FA.1.0: remove duplicate FileAttributes"
 //------------------------------------ Local dictionary search rules ------------------------------------
 
 // Rule unit: LDS.0
-rule "LDS.0.0: run local dictionary search"
+rule "LDS.0.0: Run local dictionary search"
     agenda-group "LOCAL_DICTIONARY_ADDS"
     salience -999
     when

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora_components.drl

@@ -47,82 +47,65 @@ declare GuidelineMapping
 
 //------------------------------------ Default Components rules ------------------------------------
 
-rule "StudyTitle.0.0: Study Title"
+rule "StudyTitle.0.0: First Title found"
     when
-        $titleCandidates: List() from collect (Entity(type == "study_title"))
+        $titleCandidates: List() from collect (Entity(type == "title"))
     then
-        componentCreationService.firstOrElse("StudyTitle.0.0", "Study_Title", $titleCandidates, "No study title found!");
+        componentCreationService.firstOrElse("StudyTitle.0.0", "Study_Title", $titleCandidates, "");
     end
 
 
-rule "PerformingLaboratory.1.0: Performing Laboratory"
+rule "PerformingLaboratory.1.0: Performing Laboratory name and country found in same section"
     when
-        $laboratoryName: Entity(type == "laboratory_name")
-        $laboratoryCountry: Entity(type == "laboratory_country", Math.abs($laboratoryName.startOffset - startOffset) < 80)
-        not Entity(type == "laboratory_country", Math.abs($laboratoryName.startOffset - startOffset) < Math.abs($laboratoryName.startOffset - $laboratoryCountry.startOffset))
+        $laboratoryName: Entity(type == "laboratory_name", $node: containingNode)
+        $laboratoryCountry: Entity(type == "laboratory_country", containingNode == $node)
+        not Entity(type == "laboratory_country", containingNode == $node, Math.abs($laboratoryName.startOffset - startOffset) < Math.abs($laboratoryName.startOffset - $laboratoryCountry.startOffset))
     then
-        componentCreationService.create("PerformingLaboratory.1.0", "Performing_Laboratory", $laboratoryName.getValue() + ", " + $laboratoryCountry.getValue(), "Laboratory name and country found!");
+        componentCreationService.create("PerformingLaboratory.1.0", "Performing_Laboratory", $laboratoryName.getValue() + ", " + $laboratoryCountry.getValue(), "Laboratory name and country found!", List.of($laboratoryName, $laboratoryCountry));
     end
 
-rule "PerformingLaboratory.2.0: Performing Laboratory"
+rule "PerformingLaboratory.2.0: Performing Laboratory name but no country found in same section"
     when
-        $laboratoryName: Entity(type == "laboratory_name")
-        not Entity(type == "laboratory_country", Math.abs($laboratoryName.startOffset - startOffset) < 80)
+        $laboratoryName: Entity(type == "laboratory_name", $node: containingNode)
+        not Entity(type == "laboratory_country", containingNode == $node)
     then
-        componentCreationService.create("PerformingLaboratory.2.0", "Performing_Laboratory", $laboratoryName.getValue(), "Only laboratory name found!");
+        componentCreationService.create("PerformingLaboratory.2.0", "Performing_Laboratory", $laboratoryName.getValue(), "Only laboratory name found!", List.of($laboratoryName));
     end
 
-rule "PerformingLaboratory.0.2: Performing Laboratory"
+rule "PerformingLaboratory.0.2: Performing Laboratory not found"
     salience -1
     when
-        not Component(category == "Performing_Laboratory")
+        not Component(name == "Performing_Laboratory")
     then
-        componentCreationService.create("PerformingLaboratory.0.2", "Performing_Laboratory", "n-a", "fallback");
+        componentCreationService.create("PerformingLaboratory.0.2", "Performing_Laboratory", "", "fallback");
     end
 
 
-rule "ReportNumber.0.0: Report number"
+rule "ReportNumber.0.0: First Report number found"
     when
         $reportNumberCandidates: List() from collect (Entity(type == "report_number"))
     then
-        componentCreationService.firstOrElse("ReportNumber.0.0", "Report_number", $reportNumberCandidates, "No report number found!");
+        componentCreationService.firstOrElse("ReportNumber.0.0", "Report_Number", $reportNumberCandidates, "");
     end
 
 
-rule "GLPStudy.0.0: GLP Study"
+rule "GLPStudy.0.0: GLP Study found"
     when
-        $glpStudy: Entity(type == "glp_study")
+        $glpStudyList: List(!isEmpty) from collect(Entity(type == "glp_study"))
     then
-        componentCreationService.create("GLPStudy.0.0", "GLP_study", "Yes", "Yes if present, No if not", $glpStudy);
+        componentCreationService.create("GLPStudy.0.0", "GLP_Study", "Yes", "Yes if present, No if not", $glpStudyList);
     end
 
-rule "GLPStudy.1.0: GLP Study"
+rule "GLPStudy.1.0: GLP Study not found"
     when
         not Entity(type == "glp_study")
     then
-        componentCreationService.create("GLPStudy.1.0", "GLP_study", "No", "Yes if present, No if not");
-    end
-
-rule "GLPStudy.2.0: GLP Study"
-    when
-        $first: Component(category == "GLP_study")
-        $second: Component(category == "GLP_study", value == $first.value, this != $first)
-    then
-        $first.addReferences($second.getReferences());
-        retract($second);
+        componentCreationService.create("GLPStudy.1.0", "GLP_Study", "No", "Yes if present, No if not");
     end
 
 
-rule "TestGuideline.0.0: no test guideline year found, only oecd number"
-    when
-        $guidelineNumber: Entity(type == "oecd_guideline_number")
-        not Entity(type == "oecd_guideline_year")
-    then
-        componentCreationService.create("DefaultComponents.4.0", "Test_Guidelines_1", $guidelineNumber.getValue(), "Only OECD Number found!");
-    end
-
-rule "TestGuideline.1.0: create mappings"
-salience 1
+rule "TestGuideline.0.0: create OECD number and year guideline mappings"
+    salience 2
     when
         Entity(type == "oecd_guideline_number")
         Entity(type == "oecd_guideline_year")
@@ -158,61 +141,363 @@ salience 1
         insert(new GuidelineMapping("487", "2016", "Nº 487: Micronucleus Human Lymphocytes (2016)"));
     end
 
-rule "TestGuideline.1.1: match test guidelines with mappings"
+rule "TestGuideline.0.1: match OECD number and year with guideline mappings"
+    salience 1
     when
+        not Component(name == "Test_Guidelines_1")
         GuidelineMapping($year: year, $number: number, $guideline: guideline)
         $guidelineNumber: Entity(type == "oecd_guideline_number", value == $number)
         $guidelineYear: Entity(type == "oecd_guideline_year", value == $year)
     then
         componentCreationService.create(
-           "DefaultComponents.4.1",
+           "TestGuideline.0.0",
            "Test_Guidelines_1",
            $guideline,
-           "OECD Number and guideline year mapped!"
+           "OECD Number and guideline year mapped!",
+           List.of($guidelineNumber, $guidelineYear)
         );
     end
 
-
-rule "DefaultComponents.5.0: Test Guideline 2"
+rule "TestGuideline.1.0: no guideline mapping found"
     when
-        $epaGuideLines: List() from collect (Entity(type == "epa_guideline"))
-        $ecGuideLines: List() from collect (Entity(type == "ec_guideline"))
+        not Component(name == "Test_Guidelines_1")
+        $guideLine: Entity(type == "oecd_guideline")
     then
-        componentCreationService.joining("DefaultComponents.5.0",
-            "Test_Guideline_2",
-            Stream.of(
-              $epaGuideLines.stream(),
-              $ecGuideLines.stream())
-          .flatMap(a -> a)
-          .toList()
-        );
+        componentCreationService.create("TestGuideline.2.0", "Test_Guidelines_1", $guideLine.getValue(), "No Mapping for OECD number and year found, using fallback instead!", List.of($guideLine));
+    end
+
+rule "TestGuideline.2.0: All values of EPA guideline and EC guidelines"
+    when
+        $guidelines: List() from collect (Entity(type == "epa_guideline" || type ==  "ec_guideline"))
+    then
+        componentCreationService.joining("TestGuideline.2.0", "Test_Guidelines_2", $guidelines);
     end
 
 
-rule "DefaultComponents.6.0: Experimental Starting Date"
+rule "StartDate.0.0: All experimental start dates converted to dd/MM/yyyy"
     when
-        $startDates: List(!isEmpty()) from collect (Entity(type == "experimental_start_date"))
+        $startDates: List() from collect (Entity(type == "experimental_start_date"))
     then
-        componentCreationService.convertDates("DefaultComponents.6.0", "Experimental_Starting_Date", $startDates);
+        componentCreationService.convertDates("StartDate.0.0", "Experimental_Starting_Date", $startDates);
     end
 
 
-rule "DefaultComponents.7.0: Experimental Completion Date"
+rule "CompletionDate.0.0: All experimental end dates converted to dd/MM/yyyy"
     when
-        $endDates: List(!isEmpty()) from collect (Entity(type == "experimental_end_date"))
+        $endDates: List() from collect (Entity(type == "experimental_end_date"))
     then
-        componentCreationService.convertDates("DefaultComponents.7.0", "Experimental_Completion_Date", $endDates);
+        componentCreationService.convertDates("CompletionDate.0.0", "Experimental_Completion_Date", $endDates);
     end
 
 
-rule "DefaultComponents.8.0: Certificate of analysis batch identification"
+rule "AnalysisCertificate.0.0: Unique values of certificate of analysis batch identification"
     when
-        $batchNumbers: List(!isEmpty()) from collect (Entity(type == "batch_number"))
+        $batchNumbers: List() from collect (Entity(type == "batch_number"))
     then
-        componentCreationService.joiningUnique("DefaultComponents.8.0", "Batch_Number", $batchNumbers);
+        componentCreationService.joiningUnique("AnalysisCertificate.0.0", "Certificate_of_Analysis_Batch_Identification", $batchNumbers);
     end
 
+rule "StudyConclusion.0.0: Study conclusion in first found section"
+    when
+        $oecdNumber: String() from List.of("402", "403", "404", "405", "425", "429", "436", "471")
+        FileAttribute(label == "OECD Number", value == $oecdNumber)
+        $studyConclusions: List() from collect(Entity(type == "study_conclusion"))
+    then
+        componentCreationService.joiningFromFirstSectionOnly("StudyConclusion.0.0", "Study_Conclusion", $studyConclusions, " ");
+    end
 
+rule "GuidelineDeviation.0.0: Guideline deviation as sentences"
+    when
+        $oecdNumber: String() from List.of("402", "403", "404", "405", "425", "429", "436", "471")
+        FileAttribute(label == "OECD Number", value == $oecdNumber)
+        $guidelineDeviations: List() from collect (Entity(type == "guideline_deviation"))
+    then
+        componentCreationService.joining("GuidelineDeviation.0.0", "Deviation_from_the_Guideline", $guidelineDeviations, "\n");
+    end
+
+rule "Species.0.0: First found species"
+    when
+        $oecdNumber: String() from List.of("402", "403", "404", "405", "425", "429", "436", "471")
+        FileAttribute(label == "OECD Number", value == $oecdNumber)
+        $species: List() from collect (Entity(type == "species"))
+    then
+        componentCreationService.firstOrElse("Species.0.0", "Species", $species, "");
+    end
+
+rule "Strain.0.0: First found strain"
+    when
+        $oecdNumber: String() from List.of("402", "403", "404", "405", "425", "429", "436", "471")
+        FileAttribute(label == "OECD Number", value == $oecdNumber)
+        $strain: List() from collect (Entity(type == "strain"))
+    then
+        componentCreationService.firstOrElse("Strain.0.0", "Strain", $strain, "");
+    end
+
+rule "Conclusion.0.0: Unique values of Conclusion LD50"
+    when
+        $oecdNumber: String() from List.of("402", "403", "425", "436")
+        FileAttribute(label == "OECD Number", value == $oecdNumber)
+        $conclusions: List() from collect (Entity(type == "ld50_value"))
+    then
+        componentCreationService.joiningUnique("Conclusion.0.0", "Conclusion_LD50_mg_per_kg", $conclusions);
+    end
+
+rule "Conclusion0.1.0: Greater than found"
+    when
+        $oecdNumber: String() from List.of("402", "403", "425", "436")
+        FileAttribute(label == "OECD Number", value == $oecdNumber)
+        $conclusions: List(!isEmpty()) from collect (Entity(type == "ld50_greater"))
+    then
+        componentCreationService.create("Conclusion.1.0", "Conclusion_LD50_Greater_than", "Greater than", "Entity of type 'ld50_greater' found", $conclusions);
+    end
+
+rule "Conclusion.1.1: Greater than not found"
+    when
+        $oecdNumber: String() from List.of("402", "403", "425", "436")
+        FileAttribute(label == "OECD Number", value == $oecdNumber)
+        not Entity(type == "ld50_greater")
+    then
+        componentCreationService.create("Conclusion.1.1", "Conclusion_LD50_Greater_than", "", "No entity of type 'ld50_greater' found");
+    end
+
+rule "Conclusion.2.0: Minimum confidence as unique values"
+    when
+        $oecdNumber: String() from List.of("402", "403", "425", "436")
+        FileAttribute(label == "OECD Number", value == $oecdNumber)
+        $conclusions: List() from collect (Entity(type == "confidence_minimal"))
+    then
+        componentCreationService.joiningUnique("Conclusion.2.0", "Conclusion_Minimum_Confidence", $conclusions);
+    end
+
+rule "Conclusion.3.0: Maximum confidence as unique values"
+    when
+        $oecdNumber: String() from List.of("402", "403", "425", "436")
+        FileAttribute(label == "OECD Number", value == $oecdNumber)
+        $conclusions: List() from collect (Entity(type == "confidence_maximal"))
+    then
+        componentCreationService.joiningUnique("Conclusion.3.0", "Conclusion_Maximum_Confidence", $conclusions);
+    end
+
+rule "Necropsy.0.0: Necropsy findings from longest section"
+    when
+        FileAttribute(label == "OECD Number", value == "402")
+        $necropsies: List() from collect (Entity(type == "necropsy_findings"))
+    then
+        componentCreationService.joiningFromLongestSectionOnly("Necropsy.0.0", "Necropsy_Findings", $necropsies, " ");
+    end
+
+rule "Necropsy.0.1: Necropsy findings joined with \n"
+    when
+        FileAttribute(label == "OECD Number", value == "403" || value == "436")
+        $necropsies: List() from collect (Entity(type == "necropsy_findings"))
+    then
+        componentCreationService.joining("Necropsy.0.0", "Necropsy_Findings", $necropsies, "\n");
+    end
+
+rule "Necropsy.1.0: Doses mg per kg of Bodyweight as one block"
+    when
+        FileAttribute(label == "OECD Number", value == "402")
+        $dosages: List() from collect (Entity(type == "doses_(mg_kg_bw)"))
+    then
+        componentCreationService.joining("Necropsy.1.0", "Doses_mg_per_kg_bw", $dosages, " ");
+    end
+
+rule "Necropsy.2.0: Conducted with 4 hours of exposure as one block"
+    when
+        $oecdNumber: String() from List.of("403", "436")
+        FileAttribute(label == "OECD Number", value == $oecdNumber)
+        $exposures: List() from collect (Entity(type == "4h_exposure"))
+    then
+        componentCreationService.joining("Necropsy.3.0", "Conducted_with_4_Hours_of_Exposure", $exposures, " ");
+    end
+
+rule "StudyDesign.0.0: Study design as one block"
+    when
+        $oecdNumber: String() from List.of("404", "405", "429", "406", "428", "438", "439", "474", "487")
+        FileAttribute(label == "OECD Number", value == $oecdNumber)
+        $studyDesigns: List() from collect (Entity(type == "study_design"))
+    then
+        componentCreationService.joining("StudyDesign.0.0", "Study_Design", $studyDesigns, " ");
+    end
+
+rule "Results.0.0: Results and conclusions as joined values"
+    when
+        $oecdNumber: String() from List.of("406", "428", "438", "439", "474", "487")
+        FileAttribute(label == "OECD Number", value == $oecdNumber)
+        $results: List() from collect (Entity(type == "results_and_conclusion"))
+    then
+        componentCreationService.joining("Results.0.0", "Results_and_Conclusions", $results, " ");
+    end
+
+rule "WeightBehavior.0.0: Weight change behavior as sentences"
+    when
+        FileAttribute(label == "OECD Number", value == "402")
+        $weightChanges: List() from collect (Entity(type == "weight_behavior_changes"))
+    then
+        componentCreationService.joining("WeightBehavior.0.0", "Weight_Behavior_Changes", $weightChanges, "\n");
+    end
+
+rule "MortalityStatement.0.0: Mortality statements as one block"
+    when
+        FileAttribute(label == "OECD Number", value == "402")
+        $mortalityStatements: List() from collect (Entity(type == "mortality_statement"))
+    then
+        componentCreationService.joining("MortalityStatement.0.0", "Mortality_Statement", $mortalityStatements, " ");
+    end
+
+rule "ClinicalObservations.0.0: Clinical observations as sentences"
+    when
+        FileAttribute(label == "OECD Number", value == "403")
+        $observations: List() from collect (Entity(type == "clinical_observations"))
+    then
+        componentCreationService.joining("MortalityStatement.0.0", "Clinical_Observations", $observations, "\n");
+    end
+
+rule "BodyWeight.0.0: Bodyweight changes as sentences"
+    when
+        FileAttribute(label == "OECD Number", value == "403")
+        $weightChanges: List() from collect (Entity(type == "bodyweight_changes"))
+    then
+        componentCreationService.joining("BodyWeight.0.0", "Body_Weight_Changes", $weightChanges, "\n");
+    end
+
+rule "Detailing.0.0: Detailing of reported changes as one block"
+    when
+        $oecdNumber: String() from List.of("404", "405")
+        FileAttribute(label == "OECD Number", value == $oecdNumber)
+        $detailings: List() from collect (Entity(type == "detailing"))
+    then
+        componentCreationService.joining("Detailing.0.0", "Detailing_of_Reported_Changes", $detailings, " ");
+    end
+
+rule "Sex.0.0: Male sex found"
+    when
+        $oecdNumber: String() from List.of("405", "429")
+        FileAttribute(label == "OECD Number", value == $oecdNumber)
+        $males: List(!isEmpty) from collect (Entity(type == "sex", (value.toLowerCase() == "male" || value.toLowerCase() == "males")))
+    then
+        componentCreationService.create("Sex.0.0", "Sex", "male", "male sex found", $males);
+    end
+
+rule "Sex.1.0: Female sex found"
+    when
+        $oecdNumber: String() from List.of("405", "429")
+        FileAttribute(label == "OECD Number", value == $oecdNumber)
+        $females: List(!isEmpty) from collect (Entity(type == "sex", (value.toLowerCase() == "female" || value.toLowerCase() == "females")))
+    then
+        componentCreationService.create("Sex.0.0", "Sex", "female", "female sex found", $females);
+    end
+
+rule "NumberOfAnimals.0.0: Number of animals found"
+    when
+        $oecdNumber: String() from List.of("405", "429")
+        FileAttribute(label == "OECD Number", value == $oecdNumber)
+        $numberOfAnimals: Entity(type == "number_of_animals")
+    then
+        componentCreationService.create("NumberOfAnimals.0.0", "Number_of_Animals", $numberOfAnimals.getValue(), "Number of animals found directly", $numberOfAnimals);
+    end
+
+rule "NumberOfAnimals.1.0: Count unique occurences of animals"
+    when
+        $oecdNumber: String() from List.of("405", "429")
+        FileAttribute(label == "OECD Number", value == $oecdNumber)
+        not Entity(type == "number_of_animals")
+        $animals: List() from collect (Entity(type == "animal_number"))
+    then
+        componentCreationService.uniqueValueCount("NumberOfAnimals.1.0", "Number_of_Animals", $animals);
+    end
+
+rule "ClinicalSigns.0.0: Clinical signs as sentences"
+    when
+        $oecdNumber: String() from List.of("425")
+        FileAttribute(label == "OECD Number", value == $oecdNumber)
+        $clinicalSigns: List() from collect (Entity(type == "clinical_signs"))
+    then
+        componentCreationService.joining("ClinicalSigns.0.0", "Clinical_Signs", $clinicalSigns, "\n");
+    end
+
+rule "DoseMortality.0.0: Dose mortality joined with dose from same table row"
+    when
+        $oecdNumber: String() from List.of("425")
+        FileAttribute(label == "OECD Number", value == $oecdNumber)
+        $doseMortalities: List() from collect (Entity(type == "dose_mortality" || type == "dose_mortality_dose"))
+    then
+        componentCreationService.joiningFromSameTableRow("DoseMortality.0.0", "Dose_Mortality", $doseMortalities);
+    end
+
+rule "Mortality.0.0: Mortality as one block"
+    when
+        $oecdNumber: String() from List.of("425")
+        FileAttribute(label == "OECD Number", value == $oecdNumber)
+        $mortalities: List() from collect (Entity(type == "mortality"))
+    then
+        componentCreationService.joining("Mortality.0.0", "Mortality", $mortalities, " ");
+    end
+
+rule "Dosages.0.0: First found value of Dosages"
+    when
+        $oecdNumber: String() from List.of("425")
+        FileAttribute(label == "OECD Number", value == $oecdNumber)
+        $mortalities: List() from collect (Entity(type == "dosages"))
+    then
+        componentCreationService.firstOrElse("Dosages.0.0", "Dosages", $mortalities, "");
+    end
+
+rule "PrelimResults.0.0: Preliminary test results as sentences"
+    when
+        $oecdNumber: String() from List.of("429")
+        FileAttribute(label == "OECD Number", value == $oecdNumber)
+        $results: List() from collect (Entity(type == "preliminary_test_results"))
+    then
+        componentCreationService.joining("PrelimResults.0.0", "Preliminary_Test_Results", $results, "\n");
+    end
+
+rule "TestResults.0.0: Test results as one block"
+    when
+        $oecdNumber: String() from List.of("429")
+        FileAttribute(label == "OECD Number", value == $oecdNumber)
+        $results: List() from collect (Entity(type == "test_results"))
+    then
+        componentCreationService.joining("TestResults.0.0", "Test_Results", $results, " ");
+    end
+
+rule "PositiveControl.0.0: Was the definitive study conducted with positive control"
+    when
+        $oecdNumber: String() from List.of("429")
+        FileAttribute(label == "OECD Number", value == $oecdNumber)
+        $results: List() from collect (Entity(type == "positive_control"))
+    then
+        componentCreationService.joining("PositiveControl.0.0", "Was_the_definitive_study_conducted_with_positive_control", $results, " ");
+    end
+
+rule "MainResults.0.0: Results from main study as one block"
+    when
+        $oecdNumber: String() from List.of("429")
+        FileAttribute(label == "OECD Number", value == $oecdNumber)
+        $results: List() from collect (Entity(type == "results_(main_study)"))
+    then
+        componentCreationService.joining("MainResults.0.0", "Results_Main_Study", $results, " ");
+    end
+
+rule "UsedApproach.0.0: Used approach found and mapped to 'Group'"
+    when
+        $oecdNumber: String() from List.of("429")
+        FileAttribute(label == "OECD Number", value == $oecdNumber)
+        $results: List(!isEmpty()) from collect (Entity(type == "approach_used"))
+    then
+        componentCreationService.create("UsedApproach.0.0", "What_was_the_approach_used", "Group", "'Group' when approach used is present, else 'Individual'", $results);
+    end
+
+rule "UsedApproach.1.0: Used approach not found and thus 'Individual'"
+    when
+        $oecdNumber: String() from List.of("429")
+        FileAttribute(label == "OECD Number", value == $oecdNumber)
+        not Entity(type == "approach_used")
+    then
+        componentCreationService.create("UsedApproach.1.0", "What_was_the_approach_used", "Individual", "'Group' when approach used is present, else 'Individual'");
+    end
+
+/*
 rule "DefaultComponents.999.0: Create components for all unmapped entities."
     salience -999
     when
@@ -220,16 +505,16 @@ rule "DefaultComponents.999.0: Create components for all unmapped entities."
     then
         componentCreationService.createComponentsForUnMappedEntities("DefaultComponents.999.0", $allEntities);
     end
-
-
+*/
 
 //------------------------------------ Component merging rules ------------------------------------
-
+/*
 rule "X.0.0: merge duplicate component references"
     when
         $first: Component()
-        $duplicate: Component(this != $first, category == category, value == value)
+        $duplicate: Component(this != $first, name == $first.name, value == $first.value)
     then
         $first.getReferences().addAll($duplicate.getReferences());
         retract($duplicate);
     end
+*/

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/manual_redaction_rules.drl

@@ -22,6 +22,7 @@ import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRu
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.*;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Section;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Table;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Paragraph;
@@ -123,7 +124,6 @@ rule "MAN.1.1: Apply id removals that are valid and not in forced redactions to
 
 // Rule unit: MAN.2
 rule "MAN.2.0: Apply force redaction"
-    no-loop true
     salience 128
     when
         $force: ManualForceRedaction($id: annotationId, status == AnnotationStatus.APPROVED)
@@ -132,10 +132,10 @@ rule "MAN.2.0: Apply force redaction"
         $entityToForce.getManualOverwrite().addChange($force);
         update($entityToForce);
         $entityToForce.getIntersectingNodes().forEach(node -> update(node));
+        retract($force);
     end
 
 rule "MAN.2.1: Apply force redaction to images"
-    no-loop true
     salience 128
     when
         $force: ManualForceRedaction($id: annotationId, status == AnnotationStatus.APPROVED)
@@ -144,6 +144,7 @@ rule "MAN.2.1: Apply force redaction to images"
         $imageToForce.getManualOverwrite().addChange($force);
         update($imageToForce);
         update($imageToForce.getParent());
+        retract($force);
     end
 
 
@@ -151,9 +152,9 @@ rule "MAN.2.1: Apply force redaction to images"
 rule "MAN.3.0: Apply entity recategorization"
     salience 128
     when
-        $recategorization: ManualRecategorization($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate)
+        $recategorization: ManualRecategorization($id: annotationId, $type: type, status == AnnotationStatus.APPROVED, $requestDate: requestDate)
         not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))
-        $entityToBeRecategorized: TextEntity(matchesAnnotationId($id))
+        $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type != $type)
     then
         $entityToBeRecategorized.getIntersectingNodes().forEach(node -> update(node));
         manualChangesApplicationService.recategorize($entityToBeRecategorized, $recategorization);
@@ -162,7 +163,20 @@ rule "MAN.3.0: Apply entity recategorization"
         retract($entityToBeRecategorized);
     end
 
-rule "MAN.3.1: Apply image recategorization"
+
+rule "MAN.3.1: Apply entity recategorization of same type"
+    salience 128
+    when
+        $recategorization: ManualRecategorization($id: annotationId, $type: type, status == AnnotationStatus.APPROVED, $requestDate: requestDate)
+        not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))
+        $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type == $type)
+    then
+        $entityToBeRecategorized.getManualOverwrite().addChange($recategorization);
+        retract($recategorization);
+    end
+
+
+rule "MAN.3.2: Apply image recategorization"
     salience 128
     when
         $recategorization: ManualRecategorization($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate)
@@ -175,6 +189,7 @@ rule "MAN.3.1: Apply image recategorization"
         retract($recategorization);
     end
 
+
 // Rule unit: MAN.4
 rule "MAN.4.0: Apply legal basis change"
     salience 128
@@ -193,11 +208,10 @@ rule "MAN.4.1: Apply legal basis change"
     then
         $entityToBeChanged.getManualOverwrite().addChange($legalBasisChange);
     end
-
 //------------------------------------ Local dictionary search rules ------------------------------------
 
 // Rule unit: LDS.0
-rule "LDS.0.0: run local dictionary search"
+rule "LDS.0.0: Run local dictionary search"
     agenda-group "LOCAL_DICTIONARY_ADDS"
     salience -999
     when

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl

@@ -22,6 +22,7 @@ import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRu
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.*;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Section;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Table;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Paragraph;
@@ -74,8 +75,8 @@ rule "SYN.0.0: Redact if CTL/* or BL/* was found (Non Vertebrate Study)"
         $section: Section(containsString("CTL/") || containsString("BL/"))
     then
         Stream.concat(
-                entityCreationService.byString("CTL", "must_redact", EntityType.ENTITY, $section),
-                entityCreationService.byString("BL", "must_redact", EntityType.ENTITY, $section)
+                entityCreationService.byString("CTL", "must_redact", EntityType.HINT, $section),
+                entityCreationService.byString("BL", "must_redact", EntityType.HINT, $section)
         ).forEach(entity -> entity.skip("SYN.0.0", "hint_only"));
     end
 
@@ -1023,7 +1024,7 @@ rule "X.0.0: remove Entity contained by Entity of same type"
     salience 65
     when
         $larger: TextEntity($type: type, $entityType: entityType, active())
-        $contained: TextEntity(containedBy($larger), type == $type, entityType == $entityType, this != $larger, !resized(), active())
+        $contained: TextEntity(containedBy($larger), type == $type, entityType == $entityType, this != $larger, !hasManualChanges(), active())
     then
         $contained.remove("X.0.0", "remove Entity contained by Entity of same type");
         retract($contained);
@@ -1035,7 +1036,7 @@ rule "X.1.0: merge intersecting Entities of same type"
     salience 64
     when
         $first: TextEntity($type: type, $entityType: entityType, !resized(), active())
-        $second: TextEntity(intersects($first), type == $type, entityType == $entityType, this != $first, !resized(), active())
+        $second: TextEntity(intersects($first), type == $type, entityType == $entityType, this != $first, !hasManualChanges(), active())
     then
         TextEntity mergedEntity = entityCreationService.mergeEntitiesOfSameType(List.of($first, $second), $type, $entityType, document);
         $first.remove("X.1.0", "merge intersecting Entities of same type");
@@ -1051,7 +1052,7 @@ rule "X.2.0: remove Entity of type ENTITY when contained by FALSE_POSITIVE"
     salience 64
     when
         $falsePositive: TextEntity($type: type, entityType == EntityType.FALSE_POSITIVE, active())
-        $entity: TextEntity(containedBy($falsePositive), type == $type, entityType == EntityType.ENTITY, !resized(), active())
+        $entity: TextEntity(containedBy($falsePositive), type == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges(), active())
     then
         $entity.getIntersectingNodes().forEach(node -> update(node));
         $entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE");
@@ -1064,7 +1065,7 @@ rule "X.3.0: remove Entity of type RECOMMENDATION when contained by FALSE_RECOMM
     salience 64
     when
         $falseRecommendation: TextEntity($type: type, entityType == EntityType.FALSE_RECOMMENDATION, active())
-        $recommendation: TextEntity(containedBy($falseRecommendation), type == $type, entityType == EntityType.RECOMMENDATION, !resized(), active())
+        $recommendation: TextEntity(containedBy($falseRecommendation), type == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
     then
         $recommendation.remove("X.3.0", "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION");
         retract($recommendation);
@@ -1075,8 +1076,8 @@ rule "X.3.0: remove Entity of type RECOMMENDATION when contained by FALSE_RECOMM
 rule "X.4.0: remove Entity of type RECOMMENDATION when intersected by ENTITY with same type"
     salience 256
     when
-        $entity: TextEntity($type: type, entityType == EntityType.ENTITY, active())
-        $recommendation: TextEntity(intersects($entity), type == $type, entityType == EntityType.RECOMMENDATION, !resized(), active())
+        $entity: TextEntity($type: type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
+        $recommendation: TextEntity(intersects($entity), type == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
     then
         $entity.addEngines($recommendation.getEngines());
         $recommendation.remove("X.4.0", "remove Entity of type RECOMMENDATION when intersected by ENTITY with same type");
@@ -1088,8 +1089,8 @@ rule "X.4.0: remove Entity of type RECOMMENDATION when intersected by ENTITY wit
 rule "X.5.0: remove Entity of type RECOMMENDATION when contained by ENTITY"
     salience 256
     when
-        $entity: TextEntity(entityType == EntityType.ENTITY, active())
-        $recommendation: TextEntity(containedBy($entity), entityType == EntityType.RECOMMENDATION, !resized(), active())
+        $entity: TextEntity((entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
+        $recommendation: TextEntity(containedBy($entity), entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
     then
         $recommendation.remove("X.5.0", "remove Entity of type RECOMMENDATION when contained by ENTITY");
         retract($recommendation);
@@ -1097,17 +1098,28 @@ rule "X.5.0: remove Entity of type RECOMMENDATION when contained by ENTITY"
 
 
 // Rule unit: X.6
-rule "X.6.0: remove Entity of lower rank, when intersected by entity of type ENTITY"
+rule "X.6.0: remove Entity of lower rank, when contained by entity of type ENTITY"
     salience 32
     when
-        $higherRank: TextEntity($type: type, entityType == EntityType.ENTITY, active())
-        $lowerRank: TextEntity(intersects($higherRank), type != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !resized(), active())
+        $higherRank: TextEntity($type: type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
+        $lowerRank: TextEntity(containedBy($higherRank), type != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active())
     then
         $lowerRank.getIntersectingNodes().forEach(node -> update(node));
-        $lowerRank.remove("X.6.0", "remove Entity of lower rank, when intersected by entity of type ENTITY");
+        $lowerRank.remove("X.6.0", "remove Entity of lower rank, when contained by entity of type ENTITY");
         retract($lowerRank);
     end
 
+rule "X.6.1: remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity"
+    salience 32
+    when
+        $higherRank: TextEntity($type: type, $value: value, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
+        $lowerRank: TextEntity(intersects($higherRank), type != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active(), $lowerRank.getValue().length() > $value.length())
+    then
+        $higherRank.getIntersectingNodes().forEach(node -> update(node));
+        $higherRank.remove("X.6.1", "remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity");
+        retract($higherRank);
+    end
+
 
 //------------------------------------ File attributes rules ------------------------------------
 
@@ -1125,7 +1137,7 @@ rule "FA.1.0: remove duplicate FileAttributes"
 //------------------------------------ Local dictionary search rules ------------------------------------
 
 // Rule unit: LDS.0
-rule "LDS.0.0: run local dictionary search"
+rule "LDS.0.0: Run local dictionary search"
     agenda-group "LOCAL_DICTIONARY_ADDS"
     salience -999
     when

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_v2.drl

@@ -22,6 +22,7 @@ import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRu
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.*;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Section;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Table;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Paragraph;
@@ -236,7 +237,7 @@ rule "X.0.0: remove Entity contained by Entity of same type"
     salience 65
     when
         $larger: TextEntity($type: type, $entityType: entityType, active())
-        $contained: TextEntity(containedBy($larger), type == $type, entityType == $entityType, this != $larger, !resized(), active())
+        $contained: TextEntity(containedBy($larger), type == $type, entityType == $entityType, this != $larger, !hasManualChanges(), active())
     then
         $contained.remove("X.0.0", "remove Entity contained by Entity of same type");
         retract($contained);
@@ -248,7 +249,7 @@ rule "X.1.0: merge intersecting Entities of same type"
     salience 64
     when
         $first: TextEntity($type: type, $entityType: entityType, !resized(), active())
-        $second: TextEntity(intersects($first), type == $type, entityType == $entityType, this != $first, !resized(), active())
+        $second: TextEntity(intersects($first), type == $type, entityType == $entityType, this != $first, !hasManualChanges(), active())
     then
         TextEntity mergedEntity = entityCreationService.mergeEntitiesOfSameType(List.of($first, $second), $type, $entityType, document);
         $first.remove("X.1.0", "merge intersecting Entities of same type");
@@ -264,7 +265,7 @@ rule "X.2.0: remove Entity of type ENTITY when contained by FALSE_POSITIVE"
     salience 64
     when
         $falsePositive: TextEntity($type: type, entityType == EntityType.FALSE_POSITIVE, active())
-        $entity: TextEntity(containedBy($falsePositive), type == $type, entityType == EntityType.ENTITY, !resized(), active())
+        $entity: TextEntity(containedBy($falsePositive), type == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges(), active())
     then
         $entity.getIntersectingNodes().forEach(node -> update(node));
         $entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE");
@@ -277,7 +278,7 @@ rule "X.3.0: remove Entity of type RECOMMENDATION when contained by FALSE_RECOMM
     salience 64
     when
         $falseRecommendation: TextEntity($type: type, entityType == EntityType.FALSE_RECOMMENDATION, active())
-        $recommendation: TextEntity(containedBy($falseRecommendation), type == $type, entityType == EntityType.RECOMMENDATION, !resized(), active())
+        $recommendation: TextEntity(containedBy($falseRecommendation), type == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
     then
         $recommendation.remove("X.3.0", "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION");
         retract($recommendation);
@@ -288,8 +289,8 @@ rule "X.3.0: remove Entity of type RECOMMENDATION when contained by FALSE_RECOMM
 rule "X.4.0: remove Entity of type RECOMMENDATION when intersected by ENTITY with same type"
     salience 256
     when
-        $entity: TextEntity($type: type, entityType == EntityType.ENTITY, active())
-        $recommendation: TextEntity(intersects($entity), type == $type, entityType == EntityType.RECOMMENDATION, !resized(), active())
+        $entity: TextEntity($type: type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
+        $recommendation: TextEntity(intersects($entity), type == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
     then
         $entity.addEngines($recommendation.getEngines());
         $recommendation.remove("X.4.0", "remove Entity of type RECOMMENDATION when intersected by ENTITY with same type");
@@ -301,8 +302,8 @@ rule "X.4.0: remove Entity of type RECOMMENDATION when intersected by ENTITY wit
 rule "X.5.0: remove Entity of type RECOMMENDATION when contained by ENTITY"
     salience 256
     when
-        $entity: TextEntity(entityType == EntityType.ENTITY, active())
-        $recommendation: TextEntity(containedBy($entity), entityType == EntityType.RECOMMENDATION, !resized(), active())
+        $entity: TextEntity((entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
+        $recommendation: TextEntity(containedBy($entity), entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
     then
         $recommendation.remove("X.5.0", "remove Entity of type RECOMMENDATION when contained by ENTITY");
         retract($recommendation);
@@ -310,17 +311,28 @@ rule "X.5.0: remove Entity of type RECOMMENDATION when contained by ENTITY"
 
 
 // Rule unit: X.6
-rule "X.6.0: remove Entity of lower rank, when intersected by entity of type ENTITY"
+rule "X.6.0: remove Entity of lower rank, when contained by by entity of type ENTITY"
     salience 32
     when
-        $higherRank: TextEntity($type: type, entityType == EntityType.ENTITY, active())
-        $lowerRank: TextEntity(intersects($higherRank), type != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !resized(), active())
+        $higherRank: TextEntity($type: type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
+        $lowerRank: TextEntity(containedBy($higherRank), type != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active())
     then
         $lowerRank.getIntersectingNodes().forEach(node -> update(node));
-        $lowerRank.remove("X.6.0", "remove Entity of lower rank, when intersected by entity of type ENTITY");
+        $lowerRank.remove("X.6.0", "remove Entity of lower rank, when contained by entity of type ENTITY");
         retract($lowerRank);
     end
 
+rule "X.6.1: remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity"
+    salience 32
+    when
+        $higherRank: TextEntity($type: type, $value: value, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
+        $lowerRank: TextEntity(intersects($higherRank), type != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active(), $lowerRank.getValue().length() > $value.length())
+    then
+        $higherRank.getIntersectingNodes().forEach(node -> update(node));
+        $higherRank.remove("X.6.1", "remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity");
+        retract($higherRank);
+    end
+
 
 //------------------------------------ File attributes rules ------------------------------------
 
@@ -338,7 +350,7 @@ rule "FA.1.0: remove duplicate FileAttributes"
 //------------------------------------ Local dictionary search rules ------------------------------------
 
 // Rule unit: LDS.0
-rule "LDS.0.0: run local dictionary search"
+rule "LDS.0.0: Run local dictionary search"
     agenda-group "LOCAL_DICTIONARY_ADDS"
     salience -999
     when

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/table_demo.drl

@@ -0,0 +1,448 @@
+package drools
+
+import static java.lang.String.format;
+import static com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility.anyMatch;
+import static com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility.exactMatch;
+
+import java.util.List;
+import java.util.LinkedList;
+import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.Collection;
+import java.util.stream.Stream;
+import java.util.Optional;
+
+import com.iqser.red.service.redaction.v1.server.model.document.*;
+import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
+import com.iqser.red.service.redaction.v1.server.model.document.entity.*;
+import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
+import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule;
+import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity
+import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.*;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.Section;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.Table;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.Paragraph;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.Image;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.ImageType;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.Page;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.Headline;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.SectionIdentifier;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.Footer;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.Header;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.NodeType;
+import com.iqser.red.service.redaction.v1.server.model.document.textblock.*;
+import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
+import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlockCollector;
+import com.iqser.red.service.redaction.v1.server.model.document.textblock.AtomicTextBlock;
+import com.iqser.red.service.redaction.v1.server.model.document.textblock.ConcatenatedTextBlock;
+import com.iqser.red.service.redaction.v1.server.model.NerEntities;
+import com.iqser.red.service.redaction.v1.server.model.dictionary.Dictionary;
+import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryModel;
+import com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService;
+import com.iqser.red.service.redaction.v1.server.service.ManualChangesApplicationService;
+import com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility;
+
+import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus
+
+global Document document
+global EntityCreationService entityCreationService
+global ManualChangesApplicationService manualChangesApplicationService
+global Dictionary dictionary
+
+//------------------------------------ queries ------------------------------------
+
+query "getFileAttributes"
+    $fileAttribute: FileAttribute()
+    end
+//---------------------------------------------------------------------------
+
+rule "TAB.0.0: Study Type File Attribute"
+    when
+        not FileAttribute(label == "OECD Number", valueEqualsAnyOf("402","403","404","405","425","429","436","438","439","471","487"))
+        $section: Section(containsAnyString("DATA REQUIREMENT", "TEST GUIDELINE", "MÉTODO(S) DE REFERÊNCIA(S):")
+            && containsAnyString("OECD", "EPA", "OPPTS"))
+    then
+        RedactionSearchUtility.findTextRangesByRegexIgnoreCase("(?<=OECD)(?:[\\w\\s,\\[\\]\\(\\)\\.]{1,10}|(?:.{5,40}(?:Number |Procedure |Guideline )))(4[\\d]{2})", 1 ,$section.getTextBlock()).stream()
+            .map(boundary -> $section.getTextBlock().subSequence(boundary).toString())
+            .map(value -> FileAttribute.builder().label("OECD Number").value(value).build())
+            .forEach(fileAttribute -> insert(fileAttribute));
+        RedactionSearchUtility.findTextRangesByRegexIgnoreCase("(?<=OECD).{5,40}Method (4[\\d]{2}).{1,65}(\\d{4})\\)", 1, $section.getTextBlock()).stream()
+            .map(boundary -> $section.getTextBlock().subSequence(boundary).toString())
+            .map(value -> FileAttribute.builder().label("OECD Number").value(value).build())
+            .forEach(fileAttribute -> insert(fileAttribute));
+    end
+
+rule "TAB.0.1: Guidelines"
+    when
+        $section: Section(containsAnyString("DATA REQUIREMENT", "TEST GUIDELINE", "MÉTODO(S) DE REFERÊNCIA(S):") && containsAnyString("OECD", "EPA", "OPPTS"))
+    then
+        entityCreationService.byRegex("(?<=OECD)(?:[\\w\\s,\\[\\]\\(\\)\\.]{1,10}|.{5,40}(?:Number |Procedure |Guideline ))(4[\\d]{2})", "oecd_guideline_number", EntityType.ENTITY, 1, $section)
+            .forEach(guideline -> guideline.apply("TAB.0.1", "OECD Guideline no. found"));
+        entityCreationService.byRegex("(?<=OECD)(?:[\\w\\s,\\[\\]\\(\\)\\.]{1,10}|.{5,40}(?:Number |Procedure |Guideline ))(4[\\d]{2}),?\\s\\(?(\\d{4})\\)?", "oecd_guideline_year", EntityType.ENTITY, 2, $section)
+            .forEach(guideline -> guideline.apply("TAB.0.1", "OECD Guideline year found"));
+        entityCreationService.byRegex("(?<=OECD)[\\w\\s,\\[\\]]{1,10}\\((\\d{4})\\)\\s(4[\\d]{2})", "oecd_guideline_year", EntityType.ENTITY, 1, $section)
+            .forEach(guideline -> guideline.apply("TAB.0.1", "OECD Guideline year found"));
+        entityCreationService.byRegex("(?<=OECD).{5,40}Method (4[\\d]{2}).{1,65}(\\d{4})\\)", "oecd_guideline_number", EntityType.ENTITY, 1, $section)
+            .forEach(guideline -> guideline.apply("TAB.0.1", "OECD Guideline number found"));
+        entityCreationService.byRegex("(?<=OECD).{5,40}Method (4[\\d]{2}).{1,65}(\\d{4})\\)", "oecd_guideline_year", EntityType.ENTITY, 2, $section)
+            .forEach(guideline -> guideline.apply("TAB.0.1", "OECD Guideline year found"));
+    end
+
+rule "TAB.1.0: Full Table extraction (Guideline Deviation)"
+    when
+        FileAttribute(label == "OECD Number", valueEqualsAnyOf("425"))
+        $section: Section(getHeadline().containsString("Full Table"))
+        $table: Table() from $section.getParent().streamAllSubNodesOfType(NodeType.TABLE).toList()
+        $tableCell: TableCell(!header) from $table.streamTableCells().toList()
+    then
+        entityCreationService.bySemanticNode($tableCell, "guideline_deviation", EntityType.ENTITY)
+            .ifPresent(entity -> entity.apply("TAB.1.0", "full table extracted"));
+    end
+
+rule "TAB.2.0: Individual row extraction (Clinical Signs)"
+    when
+        FileAttribute(label == "OECD Number", valueEqualsAnyOf("425"))
+        $section: Section(getHeadline().containsString("Individual Rows Extraction"))
+        $table: Table(hasHeader("Animal No."), (hasRowWithHeaderAndAnyValue("Animal No.", List.of("120-2", "120-5")))) from $section.streamChildren().toList()
+        TableCell($row: row, containsAnyString("120-2", "120-5")) from $table.streamTableCellsWithHeader("Animal No.").toList()
+        $tableCell: TableCell($row == row) from $table.streamTableCells().toList()
+    then
+        entityCreationService.bySemanticNode($tableCell, "clinical_signs", EntityType.ENTITY)
+            .ifPresent(entity -> entity.apply("TAB.2.0", "Individual row based on animal number"));
+    end
+
+rule "TAB.3.0: Individual column extraction (Strain)"
+    when
+        FileAttribute(label == "OECD Number", valueEqualsAnyOf("425"))
+        $section: Section(getHeadline().containsString("Individual Column"))
+        $table: Table(hasHeader("Sex")) from $section.streamChildren().toList()
+    then
+        $table.streamTableCellsWithHeader("Sex")
+            .map(tableCell -> entityCreationService.bySemanticNode(tableCell, "dosages", EntityType.ENTITY))
+            .filter(Optional::isPresent)
+            .map(Optional::get)
+            .forEach(redactionEntity -> redactionEntity.apply("TAB.3.0", "Individual column based on column header"));
+    end
+
+rule "TAB.4.0: Combined Columns Extraction - Sex and Dosage"
+    when
+        FileAttribute(label == "OECD Number", valueEqualsAnyOf("425"))
+        $section: Section(getHeadline().containsString("Combined Columns"))
+        $table: Table(hasHeader("Sex"), hasHeader("Dosage (mg/kg bw)")) from $section.getParent().streamAllSubNodesOfType(NodeType.TABLE).toList()
+        $maleCells: TableCell($row: row, containsAnyWordIgnoreCase("Male")) from $table.streamTableCellsWithHeader("Sex").toList()
+        $dosageCells: TableCell($row == row) from $table.streamTableCellsWithHeader("Dosage").toList()
+    then
+        entityCreationService.bySemanticNode($maleCells, "combined_male_dosage", EntityType.ENTITY)
+                    .ifPresent(entity -> entity.apply("TAB.4.0", "Dosage combined in row with male"));
+        entityCreationService.bySemanticNode($dosageCells, "combined_male_dosage", EntityType.ENTITY)
+                    .ifPresent(entity -> entity.apply("TAB.4.0", "Dosage combined in row with male"));
+    end
+
+rule "TAB.4.1: Combined Columns Extraction - Sex and Mortality"
+    when
+        FileAttribute(label == "OECD Number", valueEqualsAnyOf("425"))
+        $section: Section(getHeadline().containsString("Combined Columns"))
+        $table: Table(hasHeader("Sex"), hasHeader("Mortality")) from $section.getParent().streamAllSubNodesOfType(NodeType.TABLE).toList()
+        $femaleCells: TableCell($row: row, containsAnyWordIgnoreCase("Female")) from $table.streamTableCellsWithHeader("Sex").toList()
+        $mortalityCells: TableCell($row == row) from $table.streamTableCellsWithHeader("Mortality").toList()
+    then
+        entityCreationService.bySemanticNode($femaleCells, "combined_female_mortality", EntityType.ENTITY)
+            .ifPresent(entity -> entity.apply("TAB.4.1", "Mortality combined in row with female"));
+        entityCreationService.bySemanticNode($mortalityCells, "combined_female_mortality", EntityType.ENTITY)
+            .ifPresent(entity -> entity.apply("TAB.4.1", "Mortality combined in row with female"));
+    end
+
+rule "TAB.5.0: Targeted cell extraction"
+    when
+        FileAttribute(label == "OECD Number", valueEqualsAnyOf("425"))
+        $section: Section(getHeadline().containsString("Value Extraction"))
+        $table: Table(hasHeader("Mortality"), hasRowWithHeaderAndAnyValue("Sex", List.of("male", "Male")), hasRowWithHeaderAndValue("Mortality", "Survived")) from $section.streamChildren().toList()
+        TableCell(containsWordIgnoreCase("Male"), $row: row) from $table.streamTableCellsWithHeader("Sex").toList()
+        TableCell($row == row, containsStringIgnoreCase("Survived")) from $table.streamTableCellsWithHeader("Mortality").toList()
+        $dosageCell: TableCell($row == row) from $table.streamTableCellsWithHeader("Dosage").toList()
+    then
+        entityCreationService.bySemanticNode($dosageCell,"doses_mg_kg_bw", EntityType.ENTITY)
+            .ifPresent(entity -> entity.apply("TAB.5.0", "Dosage found in row with survived male"));
+    end
+
+rule "TAB.6.0: Targeted cell extraction (Experimental Stop date)"
+    when
+        $section: Section(getHeadline().containsString("Advanced Table Extraction"), containsAllStrings("female", "Female", "Survived", "Group 2"))
+        $table: Table(hasHeader("Group 2")) from $section.streamChildren().toList()
+        TableCell(containsWordIgnoreCase("Female"), $row: row) from $table.streamTableCellsWithHeader("Group 2").toList()
+        TableCell($row == row, containsStringIgnoreCase("Survived")) from $table.streamTableCellsWithHeader("Group 2").toList()
+        $femaleSurvived: TableCell($row == row) from $table.streamTableCellsWithHeader("Group 2").toList()
+    then
+        entityCreationService.bySemanticNode($femaleSurvived, "experiment_female_survived", EntityType.ENTITY)
+            .ifPresent(entity -> entity.apply("TAB.6.0", "Female in group to experimental start date"));
+    end
+
+rule "TAB.7.0: Indicator (Species)"
+    when
+        FileAttribute(label == "OECD Number", valueEqualsAnyOf("425"))
+        $section: Section(getHeadline().containsString("Entity-Based"))
+        $table: Table() from $section.streamAllSubNodesOfType(NodeType.TABLE).toList()
+        TableCell(isHeader(), containsString("Title"), $col: col) from $table.streamTableCells().toList()
+        TableCell(hasEntitiesOfType("vertebrate"), $row: row) from $table.streamTableCells().toList()
+        $cell: TableCell($col == col, $row == row) from $table.streamTableCells().toList()
+    then
+        entityCreationService.bySemanticNode($cell, "study_design", EntityType.ENTITY)
+            .ifPresent(redactionEntity -> redactionEntity.apply("TAB.7.0", "Vertebrate study found"));
+    end
+
+//------------------------------------ Manual redaction rules ------------------------------------
+
+// Rule unit: MAN.0
+rule "MAN.0.0: Apply manual resize redaction"
+    salience 128
+    when
+        $resizeRedaction: ManualResizeRedaction($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate)
+        not ManualResizeRedaction(annotationId == $id, requestDate.isBefore($requestDate))
+        $entityToBeResized: TextEntity(matchesAnnotationId($id))
+    then
+        manualChangesApplicationService.resizeEntityAndReinsert($entityToBeResized, $resizeRedaction);
+        retract($resizeRedaction);
+        update($entityToBeResized);
+        $entityToBeResized.getIntersectingNodes().forEach(node -> update(node));
+    end
+
+rule "MAN.0.1: Apply manual resize redaction"
+    salience 128
+    when
+        $resizeRedaction: ManualResizeRedaction($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate)
+        not ManualResizeRedaction(annotationId == $id, requestDate.isBefore($requestDate))
+        $imageToBeResized: Image(id == $id)
+    then
+        manualChangesApplicationService.resizeImage($imageToBeResized, $resizeRedaction);
+        retract($resizeRedaction);
+        update($imageToBeResized);
+        update($imageToBeResized.getParent());
+    end
+
+
+// Rule unit: MAN.1
+rule "MAN.1.0: Apply id removals that are valid and not in forced redactions to Entity"
+    salience 128
+    when
+        $idRemoval: IdRemoval($id: annotationId, status == AnnotationStatus.APPROVED)
+        $entityToBeRemoved: TextEntity(matchesAnnotationId($id))
+    then
+        $entityToBeRemoved.getManualOverwrite().addChange($idRemoval);
+        update($entityToBeRemoved);
+        retract($idRemoval);
+        $entityToBeRemoved.getIntersectingNodes().forEach(node -> update(node));
+    end
+
+rule "MAN.1.1: Apply id removals that are valid and not in forced redactions to Image"
+    salience 128
+    when
+        $idRemoval: IdRemoval($id: annotationId, status == AnnotationStatus.APPROVED)
+        $imageEntityToBeRemoved: Image($id == id)
+    then
+        $imageEntityToBeRemoved.getManualOverwrite().addChange($idRemoval);
+        update($imageEntityToBeRemoved);
+        retract($idRemoval);
+        update($imageEntityToBeRemoved.getParent());
+    end
+
+
+// Rule unit: MAN.2
+rule "MAN.2.0: Apply force redaction"
+    salience 128
+    when
+        $force: ManualForceRedaction($id: annotationId, status == AnnotationStatus.APPROVED)
+        $entityToForce: TextEntity(matchesAnnotationId($id))
+    then
+        $entityToForce.getManualOverwrite().addChange($force);
+        update($entityToForce);
+        $entityToForce.getIntersectingNodes().forEach(node -> update(node));
+        retract($force);
+    end
+
+rule "MAN.2.1: Apply force redaction to images"
+    salience 128
+    when
+        $force: ManualForceRedaction($id: annotationId, status == AnnotationStatus.APPROVED)
+        $imageToForce: Image(id == $id)
+    then
+        $imageToForce.getManualOverwrite().addChange($force);
+        update($imageToForce);
+        update($imageToForce.getParent());
+        retract($force);
+    end
+
+
+// Rule unit: MAN.3
+rule "MAN.3.0: Apply entity recategorization"
+    salience 128
+    when
+        $recategorization: ManualRecategorization($id: annotationId, $type: type, status == AnnotationStatus.APPROVED, $requestDate: requestDate)
+        not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))
+        $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type != $type)
+    then
+        $entityToBeRecategorized.getIntersectingNodes().forEach(node -> update(node));
+        manualChangesApplicationService.recategorize($entityToBeRecategorized, $recategorization);
+        retract($recategorization);
+        // Entity is copied and inserted, so the old entity needs to be retracted to avoid duplication.
+        retract($entityToBeRecategorized);
+    end
+
+rule "MAN.3.1: Apply entity recategorization of same type"
+    salience 128
+    when
+        $recategorization: ManualRecategorization($id: annotationId, $type: type, status == AnnotationStatus.APPROVED, $requestDate: requestDate)
+        not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))
+        $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type == $type)
+    then
+        $entityToBeRecategorized.getManualOverwrite().addChange($recategorization);
+        retract($recategorization);
+    end
+
+rule "MAN.3.2: Apply image recategorization"
+    salience 128
+    when
+        $recategorization: ManualRecategorization($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate)
+        not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))
+        $imageToBeRecategorized: Image($id == id)
+    then
+        manualChangesApplicationService.recategorize($imageToBeRecategorized, $recategorization);
+        update($imageToBeRecategorized);
+        update($imageToBeRecategorized.getParent());
+        retract($recategorization);
+    end
+
+
+// Rule unit: MAN.4
+rule "MAN.4.0: Apply legal basis change"
+    salience 128
+    when
+        $legalbasisChange: ManualLegalBasisChange($id: annotationId, status == AnnotationStatus.APPROVED)
+        $imageToBeRecategorized: Image($id == id)
+    then
+        $imageToBeRecategorized.getManualOverwrite().addChange($legalbasisChange);
+    end
+
+rule "MAN.4.1: Apply legal basis change"
+    salience 128
+    when
+        $legalBasisChange: ManualLegalBasisChange($id: annotationId, status == AnnotationStatus.APPROVED)
+        $entityToBeChanged: TextEntity(matchesAnnotationId($id))
+    then
+        $entityToBeChanged.getManualOverwrite().addChange($legalBasisChange);
+    end
+
+
+//------------------------------------ Entity merging rules ------------------------------------
+
+// Rule unit: X.0
+rule "X.0.0: Remove Entity contained by Entity of same type"
+    salience 65
+    when
+        $larger: TextEntity($type: type, $entityType: entityType, active())
+        $contained: TextEntity(containedBy($larger), type == $type, entityType == $entityType, this != $larger, !hasManualChanges(), active())
+    then
+        $contained.remove("X.0.0", "remove Entity contained by Entity of same type");
+        retract($contained);
+    end
+
+
+// Rule unit: X.2
+rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
+    salience 64
+    when
+        $falsePositive: TextEntity($type: type, entityType == EntityType.FALSE_POSITIVE, active())
+        $entity: TextEntity(containedBy($falsePositive), type == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges(), active())
+    then
+        $entity.getIntersectingNodes().forEach(node -> update(node));
+        $entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE");
+        retract($entity)
+    end
+
+
+// Rule unit: X.3
+rule "X.3.0: Remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"
+    salience 64
+    when
+        $falseRecommendation: TextEntity($type: type, entityType == EntityType.FALSE_RECOMMENDATION, active())
+        $recommendation: TextEntity(containedBy($falseRecommendation), type == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
+    then
+        $recommendation.remove("X.3.0", "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION");
+        retract($recommendation);
+    end
+
+
+// Rule unit: X.4
+rule "X.4.0: Remove Entity of type RECOMMENDATION when intersected by ENTITY with same type"
+    salience 256
+    when
+        $entity: TextEntity($type: type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
+        $recommendation: TextEntity(intersects($entity), type == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
+    then
+        $entity.addEngines($recommendation.getEngines());
+        $recommendation.remove("X.4.0", "remove Entity of type RECOMMENDATION when intersected by ENTITY with same type");
+        retract($recommendation);
+    end
+
+
+// Rule unit: X.5
+rule "X.5.0: Remove Entity of type RECOMMENDATION when contained by ENTITY"
+    salience 256
+    when
+        $entity: TextEntity((entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
+        $recommendation: TextEntity(containedBy($entity), entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
+    then
+        $recommendation.remove("X.5.0", "remove Entity of type RECOMMENDATION when contained by ENTITY");
+        retract($recommendation);
+    end
+
+
+// Rule unit: X.7
+rule "X.7.0: remove all images"
+    salience 512
+    when
+        $image: Image(imageType != ImageType.OCR, !hasManualChanges())
+    then
+        $image.remove("X.7.0", "remove all images");
+        retract($image);
+    end
+
+
+//------------------------------------ File attributes rules ------------------------------------
+
+// Rule unit: FA.1
+rule "FA.1.0: Remove duplicate FileAttributes"
+    salience 64
+    when
+        $fileAttribute: FileAttribute($label: label, $value: value)
+        $duplicate: FileAttribute(this != $fileAttribute, label == $label, value == $value)
+    then
+        retract($duplicate);
+    end
+
+
+//------------------------------------ Local dictionary search rules ------------------------------------
+
+// Rule unit: LDS.0
+rule "LDS.0.0: Run local dictionary search"
+    agenda-group "LOCAL_DICTIONARY_ADDS"
+    salience -999
+    when
+        $dictionaryModel: DictionaryModel(!localEntriesWithMatchedRules.isEmpty()) from dictionary.getDictionaryModels()
+    then
+        entityCreationService.bySearchImplementation($dictionaryModel.getLocalSearch(), $dictionaryModel.getType(), EntityType.RECOMMENDATION, document)
+            .forEach(entity ->  {
+                Collection<MatchedRule> matchedRules = $dictionaryModel.getLocalEntriesWithMatchedRules().get(entity.getValue());
+                entity.addMatchedRules(matchedRules);
+            });
+    end

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/table_demo_OLD.drl

@@ -0,0 +1,134 @@
+package drools
+
+import com.iqser.red.service.redaction.v1.server.redaction.model.Section
+
+global Section section
+
+
+// --------------------------------------- Your rules below this line--------------------------------------------------
+
+rule "0a: Study Type File Attribute"
+     when
+         Section(
+            !fileAttributeContainsAnyOf("OECD Number","402","403","404","405","425","429","436","438","439","471","487")
+            && (
+                text.contains("DATA REQUIREMENT")
+                || text.contains("TEST GUIDELINE")
+                || text.contains("MÉTODO(S) DE REFERÊNCIA(S):")
+            )
+            && (
+                text.contains("OECD")
+                || text.contains("EPA")
+                || text.contains("OPPTS")
+            )
+        )
+     then
+         section.addFileAttribute("OECD Number", "(?<=OECD)(?:[\\w\\s,\\[\\]\\(\\)\\.]{1,10}|(?:.{5,40}(?:Number |Procedure |Guideline )))(4[\\d]{2})", true, 1);
+         section.addFileAttribute("OECD Number", "(?<=OECD).{5,40}Method (4[\\d]{2}).{1,65}(\\d{4})\\)", true, 1);
+    end
+
+
+rule "1: Guidelines"
+     when
+         Section(
+            (
+                text.contains("DATA REQUIREMENT")
+                || text.contains("TEST GUIDELINE")
+                || text.contains("MÉTODO(S) DE REFERÊNCIA(S):")
+            )
+            && (
+                text.contains("OECD")
+                || text.contains("EPA")
+                || text.contains("OPPTS")
+            )
+        )
+     then
+         section.redactByRegEx("(?<=OECD)(?:[\\w\\s,\\[\\]\\(\\)\\.]{1,10}|.{5,40}(?:Number |Procedure |Guideline ))(4[\\d]{2})", true, 1, "oecd_guideline_number", 1, "OECD Guideline no. found", "n-a");
+         section.redactByRegEx("(?<=OECD)(?:[\\w\\s,\\[\\]\\(\\)\\.]{1,10}|.{5,40}(?:Number |Procedure |Guideline ))(4[\\d]{2}),?\\s\\(?(\\d{4})\\)?", true, 2, "oecd_guideline_year", 1, "OECD Guideline year found", "n-a");
+         section.redactByRegEx("(?<=OECD)[\\w\\s,\\[\\]]{1,10}\\((\\d{4})\\)\\s(4[\\d]{2})", true, 1, "oecd_guideline_year", 1, "OECD Guideline year found", "n-a");
+         section.redactByRegEx("(?<=OECD).{5,40}Method (4[\\d]{2}).{1,65}(\\d{4})\\)", true, 1, "oecd_guideline_number", 1, "OECD Guideline number found", "n-a");
+         section.redactByRegEx("(?<=OECD).{5,40}Method (4[\\d]{2}).{1,65}(\\d{4})\\)", true, 2, "oecd_guideline_year", 1, "OECD Guideline year found", "n-a");
+     end
+
+
+rule "2: Full Table extraction (Guideline Deviation)"
+    when
+        Section(
+            fileAttributeByLabelEqualsIgnoreCase("OECD Number","425")
+            && headlineContainsWord("Full Table")
+            && hasTableHeader("Sex")
+        )
+    then
+       section.redactSectionTextWithoutHeadLine("guideline_deviation",2,"Full table extraction into guideline deviation","n-a");
+    end
+
+rule "3: Individual row extraction (Clinical Signs)"
+    when
+        Section(
+            fileAttributeByLabelEqualsIgnoreCase("OECD Number","425")
+            && headlineContainsWord("Individual Rows")
+            && hasTableHeader("Animal No.")
+            && (rowEquals("Animal No.","120-2") || rowEquals("Animal No.","120-5"))
+        )
+    then
+       section.redactSectionTextWithoutHeadLine("clinical_signs",3,"Individual row based on animal number","n-a");
+    end
+
+rule "4: Individual column extraction (Strain)"
+    when
+        Section(
+            fileAttributeByLabelEqualsIgnoreCase("OECD Number","425")
+            && headlineContainsWord("Individual Column")
+            && hasTableHeader("Sex")
+        )
+    then
+       section.redactCell("Sex",4,"dosages",false,"Individual column based on column header","n-a");
+    end
+
+rule "5: Dose Mortality"
+    when
+        Section(
+            fileAttributeByLabelEqualsIgnoreCase("OECD Number","425")
+            && headlineContainsWord("Combined Columns")
+            && hasTableHeader("Mortality")
+            && hasTableHeader("Dosage (mg/kg bw)")
+        )
+    then
+        section.redactCell("Mortality",5,"dose_mortality",false,"Dose Mortality found.","n-a");
+        section.redactCell("Dosage (mg/kg bw)",5,"dose_mortality_dose",false,"Dose Mortality dose found.","n-a");
+    end
+
+rule "6: targeted cell extraction (Experimental Start date)"
+    when
+        Section(
+            fileAttributeByLabelEqualsIgnoreCase("OECD Number","425")
+            && headlineContainsWord("Value Extraction")
+            && hasTableHeader("Mortality")
+            && (rowEquals("Sex","male") || rowEquals("Sex","Male"))
+            && rowEquals("Mortality","Survived")
+        )
+    then
+       section.redactCell("Treatment start",6,"experimental_start_date",false,"Female deaths date to experimental start date","n-a");
+    end
+
+rule "7: targeted cell extraction (Experimental Stop date)"
+    when
+        Section(
+           isInTable()
+           && (searchText.contains("female") || searchText.contains("Female"))
+           && searchText.contains("Survived")
+        )
+    then
+        section.redactCellBelow(7,"experimental_end_date",true,false,"Female deaths date to experimental start date","n-a", "Sex", "Group 2");
+    end
+
+rule "8: Indicator (Species)"
+    when
+        Section(
+            fileAttributeByLabelEqualsIgnoreCase("OECD Number","425")
+            && headlineContainsWord("Entity-Based")
+            && matchesType("vertebrates")
+        )
+    then
+       section.redactCell("Title",8,"study_design",false,"Vertebrate study found","n-a");
+    end

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/table_demo_components.drl

@@ -0,0 +1,520 @@
+package drools
+
+import static java.lang.String.format;
+import static com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility.anyMatch;
+import static com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility.exactMatch;
+
+import java.util.List;
+import java.util.LinkedList;
+import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.Collection;
+import java.util.stream.Stream;
+import java.util.Optional;
+
+import com.iqser.red.service.redaction.v1.server.model.component.Component;
+import com.iqser.red.service.redaction.v1.server.model.component.Entity;
+import com.iqser.red.service.redaction.v1.server.service.document.ComponentCreationService;
+
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Change;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryType;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualChange;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position;
+
+import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;
+
+global ComponentCreationService componentCreationService
+
+//------------------------------------ queries ------------------------------------
+
+query "getFileAttributes"
+    $fileAttribute: FileAttribute()
+    end
+
+query "getComponents"
+    $component: Component()
+    end
+
+//------------------------------------ Guideline mapping object ------------------------------------
+
+declare GuidelineMapping
+    number: String
+    year: String
+    guideline: String
+ end
+
+//------------------------------------ Default Components rules ------------------------------------
+
+rule "StudyTitle.0.0: First Title found"
+    when
+        $titleCandidates: List() from collect (Entity(type == "title"))
+    then
+        componentCreationService.firstOrElse("StudyTitle.0.0", "Study_Title", $titleCandidates, "");
+    end
+
+
+rule "PerformingLaboratory.1.0: Performing Laboratory name and country found in same section"
+    when
+        $laboratoryName: Entity(type == "laboratory_name", $node: containingNode)
+        $laboratoryCountry: Entity(type == "laboratory_country", containingNode == $node)
+        not Entity(type == "laboratory_country", containingNode == $node, Math.abs($laboratoryName.startOffset - startOffset) < Math.abs($laboratoryName.startOffset - $laboratoryCountry.startOffset))
+    then
+        componentCreationService.create("PerformingLaboratory.1.0", "Performing_Laboratory", $laboratoryName.getValue() + ", " + $laboratoryCountry.getValue(), "Laboratory name and country found!", List.of($laboratoryName, $laboratoryCountry));
+    end
+
+rule "PerformingLaboratory.2.0: Performing Laboratory name but no country found in same section"
+    when
+        $laboratoryName: Entity(type == "laboratory_name", $node: containingNode)
+        not Entity(type == "laboratory_country", containingNode == $node)
+    then
+        componentCreationService.create("PerformingLaboratory.2.0", "Performing_Laboratory", $laboratoryName.getValue(), "Only laboratory name found!", List.of($laboratoryName));
+    end
+
+rule "PerformingLaboratory.0.2: Performing Laboratory not found"
+    salience -1
+    when
+        not Component(name == "Performing_Laboratory")
+    then
+        componentCreationService.create("PerformingLaboratory.0.2", "Performing_Laboratory", "", "fallback");
+    end
+
+
+rule "ReportNumber.0.0: First Report number found"
+    when
+        $reportNumberCandidates: List() from collect (Entity(type == "report_number"))
+    then
+        componentCreationService.firstOrElse("ReportNumber.0.0", "Report_Number", $reportNumberCandidates, "");
+    end
+
+
+rule "GLPStudy.0.0: GLP Study found"
+    when
+        $glpStudyList: List(!isEmpty) from collect(Entity(type == "glp_study"))
+    then
+        componentCreationService.create("GLPStudy.0.0", "GLP_Study", "Yes", "Yes if present, No if not", $glpStudyList);
+    end
+
+rule "GLPStudy.1.0: GLP Study not found"
+    when
+        not Entity(type == "glp_study")
+    then
+        componentCreationService.create("GLPStudy.1.0", "GLP_Study", "No", "Yes if present, No if not");
+    end
+
+
+rule "TestGuideline.0.0: create OECD number and year guideline mappings"
+    salience 2
+    when
+        Entity(type == "oecd_guideline_number")
+        Entity(type == "oecd_guideline_year")
+    then
+        insert(new GuidelineMapping("425", "2008", "Nº 425: Acute oral Toxicity - Up-and-Down Procedure (03/10/2008)"));
+        insert(new GuidelineMapping("425", "2001", "Nº 425: Acute oral Toxicity - Up-and-Down Procedure (17/12/2001)"));
+        insert(new GuidelineMapping("402", "2017", "Nº 402: Acute Dermal Toxicity (09/10/2017)"));
+        insert(new GuidelineMapping("402", "1987", "Nº 402: Acute Dermal Toxicity (24/02/1987)"));
+        insert(new GuidelineMapping("403", "2009", "Nº 403: Acute Inhalation Toxicity (08/09/2009)"));
+        insert(new GuidelineMapping("403", "1981", "Nº 403: Acute Inhalation Toxicity (12/05/1981)"));
+        insert(new GuidelineMapping("433", "2018", "Nº 433: Acute Inhalation Toxicity: Fixed Concentration Procedure (27/06/2018)"));
+        insert(new GuidelineMapping("433", "2017", "Nº 433: Acute Inhalation Toxicity: Fixed Concentration Procedure (09/10/2017)"));
+        insert(new GuidelineMapping("436", "2009", "Nº 436: Acute Inhalation Toxicity – Acute Toxic Class Method (08/09/2009)"));
+        insert(new GuidelineMapping("404", "1981", "Nº 404: Acute Dermal Irritation/Corrosion (12/05/1981)"));
+        insert(new GuidelineMapping("404", "1992", "Nº 404: Acute Dermal Irritation/Corrosion (17/07/1992)"));
+        insert(new GuidelineMapping("404", "2002", "Nº 404: Acute Dermal Irritation/Corrosion (24/04/2002)"));
+        insert(new GuidelineMapping("404", "2015", "Nº 404: Acute Dermal Irritation/Corrosion (28/07/2015)"));
+        insert(new GuidelineMapping("405", "2017", "Nº 405: Acute Eye Irritation/Corrosion (09/10/2017)"));
+        insert(new GuidelineMapping("405", "2012", "Nº 405: Acute Eye Irritation/Corrosion (02/10/2012)"));
+        insert(new GuidelineMapping("405", "2002", "Nº 405: Acute Eye Irritation/Corrosion (24/04/2002)"));
+        insert(new GuidelineMapping("405", "1987", "Nº 405: Acute Eye Irritation/Corrosion (24/02/1987)"));
+        insert(new GuidelineMapping("429", "2002", "Nº 429: Skin Sensitisation: Local Lymph Node Assay (24/04/2002)"));
+        insert(new GuidelineMapping("429", "2010", "Nº 429: Skin Sensitisation (23/07/2010)"));
+        insert(new GuidelineMapping("442A", "2018", "Nº 442A: Skin Sensitization (23/07/2018)"));
+        insert(new GuidelineMapping("442B", "2018", "Nº 442B: Skin Sensitization (27/06/2018)"));
+        insert(new GuidelineMapping("471", "1997", "Nº 471: Bacterial Reverse Mutation Test (21/07/1997)"));
+        insert(new GuidelineMapping("471", "2020", "Nº 471: Bacterial Reverse Mutation Test (26/06/2020)"));
+        insert(new GuidelineMapping("406", "1992", "Nº 406: Skin Sensitisation (1992)"));
+        insert(new GuidelineMapping("428", "2004", "Nº 428: Split-Thickness Skin test (2004)"));
+        insert(new GuidelineMapping("438", "2018", "Nº 438: Eye Irritation (26/06/2018)"));
+        insert(new GuidelineMapping("439", "2019", "Nº 439: Skin Irritation (2019)"));
+        insert(new GuidelineMapping("474", "2016", "Nº 474: Micronucleus Bone Marrow Cells Rat (2016)"));
+        insert(new GuidelineMapping("487", "2016", "Nº 487: Micronucleus Human Lymphocytes (2016)"));
+    end
+
+rule "TestGuideline.0.1: match OECD number and year with guideline mappings"
+    salience 1
+    when
+        not Component(name == "Test_Guidelines_1")
+        GuidelineMapping($year: year, $number: number, $guideline: guideline)
+        $guidelineNumber: Entity(type == "oecd_guideline_number", value == $number)
+        $guidelineYear: Entity(type == "oecd_guideline_year", value == $year)
+    then
+        componentCreationService.create(
+           "TestGuideline.0.0",
+           "Test_Guidelines_1",
+           $guideline,
+           "OECD Number and guideline year mapped!",
+           List.of($guidelineNumber, $guidelineYear)
+        );
+    end
+
+rule "TestGuideline.1.0: no guideline mapping found"
+    when
+        not Component(name == "Test_Guidelines_1")
+        $guideLine: Entity(type == "oecd_guideline")
+    then
+        componentCreationService.create("TestGuideline.2.0", "Test_Guidelines_1", $guideLine.getValue(), "No Mapping for OECD number and year found, using fallback instead!", List.of($guideLine));
+    end
+
+rule "TestGuideline.2.0: All values of EPA guideline and EC guidelines"
+    when
+        $guidelines: List() from collect (Entity(type == "epa_guideline" || type ==  "ec_guideline"))
+    then
+        componentCreationService.joining("TestGuideline.2.0", "Test_Guidelines_2", $guidelines);
+    end
+
+
+rule "StartDate.0.0: All experimental start dates converted to dd/MM/yyyy"
+    when
+        $startDates: List() from collect (Entity(type == "experimental_start_date"))
+    then
+        componentCreationService.convertDates("StartDate.0.0", "Experimental_Starting_Date", $startDates);
+    end
+
+
+rule "CompletionDate.0.0: All experimental end dates converted to dd/MM/yyyy"
+    when
+        $endDates: List() from collect (Entity(type == "experimental_end_date"))
+    then
+        componentCreationService.convertDates("CompletionDate.0.0", "Experimental_Completion_Date", $endDates);
+    end
+
+
+rule "AnalysisCertificate.0.0: Unique values of certificate of analysis batch identification"
+    when
+        $batchNumbers: List() from collect (Entity(type == "batch_number"))
+    then
+        componentCreationService.joiningUnique("AnalysisCertificate.0.0", "Certificate_of_Analysis_Batch_Identification", $batchNumbers);
+    end
+
+rule "StudyConclusion.0.0: Study conclusion in first found section"
+    when
+        $oecdNumber: String() from List.of("402", "403", "404", "405", "425", "429", "436", "471")
+        FileAttribute(label == "OECD Number", value == $oecdNumber)
+        $studyConclusions: List() from collect(Entity(type == "study_conclusion"))
+    then
+        componentCreationService.joiningFromFirstSectionOnly("StudyConclusion.0.0", "Study_Conclusion", $studyConclusions, " ");
+    end
+
+rule "GuidelineDeviation.0.0: Guideline deviation as sentences"
+    when
+        $oecdNumber: String() from List.of("402", "403", "404", "405", "425", "429", "436", "471")
+        FileAttribute(label == "OECD Number", value == $oecdNumber)
+        $guidelineDeviations: List() from collect (Entity(type == "guideline_deviation"))
+    then
+        componentCreationService.joining("GuidelineDeviation.0.0", "Deviation_from_the_Guideline", $guidelineDeviations, "\n");
+    end
+
+rule "Species.0.0: First found species"
+    when
+        $oecdNumber: String() from List.of("402", "403", "404", "405", "425", "429", "436", "471")
+        FileAttribute(label == "OECD Number", value == $oecdNumber)
+        $species: List() from collect (Entity(type == "species"))
+    then
+        componentCreationService.firstOrElse("Species.0.0", "Species", $species, "");
+    end
+
+rule "Strain.0.0: First found strain"
+    when
+        $oecdNumber: String() from List.of("402", "403", "404", "405", "425", "429", "436", "471")
+        FileAttribute(label == "OECD Number", value == $oecdNumber)
+        $strain: List() from collect (Entity(type == "strain"))
+    then
+        componentCreationService.firstOrElse("Strain.0.0", "Strain", $strain, "");
+    end
+
+rule "Conclusion.0.0: Unique values of Conclusion LD50"
+    when
+        $oecdNumber: String() from List.of("402", "403", "425", "436")
+        FileAttribute(label == "OECD Number", value == $oecdNumber)
+        $conclusions: List() from collect (Entity(type == "ld50_value"))
+    then
+        componentCreationService.joiningUnique("Conclusion.0.0", "Conclusion_LD50_mg_per_kg", $conclusions);
+    end
+
+rule "Conclusion0.1.0: Greater than found"
+    when
+        $oecdNumber: String() from List.of("402", "403", "425", "436")
+        FileAttribute(label == "OECD Number", value == $oecdNumber)
+        $conclusions: List(!isEmpty()) from collect (Entity(type == "ld50_greater"))
+    then
+        componentCreationService.create("Conclusion.1.0", "Conclusion_LD50_Greater_than", "Greater than", "Entity of type 'ld50_greater' found", $conclusions);
+    end
+
+rule "Conclusion.1.1: Greater than not found"
+    when
+        $oecdNumber: String() from List.of("402", "403", "425", "436")
+        FileAttribute(label == "OECD Number", value == $oecdNumber)
+        not Entity(type == "ld50_greater")
+    then
+        componentCreationService.create("Conclusion.1.1", "Conclusion_LD50_Greater_than", "", "No entity of type 'ld50_greater' found");
+    end
+
+rule "Conclusion.2.0: Minimum confidence as unique values"
+    when
+        $oecdNumber: String() from List.of("402", "403", "425", "436")
+        FileAttribute(label == "OECD Number", value == $oecdNumber)
+        $conclusions: List() from collect (Entity(type == "confidence_minimal"))
+    then
+        componentCreationService.joiningUnique("Conclusion.2.0", "Conclusion_Minimum_Confidence", $conclusions);
+    end
+
+rule "Conclusion.3.0: Maximum confidence as unique values"
+    when
+        $oecdNumber: String() from List.of("402", "403", "425", "436")
+        FileAttribute(label == "OECD Number", value == $oecdNumber)
+        $conclusions: List() from collect (Entity(type == "confidence_maximal"))
+    then
+        componentCreationService.joiningUnique("Conclusion.3.0", "Conclusion_Maximum_Confidence", $conclusions);
+    end
+
+rule "Necropsy.0.0: Necropsy findings from longest section"
+    when
+        FileAttribute(label == "OECD Number", value == "402")
+        $necropsies: List() from collect (Entity(type == "necropsy_findings"))
+    then
+        componentCreationService.joiningFromLongestSectionOnly("Necropsy.0.0", "Necropsy_Findings", $necropsies, " ");
+    end
+
+rule "Necropsy.0.1: Necropsy findings joined with \n"
+    when
+        FileAttribute(label == "OECD Number", value == "403" || value == "436")
+        $necropsies: List() from collect (Entity(type == "necropsy_findings"))
+    then
+        componentCreationService.joining("Necropsy.0.0", "Necropsy_Findings", $necropsies, "\n");
+    end
+
+rule "Necropsy.1.0: Doses mg per kg of Bodyweight as one block"
+    when
+        FileAttribute(label == "OECD Number", value == "402")
+        $dosages: List() from collect (Entity(type == "doses_(mg_kg_bw)"))
+    then
+        componentCreationService.joining("Necropsy.1.0", "Doses_mg_per_kg_bw", $dosages, " ");
+    end
+
+rule "Necropsy.2.0: Conducted with 4 hours of exposure as one block"
+    when
+        $oecdNumber: String() from List.of("403", "436")
+        FileAttribute(label == "OECD Number", value == $oecdNumber)
+        $exposures: List() from collect (Entity(type == "4h_exposure"))
+    then
+        componentCreationService.joining("Necropsy.3.0", "Conducted_with_4_Hours_of_Exposure", $exposures, " ");
+    end
+
+rule "StudyDesign.0.0: Study design as one block"
+    when
+        $oecdNumber: String() from List.of("404", "405", "429", "406", "428", "438", "439", "474", "487")
+        FileAttribute(label == "OECD Number", value == $oecdNumber)
+        $studyDesigns: List() from collect (Entity(type == "study_design"))
+    then
+        componentCreationService.joining("StudyDesign.0.0", "Study_Design", $studyDesigns, " ");
+    end
+
+rule "Results.0.0: Results and conclusions as joined values"
+    when
+        $oecdNumber: String() from List.of("406", "428", "438", "439", "474", "487")
+        FileAttribute(label == "OECD Number", value == $oecdNumber)
+        $results: List() from collect (Entity(type == "results_and_conclusion"))
+    then
+        componentCreationService.joining("Results.0.0", "Results_and_Conclusions", $results, " ");
+    end
+
+rule "WeightBehavior.0.0: Weight change behavior as sentences"
+    when
+        FileAttribute(label == "OECD Number", value == "402")
+        $weightChanges: List() from collect (Entity(type == "weight_behavior_changes"))
+    then
+        componentCreationService.joining("WeightBehavior.0.0", "Weight_Behavior_Changes", $weightChanges, "\n");
+    end
+
+rule "MortalityStatement.0.0: Mortality statements as one block"
+    when
+        FileAttribute(label == "OECD Number", value == "402")
+        $mortalityStatements: List() from collect (Entity(type == "mortality_statement"))
+    then
+        componentCreationService.joining("MortalityStatement.0.0", "Mortality_Statement", $mortalityStatements, " ");
+    end
+
+rule "ClinicalObservations.0.0: Clinical observations as sentences"
+    when
+        FileAttribute(label == "OECD Number", value == "403")
+        $observations: List() from collect (Entity(type == "clinical_observations"))
+    then
+        componentCreationService.joining("MortalityStatement.0.0", "Clinical_Observations", $observations, "\n");
+    end
+
+rule "BodyWeight.0.0: Bodyweight changes as sentences"
+    when
+        FileAttribute(label == "OECD Number", value == "403")
+        $weightChanges: List() from collect (Entity(type == "bodyweight_changes"))
+    then
+        componentCreationService.joining("BodyWeight.0.0", "Body_Weight_Changes", $weightChanges, "\n");
+    end
+
+rule "Detailing.0.0: Detailing of reported changes as one block"
+    when
+        $oecdNumber: String() from List.of("404", "405")
+        FileAttribute(label == "OECD Number", value == $oecdNumber)
+        $detailings: List() from collect (Entity(type == "detailing"))
+    then
+        componentCreationService.joining("Detailing.0.0", "Detailing_of_Reported_Changes", $detailings, " ");
+    end
+
+rule "Sex.0.0: Male sex found"
+    when
+        $oecdNumber: String() from List.of("405", "429")
+        FileAttribute(label == "OECD Number", value == $oecdNumber)
+        $males: List(!isEmpty) from collect (Entity(type == "sex", (value.toLowerCase() == "male" || value.toLowerCase() == "males")))
+    then
+        componentCreationService.create("Sex.0.0", "Sex", "male", "male sex found", $males);
+    end
+
+rule "Sex.1.0: Female sex found"
+    when
+        $oecdNumber: String() from List.of("405", "429")
+        FileAttribute(label == "OECD Number", value == $oecdNumber)
+        $females: List(!isEmpty) from collect (Entity(type == "sex", (value.toLowerCase() == "female" || value.toLowerCase() == "females")))
+    then
+        componentCreationService.create("Sex.0.0", "Sex", "female", "female sex found", $females);
+    end
+
+rule "NumberOfAnimals.0.0: Number of animals found"
+    when
+        $oecdNumber: String() from List.of("405", "429")
+        FileAttribute(label == "OECD Number", value == $oecdNumber)
+        $numberOfAnimals: Entity(type == "number_of_animals")
+    then
+        componentCreationService.create("NumberOfAnimals.0.0", "Number_of_Animals", $numberOfAnimals.getValue(), "Number of animals found directly", $numberOfAnimals);
+    end
+
+rule "NumberOfAnimals.1.0: Count unique occurences of animals"
+    when
+        $oecdNumber: String() from List.of("405", "429")
+        FileAttribute(label == "OECD Number", value == $oecdNumber)
+        not Entity(type == "number_of_animals")
+        $animals: List() from collect (Entity(type == "animal_number"))
+    then
+        componentCreationService.uniqueValueCount("NumberOfAnimals.1.0", "Number_of_Animals", $animals);
+    end
+
+rule "ClinicalSigns.0.0: Clinical signs as sentences"
+    when
+        $oecdNumber: String() from List.of("425")
+        FileAttribute(label == "OECD Number", value == $oecdNumber)
+        $clinicalSigns: List() from collect (Entity(type == "clinical_signs"))
+    then
+        componentCreationService.joining("ClinicalSigns.0.0", "Clinical_Signs", $clinicalSigns, "\n");
+    end
+
+rule "DoseMortality.0.0: Dose mortality joined with dose from same table row"
+    when
+        $oecdNumber: String() from List.of("425")
+        FileAttribute(label == "OECD Number", value == $oecdNumber)
+        $doseMortalities: List() from collect (Entity(type == "dose_mortality" || type == "dose_mortality_dose"))
+    then
+        componentCreationService.joiningFromSameTableRow("DoseMortality.0.0", "Dose_Mortality", $doseMortalities);
+    end
+
+rule "Mortality.0.0: Mortality as one block"
+    when
+        $oecdNumber: String() from List.of("425")
+        FileAttribute(label == "OECD Number", value == $oecdNumber)
+        $mortalities: List() from collect (Entity(type == "mortality"))
+    then
+        componentCreationService.joining("Mortality.0.0", "Mortality", $mortalities, " ");
+    end
+
+rule "Dosages.0.0: First found value of Dosages"
+    when
+        $oecdNumber: String() from List.of("425")
+        FileAttribute(label == "OECD Number", value == $oecdNumber)
+        $mortalities: List() from collect (Entity(type == "dosages"))
+    then
+        componentCreationService.firstOrElse("Dosages.0.0", "Dosages", $mortalities, "");
+    end
+
+rule "PrelimResults.0.0: Preliminary test results as sentences"
+    when
+        $oecdNumber: String() from List.of("429")
+        FileAttribute(label == "OECD Number", value == $oecdNumber)
+        $results: List() from collect (Entity(type == "preliminary_test_results"))
+    then
+        componentCreationService.joining("PrelimResults.0.0", "Preliminary_Test_Results", $results, "\n");
+    end
+
+rule "TestResults.0.0: Test results as one block"
+    when
+        $oecdNumber: String() from List.of("429")
+        FileAttribute(label == "OECD Number", value == $oecdNumber)
+        $results: List() from collect (Entity(type == "test_results"))
+    then
+        componentCreationService.joining("TestResults.0.0", "Test_Results", $results, " ");
+    end
+
+rule "PositiveControl.0.0: Was the definitive study conducted with positive control"
+    when
+        $oecdNumber: String() from List.of("429")
+        FileAttribute(label == "OECD Number", value == $oecdNumber)
+        $results: List() from collect (Entity(type == "positive_control"))
+    then
+        componentCreationService.joining("PositiveControl.0.0", "Was_the_definitive_study_conducted_with_positive_control", $results, " ");
+    end
+
+rule "MainResults.0.0: Results from main study as one block"
+    when
+        $oecdNumber: String() from List.of("429")
+        FileAttribute(label == "OECD Number", value == $oecdNumber)
+        $results: List() from collect (Entity(type == "results_(main_study)"))
+    then
+        componentCreationService.joining("MainResults.0.0", "Results_Main_Study", $results, " ");
+    end
+
+rule "UsedApproach.0.0: Used approach found and mapped to 'Group'"
+    when
+        $oecdNumber: String() from List.of("429")
+        FileAttribute(label == "OECD Number", value == $oecdNumber)
+        $results: List(!isEmpty()) from collect (Entity(type == "approach_used"))
+    then
+        componentCreationService.create("UsedApproach.0.0", "What_was_the_approach_used", "Group", "'Group' when approach used is present, else 'Individual'", $results);
+    end
+
+rule "UsedApproach.1.0: Used approach not found and thus 'Individual'"
+    when
+        $oecdNumber: String() from List.of("429")
+        FileAttribute(label == "OECD Number", value == $oecdNumber)
+        not Entity(type == "approach_used")
+    then
+        componentCreationService.create("UsedApproach.1.0", "What_was_the_approach_used", "Individual", "'Group' when approach used is present, else 'Individual'");
+    end
+
+/*
+rule "DefaultComponents.999.0: Create components for all unmapped entities."
+    salience -999
+    when
+        $allEntities: List(!isEmpty()) from collect (Entity())
+    then
+        componentCreationService.createComponentsForUnMappedEntities("DefaultComponents.999.0", $allEntities);
+    end
+*/
+
+//------------------------------------ Component merging rules ------------------------------------
+/*
+rule "X.0.0: merge duplicate component references"
+    when
+        $first: Component()
+        $duplicate: Component(this != $first, name == $first.name, value == $first.value)
+    then
+        $first.getReferences().addAll($duplicate.getReferences());
+        retract($duplicate);
+    end
+*/

redaction-service-v1/redaction-service-server-v1/src/test/resources/files/migration/def8f960580f088b975ba806dfae1f87.IMAGE_INFO.json

@@ -0,0 +1,891 @@
+{
+  "dossierId": "12778ddd-0d87-4127-90f4-a15802e8f967",
+  "fileId": "4a15b5c89a4310eaef6243fb6f1f4f6f",
+  "targetFileExtension": "ORIGIN.pdf.gz",
+  "responseFileExtension": "IMAGE_INFO.json.gz",
+  "X-TENANT-ID": "documine",
+  "data": [
+    {
+      "classification": {
+        "label": "signature",
+        "probabilities": {
+          "signature": 0.7805,
+          "other": 0.1362,
+          "logo": 0.0629,
+          "formula": 0.0204
+        }
+      },
+      "representation": "F8F3FFCFCC30F0CFCFFCF37FF",
+      "position": {
+        "x1": 0,
+        "x2": 595,
+        "y1": 0,
+        "y2": 842,
+        "pageNumber": 1
+      },
+      "geometry": {
+        "width": 595,
+        "height": 842
+      },
+      "alpha": false,
+      "filters": {
+        "geometry": {
+          "imageSize": {
+            "quotient": 1.0,
+            "tooLarge": true,
+            "tooSmall": false
+          },
+          "imageFormat": {
+            "quotient": 0.7067,
+            "tooTall": false,
+            "tooWide": false
+          }
+        },
+        "probability": {
+          "unconfident": false
+        },
+        "allPassed": false
+      }
+    },
+    {
+      "classification": {
+        "label": "signature",
+        "probabilities": {
+          "signature": 0.9955,
+          "other": 0.0023,
+          "logo": 0.0022,
+          "formula": 0.0
+        }
+      },
+      "representation": "97FFFFFFFF10608F7E18107EF",
+      "position": {
+        "x1": 0,
+        "x2": 595,
+        "y1": 0,
+        "y2": 842,
+        "pageNumber": 2
+      },
+      "geometry": {
+        "width": 595,
+        "height": 842
+      },
+      "alpha": false,
+      "filters": {
+        "geometry": {
+          "imageSize": {
+            "quotient": 1.0,
+            "tooLarge": true,
+            "tooSmall": false
+          },
+          "imageFormat": {
+            "quotient": 0.7067,
+            "tooTall": false,
+            "tooWide": false
+          }
+        },
+        "probability": {
+          "unconfident": false
+        },
+        "allPassed": false
+      }
+    },
+    {
+      "classification": {
+        "label": "signature",
+        "probabilities": {
+          "signature": 0.9998,
+          "logo": 0.0001,
+          "other": 0.0001,
+          "formula": 0.0
+        }
+      },
+      "representation": "FFFFF10EFFF3E0810608307FF",
+      "position": {
+        "x1": 0,
+        "x2": 595,
+        "y1": 0,
+        "y2": 842,
+        "pageNumber": 3
+      },
+      "geometry": {
+        "width": 595,
+        "height": 842
+      },
+      "alpha": false,
+      "filters": {
+        "geometry": {
+          "imageSize": {
+            "quotient": 1.0,
+            "tooLarge": true,
+            "tooSmall": false
+          },
+          "imageFormat": {
+            "quotient": 0.7067,
+            "tooTall": false,
+            "tooWide": false
+          }
+        },
+        "probability": {
+          "unconfident": false
+        },
+        "allPassed": false
+      }
+    },
+    {
+      "classification": {
+        "label": "signature",
+        "probabilities": {
+          "signature": 0.9998,
+          "other": 0.0001,
+          "formula": 0.0,
+          "logo": 0.0
+        }
+      },
+      "representation": "93EFF76E08F0E88F1E187C7EF",
+      "position": {
+        "x1": 0,
+        "x2": 595,
+        "y1": 0,
+        "y2": 842,
+        "pageNumber": 4
+      },
+      "geometry": {
+        "width": 595,
+        "height": 842
+      },
+      "alpha": false,
+      "filters": {
+        "geometry": {
+          "imageSize": {
+            "quotient": 1.0,
+            "tooLarge": true,
+            "tooSmall": false
+          },
+          "imageFormat": {
+            "quotient": 0.7067,
+            "tooTall": false,
+            "tooWide": false
+          }
+        },
+        "probability": {
+          "unconfident": false
+        },
+        "allPassed": false
+      }
+    },
+    {
+      "classification": {
+        "label": "signature",
+        "probabilities": {
+          "signature": 0.9997,
+          "logo": 0.0001,
+          "other": 0.0001,
+          "formula": 0.0
+        }
+      },
+      "representation": "FFFFF33EFC3060818608787FF",
+      "position": {
+        "x1": 0,
+        "x2": 595,
+        "y1": 0,
+        "y2": 842,
+        "pageNumber": 5
+      },
+      "geometry": {
+        "width": 595,
+        "height": 842
+      },
+      "alpha": false,
+      "filters": {
+        "geometry": {
+          "imageSize": {
+            "quotient": 1.0,
+            "tooLarge": true,
+            "tooSmall": false
+          },
+          "imageFormat": {
+            "quotient": 0.7067,
+            "tooTall": false,
+            "tooWide": false
+          }
+        },
+        "probability": {
+          "unconfident": false
+        },
+        "allPassed": false
+      }
+    },
+    {
+      "classification": {
+        "label": "signature",
+        "probabilities": {
+          "signature": 0.9085,
+          "logo": 0.0453,
+          "other": 0.0413,
+          "formula": 0.005
+        }
+      },
+      "representation": "FFF3F506F81060CF1E18FC7FF",
+      "position": {
+        "x1": 0,
+        "x2": 595,
+        "y1": 0,
+        "y2": 842,
+        "pageNumber": 6
+      },
+      "geometry": {
+        "width": 595,
+        "height": 842
+      },
+      "alpha": false,
+      "filters": {
+        "geometry": {
+          "imageSize": {
+            "quotient": 1.0,
+            "tooLarge": true,
+            "tooSmall": false
+          },
+          "imageFormat": {
+            "quotient": 0.7067,
+            "tooTall": false,
+            "tooWide": false
+          }
+        },
+        "probability": {
+          "unconfident": false
+        },
+        "allPassed": false
+      }
+    },
+    {
+      "classification": {
+        "label": "signature",
+        "probabilities": {
+          "signature": 0.7596,
+          "other": 0.1218,
+          "logo": 0.1073,
+          "formula": 0.0113
+        }
+      },
+      "representation": "FFFFFFF70C1070C10F3C10FFF",
+      "position": {
+        "x1": 0,
+        "x2": 595,
+        "y1": 0,
+        "y2": 842,
+        "pageNumber": 7
+      },
+      "geometry": {
+        "width": 595,
+        "height": 842
+      },
+      "alpha": false,
+      "filters": {
+        "geometry": {
+          "imageSize": {
+            "quotient": 1.0,
+            "tooLarge": true,
+            "tooSmall": false
+          },
+          "imageFormat": {
+            "quotient": 0.7067,
+            "tooTall": false,
+            "tooWide": false
+          }
+        },
+        "probability": {
+          "unconfident": false
+        },
+        "allPassed": false
+      }
+    },
+    {
+      "classification": {
+        "label": "signature",
+        "probabilities": {
+          "signature": 0.4095,
+          "other": 0.2126,
+          "formula": 0.2008,
+          "logo": 0.1771
+        }
+      },
+      "representation": "FF70C1070C1070CFD70C10FFF",
+      "position": {
+        "x1": 0,
+        "x2": 595,
+        "y1": 0,
+        "y2": 842,
+        "pageNumber": 8
+      },
+      "geometry": {
+        "width": 595,
+        "height": 842
+      },
+      "alpha": false,
+      "filters": {
+        "geometry": {
+          "imageSize": {
+            "quotient": 1.0,
+            "tooLarge": true,
+            "tooSmall": false
+          },
+          "imageFormat": {
+            "quotient": 0.7067,
+            "tooTall": false,
+            "tooWide": false
+          }
+        },
+        "probability": {
+          "unconfident": true
+        },
+        "allPassed": false
+      }
+    },
+    {
+      "classification": {
+        "label": "signature",
+        "probabilities": {
+          "signature": 0.9282,
+          "logo": 0.0402,
+          "other": 0.0305,
+          "formula": 0.0011
+        }
+      },
+      "representation": "F4F1CF170CF170C1070CF7FFF",
+      "position": {
+        "x1": 0,
+        "x2": 595,
+        "y1": 0,
+        "y2": 842,
+        "pageNumber": 9
+      },
+      "geometry": {
+        "width": 595,
+        "height": 842
+      },
+      "alpha": false,
+      "filters": {
+        "geometry": {
+          "imageSize": {
+            "quotient": 1.0,
+            "tooLarge": true,
+            "tooSmall": false
+          },
+          "imageFormat": {
+            "quotient": 0.7067,
+            "tooTall": false,
+            "tooWide": false
+          }
+        },
+        "probability": {
+          "unconfident": false
+        },
+        "allPassed": false
+      }
+    },
+    {
+      "classification": {
+        "label": "signature",
+        "probabilities": {
+          "signature": 0.9699,
+          "logo": 0.02,
+          "other": 0.0097,
+          "formula": 0.0004
+        }
+      },
+      "representation": "FF70FF4E1D3070D1070CF0FFF",
+      "position": {
+        "x1": 0,
+        "x2": 595,
+        "y1": 0,
+        "y2": 842,
+        "pageNumber": 10
+      },
+      "geometry": {
+        "width": 595,
+        "height": 842
+      },
+      "alpha": false,
+      "filters": {
+        "geometry": {
+          "imageSize": {
+            "quotient": 1.0,
+            "tooLarge": true,
+            "tooSmall": false
+          },
+          "imageFormat": {
+            "quotient": 0.7067,
+            "tooTall": false,
+            "tooWide": false
+          }
+        },
+        "probability": {
+          "unconfident": false
+        },
+        "allPassed": false
+      }
+    },
+    {
+      "classification": {
+        "label": "other",
+        "probabilities": {
+          "other": 0.9181,
+          "signature": 0.0432,
+          "formula": 0.0355,
+          "logo": 0.0032
+        }
+      },
+      "representation": "F3F1F7DF1F10F0C1070CF3FFF",
+      "position": {
+        "x1": 0,
+        "x2": 595,
+        "y1": 0,
+        "y2": 842,
+        "pageNumber": 11
+      },
+      "geometry": {
+        "width": 595,
+        "height": 842
+      },
+      "alpha": false,
+      "filters": {
+        "geometry": {
+          "imageSize": {
+            "quotient": 1.0,
+            "tooLarge": true,
+            "tooSmall": false
+          },
+          "imageFormat": {
+            "quotient": 0.7067,
+            "tooTall": false,
+            "tooWide": false
+          }
+        },
+        "probability": {
+          "unconfident": false
+        },
+        "allPassed": false
+      }
+    },
+    {
+      "classification": {
+        "label": "signature",
+        "probabilities": {
+          "signature": 0.663,
+          "other": 0.137,
+          "logo": 0.1327,
+          "formula": 0.0674
+        }
+      },
+      "representation": "B770C10F7C1070C10F0C97FFF",
+      "position": {
+        "x1": 0,
+        "x2": 595,
+        "y1": 0,
+        "y2": 842,
+        "pageNumber": 12
+      },
+      "geometry": {
+        "width": 595,
+        "height": 842
+      },
+      "alpha": false,
+      "filters": {
+        "geometry": {
+          "imageSize": {
+            "quotient": 1.0,
+            "tooLarge": true,
+            "tooSmall": false
+          },
+          "imageFormat": {
+            "quotient": 0.7067,
+            "tooTall": false,
+            "tooWide": false
+          }
+        },
+        "probability": {
+          "unconfident": false
+        },
+        "allPassed": false
+      }
+    },
+    {
+      "classification": {
+        "label": "signature",
+        "probabilities": {
+          "signature": 0.5023,
+          "other": 0.2343,
+          "logo": 0.2255,
+          "formula": 0.0379
+        }
+      },
+      "representation": "B370C10FFC1070CF3F1CF0FFF",
+      "position": {
+        "x1": 0,
+        "x2": 595,
+        "y1": 0,
+        "y2": 842,
+        "pageNumber": 13
+      },
+      "geometry": {
+        "width": 595,
+        "height": 842
+      },
+      "alpha": false,
+      "filters": {
+        "geometry": {
+          "imageSize": {
+            "quotient": 1.0,
+            "tooLarge": true,
+            "tooSmall": false
+          },
+          "imageFormat": {
+            "quotient": 0.7067,
+            "tooTall": false,
+            "tooWide": false
+          }
+        },
+        "probability": {
+          "unconfident": false
+        },
+        "allPassed": false
+      }
+    },
+    {
+      "classification": {
+        "label": "signature",
+        "probabilities": {
+          "signature": 0.9999,
+          "other": 0.0001,
+          "formula": 0.0,
+          "logo": 0.0
+        }
+      },
+      "representation": "97FFFFFFFFFFFF810608907EF",
+      "position": {
+        "x1": 0,
+        "x2": 595,
+        "y1": 0,
+        "y2": 842,
+        "pageNumber": 14
+      },
+      "geometry": {
+        "width": 595,
+        "height": 842
+      },
+      "alpha": false,
+      "filters": {
+        "geometry": {
+          "imageSize": {
+            "quotient": 1.0,
+            "tooLarge": true,
+            "tooSmall": false
+          },
+          "imageFormat": {
+            "quotient": 0.7067,
+            "tooTall": false,
+            "tooWide": false
+          }
+        },
+        "probability": {
+          "unconfident": false
+        },
+        "allPassed": false
+      }
+    },
+    {
+      "classification": {
+        "label": "signature",
+        "probabilities": {
+          "signature": 0.9099,
+          "logo": 0.0548,
+          "other": 0.0339,
+          "formula": 0.0014
+        }
+      },
+      "representation": "FFFFF10F7DDD70C1070CF77FF",
+      "position": {
+        "x1": 0,
+        "x2": 595,
+        "y1": 0,
+        "y2": 842,
+        "pageNumber": 15
+      },
+      "geometry": {
+        "width": 595,
+        "height": 842
+      },
+      "alpha": false,
+      "filters": {
+        "geometry": {
+          "imageSize": {
+            "quotient": 1.0,
+            "tooLarge": true,
+            "tooSmall": false
+          },
+          "imageFormat": {
+            "quotient": 0.7067,
+            "tooTall": false,
+            "tooWide": false
+          }
+        },
+        "probability": {
+          "unconfident": false
+        },
+        "allPassed": false
+      }
+    },
+    {
+      "classification": {
+        "label": "signature",
+        "probabilities": {
+          "signature": 0.6538,
+          "other": 0.1607,
+          "logo": 0.1489,
+          "formula": 0.0366
+        }
+      },
+      "representation": "FF70CF370CFE70C1070CB3FFF",
+      "position": {
+        "x1": 0,
+        "x2": 595,
+        "y1": 0,
+        "y2": 842,
+        "pageNumber": 16
+      },
+      "geometry": {
+        "width": 595,
+        "height": 842
+      },
+      "alpha": false,
+      "filters": {
+        "geometry": {
+          "imageSize": {
+            "quotient": 1.0,
+            "tooLarge": true,
+            "tooSmall": false
+          },
+          "imageFormat": {
+            "quotient": 0.7067,
+            "tooTall": false,
+            "tooWide": false
+          }
+        },
+        "probability": {
+          "unconfident": false
+        },
+        "allPassed": false
+      }
+    },
+    {
+      "classification": {
+        "label": "signature",
+        "probabilities": {
+          "signature": 0.9637,
+          "logo": 0.0179,
+          "other": 0.0156,
+          "formula": 0.0028
+        }
+      },
+      "representation": "FF70CF170C10F0C1070CF7FFF",
+      "position": {
+        "x1": 0,
+        "x2": 595,
+        "y1": 0,
+        "y2": 842,
+        "pageNumber": 17
+      },
+      "geometry": {
+        "width": 595,
+        "height": 842
+      },
+      "alpha": false,
+      "filters": {
+        "geometry": {
+          "imageSize": {
+            "quotient": 1.0,
+            "tooLarge": true,
+            "tooSmall": false
+          },
+          "imageFormat": {
+            "quotient": 0.7067,
+            "tooTall": false,
+            "tooWide": false
+          }
+        },
+        "probability": {
+          "unconfident": false
+        },
+        "allPassed": false
+      }
+    },
+    {
+      "classification": {
+        "label": "other",
+        "probabilities": {
+          "other": 1.0,
+          "formula": 0.0,
+          "logo": 0.0,
+          "signature": 0.0
+        }
+      },
+      "representation": "FFFFF7070E1870E18608106FF",
+      "position": {
+        "x1": 0,
+        "x2": 595,
+        "y1": 0,
+        "y2": 842,
+        "pageNumber": 18
+      },
+      "geometry": {
+        "width": 595,
+        "height": 842
+      },
+      "alpha": false,
+      "filters": {
+        "geometry": {
+          "imageSize": {
+            "quotient": 1.0,
+            "tooLarge": true,
+            "tooSmall": false
+          },
+          "imageFormat": {
+            "quotient": 0.7067,
+            "tooTall": false,
+            "tooWide": false
+          }
+        },
+        "probability": {
+          "unconfident": false
+        },
+        "allPassed": false
+      }
+    },
+    {
+      "classification": {
+        "label": "other",
+        "probabilities": {
+          "other": 1.0,
+          "formula": 0.0,
+          "logo": 0.0,
+          "signature": 0.0
+        }
+      },
+      "representation": "9B70E1870E1870E1870810EFF",
+      "position": {
+        "x1": 0,
+        "x2": 595,
+        "y1": 0,
+        "y2": 842,
+        "pageNumber": 19
+      },
+      "geometry": {
+        "width": 595,
+        "height": 842
+      },
+      "alpha": false,
+      "filters": {
+        "geometry": {
+          "imageSize": {
+            "quotient": 1.0,
+            "tooLarge": true,
+            "tooSmall": false
+          },
+          "imageFormat": {
+            "quotient": 0.7067,
+            "tooTall": false,
+            "tooWide": false
+          }
+        },
+        "probability": {
+          "unconfident": false
+        },
+        "allPassed": false
+      }
+    },
+    {
+      "classification": {
+        "label": "signature",
+        "probabilities": {
+          "signature": 0.3619,
+          "other": 0.2659,
+          "logo": 0.2169,
+          "formula": 0.1552
+        }
+      },
+      "representation": "FFFFF30FE910ECF1E708106FF",
+      "position": {
+        "x1": 0,
+        "x2": 595,
+        "y1": 0,
+        "y2": 842,
+        "pageNumber": 20
+      },
+      "geometry": {
+        "width": 595,
+        "height": 842
+      },
+      "alpha": false,
+      "filters": {
+        "geometry": {
+          "imageSize": {
+            "quotient": 1.0,
+            "tooLarge": true,
+            "tooSmall": false
+          },
+          "imageFormat": {
+            "quotient": 0.7067,
+            "tooTall": false,
+            "tooWide": false
+          }
+        },
+        "probability": {
+          "unconfident": true
+        },
+        "allPassed": false
+      }
+    },
+    {
+      "classification": {
+        "label": "signature",
+        "probabilities": {
+          "signature": 0.9984,
+          "other": 0.001,
+          "logo": 0.0006,
+          "formula": 0.0
+        }
+      },
+      "representation": "B7E9F70F3C73F1C78F3F706EF",
+      "position": {
+        "x1": 0,
+        "x2": 595,
+        "y1": 0,
+        "y2": 842,
+        "pageNumber": 21
+      },
+      "geometry": {
+        "width": 595,
+        "height": 842
+      },
+      "alpha": false,
+      "filters": {
+        "geometry": {
+          "imageSize": {
+            "quotient": 1.0,
+            "tooLarge": true,
+            "tooSmall": false
+          },
+          "imageFormat": {
+            "quotient": 0.7067,
+            "tooTall": false,
+            "tooWide": false
+          }
+        },
+        "probability": {
+          "unconfident": false
+        },
+        "allPassed": false
+      }
+    }
+  ]
+}