Merge branch 'RED-9857-bp2' into 'release/4.190.x'

RED-9857: Add new date format

See merge request redactmanager/redaction-service!481

.gitlab-ci.yml

@@ -12,7 +12,7 @@ deploy:
   script:
     - echo "Building with gradle version ${BUILDVERSION}"
     - gradle -Pversion=${BUILDVERSION} publish
-    - gradle bootBuildImage --cleanCache --publishImage -PbuildbootDockerHostNetwork=true -Pversion=${BUILDVERSION}
+    - gradle bootBuildImage --publishImage -PbuildbootDockerHostNetwork=true -Pversion=${BUILDVERSION}
     - echo "BUILDVERSION=$BUILDVERSION" >> version.env
   artifacts:
     reports:
@@ -21,3 +21,5 @@ deploy:
     - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
     - if: $CI_COMMIT_BRANCH =~ /^release/
     - if: $CI_COMMIT_TAG
+pmd:
+  allow_failure: true

redaction-service-v1/redaction-service-api-v1/build.gradle.kts

@@ -7,7 +7,7 @@ description = "redaction-service-api-v1"
 
 dependencies {
     implementation("org.springframework:spring-web:6.0.12")
-    implementation("com.iqser.red.service:persistence-service-internal-api-v1:2.207.0")
+    implementation("com.iqser.red.service:persistence-service-internal-api-v1:2.240.0")
 }
 
 publishing {

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/MigrationRequest.java

@@ -0,0 +1,17 @@
+package com.iqser.red.service.redaction.v1.model;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
+public class MigrationRequest {
+
+    String dossierId;
+    String fileId;
+
+}

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/MigrationResponse.java

@@ -0,0 +1,17 @@
+package com.iqser.red.service.redaction.v1.model;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
+public class MigrationResponse {
+
+    String dossierId;
+    String fileId;
+
+}

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/QueueNames.java

@@ -0,0 +1,14 @@
+package com.iqser.red.service.redaction.v1.model;
+
+public class QueueNames {
+
+    public static final String REDACTION_QUEUE = "redactionQueue";
+    public static final String REDACTION_DQL = "redactionDQL";
+
+    public static final String REDACTION_PRIORITY_QUEUE = "redactionPriorityQueue";
+
+    public static final String MIGRATION_QUEUE = "migrationQueue";
+    public static final String MIGRATION_DLQ = "migrationDLQ";
+    public static final String MIGRATION_RESPONSE_QUEUE = "migrationResponseQueue";
+
+}

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/UnprocessedManualEntity.java

@@ -0,0 +1,33 @@
+package com.iqser.red.service.redaction.v1.model;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.EqualsAndHashCode;
+import lombok.NoArgsConstructor;
+
+@Data
+@AllArgsConstructor
+@NoArgsConstructor
+@Builder
+@EqualsAndHashCode
+public class UnprocessedManualEntity {
+
+    private String annotationId;
+    private String textBefore;
+    private String textAfter;
+    private int startOffset;
+    private int endOffset;
+    private String closestHeadline;
+    private String section;
+    private String legalBasis;
+    private float[] color;
+    private List<Integer> containingNodeId;
+    @Builder.Default
+    private List<Position> positions = new ArrayList<>();
+}

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/resources/UnprocessedManualEntityResource.java

@@ -0,0 +1,20 @@
+package com.iqser.red.service.redaction.v1.resources;
+
+import java.util.List;
+
+import org.springframework.http.MediaType;
+import org.springframework.web.bind.annotation.PostMapping;
+import org.springframework.web.bind.annotation.RequestBody;
+import org.springframework.web.bind.annotation.RequestParam;
+
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
+import com.iqser.red.service.redaction.v1.model.UnprocessedManualEntity;
+
+public interface UnprocessedManualEntityResource {
+
+    @PostMapping(value = "/mergeUnprocessedManualEntities", produces = MediaType.APPLICATION_JSON_VALUE)
+    List<UnprocessedManualEntity> mergeUnprocessedManualEntities(@RequestParam(value = "fileId") String fileId,
+                                                                 @RequestParam(value = "dossierId") String dossierId,
+                                                                 @RequestParam(value = "dossierTemplateId") String dosserTemplateId,
+                                                                 @RequestBody ManualRedactions manualRedactions);
+}

redaction-service-v1/redaction-service-server-v1/build.gradle.kts

@@ -12,11 +12,11 @@ plugins {
 description = "redaction-service-server-v1"
 
 
-val layoutParserVersion = "0.71.0"
+val layoutParserVersion = "0.75.0"
 val jacksonVersion = "2.15.2"
 val droolsVersion = "9.44.0.Final"
 val pdfBoxVersion = "3.0.0"
-val persistenceServiceVersion = "2.210.0"
+val persistenceServiceVersion = "2.242.0"
 
 configurations {
     all {
@@ -35,7 +35,7 @@ dependencies {
 
     implementation("com.iqser.red.commons:dictionary-merge-commons:1.5.0")
     implementation("com.iqser.red.commons:storage-commons:2.45.0")
-    implementation("com.knecon.fforesight:tenant-commons:0.13.0")
+    implementation("com.knecon.fforesight:tenant-commons:0.14.0")
 
     implementation("com.fasterxml.jackson.module:jackson-module-afterburner:${jacksonVersion}")
     implementation("com.fasterxml.jackson.datatype:jackson-datatype-jsr310:${jacksonVersion}")
@@ -51,6 +51,9 @@ dependencies {
     implementation("org.springframework.cloud:spring-cloud-starter-openfeign:4.0.4")
     implementation("org.springframework.boot:spring-boot-starter-amqp:3.1.4")
 
+    implementation("com.joestelmach:natty:0.13")
+
+    testImplementation(project(":rules-management"))
     testImplementation("org.apache.pdfbox:pdfbox:${pdfBoxVersion}")
     testImplementation("org.apache.pdfbox:pdfbox-tools:${pdfBoxVersion}")
 
@@ -75,7 +78,7 @@ tasks.named<BootBuildImage>("bootBuildImage") {
     environment.put("BPE_DELIM_JAVA_TOOL_OPTIONS", " ")
     environment.put("BPE_APPEND_JAVA_TOOL_OPTIONS", "-Dfile.encoding=UTF-8")
 
-    imageName.set("nexus.knecon.com:5001/red/${project.name}:${project.version}")
+    imageName.set("nexus.knecon.com:5001/red/${project.name}")// must build image with same name always, otherwise the builder will not know which image to use as cache. DO NOT CHANGE!
     if (project.hasProperty("buildbootDockerHostNetwork")) {
         network.set("host")
     }
@@ -91,5 +94,8 @@ tasks.named<BootBuildImage>("bootBuildImage") {
             email.set(providers.gradleProperty("mavenEmail").getOrNull())
             url.set("https://nexus.knecon.com:5001/")
         }
+
+        val dockerTag = "nexus.knecon.com:5001/red/${project.name}:${project.version}"
+        tags.set(listOf(dockerTag))
     }
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/UnprocessedManualEntityController.java

@@ -0,0 +1,29 @@
+package com.iqser.red.service.redaction.v1.server.controller;
+
+import java.util.List;
+
+import org.springframework.web.bind.annotation.RestController;
+
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
+import com.iqser.red.service.redaction.v1.model.UnprocessedManualEntity;
+import com.iqser.red.service.redaction.v1.resources.UnprocessedManualEntityResource;
+import com.iqser.red.service.redaction.v1.server.service.UnprocessedChangesService;
+
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+@RestController
+@RequiredArgsConstructor
+public class UnprocessedManualEntityController implements UnprocessedManualEntityResource {
+
+    private final UnprocessedChangesService unprocessedChangesService;
+
+
+    @Override
+    public List<UnprocessedManualEntity> mergeUnprocessedManualEntities(String fileId, String dossierId, String dossierTemplateId, ManualRedactions manualRedactions) {
+
+        return unprocessedChangesService.addInformationToUnprocessedManualEntities(fileId, dossierId, dossierTemplateId, manualRedactions);
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/ManualEntity.java

@@ -4,10 +4,9 @@ import java.util.List;
 import java.util.PriorityQueue;
 
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
 import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
-import com.iqser.red.service.redaction.v1.server.model.document.entity.IEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
+import com.iqser.red.service.redaction.v1.server.model.document.entity.IEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.ManualChangeOverwrite;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule;
 
@@ -44,39 +43,6 @@ public class ManualEntity implements IEntity {
     ManualChangeOverwrite manualOverwrite = new ManualChangeOverwrite();
 
 
-    public static ManualEntity fromRedactionLogEntry(RedactionLogEntry redactionLogEntry) {
-
-        String ruleIdentifier = redactionLogEntry.getType() + "." + redactionLogEntry.getMatchedRule() + ".0";
-        List<RectangleWithPage> rectangleWithPages = redactionLogEntry.getPositions().stream().map(RectangleWithPage::fromRedactionLogRectangle).toList();
-        return ManualEntity.builder()
-                .id(redactionLogEntry.getId())
-                .value(redactionLogEntry.getValue())
-                .entityPosition(rectangleWithPages)
-                .ruleIdentifier(ruleIdentifier)
-                .reason(redactionLogEntry.getReason())
-                .legalBasis(redactionLogEntry.getLegalBasis())
-                .type(redactionLogEntry.getType())
-                .section(redactionLogEntry.getSection()).entityType(getEntityType(redactionLogEntry))
-                .applied(redactionLogEntry.isRedacted())
-                .isDictionaryEntry(redactionLogEntry.isDictionaryEntry())
-                .isDossierDictionaryEntry(redactionLogEntry.isDossierDictionaryEntry())
-                .rectangle(redactionLogEntry.isRectangle())
-                .build();
-    }
-
-
-    private static EntityType getEntityType(RedactionLogEntry redactionLogEntry) {
-
-        if (redactionLogEntry.isRecommendation()) {
-            return EntityType.RECOMMENDATION;
-        }
-        if (redactionLogEntry.isHint()) {
-            return EntityType.HINT;
-        }
-        return EntityType.ENTITY;
-    }
-
-
     public static ManualEntity fromManualRedactionEntry(ManualRedactionEntry manualRedactionEntry, boolean hint) {
 
         List<RectangleWithPage> rectangleWithPages = manualRedactionEntry.getPositions().stream().map(RectangleWithPage::fromAnnotationRectangle).toList();
@@ -90,7 +56,8 @@ public class ManualEntity implements IEntity {
                 .reason(manualRedactionEntry.getReason())
                 .legalBasis(manualRedactionEntry.getLegalBasis())
                 .type(manualRedactionEntry.getType())
-                .section(manualRedactionEntry.getSection()).entityType(hint ? EntityType.HINT : EntityType.ENTITY)
+                .section(manualRedactionEntry.getSection())
+                .entityType(hint ? EntityType.HINT : EntityType.ENTITY)
                 .applied(true)
                 .isDictionaryEntry(false)
                 .isDossierDictionaryEntry(false)

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/MigratedEntityLog.java

@@ -0,0 +1,20 @@
+package com.iqser.red.service.redaction.v1.server.model;
+
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.migration.MigratedIds;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
+public class MigratedEntityLog {
+
+    MigratedIds migratedIds;
+    EntityLog entityLog;
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/MigrationEntity.java

@@ -0,0 +1,315 @@
+package com.iqser.red.service.redaction.v1.server.model;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ChangeType;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryType;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Change;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ManualChange;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ManualRedactionType;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
+import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
+import com.iqser.red.service.redaction.v1.server.model.document.entity.IEntity;
+import com.iqser.red.service.redaction.v1.server.model.document.entity.PositionOnPage;
+import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.Image;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.ImageType;
+
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.RequiredArgsConstructor;
+
+@Data
+@AllArgsConstructor
+@RequiredArgsConstructor
+public final class MigrationEntity {
+
+    private final ManualEntity manualEntity;
+    private final RedactionLogEntry redactionLogEntry;
+    private IEntity migratedEntity;
+    private String oldId;
+    private String newId;
+
+
+    public static MigrationEntity fromRedactionLogEntry(RedactionLogEntry redactionLogEntry) {
+
+        return new MigrationEntity(createManualEntity(redactionLogEntry), redactionLogEntry);
+    }
+
+
+    public static ManualEntity createManualEntity(RedactionLogEntry redactionLogEntry) {
+
+        String ruleIdentifier = "OLD." + redactionLogEntry.getMatchedRule() + ".0";
+        List<RectangleWithPage> rectangleWithPages = redactionLogEntry.getPositions().stream().map(RectangleWithPage::fromRedactionLogRectangle).toList();
+
+        return ManualEntity.builder()
+                .id(redactionLogEntry.getId())
+                .value(redactionLogEntry.getValue())
+                .entityPosition(rectangleWithPages)
+                .ruleIdentifier(ruleIdentifier)
+                .reason(redactionLogEntry.getReason())
+                .legalBasis(redactionLogEntry.getLegalBasis())
+                .type(redactionLogEntry.getType())
+                .section(redactionLogEntry.getSection())
+                .entityType(getEntityType(redactionLogEntry))
+                .applied(redactionLogEntry.isRedacted())
+                .isDictionaryEntry(redactionLogEntry.isDictionaryEntry())
+                .isDossierDictionaryEntry(redactionLogEntry.isDossierDictionaryEntry())
+                .rectangle(redactionLogEntry.isRectangle())
+                .build();
+    }
+
+
+    private static EntityType getEntityType(RedactionLogEntry redactionLogEntry) {
+
+        if (redactionLogEntry.isRecommendation()) {
+            return EntityType.RECOMMENDATION;
+        }
+        if (redactionLogEntry.isHint()) {
+            return EntityType.HINT;
+        }
+        return EntityType.ENTITY;
+    }
+
+
+    private static com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Change toEntityLogChanges(Change change) {
+
+        return new com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Change(change.getAnalysisNumber(),
+                toEntityLogType(change.getType()),
+                change.getDateTime());
+    }
+
+
+    private static EntryType getEntryType(EntityType entityType) {
+
+        return switch (entityType) {
+            case ENTITY -> EntryType.ENTITY;
+            case HINT -> EntryType.HINT;
+            case FALSE_POSITIVE -> EntryType.FALSE_POSITIVE;
+            case RECOMMENDATION -> EntryType.RECOMMENDATION;
+            case FALSE_RECOMMENDATION -> EntryType.FALSE_RECOMMENDATION;
+        };
+    }
+
+
+    private static com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualChange toEntityLogManualChanges(ManualChange manualChange) {
+
+        return new com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualChange(toManualRedactionType(manualChange.getManualRedactionType()),
+                manualChange.getProcessedDate(),
+                manualChange.getRequestedDate(),
+                manualChange.getUserId(),
+                manualChange.getPropertyChanges());
+    }
+
+
+    private static ChangeType toEntityLogType(com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ChangeType type) {
+
+        return switch (type) {
+            case ADDED -> ChangeType.ADDED;
+            case REMOVED -> ChangeType.REMOVED;
+            case CHANGED -> ChangeType.CHANGED;
+        };
+    }
+
+
+    private static com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType toManualRedactionType(ManualRedactionType manualRedactionType) {
+
+        return switch (manualRedactionType) {
+            case ADD_LOCALLY -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.ADD_LOCALLY;
+            case ADD_TO_DICTIONARY -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.ADD_TO_DICTIONARY;
+            case REMOVE_LOCALLY -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.REMOVE_LOCALLY;
+            case REMOVE_FROM_DICTIONARY -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.REMOVE_FROM_DICTIONARY;
+            case FORCE_REDACT -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.FORCE_REDACT;
+            case FORCE_HINT -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.FORCE_HINT;
+            case RECATEGORIZE -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.RECATEGORIZE;
+            case LEGAL_BASIS_CHANGE -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.LEGAL_BASIS_CHANGE;
+            case RESIZE -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.RESIZE;
+        };
+    }
+
+
+    public EntityLogEntry toEntityLogEntry(Map<String, String> oldToNewIdMapping) {
+
+        EntityLogEntry entityLogEntry;
+        if (migratedEntity instanceof Image image) {
+            entityLogEntry = createEntityLogEntry(image);
+        } else if (migratedEntity instanceof TextEntity textEntity) {
+            entityLogEntry = createEntityLogEntry(textEntity);
+        } else if (migratedEntity instanceof ManualEntity entity) {
+            entityLogEntry = createEntityLogEntry(entity);
+        } else {
+            throw new UnsupportedOperationException("Unknown subclass " + migratedEntity.getClass());
+        }
+        entityLogEntry.setChanges(redactionLogEntry.getChanges().stream().map(MigrationEntity::toEntityLogChanges).toList());
+        entityLogEntry.setManualChanges(redactionLogEntry.getManualChanges()
+                .stream()
+                .filter(manualChange -> manualChange.getAnnotationStatus().equals(AnnotationStatus.APPROVED))
+                .map(MigrationEntity::toEntityLogManualChanges)
+                .toList());
+        entityLogEntry.setColor(redactionLogEntry.getColor());
+        entityLogEntry.setReference(migrateSetOfIds(redactionLogEntry.getReference(), oldToNewIdMapping));
+        entityLogEntry.setImportedRedactionIntersections(migrateSetOfIds(redactionLogEntry.getImportedRedactionIntersections(), oldToNewIdMapping));
+        entityLogEntry.setEngines(getMigratedEngines(redactionLogEntry));
+        if (redactionLogEntry.getLegalBasis() != null) {
+            entityLogEntry.setLegalBasis(redactionLogEntry.getLegalBasis());
+        }
+
+        return entityLogEntry;
+    }
+
+
+    private static Set<com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine> getMigratedEngines(RedactionLogEntry entry) {
+
+        if (entry.getEngines() == null) {
+            return Collections.emptySet();
+        }
+        return entry.getEngines().stream().map(MigrationEntity::toEntityLogEngine).collect(Collectors.toSet());
+    }
+
+
+    private Set<String> migrateSetOfIds(Set<String> ids, Map<String, String> oldToNewIdMapping) {
+
+        if (ids == null) {
+            return Collections.emptySet();
+        }
+        return ids.stream().map(oldToNewIdMapping::get).collect(Collectors.toSet());
+    }
+
+
+    private static com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine toEntityLogEngine(Engine engine) {
+
+        return switch (engine) {
+            case DICTIONARY -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine.DICTIONARY;
+            case NER -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine.NER;
+            case RULE -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine.RULE;
+        };
+    }
+
+
+    public EntityLogEntry createEntityLogEntry(Image image) {
+
+        String imageType = image.getImageType().equals(ImageType.OTHER) ? "image" : image.getImageType().toString().toLowerCase(Locale.ENGLISH);
+        return EntityLogEntry.builder()
+                .id(image.getId())
+                .value(image.value())
+                .type(imageType)
+                .reason(image.buildReasonWithManualChangeDescriptions())
+                .legalBasis(image.legalBasis())
+                .matchedRule(image.getMatchedRule().getRuleIdentifier().toString())
+                .dictionaryEntry(false)
+                .positions(List.of(new Position(image.getPosition(), image.getPage().getNumber())))
+                .containingNodeId(image.getTreeId())
+                .closestHeadline(image.getHeadline().getTextBlock().getSearchText())
+                .section(image.getManualOverwrite().getSection().orElse(image.getParent().toString()))
+                .imageHasTransparency(image.isTransparent())
+                .state(buildEntryState(image))
+                .entryType(redactionLogEntry.isHint() ? EntryType.IMAGE_HINT : EntryType.IMAGE)
+                .build();
+
+    }
+
+
+    public EntityLogEntry createEntityLogEntry(ManualEntity manualEntity) {
+
+        String type = manualEntity.getManualOverwrite().getType().orElse(manualEntity.getType());
+        return EntityLogEntry.builder()
+                .id(manualEntity.getId())
+                .reason(manualEntity.buildReasonWithManualChangeDescriptions())
+                .legalBasis(manualEntity.legalBasis())
+                .value(manualEntity.value())
+                .type(type)
+                .state(buildEntryState(manualEntity))
+                .entryType(buildEntryType(manualEntity))
+                .section(manualEntity.getManualOverwrite().getSection().orElse(manualEntity.getSection()))
+                .containingNodeId(Collections.emptyList())
+                .closestHeadline("")
+                .matchedRule(manualEntity.getMatchedRule().getRuleIdentifier().toString())
+                .dictionaryEntry(manualEntity.isDictionaryEntry())
+                .dossierDictionaryEntry(manualEntity.isDossierDictionaryEntry())
+                .textAfter("")
+                .textBefore("")
+                .startOffset(-1)
+                .endOffset(-1)
+                .positions(manualEntity.getManualOverwrite()
+                        .getPositions()
+                        .orElse(manualEntity.getEntityPosition())
+                        .stream()
+                        .map(entityPosition -> new Position(entityPosition.rectangle2D(), entityPosition.pageNumber()))
+                        .toList())
+                .engines(Collections.emptySet())
+                .build();
+    }
+
+
+    public EntityLogEntry createEntityLogEntry(TextEntity entity) {
+
+        assert entity.getPositionsOnPagePerPage().size() == 1;
+        PositionOnPage positionOnPage = entity.getPositionsOnPagePerPage().get(0);
+        List<Position> rectanglesPerLine = positionOnPage.getRectanglePerLine()
+                .stream()
+                .map(rectangle2D -> new Position(rectangle2D, positionOnPage.getPage().getNumber()))
+                .toList();
+        return EntityLogEntry.builder()
+                .id(positionOnPage.getId())
+                .positions(rectanglesPerLine)
+                .reason(entity.buildReasonWithManualChangeDescriptions())
+                .legalBasis(entity.legalBasis())
+                .value(entity.getManualOverwrite().getValue().orElse(entity.getMatchedRule().isWriteValueWithLineBreaks() ? entity.getValueWithLineBreaks() : entity.getValue()))
+                .type(entity.getType())
+                .section(entity.getManualOverwrite().getSection().orElse(entity.getDeepestFullyContainingNode().toString()))
+                .containingNodeId(entity.getDeepestFullyContainingNode().getTreeId())
+                .closestHeadline(entity.getDeepestFullyContainingNode().getHeadline().getTextBlock().getSearchText())
+                .matchedRule(entity.getMatchedRule().getRuleIdentifier().toString())
+                .dictionaryEntry(entity.isDictionaryEntry())
+                .textAfter(entity.getTextAfter())
+                .textBefore(entity.getTextBefore())
+                .startOffset(entity.getTextRange().start())
+                .endOffset(entity.getTextRange().end())
+                .dossierDictionaryEntry(entity.isDossierDictionaryEntry())
+                .engines(entity.getEngines() != null ? entity.getEngines() : Collections.emptySet())
+                .state(buildEntryState(entity))
+                .entryType(buildEntryType(entity))
+                .build();
+    }
+
+
+    private EntryState buildEntryState(IEntity entity) {
+
+        if (entity.applied() && entity.active()) {
+            return EntryState.APPLIED;
+        } else if (entity.skipped() && entity.active()) {
+            return EntryState.SKIPPED;
+        } else if (entity.ignored()) {
+            return EntryState.IGNORED;
+        } else {
+            return EntryState.REMOVED;
+        }
+    }
+
+
+    private EntryType buildEntryType(IEntity entity) {
+
+        if (entity instanceof TextEntity textEntity) {
+            return getEntryType(textEntity.getEntityType());
+        } else if (entity instanceof ManualEntity manualEntity) {
+            if (manualEntity.isRectangle()) {
+                return EntryType.AREA;
+            }
+            return getEntryType(manualEntity.getEntityType());
+        } else if (entity instanceof Image) {
+            return EntryType.IMAGE;
+        }
+        throw new UnsupportedOperationException(String.format("Entity subclass %s is not implemented!", entity.getClass()));
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/RectangleWithPage.java

@@ -20,13 +20,13 @@ public record RectangleWithPage(int pageNumber, Rectangle2D rectangle2D) {
 
     private static Rectangle2D toRectangle2D(Rectangle rectangle) {
 
-        return new Rectangle2D.Float(rectangle.getTopLeft().getX(), rectangle.getTopLeft().getY() + rectangle.getHeight(), rectangle.getWidth(), -rectangle.getHeight());
+        return new Rectangle2D.Float(rectangle.getTopLeft().getX(), rectangle.getTopLeft().getY(), rectangle.getWidth(), rectangle.getHeight());
     }
 
 
     private static Rectangle2D toRectangle2D(com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle rectangle) {
 
-        return new Rectangle2D.Float(rectangle.getTopLeft().getX(), rectangle.getTopLeft().getY() + rectangle.getHeight(), rectangle.getWidth(), -rectangle.getHeight());
+        return new Rectangle2D.Float(rectangle.getTopLeft().getX(), rectangle.getTopLeft().getY(), rectangle.getWidth(), rectangle.getHeight());
     }
 
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/component/Entity.java

@@ -80,7 +80,9 @@ public class Entity {
                 .legalBasis(e.getLegalBasis())
                 .imported(e.isImported())
                 .section(e.getSection())
-                .color(e.getColor()).positions(e.getPositions()).containingNode(document.getDocumentTree().getEntryById(e.getContainingNodeId()).getNode())
+                .color(e.getColor())
+                .positions(e.getPositions())
+                .containingNode(document.getDocumentTree().getEntryById(e.getContainingNodeId()).getNode())
                 .textBefore(e.getTextBefore())
                 .textAfter(e.getTextAfter())
                 .startOffset(e.getStartOffset())

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/Dictionary.java

@@ -2,23 +2,23 @@ package com.iqser.red.service.redaction.v1.server.model.dictionary;
 
 import static java.lang.String.format;
 
+import java.util.Arrays;
 import java.util.Collection;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
+import java.util.Locale;
 import java.util.Map;
 import java.util.Set;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
 import org.apache.commons.lang3.StringUtils;
 
-import com.iqser.red.service.redaction.v1.server.utils.exception.NotFoundException;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
 import com.iqser.red.service.redaction.v1.server.utils.Patterns;
+import com.iqser.red.service.redaction.v1.server.utils.exception.NotFoundException;
 
 import lombok.Data;
 import lombok.Getter;
@@ -107,11 +107,21 @@ public class Dictionary {
         if (StringUtils.isEmpty(value)) {
             throw new IllegalArgumentException(format("%s is not a valid dictionary entry", value));
         }
+        boolean isCaseInsensitive = localAccessMap.get(type).isCaseInsensitive();
         Set<MatchedRule> matchedRulesSet = new HashSet<>(matchedRules);
-        localAccessMap.get(type).getLocalEntriesWithMatchedRules().merge(value.trim(), matchedRulesSet, (set1, set2) -> Stream.concat(set1.stream(), set2.stream()).collect(Collectors.toSet()));
+
+        String cleanedValue = value;
+        if (isCaseInsensitive) {
+            cleanedValue = cleanedValue.toLowerCase(Locale.US);
+        }
+        localAccessMap.get(type)
+                .getLocalEntriesWithMatchedRules()
+                .merge(cleanedValue.trim(), matchedRulesSet, (set1, set2) -> Stream.concat(set1.stream(), set2.stream()).collect(Collectors.toSet()));
         if (alsoAddLastname) {
-            String lastname = value.split(" ")[0];
-            localAccessMap.get(type).getLocalEntriesWithMatchedRules().merge(lastname, matchedRulesSet, (set1, set2) -> Stream.concat(set1.stream(), set2.stream()).collect(Collectors.toSet()));
+            String lastname = cleanedValue.split(" ")[0];
+            localAccessMap.get(type)
+                    .getLocalEntriesWithMatchedRules()
+                    .merge(lastname, matchedRulesSet, (set1, set2) -> Stream.concat(set1.stream(), set2.stream()).collect(Collectors.toSet()));
         }
     }
 
@@ -130,16 +140,20 @@ public class Dictionary {
 
     public void addMultipleAuthorsAsRecommendation(TextEntity textEntity) {
 
-        String cleanedWord = textEntity.getValue().replaceAll(",", " ").replaceAll("  ", " ").trim() + " ";
-        Pattern pattern = Patterns.AUTHOR_TABLE_SPLITTER;
-        Matcher matcher = pattern.matcher(cleanedWord);
+        splitIntoAuthorNames(textEntity).forEach(authorName -> addLocalDictionaryEntry(textEntity.getType(), authorName, textEntity.getMatchedRuleList(), true));
 
-        while (matcher.find()) {
-            String match = matcher.group().trim();
-            if (match.length() >= 3) {
-                addLocalDictionaryEntry(textEntity.getType(), match, textEntity.getMatchedRuleList(), true);
-            }
+    }
+
+
+    public static List<String> splitIntoAuthorNames(TextEntity textEntity) {
+
+        List<String> splitAuthorNames;
+        if (textEntity.getValue().contains(",")) {
+            splitAuthorNames = Arrays.asList(textEntity.getValue().split(","));
+        } else {
+            splitAuthorNames = Arrays.asList(textEntity.getValueWithLineBreaks().split("\n"));
         }
+        return splitAuthorNames.stream().map(String::trim).filter(authorName -> Patterns.AUTHOR_NAME_PATTERN.matcher(authorName).matches()).toList();
     }
 
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/DictionaryModel.java

@@ -2,6 +2,8 @@ package com.iqser.red.service.redaction.v1.server.model.dictionary;
 
 import java.io.Serializable;
 import java.util.HashMap;
+import java.util.List;
+import java.util.Locale;
 import java.util.Set;
 import java.util.stream.Collectors;
 
@@ -109,4 +111,11 @@ public class DictionaryModel implements Serializable {
         return falseRecommendationsSearch;
     }
 
+    public Set<MatchedRule> getMatchedRulesForLocalDictionaryEntry(String value) {
+        var cleanedValue = isCaseInsensitive() ? value.toLowerCase(Locale.US) : value;
+
+        return localEntriesWithMatchedRules.get(cleanedValue);
+
+    }
+
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/DocumentTree.java

@@ -40,7 +40,10 @@ public class DocumentTree {
 
     public TextBlock buildTextBlock() {
 
-        return allEntriesInOrder().map(Entry::getNode).filter(SemanticNode::isLeaf).map(SemanticNode::getLeafTextBlock).collect(new TextBlockCollector());
+        return allEntriesInOrder().map(Entry::getNode)
+                .filter(SemanticNode::isLeaf)
+                .map(SemanticNode::getLeafTextBlock)
+                .collect(new TextBlockCollector());
     }
 
 
@@ -89,8 +92,8 @@ public class DocumentTree {
         if (treeId.isEmpty()) {
             return root != null;
         }
-        Entry entry = root.children.get(treeId.get(0));
-        for (int id : treeId.subList(1, treeId.size())) {
+        Entry entry = root;
+        for (int id : treeId) {
             if (id >= entry.children.size() || 0 > id) {
                 return false;
             }
@@ -114,13 +117,16 @@ public class DocumentTree {
 
     public Stream<SemanticNode> childNodes(List<Integer> treeId) {
 
-        return getEntryById(treeId).children.stream().map(Entry::getNode);
+        return getEntryById(treeId).children.stream()
+                .map(Entry::getNode);
     }
 
 
     public Stream<SemanticNode> childNodesOfType(List<Integer> treeId, NodeType nodeType) {
 
-        return getEntryById(treeId).children.stream().filter(entry -> entry.node.getType().equals(nodeType)).map(Entry::getNode);
+        return getEntryById(treeId).children.stream()
+                .filter(entry -> entry.node.getType().equals(nodeType))
+                .map(Entry::getNode);
     }
 
 
@@ -199,26 +205,32 @@ public class DocumentTree {
 
     public Stream<Entry> allEntriesInOrder() {
 
-        return Stream.of(root).flatMap(DocumentTree::flatten);
+        return Stream.of(root)
+                .flatMap(DocumentTree::flatten);
     }
 
 
     public Stream<Entry> allSubEntriesInOrder(List<Integer> parentId) {
 
-        return getEntryById(parentId).children.stream().flatMap(DocumentTree::flatten);
+        return getEntryById(parentId).children.stream()
+                .flatMap(DocumentTree::flatten);
     }
 
 
     @Override
     public String toString() {
 
-        return String.join("\n", allEntriesInOrder().map(Entry::toString).toList());
+        return String.join("\n",
+                           allEntriesInOrder().map(Entry::toString)
+                                   .toList());
     }
 
 
     private static Stream<Entry> flatten(Entry entry) {
 
-        return Stream.concat(Stream.of(entry), entry.children.stream().flatMap(DocumentTree::flatten));
+        return Stream.concat(Stream.of(entry),
+                             entry.children.stream()
+                                     .flatMap(DocumentTree::flatten));
     }
 
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/ManualChangeOverwrite.java

@@ -7,13 +7,15 @@ import java.util.List;
 import java.util.Map;
 import java.util.Optional;
 
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.BaseAnnotation;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
+import com.iqser.red.service.redaction.v1.server.model.RectangleWithPage;
 
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
@@ -42,6 +44,7 @@ public class ManualChangeOverwrite {
     String legalBasis;
     String section;
     String value;
+    List<RectangleWithPage> positions;
     Boolean applied;
     Boolean removed;
     Boolean ignored;
@@ -80,7 +83,18 @@ public class ManualChangeOverwrite {
 
             if (manualChange instanceof IdRemoval) {
                 applied = false;
-                ignored = true;
+                if (sortedManualChanges.get(0) instanceof ManualRedactionEntry) { // set to removed if it's a Manual Redaction
+                    removed = true;
+                } else {
+                    ignored = true;
+                }
+            }
+
+            if(manualChange instanceof ManualRedactionEntry manualRedactionEntry){
+                removed = false;
+                ignored = false;
+                applied = true;
+                legalBasis = manualRedactionEntry.getLegalBasis();
             }
 
             if (manualChange instanceof ManualForceRedaction manualForceRedaction) {
@@ -96,9 +110,12 @@ public class ManualChangeOverwrite {
                 value = manualLegalBasisChange.getValue();
             }
 
-            if (manualChange instanceof ManualResizeRedaction) {
-                // resizing logic happens in ManualChangesApplicationService.
+            if (manualChange instanceof ManualResizeRedaction manualResizeRedaction) {
+                // resizing logic for TextEntities and Images happens in ManualChangesApplicationService.
                 resized = true;
+                // This is only for not found Manual Entities.
+                value = manualResizeRedaction.getValue();
+                positions = manualResizeRedaction.getPositions().stream().map(RectangleWithPage::fromAnnotationRectangle).toList();
             }
 
             if (manualChange instanceof ManualRecategorization recategorization) {
@@ -197,4 +214,11 @@ public class ManualChangeOverwrite {
         return descriptions == null ? Collections.emptyList() : descriptions;
     }
 
+
+    public Optional<List<RectangleWithPage>> getPositions() {
+
+        calculateCurrentOverride();
+        return positions == null ? Optional.empty() : Optional.of(positions);
+    }
+
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/MatchedRule.java

@@ -31,11 +31,11 @@ public final class MatchedRule implements Comparable<MatchedRule> {
     String reason = "";
     @Builder.Default
     String legalBasis = "";
-    boolean applied;
     boolean writeValueWithLineBreaks;
+    boolean applied;
     boolean removed;
     boolean ignored;
-    boolean resized;
+
     @Builder.Default
     Set<TextEntity> references = Collections.emptySet();
 
@@ -46,6 +46,20 @@ public final class MatchedRule implements Comparable<MatchedRule> {
     }
 
 
+    public MatchedRule asSkippedIfApplied() {
+
+        if (!this.isApplied()) {
+            return this;
+        }
+        return MatchedRule.builder().ruleIdentifier(getRuleIdentifier())
+                .writeValueWithLineBreaks(this.isWriteValueWithLineBreaks())
+                .legalBasis(this.getLegalBasis())
+                .reason(this.getReason())
+                .references(this.getReferences())
+                .build();
+    }
+
+
     @Override
     public int compareTo(MatchedRule matchedRule) {
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/SemanticNode.java

@@ -86,6 +86,12 @@ public interface SemanticNode {
     }
 
 
+    /**
+     * Checks if the given page number exists in the list of pages.
+     *
+     * @param pageNumber the page number to be checked
+     * @return true if the page number exists, otherwise false
+     */
     default boolean onPage(int pageNumber) {
 
         return getPages().stream().anyMatch(page -> page.getNumber() == pageNumber);
@@ -425,6 +431,71 @@ public interface SemanticNode {
     }
 
 
+    /**
+     * Checks whether this SemanticNode contains exactly the provided String as a word.
+     * @param word - String which the TextBlock might contain
+     * @return true, if this node's TextBlock contains string
+     */
+    default boolean containsWord(String word) {
+
+        return getTextBlock().getWords().stream().anyMatch(s -> s.equals(word));
+    }
+
+
+    /**
+     * Checks whether this SemanticNode contains exactly the provided String as a word ignoring case.
+     * @param word - String which the TextBlock might contain
+     * @return true, if this node's TextBlock contains string
+     */
+    default boolean containsWordIgnoreCase(String word) {
+
+        return getTextBlock().getWords().stream().map(String::toLowerCase).anyMatch(s -> s.equals(word.toLowerCase(Locale.ENGLISH)));
+    }
+
+
+    /**
+     * Checks whether this SemanticNode contains any of the provided Strings as a word.
+     * @param words - A List of Strings which the TextBlock might contain
+     * @return true, if this node's TextBlock contains any of the provided strings
+     */
+    default boolean containsAnyWord(String... words) {
+
+        return Arrays.stream(words).anyMatch(word -> getTextBlock().getWords().stream().anyMatch(word::equals));
+    }
+
+
+    /**
+     * Checks whether this SemanticNode contains any of the provided Strings as a word ignoring case.
+     * @param words - A List of Strings which the TextBlock might contain
+     * @return true, if this node's TextBlock contains any of the provided strings
+     */
+    default boolean containsAnyWordIgnoreCase(String... words) {
+
+        return Arrays.stream(words).map(String::toLowerCase).anyMatch(word -> getTextBlock().getWords().stream().map(String::toLowerCase).anyMatch(word::equals));
+    }
+
+
+    /**
+     * Checks whether this SemanticNode contains all the provided Strings as word.
+     * @param words - A List of Strings which the TextBlock might contain
+     * @return true, if this node's TextBlock contains all the provided strings
+     */
+    default boolean containsAllWords(String... words) {
+
+        return Arrays.stream(words).allMatch(word -> getTextBlock().getWords().stream().anyMatch(word::equals));
+    }
+
+
+    /**
+     * Checks whether this SemanticNode contains all the provided Strings as word ignoring case.
+     * @param words - A List of Strings which the TextBlock might contain
+     * @return true, if this node's TextBlock contains all the provided strings
+     */
+    default boolean containsAllWordsIgnoreCase(String... words) {
+
+        return Arrays.stream(words).map(String::toLowerCase).allMatch(word -> getTextBlock().getWords().stream().map(String::toLowerCase).anyMatch(word::equals));
+    }
+
     /**
      * Checks whether this SemanticNode matches the provided regex pattern.
      *

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Table.java

@@ -172,6 +172,18 @@ public class Table implements SemanticNode {
         return streamChildrenOfType(NodeType.TABLE_CELL).map(node -> (TableCell) node);
     }
 
+    /**
+     * Streams all TableCells that contain at least one entity of a given type in this Table row-wise.
+     *
+     * @param type The type of the entity
+     * @return Stream of filtered TableCells
+     */
+    public Stream<TableCell> streamTableCellsWhichContainType(String type) {
+
+        return streamTableCells()
+                .filter(tableCell -> tableCell.getEntities().stream().filter(TextEntity::active).anyMatch(entity -> entity.getType().equals(type)));
+    }
+
 
     /**
      * Streams all TableCells in this Table which have the provided header row-wise.

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/textblock/AtomicTextBlock.java

@@ -3,11 +3,14 @@ package com.iqser.red.service.redaction.v1.server.model.document.textblock;
 import static java.lang.String.format;
 
 import java.awt.geom.Rectangle2D;
+import java.text.BreakIterator;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
+import java.util.Locale;
 import java.util.Map;
 import java.util.Set;
 import java.util.stream.Collectors;
@@ -40,6 +43,7 @@ public class AtomicTextBlock implements TextBlock {
     //string coordinates
     TextRange textRange;
     String searchText;
+    List<String> words;
     List<Integer> lineBreaks;
 
     //position coordinates
@@ -114,6 +118,20 @@ public class AtomicTextBlock implements TextBlock {
         return new TextRange(lineBreaks.get(lineNumber - 1) + textRange.start(), lineBreaks.get(lineNumber) + textRange.start());
     }
 
+    public List<String> getWords() {
+
+        if (words == null) {
+            words = new ArrayList<>();
+            BreakIterator iterator = BreakIterator.getWordInstance(Locale.ENGLISH);
+            iterator.setText(searchText);
+            int start = iterator.first();
+            for (int end = iterator.next(); end != BreakIterator.DONE; start = end, end = iterator.next()) {
+                words.add(searchText.substring(start, end));
+            }
+        }
+        return words;
+    }
+
 
     @Override
     public List<AtomicTextBlock> getAtomicTextBlocks() {

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/textblock/ConcatenatedTextBlock.java

@@ -7,6 +7,7 @@ import java.util.Collections;
 import java.util.HashMap;
 import java.util.LinkedList;
 import java.util.List;
+import java.util.Collection;
 import java.util.Map;
 import java.util.stream.Stream;
 
@@ -86,6 +87,13 @@ public class ConcatenatedTextBlock implements TextBlock {
     }
 
 
+    @Override
+    public List<String> getWords() {
+
+        return atomicTextBlocks.stream().map(AtomicTextBlock::getWords).flatMap(Collection::stream).toList();
+    }
+
+
     @Override
     public int numberOfLines() {
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/textblock/TextBlock.java

@@ -18,6 +18,7 @@ public interface TextBlock extends CharSequence {
 
     String getSearchText();
 
+    List<String> getWords();
 
     List<AtomicTextBlock> getAtomicTextBlocks();
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/drools/BasicRule.java

@@ -1,5 +1,6 @@
 package com.iqser.red.service.redaction.v1.server.model.drools;
 
+import org.drools.drl.ast.descr.AttributeDescr;
 import org.drools.drl.ast.descr.RuleDescr;
 
 import lombok.AccessLevel;
@@ -18,6 +19,7 @@ public class BasicRule {
     RuleIdentifier identifier;
     String name;
     String code;
+    String agendaGroup;
     int line;
 
 
@@ -26,7 +28,8 @@ public class BasicRule {
         RuleIdentifier identifier = RuleIdentifier.fromName(rule.getName());
         String nameWithoutIdentifier = rule.getName().replace(identifier + ":", "");
         String code = rulesString.substring(rule.getStartCharacter(), rule.getEndCharacter());
-        return new BasicRule(identifier, nameWithoutIdentifier, code, rule.getLine());
+        String agendaGroup = rule.getAttributes().getOrDefault("agenda-group", new AttributeDescr("agenda-group", "DEFAULT")).getValue();
+        return new BasicRule(identifier, nameWithoutIdentifier, code, agendaGroup, rule.getLine());
     }
 
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/drools/RuleFileBluePrint.java

@@ -1,10 +1,12 @@
 package com.iqser.red.service.redaction.v1.server.model.drools;
 
+import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
 import java.util.Objects;
 import java.util.Optional;
+import java.util.Set;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
@@ -39,6 +41,12 @@ public final class RuleFileBluePrint {
     }
 
 
+    public Set<String> getImportSplitByKeyword() {
+
+        return Arrays.stream(imports.replaceAll("\n", "").split("import")).map(String::trim).collect(Collectors.toSet());
+    }
+
+
     public List<BasicRule> findRulesByIdentifier(RuleIdentifier ruleIdentifier) {
 
         if (Objects.isNull(ruleIdentifier.unit())) {

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/queue/MessageReceiver.java

@@ -1,13 +1,11 @@
 package com.iqser.red.service.redaction.v1.server.queue;
 
-import static com.iqser.red.service.redaction.v1.server.queue.MessagingConfiguration.REDACTION_QUEUE;
+import static com.iqser.red.service.redaction.v1.model.QueueNames.REDACTION_QUEUE;
 
 import org.springframework.amqp.core.Message;
 import org.springframework.amqp.rabbit.annotation.RabbitHandler;
 import org.springframework.amqp.rabbit.annotation.RabbitListener;
 
-import com.fasterxml.jackson.core.JsonProcessingException;
-
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/queue/MessagingConfiguration.java

@@ -1,6 +1,10 @@
 package com.iqser.red.service.redaction.v1.server.queue;
 
-import lombok.RequiredArgsConstructor;
+import static com.iqser.red.service.redaction.v1.model.QueueNames.MIGRATION_DLQ;
+import static com.iqser.red.service.redaction.v1.model.QueueNames.MIGRATION_QUEUE;
+import static com.iqser.red.service.redaction.v1.model.QueueNames.REDACTION_DQL;
+import static com.iqser.red.service.redaction.v1.model.QueueNames.REDACTION_PRIORITY_QUEUE;
+import static com.iqser.red.service.redaction.v1.model.QueueNames.REDACTION_QUEUE;
 
 import org.springframework.amqp.core.Queue;
 import org.springframework.amqp.core.QueueBuilder;
@@ -8,15 +12,12 @@ import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;
 
+import lombok.RequiredArgsConstructor;
+
 @Configuration
 @RequiredArgsConstructor
 public class MessagingConfiguration {
 
-    public static final String REDACTION_QUEUE = "redactionQueue";
-    public static final String REDACTION_DQL = "redactionDQL";
-
-    public static final String REDACTION_PRIORITY_QUEUE = "redactionPriorityQueue";
-
     public static final String X_ERROR_INFO_HEADER = "x-error-message";
     public static final String X_ERROR_INFO_TIMESTAMP_HEADER = "x-error-message-timestamp";
 
@@ -37,6 +38,20 @@ public class MessagingConfiguration {
     }
 
 
+    @Bean
+    public Queue migrationQueue() {
+
+        return QueueBuilder.durable(MIGRATION_QUEUE).withArgument("x-dead-letter-exchange", "").withArgument("x-dead-letter-routing-key", MIGRATION_DLQ).maxPriority(2).build();
+    }
+
+
+    @Bean
+    public Queue migrationDLQ() {
+
+        return QueueBuilder.durable(MIGRATION_DLQ).build();
+    }
+
+
     @Bean
     public Queue redactionQueue() {
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/queue/MigrationMessageReceiver.java

@@ -0,0 +1,66 @@
+package com.iqser.red.service.redaction.v1.server.queue;
+
+import static com.iqser.red.service.redaction.v1.model.QueueNames.MIGRATION_QUEUE;
+import static com.iqser.red.service.redaction.v1.model.QueueNames.MIGRATION_RESPONSE_QUEUE;
+
+import org.springframework.amqp.core.Message;
+import org.springframework.amqp.rabbit.annotation.RabbitHandler;
+import org.springframework.amqp.rabbit.annotation.RabbitListener;
+import org.springframework.amqp.rabbit.core.RabbitTemplate;
+import org.springframework.stereotype.Service;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog;
+import com.iqser.red.service.redaction.v1.model.MigrationRequest;
+import com.iqser.red.service.redaction.v1.model.MigrationResponse;
+import com.iqser.red.service.redaction.v1.server.model.MigratedEntityLog;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
+import com.iqser.red.service.redaction.v1.server.service.RedactionLogToEntityLogMigrationService;
+import com.iqser.red.service.redaction.v1.server.service.document.DocumentGraphMapper;
+import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
+
+import lombok.AccessLevel;
+import lombok.RequiredArgsConstructor;
+import lombok.SneakyThrows;
+import lombok.experimental.FieldDefaults;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+@Service
+@RequiredArgsConstructor
+@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
+public class MigrationMessageReceiver {
+
+    ObjectMapper objectMapper;
+    RedactionLogToEntityLogMigrationService redactionLogToEntityLogMigrationService;
+    RedactionStorageService redactionStorageService;
+    RabbitTemplate rabbitTemplate;
+
+
+    @SneakyThrows
+    @RabbitHandler
+    @RabbitListener(queues = MIGRATION_QUEUE)
+    public void receiveMigrationRequest(Message message) {
+
+        MigrationRequest migrationRequest = objectMapper.readValue(message.getBody(), MigrationRequest.class);
+
+        RedactionLog redactionLog = redactionStorageService.getRedactionLog(migrationRequest.getDossierId(), migrationRequest.getFileId());
+        Document document = DocumentGraphMapper.toDocumentGraph(redactionStorageService.getDocumentData(migrationRequest.getDossierId(), migrationRequest.getFileId()));
+
+        MigratedEntityLog migratedEntityLog = redactionLogToEntityLogMigrationService.migrate(redactionLog, document);
+
+        redactionStorageService.storeObject(migrationRequest.getDossierId(), migrationRequest.getFileId(), FileType.ENTITY_LOG, migratedEntityLog.getEntityLog());
+        redactionStorageService.storeObject(migrationRequest.getDossierId(), migrationRequest.getFileId(), FileType.MIGRATED_IDS, migratedEntityLog.getMigratedIds());
+
+        sendFinished(MigrationResponse.builder().dossierId(migrationRequest.getDossierId()).fileId(migrationRequest.getFileId()).build());
+    }
+
+
+    @SneakyThrows
+    public void sendFinished(MigrationResponse migrationResponse) {
+
+        rabbitTemplate.convertAndSend(MIGRATION_RESPONSE_QUEUE, migrationResponse);
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/queue/PriorityMessageReceiver.java

@@ -1,13 +1,11 @@
 package com.iqser.red.service.redaction.v1.server.queue;
 
-import static com.iqser.red.service.redaction.v1.server.queue.MessagingConfiguration.REDACTION_PRIORITY_QUEUE;
+import static com.iqser.red.service.redaction.v1.model.QueueNames.REDACTION_PRIORITY_QUEUE;
 
 import org.springframework.amqp.core.Message;
 import org.springframework.amqp.rabbit.annotation.RabbitHandler;
 import org.springframework.amqp.rabbit.annotation.RabbitListener;
 
-import com.fasterxml.jackson.core.JsonProcessingException;
-
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/queue/RedactionMessageReceiver.java

@@ -1,8 +1,8 @@
 package com.iqser.red.service.redaction.v1.server.queue;
 
-import static com.iqser.red.service.redaction.v1.server.queue.MessagingConfiguration.REDACTION_DQL;
-import static com.iqser.red.service.redaction.v1.server.queue.MessagingConfiguration.REDACTION_PRIORITY_QUEUE;
-import static com.iqser.red.service.redaction.v1.server.queue.MessagingConfiguration.REDACTION_QUEUE;
+import static com.iqser.red.service.redaction.v1.model.QueueNames.REDACTION_DQL;
+import static com.iqser.red.service.redaction.v1.model.QueueNames.REDACTION_PRIORITY_QUEUE;
+import static com.iqser.red.service.redaction.v1.model.QueueNames.REDACTION_QUEUE;
 import static com.iqser.red.service.redaction.v1.server.queue.MessagingConfiguration.X_ERROR_INFO_HEADER;
 import static com.iqser.red.service.redaction.v1.server.queue.MessagingConfiguration.X_ERROR_INFO_TIMESTAMP_HEADER;
 import static java.lang.String.format;
@@ -23,8 +23,8 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeResu
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileErrorInfo;
 import com.iqser.red.service.redaction.v1.server.client.FileStatusProcessingUpdateClient;
 import com.iqser.red.service.redaction.v1.server.client.RulesClient;
-import com.iqser.red.service.redaction.v1.server.utils.exception.DroolsTimeoutException;
 import com.iqser.red.service.redaction.v1.server.service.AnalyzeService;
+import com.iqser.red.service.redaction.v1.server.utils.exception.DroolsTimeoutException;
 
 import lombok.RequiredArgsConstructor;
 import lombok.SneakyThrows;
@@ -118,7 +118,7 @@ public class RedactionMessageReceiver {
 
     private void sendAnalysisFailed(AnalyzeRequest analyzeRequest, boolean priority, Exception e) {
 
-        log.warn("Failed to process analyze request: {}", analyzeRequest, e);
+        log.error("Failed to process analyze request: {}", analyzeRequest, e);
         var timestamp = OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS);
         fileStatusProcessingUpdateClient.analysisFailed(analyzeRequest.getDossierId(),
                 analyzeRequest.getFileId(),
@@ -128,7 +128,7 @@ public class RedactionMessageReceiver {
 
     @RabbitHandler
     @RabbitListener(queues = REDACTION_DQL)
-    public void receiveAnalyzeRequestDQL(Message in) throws IOException {
+    public void receiveAnalyzeRequestDLQ(Message in) throws IOException {
 
         var analyzeRequest = objectMapper.readValue(in.getBody(), AnalyzeRequest.class);
         log.info("Failed to process analyze request: {}", analyzeRequest);

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalyzeService.java

@@ -18,7 +18,6 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileTyp
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentLog;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogChanges;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Comment;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.legalbasis.LegalBasis;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog;
@@ -113,7 +112,9 @@ public class AnalyzeService {
                     kieContainerCreationService.getLatestKieContainer(analyzeRequest.getDossierTemplateId(), RuleFileType.COMPONENT),
                     entityLogCreatorService.updateVersionsAndReturnChanges(previousEntityLog,
                             dictionaryIncrement.getDictionaryVersion(),
-                            analyzeRequest.getDossierTemplateId(), false), document,
+                            analyzeRequest.getDossierTemplateId(),
+                            false),
+                    document,
                     previousRedactionLog,
                     document.getNumberOfPages(),
                     dictionaryIncrement.getDictionaryVersion(),
@@ -155,7 +156,9 @@ public class AnalyzeService {
 
         return finalizeAnalysis(analyzeRequest,
                 startTime,
-                kieContainerCreationService.getLatestKieContainer(analyzeRequest.getDossierTemplateId(), RuleFileType.COMPONENT), entityLogChanges, document,
+                kieContainerCreationService.getLatestKieContainer(analyzeRequest.getDossierTemplateId(), RuleFileType.COMPONENT),
+                entityLogChanges,
+                document,
                 redactionLog,
                 document.getNumberOfPages(),
                 dictionaryIncrement.getDictionaryVersion(),
@@ -207,7 +210,11 @@ public class AnalyzeService {
                 dictionary.getVersion(),
                 kieWrapperEntityRules.rulesVersion());
 
-        return finalizeAnalysis(analyzeRequest, startTime, kieWrapperComponentRules, new EntityLogChanges(entityLog, false), document,
+        return finalizeAnalysis(analyzeRequest,
+                startTime,
+                kieWrapperComponentRules,
+                new EntityLogChanges(entityLog, false),
+                document,
                 redactionLog,
                 document.getNumberOfPages(),
                 dictionary.getVersion(),
@@ -225,8 +232,7 @@ public class AnalyzeService {
 
         List<RedactionLogEntry> newRedactionLogEntries = redactionLogCreatorService.createRedactionLog(document,
                 analyzeRequest.getDossierTemplateId(),
-                notFoundManualRedactionEntries,
-                getComments(analyzeRequest));
+                notFoundManualRedactionEntries);
 
         var importedRedactionFilteredEntries = importedRedactionService.processImportedRedactions(analyzeRequest.getDossierTemplateId(),
                 analyzeRequest.getDossierId(),
@@ -236,12 +242,18 @@ public class AnalyzeService {
 
         previousRedactionLog.getRedactionLogEntry()
                 .removeIf(entry -> sectionsToReanalyseIds.contains(entry.getSectionNumber()) && !entry.getType().equals(ImportedRedactionService.IMPORTED_REDACTION_TYPE));
+
         previousRedactionLog.getRedactionLogEntry().addAll(importedRedactionFilteredEntries);
+
         return previousRedactionLog;
     }
 
 
-    private AnalyzeResult finalizeAnalysis(AnalyzeRequest analyzeRequest, long startTime, KieWrapper kieWrapperComponentRules, EntityLogChanges entityLogChanges, Document document,
+    private AnalyzeResult finalizeAnalysis(AnalyzeRequest analyzeRequest,
+                                           long startTime,
+                                           KieWrapper kieWrapperComponentRules,
+                                           EntityLogChanges entityLogChanges,
+                                           Document document,
                                            RedactionLog redactionLog,
                                            int numberOfPages,
                                            DictionaryVersion dictionaryVersion,
@@ -284,7 +296,9 @@ public class AnalyzeService {
     }
 
 
-    private void computeComponentsWhenRulesArePresent(AnalyzeRequest analyzeRequest, KieWrapper kieWrapperComponentRules, Document document,
+    private void computeComponentsWhenRulesArePresent(AnalyzeRequest analyzeRequest,
+                                                      KieWrapper kieWrapperComponentRules,
+                                                      Document document,
                                                       Set<FileAttribute> addedFileAttributes,
                                                       EntityLogChanges entityLogChanges,
                                                       DictionaryVersion dictionaryVersion) {
@@ -293,7 +307,9 @@ public class AnalyzeService {
             return;
         }
 
-        List<Component> components = componentDroolsExecutionService.executeRules(kieWrapperComponentRules.container(), entityLogChanges.getEntityLog(), document,
+        List<Component> components = componentDroolsExecutionService.executeRules(kieWrapperComponentRules.container(),
+                entityLogChanges.getEntityLog(),
+                document,
                 addedFileAttributes.stream().toList());
         log.info("Finished component rule execution for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
 
@@ -378,8 +394,7 @@ public class AnalyzeService {
 
         List<RedactionLogEntry> redactionLogEntries = redactionLogCreatorService.createRedactionLog(document,
                 analyzeRequest.getDossierTemplateId(),
-                notFoundManualRedactionEntries,
-                getComments(analyzeRequest));
+                notFoundManualRedactionEntries);
 
         List<LegalBasis> legalBasis = legalBasisClient.getLegalBasisMapping(analyzeRequest.getDossierTemplateId());
         RedactionLog redactionLog = new RedactionLog(redactionServiceSettings.getAnalysisVersion(),
@@ -401,18 +416,6 @@ public class AnalyzeService {
     }
 
 
-    private static Map<String, List<Comment>> getComments(AnalyzeRequest analyzeRequest) {
-
-        if (analyzeRequest.getManualRedactions() == null) {
-            return Collections.emptyMap();
-        }
-        if (analyzeRequest.getManualRedactions().getComments() == null) {
-            return Collections.emptyMap();
-        }
-        return analyzeRequest.getManualRedactions().getComments();
-    }
-
-
     public List<RedactionLogLegalBasis> toRedactionLogLegalBasis(List<LegalBasis> legalBasis) {
 
         return legalBasis.stream().map(l -> new RedactionLogLegalBasis(l.getName(), l.getDescription(), l.getReason())).collect(Collectors.toList());

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/DictionaryService.java

@@ -47,7 +47,7 @@ import lombok.extern.slf4j.Slf4j;
 @RequiredArgsConstructor
 public class DictionaryService {
 
-    private static final String DEFAULT_COLOR = "#cccccc";
+    public static final String DEFAULT_COLOR = "#cccccc";
     private final DictionaryClient dictionaryClient;
 
     private final RedactionServiceSettings settings;

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityChangeLogService.java

@@ -50,6 +50,10 @@ public class EntityChangeLogService {
                     entityLogEntry.getChanges().add(new Change(analysisNumber, changeType, now));
                 }
             }
+
+            if (!previousEntity.equals(entityLogEntry)) {
+                hasChanges = true;
+            }
         }
         addRemovedEntriesAsRemoved(previousEntityLogEntries, newEntityLogEntries, analysisNumber, now);
         return hasChanges;

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java

@@ -1,5 +1,7 @@
 package com.iqser.red.service.redaction.v1.server.service;
 
+import static com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryType.IMAGE;
+
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashSet;
@@ -58,13 +60,11 @@ public class EntityLogCreatorService {
     }
 
 
-    public EntityLog createInitialEntityLog(AnalyzeRequest analyzeRequest,
-                                            Document document,
-                                            List<ManualEntity> notFoundManualRedactionEntries,
+    public EntityLog createInitialEntityLog(AnalyzeRequest analyzeRequest, Document document, List<ManualEntity> notFoundManualEntities,
                                             DictionaryVersion dictionaryVersion,
                                             long rulesVersion) {
 
-        List<EntityLogEntry> entityLogEntries = createEntityLogEntries(document, analyzeRequest.getDossierTemplateId(), notFoundManualRedactionEntries);
+        List<EntityLogEntry> entityLogEntries = createEntityLogEntries(document, analyzeRequest.getDossierTemplateId(), notFoundManualEntities);
 
         List<LegalBasis> legalBasis = legalBasisClient.getLegalBasisMapping(analyzeRequest.getDossierTemplateId());
         EntityLog entityLog = new EntityLog(redactionServiceSettings.getAnalysisVersion(),
@@ -121,10 +121,11 @@ public class EntityLogCreatorService {
 
         List<EntityLogEntry> newEntityLogEntries = createEntityLogEntries(document, analyzeRequest.getDossierTemplateId(), notFoundManualRedactionEntries);
         Set<String> newEntityIds = newEntityLogEntries.stream().map(EntityLogEntry::getId).collect(Collectors.toSet());
+
         List<EntityLogEntry> previousEntriesFromReAnalyzedSections = previousEntityLog.getEntityLogEntry()
                 .stream()
-                .filter(entry -> (newEntityIds.contains(entry.getId()) || sectionsToReanalyseIds.contains(entry.getContainingNodeId().get(0))) && !entry.getType()
-                        .equals(ImportedRedactionService.IMPORTED_REDACTION_TYPE))
+                .filter(entry -> !entry.getType()
+                        .equals(ImportedRedactionService.IMPORTED_REDACTION_TYPE) && (newEntityIds.contains(entry.getId()) || entry.getContainingNodeId().isEmpty() || sectionsToReanalyseIds.contains(entry.getContainingNodeId().get(0))))
                 .toList();
         previousEntityLog.getEntityLogEntry().removeAll(previousEntriesFromReAnalyzedSections);
 
@@ -150,6 +151,7 @@ public class EntityLogCreatorService {
         Set<String> processedIds = new HashSet<>();
         document.getEntities()
                 .stream()
+                .filter(entity -> !entity.getValue().isEmpty())
                 .filter(EntityLogCreatorService::notFalsePositiveOrFalseRecommendation)
                 .filter(entity -> !entity.removed())
                 .forEach(entityNode -> entries.addAll(toEntityLogEntries(entityNode, dossierTemplateId, processedIds)));
@@ -171,21 +173,21 @@ public class EntityLogCreatorService {
             }
             processedIds.add(positionOnPage.getId());
 
-            EntityLogEntry redactionLogEntry = createEntityLogEntry(textEntity, dossierTemplateId);
-            redactionLogEntry.setId(positionOnPage.getId());
+            EntityLogEntry entityLogEntries = createEntityLogEntry(textEntity, dossierTemplateId);
+            entityLogEntries.setId(positionOnPage.getId());
 
             List<Position> rectanglesPerLine = positionOnPage.getRectanglePerLine().stream().map(rectangle2D -> new Position(rectangle2D, positionOnPage.getPage().getNumber()))
                     .toList();
 
-            redactionLogEntry.setPositions(rectanglesPerLine);
-            redactionLogEntities.add(redactionLogEntry);
+            entityLogEntries.setPositions(rectanglesPerLine);
+            redactionLogEntities.add(entityLogEntries);
         }
 
         return redactionLogEntities;
     }
 
 
-    public EntityLogEntry createEntityLogEntry(Image image, String dossierTemplateId) {
+    private EntityLogEntry createEntityLogEntry(Image image, String dossierTemplateId) {
 
         String imageType = image.getImageType().equals(ImageType.OTHER) ? "image" : image.getImageType().toString().toLowerCase(Locale.ENGLISH);
         boolean isHint = dictionaryService.isHint(imageType, dossierTemplateId);
@@ -205,8 +207,7 @@ public class EntityLogCreatorService {
                 .section(image.getManualOverwrite().getSection().orElse(image.getParent().toString()))
                 .imageHasTransparency(image.isTransparent())
                 .manualChanges(manualChangeFactory.toManualChangeList(image.getManualOverwrite().getManualChangeLog(), isHint))
-                .state(buildEntryState(image))
-                .entryType(buildEntryType(image))
+                .state(buildEntryState(image)).entryType(isHint ? EntryType.IMAGE_HINT : EntryType.IMAGE)
                 .build();
 
     }
@@ -216,9 +217,15 @@ public class EntityLogCreatorService {
 
         String type = manualEntity.getManualOverwrite().getType().orElse(manualEntity.getType());
         boolean isHint = isHint(manualEntity.getEntityType());
-        return EntityLogEntry.builder().id(manualEntity.getId()).color(getColor(type, dossierTemplateId, manualEntity.applied(), isHint))
+        return EntityLogEntry.builder()
+                .id(manualEntity.getId())
+                .color(getColor(type, dossierTemplateId, manualEntity.applied(), isHint))
                 .reason(manualEntity.buildReasonWithManualChangeDescriptions())
-                .legalBasis(manualEntity.legalBasis()).value(manualEntity.value()).type(type).state(buildEntryState(manualEntity)).entryType(buildEntryType(manualEntity))
+                .legalBasis(manualEntity.legalBasis())
+                .value(manualEntity.value())
+                .type(type)
+                .state(buildEntryState(manualEntity))
+                .entryType(buildEntryType(manualEntity))
                 .section(manualEntity.getManualOverwrite().getSection().orElse(manualEntity.getSection()))
                 .containingNodeId(Collections.emptyList())
                 .closestHeadline("")
@@ -229,7 +236,12 @@ public class EntityLogCreatorService {
                 .textBefore("")
                 .startOffset(-1)
                 .endOffset(-1)
-                .positions(manualEntity.getEntityPosition().stream().map(entityPosition -> new Position(entityPosition.rectangle2D(), entityPosition.pageNumber())).toList())
+                .positions(manualEntity.getManualOverwrite()
+                        .getPositions()
+                        .orElse(manualEntity.getEntityPosition())
+                        .stream()
+                        .map(entityPosition -> new Position(entityPosition.rectangle2D(), entityPosition.pageNumber()))
+                        .toList())
                 .engines(Collections.emptySet())
                 .reference(Collections.emptySet())
                 .manualChanges(manualChangeFactory.toManualChangeList(manualEntity.getManualOverwrite().getManualChangeLog(), isHint))
@@ -279,6 +291,7 @@ public class EntityLogCreatorService {
             return dictionaryService.getNotRedactedColor(dossierTemplateId);
         }
         return dictionaryService.getColor(type, dossierTemplateId);
+
     }
 
 
@@ -301,12 +314,10 @@ public class EntityLogCreatorService {
         if (entity instanceof TextEntity textEntity) {
             return getEntryType(textEntity.getEntityType());
         } else if (entity instanceof ManualEntity manualEntity) {
-            if (((ManualEntity) entity).isRectangle()) {
+            if (manualEntity.isRectangle()) {
                 return EntryType.AREA;
             }
             return getEntryType(manualEntity.getEntityType());
-        } else if (entity instanceof Image) {
-            return EntryType.IMAGE;
         }
         throw new UnsupportedOperationException(String.format("Entity subclass %s is not implemented!", entity.getClass()));
     }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/ImportedRedactionService.java

@@ -1,19 +1,24 @@
 package com.iqser.red.service.redaction.v1.server.service;
 
+import java.util.ArrayList;
+import java.util.Collections;
 import java.util.HashSet;
 import java.util.List;
+import java.util.stream.Collectors;
 
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedaction;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedactions;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Point;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
 import org.springframework.stereotype.Service;
 
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryType;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Point;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.imported.ImportedRedaction;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.imported.ImportedRedactions;
+
+
 import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
 
 import io.micrometer.core.annotation.Timed;
@@ -55,22 +60,21 @@ public class ImportedRedactionService {
     @Timed("redactmanager_processImportedRedactions")
     public List<EntityLogEntry> processImportedEntities(String dossierTemplateId,
                                                         String dossierId,
-                                                        String fileId,
-                                                        List<EntityLogEntry> redactionLogEntries,
+                                                        String fileId, List<EntityLogEntry> entityLogEntries,
                                                         boolean addImportedRedactions) {
 
         ImportedRedactions importedRedactions = redactionStorageService.getImportedRedactions(dossierId, fileId);
         if (importedRedactions == null) {
-            return redactionLogEntries;
+            return entityLogEntries;
         }
 
-        redactionLogEntries.forEach(redactionLogEntry -> addIntersections(redactionLogEntry, importedRedactions));
+        entityLogEntries.forEach(redactionLogEntry -> addIntersections(redactionLogEntry, importedRedactions));
 
         if (addImportedRedactions) {
-            return addImportedRedactionsEntityLogEntries(dossierTemplateId, redactionLogEntries, importedRedactions);
+            return addImportedRedactionsEntityLogEntries(dossierTemplateId, entityLogEntries, importedRedactions);
         }
 
-        return redactionLogEntries;
+        return entityLogEntries;
     }
 
 
@@ -86,7 +90,7 @@ public class ImportedRedactionService {
                         .type(IMPORTED_REDACTION_TYPE)
                         .imported(true)
                         .redacted(true)
-                        .positions(importedRedaction.getPositions())
+                        .positions(importedRedaction.getPositions().stream().map(this::normalize).collect(Collectors.toList()))
                         .color(getColor(IMPORTED_REDACTION_TYPE, dossierTemplateId))
                         .build();
 
@@ -107,8 +111,10 @@ public class ImportedRedactionService {
                         .type(IMPORTED_REDACTION_TYPE).entryType(EntryType.ENTITY)
                         .imported(true)
                         .state(EntryState.APPLIED)
-                        .positions(toPositions(importedRedaction.getPositions()))
+                        .positions(importedRedaction.getPositions())
                         .color(getColor(IMPORTED_REDACTION_TYPE, dossierTemplateId))
+                        .containingNodeId(Collections.emptyList())
+                        .value("")
                         .build();
 
                 entityLogEntries.add(redactionLogEntry);
@@ -139,7 +145,7 @@ public class ImportedRedactionService {
             if (importedRedactions.getImportedRedactions().containsKey(rectangle.getPage())) {
                 var importedRedactionsOnPage = importedRedactions.getImportedRedactions().get(rectangle.getPage());
                 for (ImportedRedaction importedRedaction : importedRedactionsOnPage) {
-                    for (Rectangle importedRedactionPosition : importedRedaction.getPositions()) {
+                    for (Position importedRedactionPosition : importedRedaction.getPositions()) {
                         if (rectOverlap(normalizedRectangle, normalize(importedRedactionPosition))) {
                             if (redactionLogEntry.getImportedRedactionIntersections() == null) {
                                 redactionLogEntry.setImportedRedactionIntersections(new HashSet<>());
@@ -160,7 +166,7 @@ public class ImportedRedactionService {
             if (importedRedactions.getImportedRedactions().containsKey(rectangle.getPageNumber())) {
                 var importedRedactionsOnPage = importedRedactions.getImportedRedactions().get(rectangle.getPageNumber());
                 for (ImportedRedaction importedRedaction : importedRedactionsOnPage) {
-                    for (Rectangle importedRedactionPosition : importedRedaction.getPositions()) {
+                    for (Position importedRedactionPosition : importedRedaction.getPositions()) {
                         if (rectOverlap(normalizedRectangle, normalize(importedRedactionPosition))) {
                             if (redactionLogEntry.getImportedRedactionIntersections() == null) {
                                 redactionLogEntry.setImportedRedactionIntersections(new HashSet<>());

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/ManualChangesApplicationService.java

@@ -92,7 +92,7 @@ public class ManualChangesApplicationService {
 
     private static Rectangle2D toRectangle2D(com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle rect) {
 
-        return new Rectangle2D.Double(rect.getTopLeftX() - rect.getWidth(), rect.getTopLeftY() - rect.getHeight(), rect.getWidth(), rect.getHeight());
+        return new Rectangle2D.Double(rect.getTopLeftX(), rect.getTopLeftY(), rect.getWidth(), rect.getHeight());
     }
 
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RedactionChangeLogService.java

@@ -13,6 +13,7 @@ import org.springframework.stereotype.Service;
 
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Change;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ChangeType;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ManualRedactionType;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogChanges;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
@@ -102,7 +103,7 @@ public class RedactionChangeLogService {
 
         Set<RedactionLogEntry> currentExistingEntries = currentRedactionLog.getRedactionLogEntry()
                 .stream()
-                .filter(entry -> entry.getChanges().isEmpty() || !entry.lastChangeIsRemoved())
+                .filter(entry -> (entry.getChanges().isEmpty() || !entry.lastChangeIsRemoved()) && !isLastManualChangeRemove(entry))
                 .collect(Collectors.toSet());
 
         previouslyExistingEntries.forEach(currentExistingEntries::remove);
@@ -113,4 +114,12 @@ public class RedactionChangeLogService {
         return addedIds;
     }
 
+
+    private static boolean isLastManualChangeRemove(RedactionLogEntry redactionLogEntry){
+        if(redactionLogEntry.getManualChanges() == null || redactionLogEntry.getManualChanges().isEmpty()){
+            return false;
+        }
+        return redactionLogEntry.getManualChanges().get(redactionLogEntry.getManualChanges().size() -1).getManualRedactionType() == ManualRedactionType.REMOVE_LOCALLY;
+    }
+
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RedactionLogCreatorService.java

@@ -6,7 +6,6 @@ import java.util.Collections;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Locale;
-import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
 import java.util.stream.Collectors;
@@ -14,13 +13,12 @@ import java.util.stream.Collectors;
 import org.springframework.beans.BeanUtils;
 import org.springframework.stereotype.Service;
 
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Comment;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ManualChange;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ManualRedactionType;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Point;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogComment;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
 import com.iqser.red.service.redaction.v1.server.model.ManualEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
@@ -45,19 +43,18 @@ public class RedactionLogCreatorService {
     private final ManualChangeFactory manualChangeFactory;
 
 
-    public List<RedactionLogEntry> createRedactionLog(Document document,
-                                                      String dossierTemplateId,
-                                                      List<ManualEntity> notFoundManualRedactionEntries,
-                                                      Map<String, List<Comment>> comments) {
+    public List<RedactionLogEntry> createRedactionLog(Document document, String dossierTemplateId, List<ManualEntity> notFoundManualRedactionEntries) {
 
         List<RedactionLogEntry> entries = new ArrayList<>();
         Set<String> processIds = new HashSet<>();
         document.getEntities()
-                .stream().filter(RedactionLogCreatorService::notFalsePositiveOrFalseRecommendation)
+                .stream()
+                .filter(entity -> !entity.getValue().isEmpty())
+                .filter(RedactionLogCreatorService::notFalsePositiveOrFalseRecommendation)
                 .filter(IEntity::active)
-                .forEach(entityNode -> entries.addAll(toRedactionLogEntries(entityNode, dossierTemplateId, comments, processIds)));
-        document.streamAllImages().filter(image -> !image.removed()).forEach(imageNode -> entries.add(createRedactionLogEntry(imageNode, dossierTemplateId, comments)));
-        notFoundManualRedactionEntries.forEach(entityIdentifier -> entries.add(createRedactionLogEntry(entityIdentifier, dossierTemplateId, comments)));
+                .forEach(entityNode -> entries.addAll(toRedactionLogEntries(entityNode, dossierTemplateId, processIds)));
+        document.streamAllImages().filter(image -> !image.removed()).forEach(imageNode -> entries.add(createRedactionLogEntry(imageNode, dossierTemplateId)));
+        notFoundManualRedactionEntries.forEach(entityIdentifier -> entries.add(createRedactionLogEntry(entityIdentifier, dossierTemplateId)));
         return entries;
     }
 
@@ -68,7 +65,7 @@ public class RedactionLogCreatorService {
     }
 
 
-    private List<RedactionLogEntry> toRedactionLogEntries(TextEntity textEntity, String dossierTemplateId, Map<String, List<Comment>> comments, Set<String> processedIds) {
+    private List<RedactionLogEntry> toRedactionLogEntries(TextEntity textEntity, String dossierTemplateId, Set<String> processedIds) {
 
         List<RedactionLogEntry> redactionLogEntities = new ArrayList<>();
 
@@ -82,7 +79,6 @@ public class RedactionLogCreatorService {
 
             RedactionLogEntry redactionLogEntry = createRedactionLogEntry(textEntity, dossierTemplateId);
             redactionLogEntry.setId(positionOnPage.getId());
-            redactionLogEntry.setComments(buildRedactionLogComments(comments, positionOnPage.getId()));
 
             List<Rectangle> rectanglesPerLine = positionOnPage.getRectanglePerLine()
                     .stream()
@@ -97,44 +93,14 @@ public class RedactionLogCreatorService {
     }
 
 
-    private List<RedactionLogComment> buildRedactionLogComments(Map<String, List<Comment>> commentsPerId, String id) {
-
-        if (!commentsPerId.containsKey(id)) {
-            return Collections.emptyList();
-        }
-        List<Comment> comments = commentsPerId.get(id);
-        if (comments == null || comments.isEmpty()) {
-            return Collections.emptyList();
-        }
-        return toRedactionLogComments(comments);
-    }
-
-
-    private List<RedactionLogComment> toRedactionLogComments(List<Comment> comments) {
-
-        return comments.stream().map(this::toRedactionLogComment).toList();
-    }
-
-
-    private RedactionLogComment toRedactionLogComment(Comment comment) {
-
-        return new RedactionLogComment(comment.getId(),
-                comment.getUser(),
-                comment.getText(),
-                comment.getAnnotationId(),
-                comment.getFileId(),
-                comment.getDate(),
-                comment.getSoftDeletedTime());
-    }
-
-
     private RedactionLogEntry createRedactionLogEntry(TextEntity entity, String dossierTemplateId) {
 
         Set<String> referenceIds = new HashSet<>();
         entity.references().stream().filter(TextEntity::active).forEach(ref -> ref.getPositionsOnPagePerPage().forEach(pos -> referenceIds.add(pos.getId())));
         int sectionNumber = entity.getDeepestFullyContainingNode().getTreeId().isEmpty() ? 0 : entity.getDeepestFullyContainingNode().getTreeId().get(0);
         boolean isHint = isHint(entity.getEntityType());
-        return RedactionLogEntry.builder().color(getColor(entity.getType(), dossierTemplateId, entity.applied(), isHint))
+        return RedactionLogEntry.builder()
+                .color(getColor(entity.getType(), dossierTemplateId, entity.applied(), isHint))
                 .reason(entity.buildReasonWithManualChangeDescriptions())
                 .legalBasis(entity.legalBasis())
                 .value(entity.getManualOverwrite().getValue().orElse(entity.getMatchedRule().isWriteValueWithLineBreaks() ? entity.getValueWithLineBreaks() : entity.getValue()))
@@ -206,6 +172,7 @@ public class RedactionLogCreatorService {
         manualChange1.setProcessedDate(manualChange.getProcessedDate());
         manualChange1.setRequestedDate(manualChange.getRequestedDate());
         manualChange1.setPropertyChanges(manualChange.getPropertyChanges());
+        manualChange1.setAnnotationStatus(AnnotationStatus.APPROVED);
         return manualChange1;
     }
 
@@ -219,7 +186,40 @@ public class RedactionLogCreatorService {
     }
 
 
-    public RedactionLogEntry createRedactionLogEntry(ManualEntity manualEntity, String dossierTemplateId, Map<String, List<Comment>> comments) {
+    public RedactionLogEntry createRedactionLogEntry(Image image, String dossierTemplateId) {
+
+        String imageType = image.getImageType().equals(ImageType.OTHER) ? "image" : image.getImageType().toString().toLowerCase(Locale.ENGLISH);
+        boolean isHint = dictionaryService.isHint(imageType, dossierTemplateId);
+        return RedactionLogEntry.builder()
+                .id(image.getId())
+                .color(getColor(imageType, dossierTemplateId, image.applied(), isHint))
+                .isImage(true)
+                .value(image.value())
+                .type(imageType)
+                .redacted(image.applied())
+                .reason(image.buildReasonWithManualChangeDescriptions())
+                .legalBasis(image.legalBasis())
+                .matchedRule(image.getMatchedRule().getRuleIdentifier().toString())
+                .isHint(isHint)
+                .isDictionaryEntry(false)
+                .isRecommendation(false)
+                .positions(List.of(toRedactionLogRectangle(image.getPosition(), image.getPage().getNumber())))
+                .sectionNumber(image.getTreeId().get(0))
+                .section(image.getManualOverwrite().getSection().orElse(image.getParent().toString()))
+                .imageHasTransparency(image.isTransparent())
+                .manualChanges(mapManualChanges(image.getManualOverwrite(), isHint))
+                .build();
+
+    }
+
+
+    private Rectangle toRedactionLogRectangle(Rectangle2D rectangle2D, int pageNumber) {
+
+        return new Rectangle(new Point((float) rectangle2D.getMinX(), (float) rectangle2D.getMinY()), (float) rectangle2D.getWidth(), (float) rectangle2D.getHeight(), pageNumber);
+    }
+
+
+    public RedactionLogEntry createRedactionLogEntry(ManualEntity manualEntity, String dossierTemplateId) {
 
         String type = manualEntity.getManualOverwrite().getType().orElse(manualEntity.getType());
         boolean isHint = isHint(manualEntity.getEntityType());
@@ -244,48 +244,16 @@ public class RedactionLogCreatorService {
                 .textBefore("")
                 .startOffset(-1)
                 .endOffset(-1)
-                .positions(manualEntity.getEntityPosition()
+                .positions(manualEntity.getManualOverwrite()
+                        .getPositions()
+                        .orElse(manualEntity.getEntityPosition())
                         .stream()
                         .map(entityPosition -> toRedactionLogRectangle(entityPosition.rectangle2D(), entityPosition.pageNumber()))
-                        .collect(Collectors.toList()))
+                        .toList())
                 .engines(Collections.emptySet())
                 .reference(Collections.emptySet())
                 .manualChanges(mapManualChanges(manualEntity.getManualOverwrite(), isHint))
-                .comments(buildRedactionLogComments(comments, manualEntity.getId()))
                 .build();
     }
 
-
-    private Rectangle toRedactionLogRectangle(Rectangle2D rectangle2D, int pageNumber) {
-
-        return new Rectangle(new Point((float) rectangle2D.getMinX(), (float) rectangle2D.getMinY()), (float) rectangle2D.getWidth(), (float) rectangle2D.getHeight(), pageNumber);
-    }
-
-
-    public RedactionLogEntry createRedactionLogEntry(Image image, String dossierTemplateId, Map<String, List<Comment>> comments) {
-
-        String imageType = image.getImageType().equals(ImageType.OTHER) ? "image" : image.getImageType().toString().toLowerCase(Locale.ENGLISH);
-        boolean isHint = dictionaryService.isHint(imageType, dossierTemplateId);
-        return RedactionLogEntry.builder()
-                .id(image.getId()).color(getColor(imageType, dossierTemplateId, image.applied(), isHint))
-                .isImage(true)
-                .value(image.value())
-                .type(imageType)
-                .redacted(image.applied())
-                .reason(image.buildReasonWithManualChangeDescriptions())
-                .legalBasis(image.legalBasis())
-                .matchedRule(image.getMatchedRule().getRuleIdentifier().toString())
-                .isHint(isHint)
-                .isDictionaryEntry(false)
-                .isRecommendation(false)
-                .positions(List.of(toRedactionLogRectangle(image.getPosition(), image.getPage().getNumber())))
-                .sectionNumber(image.getTreeId().get(0))
-                .section(image.getManualOverwrite().getSection().orElse(image.getParent().toString()))
-                .imageHasTransparency(image.isTransparent())
-                .manualChanges(mapManualChanges(image.getManualOverwrite(), isHint))
-                .comments(buildRedactionLogComments(comments, image.getId()))
-                .build();
-
-    }
-
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RedactionLogToEntityLogMigrationService.java

@@ -0,0 +1,187 @@
+package com.iqser.red.service.redaction.v1.server.service;
+
+import java.util.Collection;
+import java.util.Comparator;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import org.springframework.stereotype.Service;
+
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogLegalBasis;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.migration.MigratedIds;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogLegalBasis;
+import com.iqser.red.service.redaction.v1.server.model.ManualEntity;
+import com.iqser.red.service.redaction.v1.server.model.MigratedEntityLog;
+import com.iqser.red.service.redaction.v1.server.model.MigrationEntity;
+import com.iqser.red.service.redaction.v1.server.model.RectangleWithPage;
+import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
+import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.Image;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
+import com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService;
+import com.iqser.red.service.redaction.v1.server.service.document.EntityEnrichmentService;
+import com.iqser.red.service.redaction.v1.server.service.document.EntityFindingUtility;
+import com.iqser.red.service.redaction.v1.server.utils.MigratedIdsCollector;
+
+import lombok.AccessLevel;
+import lombok.RequiredArgsConstructor;
+import lombok.experimental.FieldDefaults;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+@Service
+@RequiredArgsConstructor
+@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
+//TODO: remove this, once the migration is done
+public class RedactionLogToEntityLogMigrationService {
+
+    private static final double MATCH_THRESHOLD = 10;
+    EntityFindingUtility entityFindingUtility;
+    EntityEnrichmentService entityEnrichmentService;
+
+
+    public MigratedEntityLog migrate(RedactionLog redactionLog, Document document) {
+
+        List<MigrationEntity> entitiesToMigrate = calculateMigrationEntitiesFromRedactionLog(redactionLog, document);
+        MigratedIds migratedIds = entitiesToMigrate.stream().collect(new MigratedIdsCollector());
+
+        EntityLog entityLog = new EntityLog();
+        entityLog.setAnalysisNumber(redactionLog.getAnalysisNumber());
+        entityLog.setRulesVersion(redactionLog.getRulesVersion());
+        entityLog.setDictionaryVersion(redactionLog.getDictionaryVersion());
+        entityLog.setDossierDictionaryVersion(redactionLog.getDossierDictionaryVersion());
+        entityLog.setLegalBasisVersion(redactionLog.getLegalBasisVersion());
+        entityLog.setAnalysisVersion(redactionLog.getAnalysisVersion());
+        entityLog.setLegalBasis(redactionLog.getLegalBasis().stream().map(RedactionLogToEntityLogMigrationService::toEntityLogLegalBasis).toList());
+
+        Map<String, String> oldToNewIDMapping = migratedIds.buildOldToNewMapping();
+        entityLog.setEntityLogEntry(entitiesToMigrate.stream().map(migrationEntity -> migrationEntity.toEntityLogEntry(oldToNewIDMapping)).toList());
+
+        if (redactionLog.getRedactionLogEntry().size() != entityLog.getEntityLogEntry().size()) {
+            String message = String.format("Not all entities have been found during the migration redactionLog has %d entries and new entityLog %d",
+                    redactionLog.getRedactionLogEntry().size(),
+                    entityLog.getEntityLogEntry().size());
+            log.error(message);
+            throw new AssertionError(message);
+        }
+
+        return new MigratedEntityLog(migratedIds, entityLog);
+    }
+
+
+    private List<MigrationEntity> calculateMigrationEntitiesFromRedactionLog(RedactionLog redactionLog, Document document) {
+
+        List<MigrationEntity> images = getImageBasedMigrationEntities(redactionLog, document);
+        List<MigrationEntity> textMigrationEntities = getTextBasedMigrationEntities(redactionLog, document);
+        return Stream.of(textMigrationEntities.stream(), images.stream()).flatMap(Function.identity()).toList();
+    }
+
+
+    private static EntityLogLegalBasis toEntityLogLegalBasis(RedactionLogLegalBasis redactionLogLegalBasis) {
+
+        return new EntityLogLegalBasis(redactionLogLegalBasis.getName(), redactionLogLegalBasis.getDescription(), redactionLogLegalBasis.getReason());
+    }
+
+
+    private List<MigrationEntity> getImageBasedMigrationEntities(RedactionLog redactionLog, Document document) {
+
+        List<Image> images = document.streamAllImages().collect(Collectors.toList());
+        List<RedactionLogEntry> redactionLogImages = redactionLog.getRedactionLogEntry().stream().filter(RedactionLogEntry::isImage).toList();
+        List<MigrationEntity> migrationEntities = new LinkedList<>();
+        for (RedactionLogEntry redactionLogImage : redactionLogImages) {
+            List<RectangleWithPage> imagePositions = redactionLogImage.getPositions().stream().map(RectangleWithPage::fromRedactionLogRectangle).toList();
+            assert imagePositions.size() == 1;
+            Image closestImage = images.stream()
+                    .filter(image -> image.onPage(redactionLogImage.getPositions().get(0).getPage()))
+                    .min(Comparator.comparingDouble(image -> entityFindingUtility.calculateDistance(image.getPosition(), imagePositions.get(0).rectangle2D())))
+                    .orElseThrow(() -> new RuntimeException("Image from redaction log not found: " + redactionLogImage));
+
+            double minDistance = entityFindingUtility.calculateDistance(closestImage.getPosition(), imagePositions.get(0).rectangle2D());
+            if (minDistance > MATCH_THRESHOLD) {
+                throw new RuntimeException(String.format("Closest image has a distance of %.2f which is higher than the allowed %.2f", minDistance, MATCH_THRESHOLD));
+            }
+
+            images.remove(closestImage);
+
+            String ruleIdentifier = "OLDIMG." + redactionLogImage.getMatchedRule() + ".0";
+            if (redactionLogImage.lastChangeIsRemoved()) {
+                closestImage.remove(ruleIdentifier, redactionLogImage.getReason());
+            } else if (redactionLogImage.isRedacted()) {
+                closestImage.apply(ruleIdentifier, redactionLogImage.getReason(), redactionLogImage.getLegalBasis());
+            } else {
+                closestImage.skip(ruleIdentifier, redactionLogImage.getReason());
+            }
+            migrationEntities.add(new MigrationEntity(null, redactionLogImage, closestImage, redactionLogImage.getId(), closestImage.getId()));
+        }
+        return migrationEntities;
+    }
+
+
+    private List<MigrationEntity> getTextBasedMigrationEntities(RedactionLog redactionLog, Document document) {
+
+        List<MigrationEntity> entitiesToMigrate = redactionLog.getRedactionLogEntry()
+                .stream()
+                .filter(redactionLogEntry -> !redactionLogEntry.isImage())
+                .map(MigrationEntity::fromRedactionLogEntry)
+                .peek(migrationEntity -> {
+                    if (migrationEntity.getRedactionLogEntry().lastChangeIsRemoved()) {
+                        migrationEntity.getManualEntity().remove(migrationEntity.getManualEntity().getRuleIdentifier(), migrationEntity.getRedactionLogEntry().getReason());
+                    } else if (migrationEntity.getManualEntity().isApplied()) {
+                        migrationEntity.getManualEntity()
+                                .apply(migrationEntity.getManualEntity().getRuleIdentifier(),
+                                        migrationEntity.getManualEntity().getReason(),
+                                        migrationEntity.getManualEntity().getLegalBasis());
+                    } else {
+                        migrationEntity.getManualEntity().skip(migrationEntity.getManualEntity().getRuleIdentifier(), migrationEntity.getManualEntity().getReason());
+                    }
+                })
+                .toList();
+
+        Map<String, List<TextEntity>> tempEntitiesByValue = entityFindingUtility.findAllPossibleEntitiesAndGroupByValue(document,
+                entitiesToMigrate.stream().map(MigrationEntity::getManualEntity).toList());
+
+        for (MigrationEntity migrationEntity : entitiesToMigrate) {
+            Optional<TextEntity> optionalTextEntity = entityFindingUtility.findClosestEntityAndReturnEmptyIfNotFound(migrationEntity.getManualEntity(),
+                    tempEntitiesByValue,
+                    MATCH_THRESHOLD);
+
+            if (optionalTextEntity.isEmpty()) {
+                migrationEntity.setMigratedEntity(migrationEntity.getManualEntity());
+                migrationEntity.setOldId(migrationEntity.getManualEntity().getId());
+                migrationEntity.setNewId(migrationEntity.getManualEntity().getId());
+                continue;
+            }
+
+            TextEntity entity = createCorrectEntity(migrationEntity.getManualEntity(), document, optionalTextEntity.get().getTextRange());
+            migrationEntity.setMigratedEntity(entity);
+            migrationEntity.setOldId(migrationEntity.getManualEntity().getId());
+            migrationEntity.setNewId(entity.getPositionsOnPagePerPage().get(0).getId()); // Can only be on one page, since redactionLogEntries can only be on one page
+        }
+
+        tempEntitiesByValue.values().stream().flatMap(Collection::stream).forEach(TextEntity::removeFromGraph);
+        return entitiesToMigrate;
+    }
+
+
+    private TextEntity createCorrectEntity(ManualEntity manualEntity, SemanticNode node, TextRange closestTextRange) {
+
+        EntityCreationService entityCreationService = new EntityCreationService(entityEnrichmentService);
+        TextEntity correctEntity = entityCreationService.forceByTextRange(closestTextRange, manualEntity.getType(), manualEntity.getEntityType(), node);
+
+        correctEntity.addMatchedRules(manualEntity.getMatchedRuleList());
+        correctEntity.setDictionaryEntry(manualEntity.isDictionaryEntry());
+        correctEntity.setDossierDictionaryEntry(manualEntity.isDossierDictionaryEntry());
+        correctEntity.getManualOverwrite().addChanges(manualEntity.getManualOverwrite().getManualChangeLog());
+        return correctEntity;
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/UnprocessedChangesService.java

@@ -0,0 +1,114 @@
+package com.iqser.red.service.redaction.v1.server.service;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import org.springframework.stereotype.Service;
+
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
+import com.iqser.red.service.redaction.v1.model.UnprocessedManualEntity;
+import com.iqser.red.service.redaction.v1.server.model.ManualEntity;
+import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
+import com.iqser.red.service.redaction.v1.server.service.document.DocumentGraphMapper;
+import com.iqser.red.service.redaction.v1.server.service.document.ManualEntityCreationService;
+import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
+
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+@Service
+@RequiredArgsConstructor
+public class UnprocessedChangesService {
+
+    private final RedactionStorageService redactionStorageService;
+    private final DictionaryService dictionaryService;
+    private final ManualEntityCreationService manualEntityCreationService;
+
+
+    public List<UnprocessedManualEntity> addInformationToUnprocessedManualEntities(String fileId, String dossierId, String dossierTemplateId, ManualRedactions manualRedactions) {
+
+        // FIXME, because of the color if you start redaction-service a no processing is performed, a nullpointer will be throw when getting color.
+        // The entire concept needs to be rethought, because it is bullshit.
+        dictionaryService.updateDictionary(dossierTemplateId, dossierId);
+
+        Document document = DocumentGraphMapper.toDocumentGraph(redactionStorageService.getDocumentData(dossierId, fileId));
+        List<UnprocessedManualEntity> unprocessedManualEntities = new ArrayList<>();
+        Set<String> annotationIds = manualRedactions.getEntriesToAdd().stream().map(ManualRedactionEntry::getAnnotationId).collect(Collectors.toSet());
+
+        List<ManualEntity> notFoundManualEntities = manualEntityCreationService.toTextEntity(manualEntitiesConverter(manualRedactions), document);
+
+        document.getEntities().forEach(textEntity -> {
+            Set<String> processedIds = new HashSet<>();
+            for (var positionsOnPerPage : textEntity.getPositionsOnPagePerPage()) {
+                if (processedIds.contains(positionsOnPerPage.getId())) {
+                    continue;
+                }
+                processedIds.add(positionsOnPerPage.getId());
+                List<Position> positions = positionsOnPerPage.getRectanglePerLine().stream().map(rectangle2D -> new Position(rectangle2D, positionsOnPerPage.getPage().getNumber()))
+                        .toList();
+                unprocessedManualEntities.add(UnprocessedManualEntity.builder()
+                        .annotationId(annotationIds.stream().filter(textEntity::matchesAnnotationId).findFirst().orElse(""))
+                        .closestHeadline(textEntity.getDeepestFullyContainingNode().getHeadline().getTextBlock().getSearchText())
+                        .color(getColor(textEntity.getType(), dossierTemplateId, textEntity.applied(), textEntity.getEntityType().equals(EntityType.HINT)))
+                        .startOffset(textEntity.getTextRange().start())
+                        .endOffset(textEntity.getTextRange().end())
+                        .containingNodeId(textEntity.getDeepestFullyContainingNode().getTreeId())
+                        .textBefore(textEntity.getTextBefore())
+                        .textAfter(textEntity.getTextAfter())
+                        .section(textEntity.getManualOverwrite().getSection().orElse(textEntity.getDeepestFullyContainingNode().toString()))
+                        .positions(positions)
+                        .legalBasis(textEntity.legalBasis())
+                        .build());
+            }
+
+        });
+
+        notFoundManualEntities.forEach(manualEntity -> {
+            unprocessedManualEntities.add(UnprocessedManualEntity.builder()
+                    .annotationId(manualEntity.getId())
+                    .color(getColor(manualEntity.getType(), dossierTemplateId, manualEntity.applied(), manualEntity.getEntityType().equals(EntityType.HINT)))
+                    .closestHeadline("")
+                    .startOffset(-1)
+                    .endOffset(-1)
+                    .containingNodeId(Collections.emptyList())
+                    .textAfter("")
+                    .textBefore("")
+                    .section("")
+                    .positions(manualEntity.getManualOverwrite()
+                            .getPositions()
+                            .orElse(manualEntity.getEntityPosition())
+                            .stream()
+                            .map(entityPosition -> new Position(entityPosition.rectangle2D(), entityPosition.pageNumber()))
+                            .toList())
+                    .build());
+        });
+
+        return unprocessedManualEntities;
+    }
+
+
+    private float[] getColor(String type, String dossierTemplateId, boolean isApplied, boolean isHint) {
+
+        if (!isApplied && !isHint) {
+            return dictionaryService.getNotRedactedColor(dossierTemplateId);
+        }
+        return dictionaryService.getColor(type, dossierTemplateId);
+    }
+
+
+    private List<ManualEntity> manualEntitiesConverter(ManualRedactions manualRedactions) {
+
+        return manualRedactions.getEntriesToAdd().stream()
+                .map(manualRedactionEntry -> ManualEntity.fromManualRedactionEntry(manualRedactionEntry,
+                        manualRedactionEntry.getType() != null && manualRedactionEntry.getType().equals("hint_only"))).toList();
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/ComponentCreationService.java

@@ -19,6 +19,7 @@ import org.kie.api.runtime.KieSession;
 
 import com.iqser.red.service.redaction.v1.server.model.component.Component;
 import com.iqser.red.service.redaction.v1.server.model.component.Entity;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.Paragraph;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Table;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell;
@@ -102,7 +103,11 @@ public class ComponentCreationService {
 
         referencedEntities.addAll(references);
 
-        kieSession.insert(Component.builder().matchedRule(RuleIdentifier.fromString(ruleIdentifier)).name(name).value(value).valueDescription(valueDescription)
+        kieSession.insert(Component.builder()
+                .matchedRule(RuleIdentifier.fromString(ruleIdentifier))
+                .name(name)
+                .value(value)
+                .valueDescription(valueDescription)
                 .references(new LinkedList<>(references))
                 .build());
     }
@@ -293,6 +298,30 @@ public class ComponentCreationService {
     }
 
 
+    /**
+     * Counts the distinct rows where any of the provided entities occur and creates a component for each distinct table with its respective row count.
+     *
+     * @param ruleIdentifier the identifier of the rule
+     * @param name           the name of the record
+     * @param entities       the collection of entities to count row values from
+     */
+    public void rowValueCount(String ruleIdentifier, String name, Collection<Entity> entities) {
+
+        entities.stream().collect(Collectors.groupingBy(this::getFirstTable)).forEach((optionalTable, groupedEntities) -> {
+
+            if (optionalTable.isEmpty()) {
+                return;
+            }
+
+            long count = groupedEntities.stream()
+                    .collect(Collectors.groupingBy(entity -> getFirstTableCell(entity).map(TableCell::getRow).orElse(-1)))
+                    .size();
+
+            create(ruleIdentifier, name, String.valueOf(count), "Count rows with values in the entity references in same table", entities);
+        });
+    }
+
+
     /**
      * Creates a component for each sentence in the collection of entities.
      *
@@ -354,7 +383,9 @@ public class ComponentCreationService {
      */
     public void createComponentsForUnMappedEntities(String ruleIdentifier, Collection<Entity> entities) {
 
-        entities.stream().filter(entity -> !referencedEntities.contains(entity)).sorted(EntityComparators.first())
+        entities.stream()
+                .filter(entity -> !referencedEntities.contains(entity))
+                .sorted(EntityComparators.first())
                 .forEach(entity -> create(ruleIdentifier, entity.getType(), entity.getValue(), "Unmapped Entity", List.of(entity)));
     }
 
@@ -421,10 +452,6 @@ public class ComponentCreationService {
                 groupedEntities.forEach(entity -> create(ruleIdentifier, name, entity.getValue(), valueDescription, entity));
             }
 
-            groupedEntities.stream()
-                    .filter(entity -> !(entity.getContainingNode() instanceof TableCell))
-                    .forEach(entity -> create(ruleIdentifier, name, entity.getValue(), valueDescription, entity));
-
             groupedEntities.stream()
                     .filter(entity -> entity.getContainingNode() instanceof TableCell)
                     .collect(Collectors.groupingBy(entity -> ((TableCell) entity.getContainingNode()).getRow()))
@@ -434,7 +461,7 @@ public class ComponentCreationService {
                     .map(Map.Entry::getValue)
                     .forEach(entitiesInSameRow -> create(ruleIdentifier,
                             name,
-                            entitiesInSameRow.stream().sorted(Comparator.comparing(Entity::getType).reversed()).map(Entity::getValue).collect(Collectors.joining(", ")),
+                            entitiesInSameRow.stream().sorted(EntityComparators.first()).map(Entity::getValue).collect(Collectors.joining(", ")),
                             valueDescription,
                             entitiesInSameRow));
         });
@@ -455,6 +482,20 @@ public class ComponentCreationService {
     }
 
 
+    private Optional<TableCell> getFirstTableCell(Entity entity) {
+
+        SemanticNode node = entity.getContainingNode();
+        while (!(node instanceof TableCell)) {
+            if (!node.hasParent()) {
+                return Optional.empty();
+            }
+            node = node.getParent();
+        }
+
+        return Optional.of((TableCell) node);
+    }
+
+
     /**
      * Creates a new component with the given rule identifier, name, value, and value description.
      * If the component is part of a table, it also takes a list of entities that belong to the same table row.
@@ -479,7 +520,9 @@ public class ComponentCreationService {
      */
     public void create(String ruleIdentifier, String name, String value) {
 
-        kieSession.insert(Component.builder().matchedRule(RuleIdentifier.fromString(ruleIdentifier)).name(name)
+        kieSession.insert(Component.builder()
+                .matchedRule(RuleIdentifier.fromString(ruleIdentifier))
+                .name(name)
                 .value(value)
                 .valueDescription("")
                 .references(Collections.emptyList())

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityCreationService.java

@@ -58,6 +58,8 @@ public class EntityCreationService {
 
     public Stream<TextEntity> betweenStrings(String start, String stop, String type, EntityType entityType, SemanticNode node) {
 
+        checkIfBothStartAndEndAreEmpty(start, stop);
+
         List<TextRange> startTextRanges = RedactionSearchUtility.findTextRangesByString(start, node.getTextBlock());
         List<TextRange> stopTextRanges = RedactionSearchUtility.findTextRangesByString(stop, node.getTextBlock());
 
@@ -67,6 +69,8 @@ public class EntityCreationService {
 
     public Stream<TextEntity> betweenStringsIgnoreCase(String start, String stop, String type, EntityType entityType, SemanticNode node) {
 
+        checkIfBothStartAndEndAreEmpty(start, stop);
+
         List<TextRange> startBoundaries = RedactionSearchUtility.findTextRangesByStringIgnoreCase(start, node.getTextBlock());
         List<TextRange> stopBoundaries = RedactionSearchUtility.findTextRangesByStringIgnoreCase(stop, node.getTextBlock());
 
@@ -76,6 +80,8 @@ public class EntityCreationService {
 
     public Stream<TextEntity> betweenStringsIncludeStart(String start, String stop, String type, EntityType entityType, SemanticNode node) {
 
+        checkIfBothStartAndEndAreEmpty(start, stop);
+
         List<TextRange> startBoundaries = RedactionSearchUtility.findTextRangesByString(start, node.getTextBlock());
         List<TextRange> stopBoundaries = RedactionSearchUtility.findTextRangesByString(stop, node.getTextBlock());
 
@@ -90,6 +96,8 @@ public class EntityCreationService {
 
     public Stream<TextEntity> betweenStringsIncludeStartIgnoreCase(String start, String stop, String type, EntityType entityType, SemanticNode node) {
 
+        checkIfBothStartAndEndAreEmpty(start, stop);
+
         List<TextRange> startBoundaries = RedactionSearchUtility.findTextRangesByStringIgnoreCase(start, node.getTextBlock());
         List<TextRange> stopBoundaries = RedactionSearchUtility.findTextRangesByStringIgnoreCase(stop, node.getTextBlock());
 
@@ -104,6 +112,8 @@ public class EntityCreationService {
 
     public Stream<TextEntity> betweenStringsIncludeEnd(String start, String stop, String type, EntityType entityType, SemanticNode node) {
 
+        checkIfBothStartAndEndAreEmpty(start, stop);
+
         List<TextRange> startBoundaries = RedactionSearchUtility.findTextRangesByString(start, node.getTextBlock());
         List<TextRange> stopBoundaries = RedactionSearchUtility.findTextRangesByString(stop, node.getTextBlock());
 
@@ -118,6 +128,8 @@ public class EntityCreationService {
 
     public Stream<TextEntity> betweenStringsIncludeEndIgnoreCase(String start, String stop, String type, EntityType entityType, SemanticNode node) {
 
+        checkIfBothStartAndEndAreEmpty(start, stop);
+
         List<TextRange> startBoundaries = RedactionSearchUtility.findTextRangesByStringIgnoreCase(start, node.getTextBlock());
         List<TextRange> stopBoundaries = RedactionSearchUtility.findTextRangesByStringIgnoreCase(stop, node.getTextBlock());
 
@@ -132,6 +144,8 @@ public class EntityCreationService {
 
     public Stream<TextEntity> betweenStringsIncludeStartAndEnd(String start, String stop, String type, EntityType entityType, SemanticNode node) {
 
+        checkIfBothStartAndEndAreEmpty(start, stop);
+
         List<TextRange> startBoundaries = RedactionSearchUtility.findTextRangesByString(start, node.getTextBlock());
         List<TextRange> stopBoundaries = RedactionSearchUtility.findTextRangesByString(stop, node.getTextBlock());
 
@@ -150,6 +164,8 @@ public class EntityCreationService {
 
     public Stream<TextEntity> betweenStringsIncludeStartAndEndIgnoreCase(String start, String stop, String type, EntityType entityType, SemanticNode node) {
 
+        checkIfBothStartAndEndAreEmpty(start, stop);
+
         List<TextRange> startBoundaries = RedactionSearchUtility.findTextRangesByStringIgnoreCase(start, node.getTextBlock());
         List<TextRange> stopBoundaries = RedactionSearchUtility.findTextRangesByStringIgnoreCase(stop, node.getTextBlock());
 
@@ -168,6 +184,8 @@ public class EntityCreationService {
 
     public Stream<TextEntity> shortestBetweenAnyString(List<String> starts, List<String> stops, String type, EntityType entityType, SemanticNode node) {
 
+        checkIfBothStartAndEndAreEmpty(starts, stops);
+
         List<TextRange> startTextRanges = RedactionSearchUtility.findTextRangesByList(starts, node.getTextBlock());
         List<TextRange> stopTextRanges = RedactionSearchUtility.findTextRangesByList(stops, node.getTextBlock());
 
@@ -177,12 +195,24 @@ public class EntityCreationService {
 
     public Stream<TextEntity> shortestBetweenAnyStringIgnoreCase(List<String> starts, List<String> stops, String type, EntityType entityType, SemanticNode node) {
 
+        checkIfBothStartAndEndAreEmpty(starts, stops);
+
         List<TextRange> startTextRanges = RedactionSearchUtility.findTextRangesByListIgnoreCase(starts, node.getTextBlock());
         List<TextRange> stopTextRanges = RedactionSearchUtility.findTextRangesByListIgnoreCase(stops, node.getTextBlock());
 
         return betweenTextRanges(startTextRanges, stopTextRanges, type, entityType, node);
     }
 
+    public Stream<TextEntity> shortestBetweenAnyStringIgnoreCase(List<String> starts, List<String> stops, String type, EntityType entityType, SemanticNode node, int limit) {
+
+        checkIfBothStartAndEndAreEmpty(starts, stops);
+
+        List<TextRange> startTextRanges = RedactionSearchUtility.findTextRangesByListIgnoreCase(starts, node.getTextBlock());
+        List<TextRange> stopTextRanges = RedactionSearchUtility.findTextRangesByListIgnoreCase(stops, node.getTextBlock());
+
+        return betweenTextRanges(startTextRanges, stopTextRanges, type, entityType, node, limit);
+    }
+
 
     public Stream<TextEntity> betweenRegexes(String regexStart, String regexStop, String type, EntityType entityType, SemanticNode node) {
 
@@ -203,14 +233,17 @@ public class EntityCreationService {
         return betweenTextRanges(startBoundaries, stopBoundaries, type, entityType, node);
     }
 
-
     public Stream<TextEntity> betweenTextRanges(List<TextRange> startBoundaries, List<TextRange> stopBoundaries, String type, EntityType entityType, SemanticNode node) {
 
-        if (startBoundaries.isEmpty() || stopBoundaries.isEmpty()) {
-            return Stream.empty();
-        }
+        return betweenTextRanges(startBoundaries, stopBoundaries, type, entityType, node,0);
+    }
+
+
+    public Stream<TextEntity> betweenTextRanges(List<TextRange> startBoundaries, List<TextRange> stopBoundaries, String type, EntityType entityType, SemanticNode node, int limit) {
+
         List<TextRange> entityBoundaries = findNonOverlappingBoundariesBetweenBoundariesWithMinimalDistances(startBoundaries, stopBoundaries);
         return entityBoundaries.stream()
+                .filter(range -> (limit == 0 || range.length() <= limit))
                 .map(textRange -> textRange.trim(node.getTextBlock()))
                 .filter(textRange -> isValidEntityTextRange(node.getTextBlock(), textRange))
                 .map(textRange -> byTextRange(textRange, type, entityType, node))
@@ -249,6 +282,18 @@ public class EntityCreationService {
     }
 
 
+    private void checkIfBothStartAndEndAreEmpty(String start, String end) {
+        checkIfBothStartAndEndAreEmpty(List.of(start), List.of(end));
+    }
+
+
+    private <T> void checkIfBothStartAndEndAreEmpty(List<T> start, List<T> end) {
+        if ((start == null || start.isEmpty()) && (end == null || end.isEmpty())) {
+            throw new IllegalArgumentException("Start and end values are empty!");
+        }
+    }
+
+
     public Stream<TextEntity> bySearchImplementation(SearchImplementation searchImplementation, String type, EntityType entityType, SemanticNode node) {
 
         return searchImplementation.getBoundaries(node.getTextBlock(), node.getTextRange())
@@ -554,6 +599,9 @@ public class EntityCreationService {
             throw new IllegalArgumentException(String.format("%s is not in the %s of the provided semantic node %s", textRange, node.getTextRange(), node));
         }
         TextRange trimmedTextRange = textRange.trim(node.getTextBlock());
+        if (trimmedTextRange.length() == 0){
+            return Optional.empty();
+        }
         TextEntity entity = TextEntity.initialEntityNode(trimmedTextRange, type, entityType);
         if (node.getEntities().contains(entity)) {
             return node.getEntities().stream().filter(entity::equals).peek(e -> e.addEngines(engines)).findAny();

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityFindingUtility.java

@@ -0,0 +1,164 @@
+package com.iqser.red.service.redaction.v1.server.service.document;
+
+import static java.lang.String.format;
+import static java.util.stream.Collectors.groupingBy;
+
+import java.awt.geom.Rectangle2D;
+import java.util.Collection;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+
+import com.iqser.red.service.redaction.v1.server.model.ManualEntity;
+import com.iqser.red.service.redaction.v1.server.model.RectangleWithPage;
+import com.iqser.red.service.redaction.v1.server.model.dictionary.SearchImplementation;
+import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
+import com.iqser.red.service.redaction.v1.server.model.document.entity.PositionOnPage;
+import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.Page;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
+import com.iqser.red.service.redaction.v1.server.service.DictionaryService;
+
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+@Service
+public class EntityFindingUtility {
+
+    EntityCreationService entityCreationService;
+    DictionaryService dictionaryService;
+
+
+    @Autowired
+    public EntityFindingUtility(EntityEnrichmentService entityEnrichmentService, DictionaryService dictionaryService) {
+
+        entityCreationService = new EntityCreationService(entityEnrichmentService);
+        this.dictionaryService = dictionaryService;
+    }
+
+
+    public Optional<TextEntity> findClosestEntityAndReturnEmptyIfNotFound(ManualEntity identifier, Map<String, List<TextEntity>> entitiesWithSameValue, double matchThreshold) {
+
+        List<TextEntity> possibleEntities = entitiesWithSameValue.get(identifier.getValue().toLowerCase(Locale.ENGLISH));
+
+        if (entityIdentifierValueNotFound(possibleEntities)) {
+            log.warn("Entity could not be created with identifier: {}, due to the value {} not being found anywhere.", identifier, identifier.getValue());
+            return Optional.empty();
+        }
+
+        Optional<TextEntity> optionalClosestEntity = possibleEntities.stream()
+                .filter(entity -> pagesMatch(entity, identifier.getEntityPosition()))
+                .min(Comparator.comparingDouble(entity -> calculateMinDistance(identifier.getEntityPosition(), entity)));
+
+        if (optionalClosestEntity.isEmpty()) {
+            log.warn("No Entity with value {} found on page {}", identifier.getValue(), identifier.getEntityPosition());
+            return Optional.empty();
+        }
+
+        TextEntity closestEntity = optionalClosestEntity.get();
+        double distance = calculateMinDistance(identifier.getEntityPosition(), closestEntity);
+        if (distance > matchThreshold) {
+            log.warn("For entity {} on page {} with positions {} distance to closest found entity is {} and therefore higher than the threshold of {}",
+                    identifier.getValue(),
+                    identifier.getEntityPosition().get(0).pageNumber(),
+                    identifier.getEntityPosition().stream().map(RectangleWithPage::rectangle2D).toList(),
+                    distance,
+                    matchThreshold);
+            return Optional.empty();
+        }
+
+        return Optional.of(closestEntity);
+    }
+
+
+    private static boolean entityIdentifierValueNotFound(List<TextEntity> possibleEntities) {
+
+        return possibleEntities == null || possibleEntities.isEmpty();
+    }
+
+
+    private static boolean pagesMatch(TextEntity entity, List<RectangleWithPage> originalPositions) {
+
+        Set<Integer> entityPageNumbers = entity.getPositionsOnPagePerPage().stream().map(PositionOnPage::getPage).map(Page::getNumber).collect(Collectors.toSet());
+        Set<Integer> originalPageNumbers = originalPositions.stream().map(RectangleWithPage::pageNumber).collect(Collectors.toSet());
+        return entityPageNumbers.containsAll(originalPageNumbers);
+    }
+
+
+    private double calculateMinDistance(List<RectangleWithPage> originalPositions, TextEntity entity) {
+
+        if (originalPositions.size() != countRectangles(entity)) {
+            return Double.MAX_VALUE;
+        }
+        return originalPositions.stream()
+                .mapToDouble(rectangleWithPage -> calculateMinDistancePerRectangle(entity, rectangleWithPage.pageNumber(), rectangleWithPage.rectangle2D()))
+                .sum();
+    }
+
+
+    private static long countRectangles(TextEntity entity) {
+
+        return entity.getPositionsOnPagePerPage().stream().mapToLong(redactionPosition -> redactionPosition.getRectanglePerLine().size()).sum();
+    }
+
+
+    private double calculateMinDistancePerRectangle(TextEntity entity, int pageNumber, Rectangle2D originalRectangle) {
+
+        return entity.getPositionsOnPagePerPage()
+                .stream()
+                .filter(redactionPosition -> redactionPosition.getPage().getNumber() == pageNumber)
+                .map(PositionOnPage::getRectanglePerLine)
+                .flatMap(Collection::stream)
+                .mapToDouble(rectangle -> calculateDistance(rectangle, originalRectangle))
+                .min()
+                .orElse(Double.MAX_VALUE);
+    }
+
+
+    public double calculateDistance(Rectangle2D rectangle1, Rectangle2D rectangle2) {
+
+        // mirrored coordinates safe comparison
+        double minX1 = Math.min(rectangle1.getMinX(), rectangle1.getMaxX());
+        double minY1 = Math.min(rectangle1.getMinY(), rectangle1.getMaxY());
+        double maxX1 = Math.max(rectangle1.getMinX(), rectangle1.getMaxX());
+        double maxY1 = Math.max(rectangle1.getMinY(), rectangle1.getMaxY());
+
+        double minX2 = Math.min(rectangle2.getMinX(), rectangle2.getMaxX());
+        double minY2 = Math.min(rectangle2.getMinY(), rectangle2.getMaxY());
+        double maxX2 = Math.max(rectangle2.getMinX(), rectangle2.getMaxX());
+        double maxY2 = Math.max(rectangle2.getMinY(), rectangle2.getMaxY());
+
+        return Math.abs(minX1 - minX2) //
+                + Math.abs(minY1 - minY2) //
+                + Math.abs(maxX1 - maxX2) //
+                + Math.abs(maxY1 - maxY2);
+    }
+
+
+    public Map<String, List<TextEntity>> findAllPossibleEntitiesAndGroupByValue(SemanticNode node, List<ManualEntity> manualEntities) {
+
+        Set<Integer> pageNumbers = manualEntities.stream().flatMap(entry -> entry.getEntityPosition().stream().map(RectangleWithPage::pageNumber)).collect(Collectors.toSet());
+        Set<String> entryValues = manualEntities.stream().map(ManualEntity::getValue).map(String::toLowerCase).collect(Collectors.toSet());
+
+        if (!pageNumbers.stream().allMatch(node::onPage)) {
+            throw new IllegalArgumentException(format("SemanticNode \"%s\" does not contain these pages %s, it has pages: %s",
+                    node,
+                    pageNumbers.stream().filter(pageNumber -> !node.onPage(pageNumber)).toList(),
+                    node.getPages()));
+        }
+        SearchImplementation searchImplementation = new SearchImplementation(entryValues, true);
+
+        return searchImplementation.getBoundaries(node.getTextBlock(), node.getTextRange())
+                .stream()
+                .map(boundary -> entityCreationService.forceByTextRange(boundary, "temp", EntityType.ENTITY, node))
+                .collect(groupingBy(entity -> entity.getValue().toLowerCase(Locale.ROOT)));
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/ManualEntityCreationService.java

@@ -1,33 +1,22 @@
 package com.iqser.red.service.redaction.v1.server.service.document;
 
-import static java.lang.String.format;
-import static java.util.stream.Collectors.groupingBy;
-
-import java.awt.geom.Rectangle2D;
 import java.util.ArrayList;
 import java.util.Collection;
-import java.util.Comparator;
 import java.util.LinkedList;
 import java.util.List;
-import java.util.Locale;
 import java.util.Map;
 import java.util.Optional;
 import java.util.Set;
-import java.util.stream.Collectors;
 
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
 
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog;
 import com.iqser.red.service.redaction.v1.server.model.ManualEntity;
-import com.iqser.red.service.redaction.v1.server.model.RectangleWithPage;
-import com.iqser.red.service.redaction.v1.server.model.dictionary.SearchImplementation;
 import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.PositionOnPage;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
-import com.iqser.red.service.redaction.v1.server.model.document.nodes.Page;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
 import com.iqser.red.service.redaction.v1.server.service.DictionaryService;
 
@@ -41,31 +30,20 @@ import lombok.extern.slf4j.Slf4j;
 public class ManualEntityCreationService {
 
     static double MATCH_THRESHOLD = 5; // Is compared to the sum of distances in pdf coordinates for each corner of the bounding box of the entities
+    EntityFindingUtility entityFindingUtility;
     EntityCreationService entityCreationService;
     DictionaryService dictionaryService;
 
 
     @Autowired
-    public ManualEntityCreationService(EntityEnrichmentService entityEnrichmentService, DictionaryService dictionaryService) {
+    public ManualEntityCreationService(EntityEnrichmentService entityEnrichmentService, DictionaryService dictionaryService, EntityFindingUtility entityFindingUtility) {
 
+        this.entityFindingUtility = entityFindingUtility;
         entityCreationService = new EntityCreationService(entityEnrichmentService);
         this.dictionaryService = dictionaryService;
     }
 
 
-    public List<ManualEntity> toRedactionEntity(RedactionLog redactionLog, SemanticNode node) {
-
-        List<ManualEntity> manualEntities = redactionLog.getRedactionLogEntry().stream().map(ManualEntity::fromRedactionLogEntry).peek(manualEntity -> {
-            if (manualEntity.isApplied()) {
-                manualEntity.apply(manualEntity.getRuleIdentifier(), manualEntity.getReason(), manualEntity.getLegalBasis());
-            } else {
-                manualEntity.skip(manualEntity.getRuleIdentifier(), manualEntity.getReason());
-            }
-        }).toList();
-        return toRedactionEntity(manualEntities, node);
-    }
-
-
     public List<ManualEntity> createRedactionEntitiesIfFoundAndReturnNotFoundEntries(Set<ManualRedactionEntry> manualRedactionEntries,
                                                                                      SemanticNode node,
                                                                                      String dossierTemplateId) {
@@ -73,24 +51,26 @@ public class ManualEntityCreationService {
         List<ManualEntity> manualEntities = manualRedactionEntries.stream()
                 .filter(manualRedactionEntry -> !(manualRedactionEntry.isAddToDictionary() || manualRedactionEntry.isAddToDossierDictionary()))
                 .map(manualRedactionEntry -> ManualEntity.fromManualRedactionEntry(manualRedactionEntry,
-                        dictionaryService.isHint(manualRedactionEntry.getType(), dossierTemplateId)))
-                .peek(manualEntity -> manualEntity.apply("MAN.5.0", "manual entries are applied by default", manualEntity.getLegalBasis()))
+                        dictionaryService.isHint(manualRedactionEntry.getType(), dossierTemplateId))).peek(manualEntity -> {
+                    if (manualEntity.getEntityType().equals(EntityType.HINT)) {
+                        manualEntity.skip("MAN.5.1", "manual hint is skipped by default");
+                    } else {
+                        manualEntity.apply("MAN.5.0", "manual entries are applied by default", manualEntity.getLegalBasis());
+                    }
+                })
                 .toList();
 
-        return toRedactionEntity(manualEntities, node);
+        return toTextEntity(manualEntities, node);
     }
 
 
-    private List<ManualEntity> toRedactionEntity(List<ManualEntity> manualEntities, SemanticNode node) {
+    public List<ManualEntity> toTextEntity(List<ManualEntity> manualEntities, SemanticNode node) {
 
-        Set<Integer> pageNumbers = manualEntities.stream().flatMap(entry -> entry.getEntityPosition().stream().map(RectangleWithPage::pageNumber)).collect(Collectors.toSet());
-        Set<String> entryValues = manualEntities.stream().map(ManualEntity::getValue).map(String::toLowerCase).collect(Collectors.toSet());
-
-        Map<String, List<TextEntity>> tempEntitiesByValue = findAllPossibleEntitiesAndGroupByValue(node, pageNumbers, entryValues);
+        Map<String, List<TextEntity>> tempEntitiesByValue = entityFindingUtility.findAllPossibleEntitiesAndGroupByValue(node, manualEntities);
 
         List<ManualEntity> notFoundManualEntities = new LinkedList<>();
         for (ManualEntity manualEntity : manualEntities) {
-            Optional<TextEntity> optionalRedactionEntity = findClosestEntityAndReturnEmptyIfNotFound(manualEntity, tempEntitiesByValue);
+            Optional<TextEntity> optionalRedactionEntity = entityFindingUtility.findClosestEntityAndReturnEmptyIfNotFound(manualEntity, tempEntitiesByValue, MATCH_THRESHOLD);
             if (optionalRedactionEntity.isEmpty()) {
                 notFoundManualEntities.add(manualEntity);
                 continue;
@@ -105,9 +85,9 @@ public class ManualEntityCreationService {
     /**
      * Deletes the temp Entity and creates a RedactionEntity with correct values, based on the given parameters.
      *
-     * @param manualEntity The entity identifier for the RedactionEntity.
+     * @param manualEntity     The entity identifier for the RedactionEntity.
      * @param node             The SemanticNode associated with the RedactionEntity.
-     * @param closestTextRange  The closest Boundary to the RedactionEntity.
+     * @param closestTextRange The closest Boundary to the RedactionEntity.
      */
     private void createCorrectEntity(ManualEntity manualEntity, SemanticNode node, TextRange closestTextRange) {
 
@@ -118,7 +98,6 @@ public class ManualEntityCreationService {
         correctEntity.setDossierDictionaryEntry(manualEntity.isDossierDictionaryEntry());
         correctEntity.getManualOverwrite().addChanges(manualEntity.getManualOverwrite().getManualChangeLog());
 
-
         // AnnotationIds must match the IDs in the add requests, or comments break. Maybe think about migrating IDs on the fly!
         List<PositionOnPage> redactionPositionsWithIdOfManualOnPage = new ArrayList<>(correctEntity.getPositionsOnPagePerPage().size());
         for (PositionOnPage positionOnPage : correctEntity.getPositionsOnPagePerPage()) {
@@ -127,123 +106,4 @@ public class ManualEntityCreationService {
         correctEntity.setPositionsOnPagePerPage(redactionPositionsWithIdOfManualOnPage);
     }
 
-
-    private Optional<TextEntity> findClosestEntityAndReturnEmptyIfNotFound(ManualEntity identifier, Map<String, List<TextEntity>> entitiesWithSameValue) {
-
-        List<TextEntity> possibleEntities = entitiesWithSameValue.get(identifier.getValue().toLowerCase(Locale.ENGLISH));
-
-        if (entityIdentifierValueNotFound(possibleEntities)) {
-            log.warn("Entity could not be created with identifier: {}, due to the value {} not being found anywhere.", identifier, identifier.getValue());
-            return Optional.empty();
-        }
-
-        Optional<TextEntity> optionalClosestEntity = possibleEntities.stream()
-                .filter(entity -> pagesMatch(entity, identifier.getEntityPosition()))
-                .min(Comparator.comparingDouble(entity -> calculateMinDistance(identifier.getEntityPosition(), entity)));
-
-        if (optionalClosestEntity.isEmpty()) {
-            log.warn("No Entity with value {} found on page {}", identifier.getValue(), identifier.getEntityPosition());
-            return Optional.empty();
-        }
-
-        TextEntity closestEntity = optionalClosestEntity.get();
-        double distance = calculateMinDistance(identifier.getEntityPosition(), closestEntity);
-        if (distance > MATCH_THRESHOLD) {
-            log.warn(format("Distance to closest found entity is %.2f and therefore higher than the threshold of %.2f for \n%s \n%s",
-                    distance,
-                    MATCH_THRESHOLD,
-                    identifier.getEntityPosition(),
-                    closestEntity.getPositionsOnPagePerPage()));
-            return Optional.empty();
-        }
-
-        return Optional.of(closestEntity);
-    }
-
-
-    private static boolean entityIdentifierValueNotFound(List<TextEntity> possibleEntities) {
-
-        return possibleEntities == null || possibleEntities.isEmpty();
-    }
-
-
-    private Map<String, List<TextEntity>> findAllPossibleEntitiesAndGroupByValue(SemanticNode node, Set<Integer> pageNumbers, Set<String> entryValues) {
-
-        if (!pageNumbers.stream().allMatch(node::onPage)) {
-            throw new IllegalArgumentException(format("SemanticNode \"%s\" does not contain these pages %s, it has pages: %s",
-                    node,
-                    pageNumbers.stream().filter(pageNumber -> !node.onPage(pageNumber)).toList(),
-                    node.getPages()));
-        }
-        SearchImplementation searchImplementation = new SearchImplementation(entryValues, true);
-
-        return searchImplementation.getBoundaries(node.getTextBlock(), node.getTextRange())
-                .stream().map(boundary -> entityCreationService.forceByTextRange(boundary, "temp", EntityType.ENTITY, node))
-                .collect(groupingBy(entity -> entity.getValue().toLowerCase(Locale.ROOT)));
-    }
-
-
-    private static boolean allValuesFound(Map<String, List<TextEntity>> entitiesByValue, Set<String> entryValues) {
-
-        return entitiesByValue.keySet().equals(entryValues);
-    }
-
-
-    private static boolean pagesMatch(TextEntity entity, List<RectangleWithPage> originalPositions) {
-
-        Set<Integer> entityPageNumbers = entity.getPositionsOnPagePerPage().stream().map(PositionOnPage::getPage).map(Page::getNumber).collect(Collectors.toSet());
-        Set<Integer> originalPageNumbers = originalPositions.stream().map(RectangleWithPage::pageNumber).collect(Collectors.toSet());
-        return entityPageNumbers.containsAll(originalPageNumbers);
-    }
-
-
-    private double calculateMinDistance(List<RectangleWithPage> originalPositions, TextEntity entity) {
-
-        if (originalPositions.size() != countRectangles(entity)) {
-            return Double.MAX_VALUE;
-        }
-        return originalPositions.stream()
-                .mapToDouble(rectangleWithPage -> calculateMinDistancePerRectangle(entity, rectangleWithPage.pageNumber(), rectangleWithPage.rectangle2D()))
-                .sum();
-    }
-
-
-    private static long countRectangles(TextEntity entity) {
-
-        return entity.getPositionsOnPagePerPage().stream().mapToLong(redactionPosition -> redactionPosition.getRectanglePerLine().size()).sum();
-    }
-
-
-    private double calculateMinDistancePerRectangle(TextEntity entity, int pageNumber, Rectangle2D originalRectangle) {
-
-        return entity.getPositionsOnPagePerPage()
-                .stream()
-                .filter(redactionPosition -> redactionPosition.getPage().getNumber() == pageNumber)
-                .map(PositionOnPage::getRectanglePerLine)
-                .flatMap(Collection::stream)
-                .mapToDouble(rectangle -> calculateDistance(rectangle, originalRectangle))
-                .min()
-                .orElse(Double.MAX_VALUE);
-    }
-
-
-    private double calculateDistance(Rectangle2D rectangle, Rectangle2D rectangle2D) {
-
-        // mirrored coordinates safe comparison
-        double minX1 = Math.min(rectangle.getMinX(), rectangle.getMaxX());
-        double minY1 = Math.min(rectangle.getMinY(), rectangle.getMaxY());
-        double maxX1 = Math.max(rectangle.getMinX(), rectangle.getMaxX());
-        double maxY1 = Math.max(rectangle.getMinY(), rectangle.getMaxY());
-
-        double minX2 = Math.min(rectangle2D.getMinX(), rectangle2D.getMaxX());
-        double minY2 = Math.min(rectangle2D.getMinY(), rectangle2D.getMaxY());
-        double maxX2 = Math.max(rectangle2D.getMinX(), rectangle2D.getMaxX());
-        double maxY2 = Math.max(rectangle2D.getMinY(), rectangle2D.getMaxY());
-
-        return Math.abs(minX1 - minX2) //
-                + Math.abs(minY1 - minY2) //
-                + Math.abs(maxX1 - maxX2) //
-                + Math.abs(maxY1 - maxY2);
-    }
-
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/DroolsSyntaxValidationService.java

@@ -1,18 +1,21 @@
 package com.iqser.red.service.redaction.v1.server.service.drools;
 
 import java.util.List;
+import java.util.Set;
+import java.util.regex.Pattern;
 import java.util.stream.Collectors;
 
+import org.drools.drl.parser.DroolsParserException;
 import org.kie.api.builder.KieBuilder;
 import org.kie.api.builder.Message;
 import org.springframework.stereotype.Service;
 
+import com.google.common.collect.Sets;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType;
 import com.iqser.red.service.redaction.v1.model.DroolsSyntaxErrorMessage;
 import com.iqser.red.service.redaction.v1.model.DroolsSyntaxValidation;
 import com.iqser.red.service.redaction.v1.model.RuleValidationModel;
 import com.iqser.red.service.redaction.v1.server.model.drools.BasicQuery;
-import com.iqser.red.service.redaction.v1.server.model.drools.BasicRule;
 import com.iqser.red.service.redaction.v1.server.model.drools.RuleFileBluePrint;
 import com.iqser.red.service.redaction.v1.server.storage.RuleManagementResources;
 
@@ -22,6 +25,8 @@ import lombok.SneakyThrows;
 @Service
 @RequiredArgsConstructor
 public class DroolsSyntaxValidationService {
+    
+    private static final Pattern allowedImportsPattern = Pattern.compile("^(?:import\\s+static\\s+)?(?:import\\s+)?(?:com\\.knecon\\.fforesight|com\\.iqser\\.red)\\..*;$");
 
     private final KieContainerCreationService kieContainerCreationService;
 
@@ -29,14 +34,20 @@ public class DroolsSyntaxValidationService {
     @SneakyThrows
     public DroolsSyntaxValidation testRules(RuleValidationModel rules) {
 
-        DroolsSyntaxValidation customDroolsSyntaxValidation = buildCustomDroolsSyntaxValidation(rules.getRulesString(), RuleFileType.valueOf(rules.getRuleFileType()));
+        DroolsSyntaxValidation customDroolsSyntaxValidation;
+        try {
+            customDroolsSyntaxValidation = buildCustomDroolsSyntaxValidation(rules.getRulesString(), RuleFileType.valueOf(rules.getRuleFileType()));
+        } catch (DroolsParserException e) {
+            // this means the parser could not parse the file at all. In this case use drools compiler only as it will return useful error messages.
+            customDroolsSyntaxValidation = new DroolsSyntaxValidation();
+        }
         DroolsSyntaxValidation droolsCompilerSyntaxValidation = buildDroolsCompilerSyntaxValidation(rules);
         droolsCompilerSyntaxValidation.getDroolsSyntaxErrorMessages().addAll(customDroolsSyntaxValidation.getDroolsSyntaxErrorMessages());
         return droolsCompilerSyntaxValidation;
     }
 
 
-    private DroolsSyntaxValidation buildCustomDroolsSyntaxValidation(String ruleString, RuleFileType ruleFileType) {
+    private DroolsSyntaxValidation buildCustomDroolsSyntaxValidation(String ruleString, RuleFileType ruleFileType) throws DroolsParserException {
 
         RuleFileBluePrint ruleFileBluePrint = RuleFileParser.buildBluePrintFromRulesString(ruleString);
         DroolsSyntaxValidation customSyntaxValidation = ruleFileBluePrint.getDroolsSyntaxValidation();
@@ -46,7 +57,7 @@ public class DroolsSyntaxValidationService {
             case COMPONENT -> RuleFileParser.buildBluePrintFromRulesString(RuleManagementResources.getBaseComponentRuleFileString());
         };
 
-        if (!ruleFileBluePrint.getImports().equals(baseRuleFileBluePrint.getImports())) {
+        if (!importsAreValid(baseRuleFileBluePrint, ruleFileBluePrint)) {
             customSyntaxValidation.getDroolsSyntaxErrorMessages()
                     .add(DroolsSyntaxErrorMessage.builder()
                             .line(ruleFileBluePrint.getImportLine())
@@ -56,8 +67,10 @@ public class DroolsSyntaxValidationService {
         }
         if (!ruleFileBluePrint.getGlobals().equals(baseRuleFileBluePrint.getGlobals())) {
             customSyntaxValidation.getDroolsSyntaxErrorMessages()
-                    .add(DroolsSyntaxErrorMessage.builder().line(ruleFileBluePrint.getGlobalsLine())
-                            .column(0).message(String.format("Changing the globals is not allowed! Must be: %n%s", baseRuleFileBluePrint.getGlobals()))
+                    .add(DroolsSyntaxErrorMessage.builder()
+                            .line(ruleFileBluePrint.getGlobalsLine())
+                            .column(0)
+                            .message(String.format("Changing the globals is not allowed! Must be: %n%s", baseRuleFileBluePrint.getGlobals()))
                             .build());
         }
         baseRuleFileBluePrint.getQueries().forEach(basicQuery -> {
@@ -70,23 +83,33 @@ public class DroolsSyntaxValidationService {
                                 .build());
             }
         });
-        baseRuleFileBluePrint.streamAllRules().forEach(basicRule -> {
-            if (!validateRuleIsPresent(basicRule, ruleFileBluePrint)) {
-                customSyntaxValidation.getDroolsSyntaxErrorMessages()
-                        .add(DroolsSyntaxErrorMessage.builder()
-                                .line(basicRule.getLine())
-                                .column(0)
-                                .message(String.format("Changing or removing the rule %s is not allowed! Must be: %n%s", basicRule.getName(), basicRule.getCode()))
+        if (ruleFileType.equals(RuleFileType.ENTITY)) {
+            String requiredAgendaGroup = "LOCAL_DICTIONARY_ADDS";
+            if (!validateAgendaGroupIsPresent(ruleFileBluePrint, requiredAgendaGroup)) {
+                customSyntaxValidation.getDroolsSyntaxErrorMessages().add(DroolsSyntaxErrorMessage.builder().line(0)
+                                .column(0).message(String.format("At least one rule with Agenda-Group '%s' required!", requiredAgendaGroup))
                                 .build());
             }
-        });
+        }
         return customSyntaxValidation;
     }
 
 
-    private static boolean validateRuleIsPresent(BasicRule basicRule, RuleFileBluePrint ruleFileBluePrint) {
+    private boolean validateAgendaGroupIsPresent(RuleFileBluePrint ruleFileBluePrint, String agendaGroupName) {
 
-        return ruleFileBluePrint.findRulesByIdentifier(basicRule.getIdentifier()).stream().anyMatch(otherRule -> otherRule.getCode().equals(basicRule.getCode()));
+        return ruleFileBluePrint.streamAllRules().anyMatch(basicRule -> basicRule.getAgendaGroup().equals(agendaGroupName));
+    }
+
+
+    private boolean importsAreValid(RuleFileBluePrint baseRuleFileBluePrint, RuleFileBluePrint ruleFileBluePrint) {
+
+        // imports may shrink, but not add anything new!
+        Set<String> baseImports = baseRuleFileBluePrint.getImportSplitByKeyword();
+        Set<String> imports = ruleFileBluePrint.getImportSplitByKeyword();
+
+        Set<String> additionalImports = Sets.difference(imports, baseImports);
+
+        return additionalImports.stream().allMatch(additionalImport -> allowedImportsPattern.matcher(additionalImport).matches());
     }
 
 
@@ -101,7 +124,9 @@ public class DroolsSyntaxValidationService {
         var versionId = System.currentTimeMillis();
         var testRules = "test-rules";
         KieBuilder kieBuilder = kieContainerCreationService.registerNewKieContainerVersion(testRules,
-                versionId, rules.getRulesString(), RuleFileType.valueOf(rules.getRuleFileType()));
+                versionId,
+                rules.getRulesString(),
+                RuleFileType.valueOf(rules.getRuleFileType()));
         return buildDroolsCompilerSyntaxValidation(kieBuilder);
     }
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/EntityDroolsExecutionService.java

@@ -7,8 +7,6 @@ import java.util.concurrent.ExecutionException;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
 
-import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType;
-import com.iqser.red.service.redaction.v1.server.utils.exception.DroolsTimeoutException;
 import org.kie.api.runtime.KieContainer;
 import org.kie.api.runtime.KieSession;
 import org.kie.api.runtime.rule.QueryResults;
@@ -16,6 +14,7 @@ import org.kie.api.runtime.rule.QueryResultsRow;
 import org.springframework.stereotype.Service;
 
 import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
 import com.iqser.red.service.redaction.v1.server.RedactionServiceSettings;
 import com.iqser.red.service.redaction.v1.server.model.NerEntities;
@@ -25,6 +24,7 @@ import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNo
 import com.iqser.red.service.redaction.v1.server.service.ManualChangesApplicationService;
 import com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService;
 import com.iqser.red.service.redaction.v1.server.service.document.EntityEnrichmentService;
+import com.iqser.red.service.redaction.v1.server.utils.exception.DroolsTimeoutException;
 
 import io.micrometer.core.annotation.Timed;
 import lombok.AccessLevel;
@@ -81,7 +81,7 @@ public class EntityDroolsExecutionService {
         fileAttributes.stream().filter(f -> f.getValue() != null).forEach(kieSession::insert);
 
         if (manualRedactions != null) {
-            manualRedactions.getResizeRedactions().forEach(kieSession::insert);
+            manualRedactions.getResizeRedactions().stream().filter(manualResizeRedaction -> !manualResizeRedaction.getUpdateDictionary()).forEach(kieSession::insert);
             manualRedactions.getRecategorizations().forEach(kieSession::insert);
             manualRedactions.getEntriesToAdd().forEach(kieSession::insert);
             manualRedactions.getForceRedactions().forEach(kieSession::insert);

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/RedactionStorageService.java

@@ -3,6 +3,9 @@ package com.iqser.red.service.redaction.v1.server.storage;
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.InputStream;
+import java.util.stream.Collectors;
+
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedactions;
 
 import org.springframework.stereotype.Service;
 
@@ -10,11 +13,9 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.imported.ImportedRedactions;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.SectionGrid;
 import com.iqser.red.service.redaction.v1.server.client.model.NerEntitiesModel;
-import com.iqser.red.service.redaction.v1.server.utils.exception.NotFoundException;
 import com.iqser.red.service.redaction.v1.server.model.document.DocumentData;
+import com.iqser.red.service.redaction.v1.server.utils.exception.NotFoundException;
 import com.iqser.red.storage.commons.exception.StorageObjectDoesNotExist;
 import com.iqser.red.storage.commons.service.StorageService;
 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPage;
@@ -91,7 +92,11 @@ public class RedactionStorageService {
     public RedactionLog getRedactionLog(String dossierId, String fileId) {
 
         try {
-            return storageService.readJSONObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.REDACTION_LOG), RedactionLog.class);
+            RedactionLog redactionLog = storageService.readJSONObject(TenantContext.getTenantId(),
+                    StorageIdUtils.getStorageId(dossierId, fileId, FileType.REDACTION_LOG),
+                    RedactionLog.class);
+            redactionLog.setRedactionLogEntry(redactionLog.getRedactionLogEntry().stream().filter(entry -> !entry.getValue().isEmpty()).collect(Collectors.toList()));
+            return redactionLog;
         } catch (StorageObjectDoesNotExist e) {
             log.debug("RedactionLog not available.");
             return null;
@@ -99,13 +104,16 @@ public class RedactionStorageService {
 
     }
 
+
     @Timed("redactmanager_getRedactionLog")
     public EntityLog getEntityLog(String dossierId, String fileId) {
 
         try {
-            return storageService.readJSONObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.ENTITY_LOG), EntityLog.class);
+            EntityLog entityLog = storageService.readJSONObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.ENTITY_LOG), EntityLog.class);
+            entityLog.setEntityLogEntry(entityLog.getEntityLogEntry().stream().filter(entry -> !entry.getValue().isEmpty()).collect(Collectors.toList()));
+            return entityLog;
         } catch (StorageObjectDoesNotExist e) {
-            log.debug("RedactionLog not available.");
+            log.debug("EntityLog not available.");
             return null;
         }
 
@@ -117,10 +125,18 @@ public class RedactionStorageService {
 
         try {
             return DocumentData.builder()
-                    .documentStructure(storageService.readJSONObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_STRUCTURE), DocumentStructure.class))
-                    .documentTextData(storageService.readJSONObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_TEXT), DocumentTextData[].class))
-                    .documentPositionData(storageService.readJSONObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_POSITION), DocumentPositionData[].class))
-                    .documentPages(storageService.readJSONObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_PAGES), DocumentPage[].class))
+                    .documentStructure(storageService.readJSONObject(TenantContext.getTenantId(),
+                            StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_STRUCTURE),
+                            DocumentStructure.class))
+                    .documentTextData(storageService.readJSONObject(TenantContext.getTenantId(),
+                            StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_TEXT),
+                            DocumentTextData[].class))
+                    .documentPositionData(storageService.readJSONObject(TenantContext.getTenantId(),
+                            StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_POSITION),
+                            DocumentPositionData[].class))
+                    .documentPages(storageService.readJSONObject(TenantContext.getTenantId(),
+                            StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_PAGES),
+                            DocumentPage[].class))
                     .build();
         } catch (StorageObjectDoesNotExist e) {
             log.debug("DocumentData not available.");
@@ -140,17 +156,6 @@ public class RedactionStorageService {
     }
 
 
-    @Timed("redactmanager_getSectionGrid")
-    public SectionGrid getSectionGrid(String dossierId, String fileId) {
-
-        try {
-            return storageService.readJSONObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.SECTION_GRID), SectionGrid.class);
-        } catch (StorageObjectDoesNotExist e) {
-            throw new NotFoundException("Section Grid is not available.");
-        }
-    }
-
-
     public ComponentLog getComponentLog(String dossierId, String fileId) {
 
         try {

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/utils/DateConverter.java

@@ -1,11 +1,25 @@
 package com.iqser.red.service.redaction.v1.server.utils;
 
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
 import java.text.DateFormat;
 import java.text.SimpleDateFormat;
+import java.time.LocalDate;
+import java.time.ZoneId;
+import java.time.format.DateTimeFormatter;
+import java.time.format.DateTimeFormatterBuilder;
+import java.time.format.DateTimeParseException;
+import java.time.format.ResolverStyle;
+import java.time.temporal.ChronoField;
+import java.util.Arrays;
 import java.util.Date;
 import java.util.List;
 import java.util.Locale;
+import java.util.Objects;
 import java.util.Optional;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 
 import lombok.AccessLevel;
 import lombok.experimental.FieldDefaults;
@@ -17,39 +31,123 @@ import lombok.extern.slf4j.Slf4j;
 @FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
 public class DateConverter {
 
-    static List<SimpleDateFormat> formats = List.of(new SimpleDateFormat("dd MMM yy", Locale.ENGLISH),
-            new SimpleDateFormat("dd MM yyyy", Locale.ENGLISH),
-            new SimpleDateFormat("dd MM yyyy.", Locale.ENGLISH),
-            new SimpleDateFormat("dd MMMM yyyy", Locale.ENGLISH),
-            new SimpleDateFormat("MMMM dd, yyyy", Locale.ENGLISH),
-            new SimpleDateFormat("dd-MMM-yyyy", Locale.ENGLISH));
+    private static DateTimeFormatter DATE_TIME_FORMATTER;
+    private static final List<Locale> LOCALES = Arrays.asList(Locale.UK, Locale.US);
+    private static int BASE_YEAR = 1950; // base year 1950 means, that "yy" will be interpreted in range 1950-2049
 
 
     public Optional<Date> parseDate(String dateAsString) {
 
-        Date date = null;
-        for (SimpleDateFormat format : formats) {
+        DateTimeFormatter formatter = getDateTimeFormatter();
+        String cleanDate = dateAsString.trim();
+        cleanDate = removeTrailingDot(cleanDate);
 
+        for (Locale locale : LOCALES) {
             try {
-                date = format.parse(dateAsString);
-                break;
-            } catch (Exception e) {
-                log.warn("Failed to parse date from string {}. \n{}", dateAsString, e.getMessage());
-                // ignore, try next...
+                return convertToDate(locale, cleanDate, formatter);
+            } catch (DateTimeParseException e) {
+                try {
+                    Optional<String> extractedDate = DateExtractorNatty.extractDate(cleanDate);
+                    if (extractedDate.isEmpty()) {
+                        log.warn("Failed to extract a valid date from value: {}", cleanDate);
+                        return Optional.empty();
+                    } else {
+                        cleanDate = extractedDate.get();
+                        return convertToDate(locale, cleanDate, formatter);
+                    }
+                } catch (DateTimeParseException exception) {
+                    log.debug("Failed to parse date: {} with locale: {}", cleanDate, locale);
+                }
             }
         }
-        if (date == null) {
-            return Optional.empty();
-        }
-        return Optional.of(date);
+
+        log.warn("Failed to parse date: {}", cleanDate);
+        return Optional.empty();
+
     }
 
 
     public String convertDate(Date date, String resultFormat) {
 
-        DateFormat resultDateFormat = new SimpleDateFormat(resultFormat, Locale.ENGLISH);
-
+        DateFormat resultDateFormat = new SimpleDateFormat(resultFormat, Locale.UK);
         return resultDateFormat.format(date);
     }
 
+
+    private Optional<Date> convertToDate(Locale locale, String cleanDate, DateTimeFormatter formatter) {
+
+        LocalDate localDate = LocalDate.parse(cleanDate, formatter.withLocale(locale));
+        Date date = Date.from(localDate.atStartOfDay(ZoneId.systemDefault()).toInstant());
+        return Optional.of(date);
+    }
+
+
+    private DateTimeFormatter getDateTimeFormatter() {
+
+        if (DATE_TIME_FORMATTER == null) {
+            DATE_TIME_FORMATTER = createFormatterFromResource();
+        }
+        return DATE_TIME_FORMATTER;
+    }
+
+
+    private DateTimeFormatter createFormatterFromResource() {
+
+        DateTimeFormatterBuilder builder = new DateTimeFormatterBuilder();
+        builder.parseCaseInsensitive();
+        try (BufferedReader reader = new BufferedReader(new InputStreamReader(Objects.requireNonNull(DateConverter.class.getResourceAsStream("/date_formats.txt"))))) {
+            String line;
+            while ((line = reader.readLine()) != null) {
+                String pattern = line.trim();
+                if (!pattern.isEmpty()) {
+                    if (hasTwoDigitsForYear(pattern)) {
+                        builder.appendOptional(setBaseYear(pattern));
+                    } else {
+                        builder.appendOptional(DateTimeFormatter.ofPattern(pattern, Locale.UK));
+                    }
+                }
+            }
+        } catch (IOException e) {
+            throw new RuntimeException("Error reading date format file: " + e.getMessage(), e);
+        }
+        return builder.toFormatter().withResolverStyle(ResolverStyle.SMART).withLocale(Locale.UK);
+    }
+
+
+    private boolean hasTwoDigitsForYear(String input) {
+        // Regex to match any string with exactly two 'y' characters
+        Pattern pattern = Pattern.compile("^[^y]*(y[^y]*){2}$");
+        Matcher matcher = pattern.matcher(input);
+
+        return matcher.matches();
+
+    }
+
+
+    private DateTimeFormatter setBaseYear(String pattern) {
+
+        DateTimeFormatterBuilder builder = new DateTimeFormatterBuilder();
+        if (pattern.startsWith("yy")) {
+            String editedPattern = pattern.substring(2);
+            builder.appendValueReduced(ChronoField.YEAR_OF_ERA, 2, 2, BASE_YEAR).appendPattern(editedPattern).toFormatter();
+        } else if (pattern.endsWith("yy")) {
+            String editedPattern = pattern.substring(0, pattern.length() - 2);
+            builder.appendPattern(editedPattern).appendValueReduced(ChronoField.YEAR_OF_ERA, 2, 2, BASE_YEAR).toFormatter();
+        } else {
+            throw new RuntimeException("Date format not supported: " + pattern);
+        }
+        return builder.toFormatter();
+    }
+
+
+    private String removeTrailingDot(String dateAsString) {
+
+        String str = dateAsString;
+        if (str != null && !str.isEmpty() && str.charAt(str.length() - 1) == '.') {
+            str = str.substring(0, str.length() - 1);
+        }
+
+        return str;
+    }
+
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/utils/DateExtractorNatty.java

@@ -0,0 +1,26 @@
+package com.iqser.red.service.redaction.v1.server.utils;
+
+import java.util.List;
+import java.util.Optional;
+
+import com.joestelmach.natty.DateGroup;
+import com.joestelmach.natty.Parser;
+
+import lombok.experimental.UtilityClass;
+
+@UtilityClass
+public class DateExtractorNatty {
+
+    public Optional<String> extractDate(String text) {
+
+        Parser parser = new Parser();
+        List<DateGroup> groups = parser.parse(text);
+        if (!groups.isEmpty()) {
+            DateGroup group = groups.get(0);
+            String dateText = group.getText();
+            return Optional.of(dateText);
+        }
+        return Optional.empty();
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/utils/MigratedIdsCollector.java

@@ -0,0 +1,54 @@
+package com.iqser.red.service.redaction.v1.server.utils;
+
+import java.util.LinkedList;
+import java.util.Set;
+import java.util.function.BiConsumer;
+import java.util.function.BinaryOperator;
+import java.util.function.Function;
+import java.util.function.Supplier;
+import java.util.stream.Collector;
+
+import com.google.common.base.Functions;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.migration.MigratedIds;
+import com.iqser.red.service.redaction.v1.server.model.MigrationEntity;
+
+public class MigratedIdsCollector implements Collector<MigrationEntity, MigratedIds, MigratedIds> {
+
+    @Override
+    public Supplier<MigratedIds> supplier() {
+
+        return () -> new MigratedIds(new LinkedList<>());
+    }
+
+
+    @Override
+    public BiConsumer<MigratedIds, MigrationEntity> accumulator() {
+
+        return (migratedIds, migrationEntity) -> migratedIds.addMapping(migrationEntity.getOldId(), migrationEntity.getNewId());
+    }
+
+
+    @Override
+    public BinaryOperator<MigratedIds> combiner() {
+
+        return (migratedIds, migratedIds2) -> {
+            migratedIds.getMappings().addAll(migratedIds2.getMappings());
+            return migratedIds;
+        };
+    }
+
+
+    @Override
+    public Function<MigratedIds, MigratedIds> finisher() {
+
+        return Functions.identity();
+    }
+
+
+    @Override
+    public Set<Characteristics> characteristics() {
+
+        return Set.of(Characteristics.IDENTITY_FINISH, Characteristics.CONCURRENT, Characteristics.UNORDERED);
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/utils/Patterns.java

@@ -11,8 +11,7 @@ public final class Patterns {
 
     public static final Map<String, Pattern> patternCache = new HashMap<>();
 
-    public static final Pattern AUTHOR_TABLE_SPLITTER = Pattern.compile(
-            "(((((di)|(van)) )|[A-Z]’)?[A-ZÄÖÜ][\\wäöüéèê]{2,500}( ?[A-ZÄÖÜ]{1,2}\\.){1,3})|(((((di)|(van)) )|[A-Z]’)?[A-ZÄÖÜ][\\wäöüéèê]{2,500}( ?[A-ZÄÖÜ]{1,2} ){1,3})");
+    public static final Pattern AUTHOR_NAME_PATTERN = Pattern.compile("^(?!\\w*[.]$)\\p{Lu}\\p{L}*[.\\p{L}\\s]*$");
 
 
     public Pattern getCompiledPattern(String pattern, boolean caseInsensitive) {

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/utils/RedactionSearchUtility.java

@@ -132,6 +132,12 @@ public class RedactionSearchUtility {
         return getTextRangesByPatternWithLineBreaks(textBlock, group, pattern);
     }
 
+    public static List<TextRange> findTextRangesByRegexIgnoreCase(String regexPattern, TextBlock textBlock) {
+
+        Pattern pattern = Patterns.getCompiledPattern(regexPattern, true);
+        return getTextRangesByPattern(textBlock, 0, pattern);
+    }
+
 
     public static List<TextRange> findTextRangesByRegexIgnoreCase(String regexPattern, int group, TextBlock textBlock) {
 
@@ -145,7 +151,9 @@ public class RedactionSearchUtility {
         Matcher matcher = pattern.matcher(textBlock.subSequence(textBlock.getTextRange()));
         List<TextRange> boundaries = new LinkedList<>();
         while (matcher.find()) {
-            boundaries.add(new TextRange(matcher.start(group) + textBlock.getTextRange().start(), matcher.end(group) + textBlock.getTextRange().start()));
+            if (!matcher.group(group).isBlank()) {
+                boundaries.add(new TextRange(matcher.start(group) + textBlock.getTextRange().start(), matcher.end(group) + textBlock.getTextRange().start()));
+            }
         }
         return boundaries;
     }
@@ -157,7 +165,9 @@ public class RedactionSearchUtility {
         Matcher matcher = pattern.matcher(searchTextWithLineBreaks);
         List<TextRange> boundaries = new LinkedList<>();
         while (matcher.find()) {
-            boundaries.add(new TextRange(matcher.start(group) + textBlock.getTextRange().start(), matcher.end(group) + textBlock.getTextRange().start()));
+            if (!matcher.group(group).isBlank()) {
+                boundaries.add(new TextRange(matcher.start(group) + textBlock.getTextRange().start(), matcher.end(group) + textBlock.getTextRange().start()));
+            }
         }
         return boundaries;
     }

redaction-service-v1/redaction-service-server-v1/src/main/resources/application-dev.yaml

@@ -1,5 +1,5 @@
 server:
-  port: 8083
+  port: 8077
 
 persistence-service.url: "http://localhost:8085"
 tenant-user-management-service.url: "http://localhost:8091/internal"

redaction-service-v1/redaction-service-server-v1/src/main/resources/date_formats.txt

@@ -0,0 +1,79 @@
+yyyy MMM dd
+yyyy MMM d
+yyyy MM dd
+yyyy MM d
+dd-MMM-yyyy
+d-MMM-yyyy
+dd MMM yyyy
+d MMM yyyy
+dd MMM yy
+d MMM yy
+dd MM yyyy
+d MM yyyy
+dd MMMM yyyy
+d MMMM yyyy
+MMMM dd, yyyy
+MMMM d, yyyy
+MMMM, d yyyy
+MMMM d,yyyy
+dd.MM.yyyy
+d.MM.yyyy
+yyyy/MM/dd
+yyyy/MM/d
+yyyy-MM-dd
+yyyy-MM-d
+dd-MM-yyyy
+d-MM-yyyy
+MMM dd, yyyy
+MMM d, yyyy
+dd['.'] MMM yyyy
+d['.'] MMM yyyy
+dd['th']['st']['nd']['rd'] 'of' MMMM, yyyy
+d['th']['st']['nd']['rd'] 'of' MMMM, yyyy
+d['st']['nd']['rd']['th'] MMMM yyyy
+MMMM dd['th']['st']['nd']['rd'], yyyy
+MMMM d['th']['st']['nd']['rd'], yyyy
+yyyy, MMMM dd
+yyyy, MMMM d
+yyyy.MM.dd
+yyyy.MM.d
+dd-MM-yy
+d-MM-yy
+dd/MM/yy
+d/MM/yy
+MMMM dd, yy
+MMMM d, yy
+dd MMMM, yy
+d MMMM, yy
+dd['th']['st']['nd']['rd'] MMM yyyy
+d['th']['st']['nd']['rd'] MMM yyyy
+MMM dd['th']['st']['nd']['rd'], yy
+MMM d['th']['st']['nd']['rd'], yy
+yyyy-MMM-dd
+yyyy-MMM-d
+MMM-dd-yyyy
+MMM-d-yyyy
+dd.MMM.yyyy
+d.MMM.yyyy
+dd.MMMM.yyyy
+d.MMMM.yyyy
+dd.MMM.yy
+d.MMM.yy
+dd.MMMM.yy
+d.MMMM.yy
+dd.MMM-yyyy
+d.MMM-yyyy
+dd.MMMM-yyyy
+d.MMMM-yyyy
+dd. MMM yy
+d. MMM yy
+dd['th']['st']['nd']['rd'] MMMM yy
+d['th']['st']['nd']['rd'] MMMM yy
+dd['th']['st']['nd']['rd'] MMMM yyyy
+d['th']['st']['nd']['rd'] MMMM yyyy
+dd.MM.yy
+d.MM.yy
+dd MMM. yyyy
+d MMM. yyyy
+d-MMMM-yyyy
+dd-MMMM-yyyy

redaction-service-v1/redaction-service-server-v1/src/main/resources/drools/base_rules.drl

@@ -22,6 +22,7 @@ import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRu
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.*;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Section;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Table;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Paragraph;
@@ -73,7 +74,7 @@ query "getFileAttributes"
 //------------------------------------ Local dictionary search rules ------------------------------------
 
 // Rule unit: LocalDictionarySearch.0
-rule "LDS.0.0: run local dictionary search"
+rule "LDS.0.0: Run local dictionary search"
     agenda-group "LOCAL_DICTIONARY_ADDS"
     salience -999
     when

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AbstractRedactionIntegrationTest.java

@@ -1,5 +1,28 @@
 package com.iqser.red.service.redaction.v1.server;
 
+import static org.mockito.Mockito.when;
+
+import java.io.File;
+import java.io.InputStream;
+import java.net.URL;
+import java.nio.file.Files;
+import java.time.OffsetDateTime;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import org.junit.jupiter.api.AfterEach;
+import org.mockito.stubbing.Answer;
+import org.springframework.amqp.rabbit.core.RabbitTemplate;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.test.mock.mockito.MockBean;
+import org.springframework.core.io.ClassPathResource;
+
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.iqser.red.service.dictionarymerge.commons.DictionaryEntry;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
@@ -12,33 +35,19 @@ import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient;
 import com.iqser.red.service.redaction.v1.server.client.RulesClient;
 import com.iqser.red.service.redaction.v1.server.controller.RedactionController;
 import com.iqser.red.service.redaction.v1.server.service.AnalyzeService;
-import com.iqser.red.service.redaction.v1.server.utils.ResourceLoader;
-import com.iqser.red.service.redaction.v1.server.utils.TextNormalizationUtilities;
 import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
 import com.iqser.red.service.redaction.v1.server.utils.LayoutParsingRequestProvider;
+import com.iqser.red.service.redaction.v1.server.utils.ResourceLoader;
+import com.iqser.red.service.redaction.v1.server.utils.TextNormalizationUtilities;
 import com.iqser.red.storage.commons.service.StorageService;
+import com.iqser.red.storage.commons.utils.FileSystemBackedStorageService;
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingFinishedEvent;
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
 import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingPipeline;
 import com.knecon.fforesight.tenantcommons.TenantContext;
 import com.knecon.fforesight.tenantcommons.TenantsClient;
+
 import lombok.SneakyThrows;
-import org.junit.jupiter.api.AfterEach;
-import org.mockito.stubbing.Answer;
-import org.springframework.amqp.rabbit.core.RabbitTemplate;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.boot.test.mock.mockito.MockBean;
-import org.springframework.core.io.ClassPathResource;
-
-import java.io.File;
-import java.io.InputStream;
-import java.net.URL;
-import java.nio.file.Files;
-import java.time.OffsetDateTime;
-import java.util.*;
-import java.util.stream.Collectors;
-
-import static org.mockito.Mockito.when;
 
 public abstract class AbstractRedactionIntegrationTest {
 
@@ -431,7 +440,6 @@ public abstract class AbstractRedactionIntegrationTest {
     @SneakyThrows
     protected LayoutParsingFinishedEvent analyzeDocumentStructure(LayoutParsingType layoutParsingType, AnalyzeRequest request) {
 
-
         return layoutParsingPipeline.parseLayoutAndSaveFilesToStorage(LayoutParsingRequestProvider.build(layoutParsingType, request));
     }
 

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AnalysisTest.java

@@ -0,0 +1,122 @@
+package com.iqser.red.service.redaction.v1.server;
+
+import static org.mockito.Mockito.when;
+
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.util.List;
+
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Disabled;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
+import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration;
+import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.ComponentScan;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.context.annotation.FilterType;
+import org.springframework.context.annotation.Import;
+import org.springframework.context.annotation.Primary;
+import org.springframework.test.context.junit.jupiter.SpringExtension;
+
+import com.iqser.red.commons.jackson.ObjectMapperFactory;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeResult;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type;
+import com.iqser.red.service.redaction.v1.server.annotate.AnnotateRequest;
+import com.iqser.red.service.redaction.v1.server.annotate.AnnotateResponse;
+import com.iqser.red.service.redaction.v1.server.redaction.utils.OsUtils;
+import com.iqser.red.storage.commons.StorageAutoConfiguration;
+import com.iqser.red.storage.commons.service.StorageService;
+import com.iqser.red.storage.commons.utils.FileSystemBackedStorageService;
+import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
+import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingServiceProcessorConfiguration;
+import com.knecon.fforesight.tenantcommons.TenantContext;
+
+@ExtendWith(SpringExtension.class)
+@SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT, properties = {"application.type=DocuMine"})
+@Import(AnalysisTest.RedactionIntegrationTestConfiguration.class)
+public class AnalysisTest extends AbstractRedactionIntegrationTest {
+
+    private static final String RULES = loadFromClassPath("drools/test_rules.drl");
+    private static final String COMPONENT_RULES = loadFromClassPath("drools/test_components.drl");
+
+
+    @Test
+    @Disabled
+    public void analyzeTableDemoFile() throws IOException {
+
+        AnalyzeRequest request = uploadFileToStorage("files/Metolachlor/S-Metolachlor_RAR_01_Volume_1_2018-09-06.pdf");
+
+        System.out.println("Start Full integration test");
+        analyzeDocumentStructure(LayoutParsingType.DOCUMINE, request);
+        System.out.println("Finished structure analysis");
+        AnalyzeResult result = analyzeService.analyze(request);
+        System.out.println("Finished analysis");
+        var entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
+        var componentLog = redactionStorageService.getComponentLog(TEST_DOSSIER_ID, TEST_FILE_ID);
+        AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder().dossierId(TEST_DOSSIER_ID).fileId(TEST_FILE_ID).build());
+
+        String outputFileName = OsUtils.getTemporaryDirectory() + "/Documine.pdf";
+
+        try (FileOutputStream fileOutputStream = new FileOutputStream(outputFileName)) {
+            fileOutputStream.write(annotateResponse.getDocument());
+        }
+    }
+
+
+    @Configuration
+    @EnableAutoConfiguration(exclude = {RabbitAutoConfiguration.class})
+    @Import(LayoutParsingServiceProcessorConfiguration.class)
+    @ComponentScan(excludeFilters = {@ComponentScan.Filter(type = FilterType.ASSIGNABLE_TYPE, value = StorageAutoConfiguration.class)})
+    static class RedactionIntegrationTestConfiguration {
+
+        @Bean
+        @Primary
+        public StorageService inmemoryStorage() {
+
+            return new FileSystemBackedStorageService(ObjectMapperFactory.create());
+        }
+
+    }
+
+
+    @BeforeEach
+    public void stubClients() {
+
+        TenantContext.setTenantId("documine");
+
+        when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.ENTITY)).thenReturn(System.currentTimeMillis());
+        when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.ENTITY)).thenReturn(JSONPrimitive.of(RULES));
+        when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.COMPONENT)).thenReturn(System.currentTimeMillis());
+        when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.COMPONENT)).thenReturn(JSONPrimitive.of(COMPONENT_RULES));
+
+        loadDictionaryForTest();
+        loadTypeForTest();
+        loadNerForTest();
+        when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L);
+        when(dictionaryClient.getAllTypesForDossierTemplate(TEST_DOSSIER_TEMPLATE_ID, false)).thenReturn(getTypeResponse());
+
+        when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L);
+        when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, false)).thenReturn(List.of(Type.builder()
+                .id(DOSSIER_REDACTIONS_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID)
+                .type(DOSSIER_REDACTIONS_INDICATOR)
+                .dossierTemplateId(TEST_DOSSIER_ID)
+                .hexColor("#ffe187")
+                .isHint(hintTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
+                .isCaseInsensitive(caseInSensitiveMap.get(DOSSIER_REDACTIONS_INDICATOR))
+                .isRecommendation(recommendationTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
+                .rank(rankTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
+                .build()));
+
+        mockDictionaryCalls(null);
+
+        when(dictionaryClient.getColors(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(colors);
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DictionaryServiceTest.java

@@ -28,6 +28,7 @@ import org.springframework.context.annotation.Primary;
 import org.springframework.test.context.junit.jupiter.SpringExtension;
 
 import com.google.common.collect.Sets;
+import com.iqser.red.commons.jackson.ObjectMapperFactory;
 import com.iqser.red.service.dictionarymerge.commons.DictionaryEntry;
 import com.iqser.red.service.dictionarymerge.commons.DictionaryEntryModel;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.configuration.Colors;
@@ -36,6 +37,7 @@ import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
 import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryVersion;
 import com.iqser.red.service.redaction.v1.server.service.DictionaryService;
 import com.iqser.red.storage.commons.service.StorageService;
+import com.iqser.red.storage.commons.utils.FileSystemBackedStorageService;
 import com.knecon.fforesight.tenantcommons.TenantContext;
 import com.knecon.fforesight.tenantcommons.TenantsClient;
 
@@ -67,7 +69,7 @@ public class DictionaryServiceTest {
         @Primary
         public StorageService inmemoryStorage() {
 
-            return new FileSystemBackedStorageService();
+            return new FileSystemBackedStorageService(ObjectMapperFactory.create());
         }
 
     }

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DocumineFloraTest.java

@@ -1,11 +1,16 @@
 package com.iqser.red.service.redaction.v1.server;
 
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
 import static org.mockito.Mockito.when;
 
 import java.io.FileOutputStream;
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.List;
 
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
+import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.Test;
@@ -19,8 +24,10 @@ import org.springframework.context.annotation.Configuration;
 import org.springframework.context.annotation.FilterType;
 import org.springframework.context.annotation.Import;
 import org.springframework.context.annotation.Primary;
+import org.springframework.core.io.ClassPathResource;
 import org.springframework.test.context.junit.jupiter.SpringExtension;
 
+import com.iqser.red.commons.jackson.ObjectMapperFactory;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeResult;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType;
@@ -31,6 +38,7 @@ import com.iqser.red.service.redaction.v1.server.annotate.AnnotateResponse;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.OsUtils;
 import com.iqser.red.storage.commons.StorageAutoConfiguration;
 import com.iqser.red.storage.commons.service.StorageService;
+import com.iqser.red.storage.commons.utils.FileSystemBackedStorageService;
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
 import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingServiceProcessorConfiguration;
 import com.knecon.fforesight.tenantcommons.TenantContext;
@@ -48,7 +56,11 @@ public class DocumineFloraTest extends AbstractRedactionIntegrationTest {
 //    @Disabled
     public void titleExtraction() throws IOException {
 
-        AnalyzeRequest request = uploadFileToStorage("files/Documine/Flora/402Study-ocred.pdf");
+        AnalyzeRequest request = uploadFileToStorage("files/ImportedRedactions/18 Chlorothalonil RAR 08 Volume 3CA B 6a Oct 2017.pdf");
+
+        ClassPathResource importedRedactionClasspathResource = new ClassPathResource("files/ImportedRedactions/18 Chlorothalonil RAR 08 Volume 3CA B 6a Oct 2017.IMPORTED_REDACTIONS.json");
+        storageService.storeObject(TenantContext.getTenantId(), RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.IMPORTED_REDACTIONS), importedRedactionClasspathResource.getInputStream());
+
 
 //        AnalyzeRequest request = prepareStorage("files/Documine/Flora/ProblemDocs/SOLICITA_VICTRATO-GOLD-II_Item 21_Mutacao_Genica (1).pdf",
 //                "files/Documine/Flora/ProblemDocs/SOLICITA_VICTRATO-GOLD-II_Item 21_Mutacao_Genica (1).TABLES.json");
@@ -97,6 +109,60 @@ public class DocumineFloraTest extends AbstractRedactionIntegrationTest {
     }
 
 
+    @Test
+    public void testConvertingVariousDateFormats() {
+
+        AnalyzeRequest request = uploadFileToStorage("files/dates/date_formats.pdf");
+        System.out.println("Start Full integration test");
+        analyzeDocumentStructure(LayoutParsingType.DOCUMINE, request);
+        System.out.println("Finished structure analysis");
+        analyzeService.analyze(request);
+        System.out.println("Finished analysis");
+
+        var componentLog = redactionStorageService.getComponentLog(TEST_DOSSIER_ID, TEST_FILE_ID);
+        assertNotNull(componentLog);
+
+        var experimentalDates = componentLog.getComponentLogEntries()
+                .stream()
+                .filter(c -> c.getName().equals("Experimental_Starting_Date"))
+                .findFirst()
+                .get();
+        assertNotNull(experimentalDates);
+
+        List<String> expectedDates = new ArrayList<>();
+        expectedDates.add("10/01/2022");
+        expectedDates.add("01/01/2022");
+        expectedDates.add("08/09/2024");
+        expectedDates.add("03/01/2022");
+        expectedDates.add("03/08/1992");
+        expectedDates.add("13/08/1992");
+        expectedDates.add("27/02/1992");
+        expectedDates.add("27/10/1989");
+        expectedDates.add("07/10/1989");
+        expectedDates.add("21/08/1998");
+        expectedDates.add("02/08/1998");
+        expectedDates.add("01/05/1988");
+        expectedDates.add("02/06/2003");
+        expectedDates.add("03/09/2005");
+        expectedDates.add("06/09/2005");
+        expectedDates.add("17/08/2005");
+        expectedDates.add("22/08/2035");
+
+        String dates = experimentalDates.getComponentValues()
+                .get(0).getValue();
+        String[] dateArray = dates.split(", ");
+        boolean allEqual = true;
+        for (String date : dateArray) {
+            if (!expectedDates.contains(date)) {
+                allEqual = false;
+                break;
+            }
+        }
+
+        assertTrue(allEqual);
+    }
+
+
     @Test
 //    @Disabled
     public void testTopOfPage13InNotHeader() throws IOException {
@@ -134,7 +200,7 @@ public class DocumineFloraTest extends AbstractRedactionIntegrationTest {
         @Primary
         public StorageService inmemoryStorage() {
 
-            return new FileSystemBackedStorageService();
+            return new FileSystemBackedStorageService(ObjectMapperFactory.create());
         }
 
     }

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/FileSystemBackedStorageService.java

@@ -1,128 +0,0 @@
-package com.iqser.red.service.redaction.v1.server;
-
-import static java.io.File.createTempFile;
-
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileOutputStream;
-import java.io.InputStream;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.stream.Collectors;
-
-import org.apache.commons.io.IOUtils;
-import org.springframework.core.io.InputStreamResource;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.iqser.red.commons.jackson.ObjectMapperFactory;
-import com.iqser.red.storage.commons.exception.StorageObjectDoesNotExist;
-import com.iqser.red.storage.commons.service.StorageService;
-
-import lombok.SneakyThrows;
-
-public class FileSystemBackedStorageService implements StorageService {
-
-    private final Map<String, File> dataMap = new HashMap<>();
-
-
-    public FileSystemBackedStorageService() {
-
-    }
-
-
-    @SneakyThrows
-    @Override
-    public void downloadTo(String tenantId, String objectId, File destinationFile) {
-
-        var res = dataMap.get(objectId);
-        if (res == null) {
-            throw new StorageObjectDoesNotExist(new RuntimeException());
-        }
-
-        IOUtils.copy(new FileInputStream(res), new FileOutputStream(destinationFile));
-    }
-
-    @SneakyThrows
-    public InputStreamResource getObject(String tenantId, String objectId) {
-        return new InputStreamResource(new FileInputStream(dataMap.get(objectId)));
-    }
-
-
-    @Override
-    public void deleteObject(String tenantId, String objectId) {
-
-        dataMap.remove(objectId);
-    }
-
-
-    @Override
-    public boolean objectExists(String tenantId, String objectId) {
-
-        return dataMap.containsKey(objectId);
-    }
-
-
-    @Override
-    @SneakyThrows
-    public <T> void storeJSONObject(String tenantId, String objectId, T any) {
-
-        File tempFile = createTempFile("test", ".tmp");
-        getMapper().writeValue(new FileOutputStream(tempFile), any);
-        dataMap.put(objectId, tempFile);
-    }
-
-
-    private ObjectMapper getMapper() {
-
-        return ObjectMapperFactory.create();
-    }
-
-
-    @Override
-    @SneakyThrows
-    public <T> T readJSONObject(String tenantId, String objectId, Class<T> clazz) {
-
-        if (dataMap.get(objectId) == null || !dataMap.get(objectId).exists()) {
-            throw new StorageObjectDoesNotExist("Stored object not found");
-        }
-        return getMapper().readValue(new FileInputStream(dataMap.get(objectId)), clazz);
-    }
-
-
-    public List<String> listPaths() {
-
-        return new ArrayList<>(dataMap.keySet());
-    }
-
-
-    public List<String> listFilePaths() {
-
-        return dataMap.values().stream().map(File::getAbsolutePath).collect(Collectors.toList());
-    }
-
-
-    @Override
-    @SneakyThrows
-    public void storeObject(String tenantId, String objectId, InputStream stream) {
-
-        File tempFile = createTempFile("test", ".tmp");
-
-        try (var fileOutputStream = new FileOutputStream(tempFile)) {
-            IOUtils.copy(stream, fileOutputStream);
-        }
-
-        dataMap.put(objectId, tempFile);
-    }
-
-
-    public void clearStorage() {
-
-        this.dataMap.forEach((k, v) -> {
-            v.delete();
-        });
-        this.dataMap.clear();
-    }
-
-}

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/MigrationIntegrationTest.java

@@ -0,0 +1,189 @@
+package com.iqser.red.service.redaction.v1.server;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.nio.file.Path;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import org.junit.jupiter.api.Disabled;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.context.annotation.Import;
+import org.springframework.core.io.ClassPathResource;
+import org.springframework.test.context.junit.jupiter.SpringExtension;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.migration.MigratedIds;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
+import com.iqser.red.service.redaction.v1.server.annotate.AnnotateRequest;
+import com.iqser.red.service.redaction.v1.server.annotate.AnnotateResponse;
+import com.iqser.red.service.redaction.v1.server.document.graph.BuildDocumentIntegrationTest;
+import com.iqser.red.service.redaction.v1.server.model.MigratedEntityLog;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
+import com.iqser.red.service.redaction.v1.server.redaction.utils.OsUtils;
+import com.iqser.red.service.redaction.v1.server.service.RedactionLogToEntityLogMigrationService;
+
+import lombok.SneakyThrows;
+
+@Disabled
+@ExtendWith(SpringExtension.class)
+@SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
+@Import(MigrationIntegrationTest.TestConfiguration.class)
+public class MigrationIntegrationTest extends BuildDocumentIntegrationTest {
+
+    @Autowired
+    RedactionLogToEntityLogMigrationService redactionLogToEntityLogMigrationService;
+
+    @Autowired
+    ObjectMapper mapper;
+
+
+    @Test
+    @SneakyThrows
+    public void testSave() {
+
+        MigratedIds ids = new MigratedIds(new LinkedList<>());
+        ids.addMapping("123", "321");
+        ids.addMapping("123", "321");
+        ids.addMapping("123", "321");
+        ids.addMapping("123", "321");
+        ids.addMapping("123", "321");
+
+        mapper.writeValue(new FileOutputStream("/tmp/testIds.json"), ids);
+        var ids2 = mapper.readValue(new FileInputStream("/tmp/testIds.json"), MigratedIds.class);
+        assert ids2.getMappings().size() == 5;
+    }
+
+    @Test
+    @SneakyThrows
+    public void testMigration() {
+
+        String fileName = "files/migration/def8f960580f088b975ba806dfae1f87.ORIGIN.pdf";
+        String imageFileName = "files/migration/def8f960580f088b975ba806dfae1f87.IMAGE_INFO.json";
+        String tableFileName = "files/migration/def8f960580f088b975ba806dfae1f87.TABLES.json";
+
+        Document document = buildGraph(fileName, imageFileName, tableFileName);
+        RedactionLog redactionLog;
+        try (var in = new ClassPathResource("files/migration/def8f960580f088b975ba806dfae1f87.REDACTION_LOG.json").getInputStream()) {
+            redactionLog = mapper.readValue(in, RedactionLog.class);
+        }
+        MigratedEntityLog migratedEntityLog = redactionLogToEntityLogMigrationService.migrate(redactionLog, document);
+
+        redactionStorageService.storeObject(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.ENTITY_LOG, migratedEntityLog.getEntityLog());
+        assertEquals(redactionLog.getRedactionLogEntry().size(), migratedEntityLog.getEntityLog().getEntityLogEntry().size());
+        assertEquals(redactionLog.getRedactionLogEntry().size(), migratedEntityLog.getMigratedIds().getMappings().size());
+        EntityLog entityLog = migratedEntityLog.getEntityLog();
+        assertEquals(redactionLog.getAnalysisNumber(), entityLog.getAnalysisNumber());
+        assertEquals(redactionLog.getAnalysisVersion(), entityLog.getAnalysisVersion());
+        assertEquals(redactionLog.getDictionaryVersion(), entityLog.getDictionaryVersion());
+        assertEquals(redactionLog.getDossierDictionaryVersion(), entityLog.getDossierDictionaryVersion());
+        assertEquals(redactionLog.getLegalBasisVersion(), entityLog.getLegalBasisVersion());
+        assertEquals(redactionLog.getRulesVersion(), entityLog.getRulesVersion());
+        assertEquals(redactionLog.getLegalBasis().size(), entityLog.getLegalBasis().size());
+
+        Map<String, String> migratedIds = migratedEntityLog.getMigratedIds().buildOldToNewMapping();
+        migratedIds.forEach((oldId, newId) -> assertEntryIsEqual(oldId, newId, redactionLog, entityLog, migratedIds));
+
+        AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder().dossierId(TEST_DOSSIER_ID).fileId(TEST_FILE_ID).build());
+        File outputFile = Path.of(OsUtils.getTemporaryDirectory()).resolve(Path.of(fileName.replaceAll(".pdf", "_MIGRATED.pdf")).getFileName()).toFile();
+
+        try (FileOutputStream fileOutputStream = new FileOutputStream(outputFile)) {
+            fileOutputStream.write(annotateResponse.getDocument());
+        }
+    }
+
+
+    private void assertEntryIsEqual(String oldId, String newId, RedactionLog redactionLog, EntityLog entityLog, Map<String, String> oldToNewMapping) {
+
+        RedactionLogEntry redactionLogEntry = redactionLog.getRedactionLogEntry().stream().filter(entry -> entry.getId().equals(oldId)).findAny().orElseThrow();
+        EntityLogEntry entityLogEntry = entityLog.getEntityLogEntry().stream().filter(entry -> entry.getId().equals(newId)).findAny().orElseThrow();
+
+        if (!redactionLogEntry.isImage()) {
+            assertEquals(redactionLogEntry.getValue().toLowerCase(Locale.ENGLISH), entityLogEntry.getValue().toLowerCase(Locale.ENGLISH));
+        }
+        assertEquals(redactionLogEntry.getChanges().size(), entityLogEntry.getChanges().size());
+        assertEquals(redactionLogEntry.getManualChanges().size(), entityLogEntry.getManualChanges().size());
+        assertEquals(redactionLogEntry.getPositions().size(), entityLogEntry.getPositions().size());
+        assertTrue(positionsAlmostEqual(redactionLogEntry.getPositions(), entityLogEntry.getPositions()));
+        assertEquals(redactionLogEntry.getColor(), entityLogEntry.getColor());
+        assertEqualsNullSafe(redactionLogEntry.getLegalBasis(), entityLogEntry.getLegalBasis());
+        assertEqualsNullSafe(redactionLogEntry.getReason(), entityLogEntry.getReason());
+        assertReferencesEqual(redactionLogEntry.getReference(), entityLogEntry.getReference(), oldToNewMapping);
+        assertEquals(redactionLogEntry.isDictionaryEntry(), entityLogEntry.isDictionaryEntry());
+        assertEquals(redactionLogEntry.isDossierDictionaryEntry(), entityLogEntry.isDossierDictionaryEntry());
+        assertEquals(redactionLogEntry.getEngines().stream().map(Enum::name).collect(Collectors.toSet()),
+                entityLogEntry.getEngines().stream().map(Enum::name).collect(Collectors.toSet()));
+    }
+
+
+    private boolean positionsAlmostEqual(List<Rectangle> positions1, List<Position> positions2) {
+
+        double tolerance = 2;
+        for (int i = 0; i < positions1.size(); i++) {
+            Rectangle p1 = positions1.get(0);
+            Position p2 = positions2.get(0);
+            if (p1.getPage() != p2.getPageNumber()) {
+                return false;
+            }
+            if (Math.abs(p1.getHeight() - p2.getRectangle()[3]) > tolerance) {
+                return false;
+            }
+            if (Math.abs(p1.getWidth() - p2.getRectangle()[2]) > tolerance) {
+                return false;
+            }
+            if (Math.abs(p1.getTopLeft().getX() - p2.getRectangle()[0]) > tolerance) {
+                return false;
+            }
+            if (Math.abs(p1.getTopLeft().getY() - p2.getRectangle()[1]) > tolerance) {
+                return false;
+            }
+        }
+        return true;
+    }
+
+
+    private void assertEqualsNullSafe(String string1, String string2) {
+
+        if (Objects.isNull(string1) && Objects.isNull(string2)) {
+            assertEquals(string1, string2);
+        } else if (Objects.isNull(string1)) {
+            assertEquals("", string2);
+        } else if (Objects.isNull(string2)) {
+            assertEquals("", string1);
+        } else {
+            assertEquals(string1, string2);
+        }
+
+    }
+
+
+    private void assertReferencesEqual(Set<String> reference, Set<String> reference1, Map<String, String> oldToNewMapping) {
+
+        if (reference == null) {
+            assertTrue(reference1.isEmpty());
+            return;
+        }
+
+        assertEquals(reference.stream().map(oldToNewMapping::get).collect(Collectors.toSet()), reference1);
+
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionAcceptanceTest.java

@@ -1,5 +1,6 @@
 package com.iqser.red.service.redaction.v1.server;
 
+import static org.assertj.core.api.Assertions.assertThat;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.mockito.Mockito.when;
 
@@ -24,6 +25,7 @@ import org.springframework.context.annotation.Import;
 import org.springframework.context.annotation.Primary;
 import org.springframework.test.context.junit.jupiter.SpringExtension;
 
+import com.iqser.red.commons.jackson.ObjectMapperFactory;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeResult;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType;
@@ -41,6 +43,7 @@ import com.iqser.red.service.redaction.v1.server.annotate.AnnotateResponse;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.OsUtils;
 import com.iqser.red.storage.commons.StorageAutoConfiguration;
 import com.iqser.red.storage.commons.service.StorageService;
+import com.iqser.red.storage.commons.utils.FileSystemBackedStorageService;
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
 import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingServiceProcessorConfiguration;
 import com.knecon.fforesight.tenantcommons.TenantContext;
@@ -62,7 +65,7 @@ public class RedactionAcceptanceTest extends AbstractRedactionIntegrationTest {
         @Primary
         public StorageService inmemoryStorage() {
 
-            return new FileSystemBackedStorageService();
+            return new FileSystemBackedStorageService(ObjectMapperFactory.create());
         }
 
     }
@@ -101,6 +104,25 @@ public class RedactionAcceptanceTest extends AbstractRedactionIntegrationTest {
     }
 
 
+    @Test
+    public void authorNameSplittingTest() {
+
+        AnalyzeRequest request = uploadFileToStorage("files/Minimal Examples/tableWithAuthors.pdf");
+
+        analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request);
+        AnalyzeResult result = analyzeService.analyze(request);
+
+        var entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
+
+        var recommendations = entityLog.getEntityLogEntry()
+                .stream()
+                .filter(entityLogEntry -> entityLogEntry.getEntryType().equals(EntryType.RECOMMENDATION))
+                .map(EntityLogEntry::getValue)
+                .toList();
+
+        assertThat(recommendations).containsExactlyInAnyOrder("Michael N.", "Funnarie B.", "Feuer A.");
+    }
+
     @Test
     public void acceptanceTests() throws IOException {
 
@@ -146,7 +168,8 @@ public class RedactionAcceptanceTest extends AbstractRedactionIntegrationTest {
         return redactionLog.getEntityLogEntry()
                 .stream()
                 .filter(entry -> entry.getType().equals(type))
-                .filter(entry -> entry.getValue().equals(value)).filter(entry -> entry.getContainingNodeId().get(0).equals(sectionNumber.get(0)));
+                .filter(entry -> entry.getValue().equals(value))
+                .filter(entry -> entry.getContainingNodeId().get(0).equals(sectionNumber.get(0)));
     }
 
 

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java

@@ -41,6 +41,7 @@ import org.springframework.core.io.ClassPathResource;
 import org.springframework.test.context.junit.jupiter.SpringExtension;
 
 import com.fasterxml.jackson.core.type.TypeReference;
+import com.iqser.red.commons.jackson.ObjectMapperFactory;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeResult;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;
@@ -49,7 +50,6 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryType;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Comment;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;
@@ -72,6 +72,7 @@ import com.iqser.red.service.redaction.v1.server.service.document.DocumentGraphM
 import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
 import com.iqser.red.storage.commons.StorageAutoConfiguration;
 import com.iqser.red.storage.commons.service.StorageService;
+import com.iqser.red.storage.commons.utils.FileSystemBackedStorageService;
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
 import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingServiceProcessorConfiguration;
 import com.knecon.fforesight.tenantcommons.TenantContext;
@@ -95,7 +96,7 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
         @Primary
         public StorageService inmemoryStorage() {
 
-            return new FileSystemBackedStorageService();
+            return new FileSystemBackedStorageService(ObjectMapperFactory.create());
         }
 
     }
@@ -195,6 +196,55 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
     }
 
 
+    @Test
+    public void manualResizeTest() throws IOException {
+
+//        c6be5277f5ee60dc3d83527798b7fe02
+//        Rectangle(topLeft=Point(x=236.8, y=182.90005), width=70.89604, height=12.642, page=7)
+
+//        {
+//            "page": 7,
+//                "topLeft": {
+//            "x": 200.8,
+//                    "y": 332.364
+//        },
+//            "height": 15.408000000000015,
+//                "width": 40.584
+//        }
+
+        AnalyzeRequest request = uploadFileToStorage("files/new/crafted document.pdf");
+
+        ManualRedactions manualRedactions = ManualRedactions.builder()
+                .resizeRedactions(Set.of(ManualResizeRedaction.builder()
+                        .annotationId("c6be5277f5ee60dc3d83527798b7fe02")
+                        .value("Dr. Alan")
+                        .positions(List.of(new Rectangle(236.8f, 182.90005f, 40.584f, 12.642f, 7)))
+                        .status(AnnotationStatus.APPROVED)
+                        .requestDate(OffsetDateTime.now())
+                        .updateDictionary(false)
+                        .build()))
+                .build();
+
+        request.setManualRedactions(manualRedactions);
+
+        System.out.println("Start Full integration test");
+        analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request);
+        System.out.println("Finished structure analysis");
+        AnalyzeResult result = analyzeService.analyze(request);
+        System.out.println("Finished analysis");
+        var redactionLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
+
+        AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder().dossierId(TEST_DOSSIER_ID).fileId(TEST_FILE_ID).build());
+
+        String outputFileName = OsUtils.getTemporaryDirectory() + "/Annotated.pdf";
+
+        try (FileOutputStream fileOutputStream = new FileOutputStream(outputFileName)) {
+            fileOutputStream.write(annotateResponse.getDocument());
+        }
+
+    }
+
+
     @Test
     public void redactionExpansionOverlap() {
 
@@ -213,10 +263,38 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
     }
 
 
+    @Test
+    public void importedRedactionsTest() throws IOException {
+
+        ClassPathResource importedRedactionClasspathResource = new ClassPathResource(
+                "files/ImportedRedactions/18 Chlorothalonil RAR 08 Volume 3CA B 6a Oct 2017.IMPORTED_REDACTIONS.json");
+        storageService.storeObject(TenantContext.getTenantId(),
+                RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.IMPORTED_REDACTIONS),
+                importedRedactionClasspathResource.getInputStream());
+
+        AnalyzeRequest request = uploadFileToStorage("files/ImportedRedactions/18 Chlorothalonil RAR 08 Volume 3CA B 6a Oct 2017.pdf");
+        System.out.println("Start Full integration test");
+        analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request);
+        System.out.println("Finished structure analysis");
+        AnalyzeResult result = analyzeService.analyze(request);
+        System.out.println("Finished analysis");
+        var entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
+
+        AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder().dossierId(TEST_DOSSIER_ID).fileId(TEST_FILE_ID).build());
+
+        String outputFileName = OsUtils.getTemporaryDirectory() + "/Annotated.pdf";
+
+        try (FileOutputStream fileOutputStream = new FileOutputStream(outputFileName)) {
+            fileOutputStream.write(annotateResponse.getDocument());
+        }
+
+    }
+
+
     @Test
     public void titleExtraction() throws IOException {
 
-        AnalyzeRequest request = uploadFileToStorage("files/Metolachlor/S-Metolachlor_RAR_02_Volume_2_2018-09-06.pdf");
+        AnalyzeRequest request = uploadFileToStorage("files/migration/def8f960580f088b975ba806dfae1f87.ORIGIN.pdf");
         System.out.println("Start Full integration test");
         analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request);
         System.out.println("Finished structure analysis");
@@ -522,7 +600,8 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
         }
 
         List<String> valuesInDocument = entityLog.getEntityLogEntry()
-                .stream().filter(e -> !e.getEntryType().equals(EntryType.IMAGE))
+                .stream()
+                .filter(e -> !e.getEntryType().equals(EntryType.IMAGE))
                 .map(redactionLogEntry -> new TextRange(redactionLogEntry.getStartOffset(), redactionLogEntry.getEndOffset()))
                 .map(boundary -> documentGraph.getTextBlock().subSequence(boundary).toString())
                 .toList();
@@ -840,19 +919,19 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
         AnalyzeRequest request = uploadFileToStorage("files/new/S157.pdf");
 
         ManualRedactions manualRedactions = new ManualRedactions();
-
-        manualRedactions.getResizeRedactions()
-                .add(ManualResizeRedaction.builder()
-                        .annotationId("ca2b437e2480a4b5966cb8386020d454")
-                        .fileId(TEST_FILE_ID)
-                        .status(AnnotationStatus.APPROVED)
-                        .requestDate(OffsetDateTime.of(2022, 05, 23, 8, 30, 07, 475479, ZoneOffset.UTC))
-                        .processedDate(OffsetDateTime.of(2022, 05, 23, 8, 30, 07, 483651, ZoneOffset.UTC))
-                        .value("Bera P")
-                        .positions(List.of(new Rectangle(382.12485f, 235.94246f, 8.768621f, -21.632504f, 1)))
-                        .textBefore("")
-                        .textAfter("")
-                        .build());
+        ManualResizeRedaction manualResizeRedaction = ManualResizeRedaction.builder()
+                .annotationId("ca2b437e2480a4b5966cb8386020d454")
+                .fileId(TEST_FILE_ID)
+                .status(AnnotationStatus.APPROVED)
+                .requestDate(OffsetDateTime.of(2022, 05, 23, 8, 30, 07, 475479, ZoneOffset.UTC))
+                .processedDate(OffsetDateTime.of(2022, 05, 23, 8, 30, 07, 483651, ZoneOffset.UTC))
+                .value("Bera P")
+                .positions(List.of(new Rectangle(382.12485f, 235.94246f, 8.768621f, -21.632504f, 1)))
+                .textBefore("")
+                .updateDictionary(false)
+                .textAfter("")
+                .build();
+        manualRedactions.getResizeRedactions().add(manualResizeRedaction);
 
         request.setManualRedactions(manualRedactions);
 
@@ -891,7 +970,11 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
 
         redactionLog.getEntityLogEntry().forEach(entry -> {
             if (!entry.getEntryType().equals(EntryType.HINT)) {
-                assertThat(entry.getReason()).isEqualTo("Not redacted because it's row does not belong to a vertebrate study");
+                if (entry.getType().equals("CBI_author")) {
+                    assertThat(entry.getReason()).isEqualTo("Not redacted because it's row does not belong to a vertebrate study");
+                } else if (entry.getType().equals("CBI_address")) {
+                    assertThat(entry.getReason()).isEqualTo("No vertebrate found");
+                }
             }
         });
     }
@@ -929,7 +1012,6 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
 
         String manualAddId = UUID.randomUUID().toString();
 
-        Comment comment = Comment.builder().date(OffsetDateTime.now()).user("TEST_USER").text("This is a comment test").build();
         manualRedactions.setIdsToRemove(Set.of(IdRemoval.builder()
                 .annotationId("5b940b2cb401ed9f5be6fc24f6e77bcf")
                 .fileId("fileId")
@@ -946,10 +1028,6 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
                 .processedDate(OffsetDateTime.now())
                 .build()));
 
-        manualRedactions.getComments().put("e5be0f1d941bbb92a068e198648d06c4", List.of(comment));
-        manualRedactions.getComments().put("0836727c3508a0b2ea271da69c04cc2f", List.of(comment));
-        manualRedactions.getComments().put(manualAddId, List.of(comment));
-
         ManualRedactionEntry manualRedactionEntry = new ManualRedactionEntry();
         manualRedactionEntry.setAnnotationId(manualAddId);
         manualRedactionEntry.setFileId("fileId");
@@ -996,6 +1074,7 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
                         Rectangle.builder().topLeftX(307.89517f).topLeftY(327.567f).width(7.648041f).height(61.670967f).page(1).build(),
                         Rectangle.builder().topLeftX(316.99985f).topLeftY(327.567f).width(7.648041f).height(38.104286f).page(1).build()))
                 .status(AnnotationStatus.APPROVED)
+                .updateDictionary(false)
                 .build()));
 
         analyzeService.reanalyze(request);
@@ -1037,6 +1116,7 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
 
 
     @Test
+    @Disabled
     public void testImportedRedactions() throws IOException {
 
         String outputFileName = OsUtils.getTemporaryDirectory() + "/ImportedRedactions.pdf";

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationV2Test.java

@@ -20,6 +20,7 @@ import org.springframework.context.annotation.Import;
 import org.springframework.context.annotation.Primary;
 import org.springframework.test.context.junit.jupiter.SpringExtension;
 
+import com.iqser.red.commons.jackson.ObjectMapperFactory;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine;
@@ -29,6 +30,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSON
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type;
 import com.iqser.red.storage.commons.StorageAutoConfiguration;
 import com.iqser.red.storage.commons.service.StorageService;
+import com.iqser.red.storage.commons.utils.FileSystemBackedStorageService;
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
 import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingServiceProcessorConfiguration;
 import com.knecon.fforesight.tenantcommons.TenantContext;
@@ -53,7 +55,7 @@ public class RedactionIntegrationV2Test extends AbstractRedactionIntegrationTest
         @Primary
         public StorageService inmemoryStorage() {
 
-            return new FileSystemBackedStorageService();
+            return new FileSystemBackedStorageService(ObjectMapperFactory.create());
         }
 
     }

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RulesTest.java

@@ -57,6 +57,7 @@ import org.springframework.test.context.junit.jupiter.SpringExtension;
 import com.fasterxml.jackson.databind.DeserializationFeature;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
+import com.iqser.red.commons.jackson.ObjectMapperFactory;
 import com.iqser.red.service.dictionarymerge.commons.DictionaryEntry;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType;
@@ -85,6 +86,7 @@ import com.iqser.red.service.redaction.v1.server.utils.ResourceLoader;
 import com.iqser.red.service.redaction.v1.server.utils.TextNormalizationUtilities;
 import com.iqser.red.storage.commons.StorageAutoConfiguration;
 import com.iqser.red.storage.commons.service.StorageService;
+import com.iqser.red.storage.commons.utils.FileSystemBackedStorageService;
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingFinishedEvent;
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
 import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingPipeline;
@@ -837,7 +839,7 @@ public class RulesTest {
         @Primary
         public StorageService inmemoryStorage() {
 
-            return new FileSystemBackedStorageService();
+            return new FileSystemBackedStorageService(ObjectMapperFactory.create());
         }
 
     }

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/annotate/AnnotationService.java

@@ -12,10 +12,12 @@ import java.util.stream.Stream;
 
 import org.apache.pdfbox.Loader;
 import org.apache.pdfbox.pdmodel.PDDocument;
+import org.apache.pdfbox.pdmodel.PDDocumentCatalog;
 import org.apache.pdfbox.pdmodel.PDPage;
 import org.apache.pdfbox.pdmodel.common.PDRectangle;
 import org.apache.pdfbox.pdmodel.graphics.color.PDColor;
 import org.apache.pdfbox.pdmodel.graphics.color.PDDeviceRGB;
+import org.apache.pdfbox.pdmodel.graphics.optionalcontent.PDOptionalContentProperties;
 import org.apache.pdfbox.pdmodel.interactive.annotation.PDAnnotation;
 import org.apache.pdfbox.pdmodel.interactive.annotation.PDAnnotationHighlight;
 import org.springframework.stereotype.Service;
@@ -27,7 +29,6 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryType;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
-import com.iqser.red.service.redaction.v1.server.service.DictionaryService;
 import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
 import com.iqser.red.service.redaction.v1.server.utils.RectangleTransformations;
 
@@ -39,11 +40,10 @@ import lombok.SneakyThrows;
 public class AnnotationService {
 
     private final RedactionStorageService redactionStorageService;
-    private final DictionaryService dictionaryService;
 
 
     /**
-     * This method draws a PDF Text Markup Annotation for each RedactionLogEntry and the section grid.
+     * This method draws a PDF Text Markup Annotation for each EntityLogEntry and the Viewer document.
      * A Text Markup Annotation has two main functionalities.
      * First, it highlights text with a color and second, when hovered over in a PDF Viewer it displays a popup message.
      * Where the Popup message appears is defined by the Rectangle provided to the Annotation.
@@ -60,21 +60,24 @@ public class AnnotationService {
     public AnnotateResponse annotate(AnnotateRequest annotateRequest) {
 
         var storedObjectFile = redactionStorageService.getStoredObjectFile(RedactionStorageService.StorageIdUtils.getStorageId(annotateRequest.getDossierId(),
-                annotateRequest.getFileId(), FileType.VIEWER_DOCUMENT));
+                annotateRequest.getFileId(),
+                FileType.VIEWER_DOCUMENT));
 
         var entityLog = redactionStorageService.getEntityLog(annotateRequest.getDossierId(), annotateRequest.getFileId());
 
         try (PDDocument pdDocument = Loader.loadPDF(storedObjectFile)) {
             pdDocument.setAllSecurityToBeRemoved(true);
 
-            dictionaryService.updateDictionary(annotateRequest.getDossierTemplateId(), annotateRequest.getDossierId());
             annotate(pdDocument, entityLog);
 
+            PDDocumentCatalog catalog = pdDocument.getDocumentCatalog();
+            PDOptionalContentProperties ocprops = catalog.getOCProperties();
+            ocprops.setGroupEnabled("Layout grid", true);
+
             try (ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream()) {
                 pdDocument.save(byteArrayOutputStream);
                 return AnnotateResponse.builder().document(byteArrayOutputStream.toByteArray()).build();
             }
-
         }
     }
 
@@ -173,6 +176,7 @@ public class AnnotationService {
         return rectangles.stream().map(Position::toRectangle2D).flatMap(AnnotationService::toQuadPoints).toList();
     }
 
+
     public static Stream<Double> toQuadPoints(Rectangle2D rectangle) {
 
         double x1 = rectangle.getMinX();

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/date/DateConverterTest.java

@@ -0,0 +1,66 @@
+package com.iqser.red.service.redaction.v1.server.date;
+
+import java.util.Arrays;
+import java.util.Date;
+import java.util.List;
+import java.util.Optional;
+
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import org.junit.jupiter.api.Test;
+
+import com.iqser.red.service.redaction.v1.server.utils.DateConverter;
+
+public class DateConverterTest {
+
+    @Test
+    public void testDateConverter() {
+
+        List<String> goldenStandardDates = Arrays.asList("3 Jun 08",
+                                                         "09. Apr 09",
+                                                         "07-Sep-2010",
+                                                         "26-FEB-2008",
+                                                         "30-APR-2008",
+                                                         "30-apr-2008",
+                                                         "30-Apr-2008",
+                                                         "1 Apr 08",
+                                                         "26-FEB-2008",
+                                                         "19-MAR-2008",
+                                                         "1 Apr 08",
+                                                         "27-MAR-2008",
+                                                         "06-MAY-2008",
+                                                         "3 Apr 08",
+                                                         "12-MAR-2008",
+                                                         "08-APR-2008",
+                                                         "1 Apr 08",
+                                                         "4 Apr 08",
+                                                         "13 November 2017 (animal 1)",
+                                                         "16 November 2017 (animal 1)",
+                                                         "27 March 2018 (animal 1 - 5000 mg/kg bw)",
+                                                         "10 April 2018 (animal 1 - 5000 mg/kg bw)",
+                                                         "13 November 2017 (animal 1)",
+                                                         "16 November 2017 (animal 1)",
+                                                         "28 March 2018 (animal 1 - 5000 mg/kg bw)",
+                                                         "28 March 2018 (animal1 - 5000 mg/kg bw)",
+                                                         "28 August 2018 (animal 1)",
+                                                         "October, 27 1989",
+                                                         "October, 7 1989",
+                                                         "August 21,1998",
+                                                         "August 2,1998",
+                                                         "1st May 1988",
+                                                         "2nd June 2003",
+                                                         "3rd September 2005",
+                                                         "6th September 2005",
+                                                         "17th August 2005",
+                                                         "22nd August 2035",
+                                                         "12-January-2023",
+                                                         "2-January-2023");
+
+
+
+        for (String dateStr : goldenStandardDates) {
+            Optional<Date> parsedDate = DateConverter.parseDate(dateStr);
+            assertTrue(parsedDate.isPresent(), "Failed to parse date: " + dateStr);
+        }
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/date/DateExtractorNattyTest.java

@@ -0,0 +1,28 @@
+package com.iqser.red.service.redaction.v1.server.date;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.util.Optional;
+
+import org.junit.jupiter.api.Test;
+
+import com.iqser.red.service.redaction.v1.server.utils.DateExtractorNatty;
+
+public class DateExtractorNattyTest {
+
+    @Test
+    public void testExtractDate() {
+
+        String[] testStrings = {"13 November 2017 (animal 1)", "16 November 2017 (animal 1)", "27 March 2018 (animal 1 - 5000 mg/kg bw)", "10 April 2018 (animal 1 - 5000 mg/kg bw)", "13 November 2017 (animal 1)", "16 November 2017 (animal 1)", "28 March 2018 (animal 1 - 5000 mg/kg bw)", "28 March 2018 (animal1 - 5000 mg/kg bw)", "28 August 2018 (animal 1)", "31 August 2018 (animal 1)"};
+
+        String[] expectedDates = {"13 November 2017", "16 November 2017", "27 March 2018", "10 April 2018", "13 November 2017", "16 November 2017", "28 March 2018", "28 March 2018", "28 August 2018", "31 August 2018"};
+
+        for (int i = 0; i < testStrings.length; i++) {
+            Optional<String> extractedDate = DateExtractorNatty.extractDate(testStrings[i]);
+            assertTrue(extractedDate.isPresent(), "No date found in: " + testStrings[i]);
+            assertEquals(expectedDates[i], extractedDate.get(), "Failed to extract correct date from: " + testStrings[i]);
+        }
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/BuildDocumentIntegrationTest.java

@@ -12,14 +12,15 @@ import org.springframework.context.annotation.Import;
 import org.springframework.context.annotation.Primary;
 import org.springframework.test.context.junit.jupiter.SpringExtension;
 
+import com.iqser.red.commons.jackson.ObjectMapperFactory;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
 import com.iqser.red.service.redaction.v1.server.AbstractRedactionIntegrationTest;
 import com.iqser.red.service.redaction.v1.server.Application;
-import com.iqser.red.service.redaction.v1.server.FileSystemBackedStorageService;
 import com.iqser.red.service.redaction.v1.server.service.document.DocumentGraphMapper;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
 import com.iqser.red.storage.commons.StorageAutoConfiguration;
 import com.iqser.red.storage.commons.service.StorageService;
+import com.iqser.red.storage.commons.utils.FileSystemBackedStorageService;
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
 import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingServiceProcessorConfiguration;
 
@@ -40,11 +41,12 @@ public class BuildDocumentIntegrationTest extends AbstractRedactionIntegrationTe
         @Primary
         public StorageService inmemoryStorage() {
 
-            return new FileSystemBackedStorageService();
+            return new FileSystemBackedStorageService(ObjectMapperFactory.create());
         }
 
     }
 
+
     @SneakyThrows
     protected Document buildGraph(String filename) {
 
@@ -64,6 +66,20 @@ public class BuildDocumentIntegrationTest extends AbstractRedactionIntegrationTe
     }
 
 
+    @SneakyThrows
+    protected Document buildGraph(String filename, String imageFileName, String tableFileName) {
+
+        String fileNameWithPdf = filename;
+        if (!filename.endsWith(".pdf")) {
+            fileNameWithPdf = filename + ".pdf";
+        }
+        AnalyzeRequest request = prepareStorage(fileNameWithPdf, tableFileName, imageFileName);
+
+        analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request);
+        return DocumentGraphMapper.toDocumentGraph(redactionStorageService.getDocumentData(request.getDossierId(), request.getFileId()));
+    }
+
+
     @SneakyThrows
     protected Document buildGraphNoImages(String filename) {
 

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/MigrationPocTest.java

@@ -1,164 +0,0 @@
-package com.iqser.red.service.redaction.v1.server.document.graph;
-
-import static java.util.stream.Collectors.toMap;
-import static org.junit.jupiter.api.Assertions.assertTrue;
-import static org.mockito.Mockito.doThrow;
-import static org.mockito.Mockito.when;
-
-import java.io.IOException;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-import java.util.stream.Stream;
-
-import org.drools.io.ClassPathResource;
-import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.Disabled;
-import org.junit.jupiter.api.Test;
-import org.springframework.beans.factory.annotation.Autowired;
-
-import com.google.common.base.Functions;
-import com.google.common.collect.Sets;
-import com.iqser.red.commons.jackson.ObjectMapperFactory;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeResult;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
-import com.iqser.red.service.redaction.v1.server.service.document.DocumentGraphMapper;
-import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
-import com.iqser.red.service.redaction.v1.server.service.document.ManualEntityCreationService;
-import com.iqser.red.service.redaction.v1.server.model.ManualEntity;
-import com.iqser.red.service.redaction.v1.server.service.RedactionLogCreatorService;
-import com.iqser.red.service.redaction.v1.server.utils.ExceptionProvider;
-import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
-import com.knecon.fforesight.tenantcommons.TenantContext;
-
-import lombok.SneakyThrows;
-
-public class MigrationPocTest extends BuildDocumentIntegrationTest {
-
-    private static final String RULES = loadFromClassPath("drools/rules.drl");
-
-    @Autowired
-    private ManualEntityCreationService redactionLogAdapter;
-    @Autowired
-    private RedactionLogCreatorService redactionLogCreatorService;
-
-
-    @BeforeEach
-    public void stubClients() {
-
-        TenantContext.setTenantId("redaction");
-        when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.ENTITY)).thenReturn(System.currentTimeMillis());
-        when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.ENTITY)).thenReturn(JSONPrimitive.of(RULES));
-        when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.COMPONENT)).thenReturn(-1L);
-
-        loadDictionaryForTest();
-        loadTypeForTest();
-        loadNerForTest();
-        when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L);
-        when(dictionaryClient.getAllTypesForDossierTemplate(TEST_DOSSIER_TEMPLATE_ID, false)).thenReturn(getTypeResponse());
-
-        when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L);
-        when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, false)).thenReturn(List.of(Type.builder()
-                .id(DOSSIER_REDACTIONS_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID)
-                .type(DOSSIER_REDACTIONS_INDICATOR)
-                .dossierTemplateId(TEST_DOSSIER_ID)
-                .hexColor("#ffe187")
-                .isHint(hintTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
-                .isCaseInsensitive(caseInSensitiveMap.get(DOSSIER_REDACTIONS_INDICATOR))
-                .isRecommendation(recommendationTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
-                .rank(rankTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
-                .build()));
-
-        mockDictionaryCalls(null);
-
-        when(dictionaryClient.getColors(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(colors);
-    }
-
-
-    @Test
-    @Disabled // Enable if you fix the TODO in EntityCreationService
-    @SneakyThrows
-    public void testMigration() {
-
-        AnalyzeRequest request = uploadFileToStorage("files/new/crafted document.pdf");
-        analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request);
-        AnalyzeResult result = analyzeService.analyze(request);
-        var newRedactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID);
-
-        var originalRedactionLog = getOriginalRedactionLog();
-
-        // IMPORTANT: always use the graph which is mapped from the DocumentData, since rounding errors occur during storage.
-        Document document = DocumentGraphMapper.toDocumentGraph(redactionStorageService.getDocumentData(request.getDossierId(), request.getFileId()));
-        List<ManualEntity> notFoundManualRedactionEntries = redactionLogAdapter.toRedactionEntity(originalRedactionLog, document);
-
-        var migratedRedactionLogEntries = redactionLogCreatorService.createRedactionLog(document, TEST_DOSSIER_TEMPLATE_ID, notFoundManualRedactionEntries, Collections.emptyMap());
-
-        Map<String, RedactionLogEntry> migratedIds = migratedRedactionLogEntries.stream().collect(toMap(RedactionLogEntry::getId, Functions.identity()));
-        Map<String, RedactionLogEntry> newIds = newRedactionLog.getRedactionLogEntry().stream().collect(toMap(RedactionLogEntry::getId, Functions.identity()));
-
-        logPrecision(migratedIds, newIds);
-        logRecall(migratedIds, newIds);
-    }
-
-
-    private static void logPrecision(Map<String, RedactionLogEntry> migratedIds, Map<String, RedactionLogEntry> newIds) {
-
-        var precision = computePrecision(migratedIds, newIds);
-        System.out.printf("precision %.2f\n", precision);
-        System.out.println("New Entries");
-        getAddedEntries(migratedIds, newIds).forEach(System.out::println);
-        assertTrue(precision >= 0.85);
-        System.out.println();
-    }
-
-
-    private static void logRecall(Map<String, RedactionLogEntry> migratedIds, Map<String, RedactionLogEntry> newIds) {
-
-        var recall = computeRecall(migratedIds, newIds);
-        System.out.printf("recall %.2f\n", recall);
-        System.out.println("Missing entries");
-        getMissingEntries(migratedIds, newIds).forEach(System.out::println);
-        assertTrue(recall >= 0.85);
-        System.out.println();
-    }
-
-
-    private static Stream<RedactionLogEntry> getMissingEntries(Map<String, RedactionLogEntry> migratedIds, Map<String, RedactionLogEntry> newIds) {
-
-        return migratedIds.entrySet().stream().filter(entry -> !newIds.containsKey(entry.getKey())).map(Map.Entry::getValue);
-    }
-
-
-    private static Stream<RedactionLogEntry> getAddedEntries(Map<String, RedactionLogEntry> migratedIds, Map<String, RedactionLogEntry> newIds) {
-
-        return newIds.entrySet().stream().filter(entry -> !migratedIds.containsKey(entry.getKey())).map(Map.Entry::getValue);
-    }
-
-
-    private static double computePrecision(Map<String, RedactionLogEntry> migratedIds, Map<String, RedactionLogEntry> newIds) {
-
-        return (double) Sets.intersection(newIds.keySet(), migratedIds.keySet()).size() / (double) newIds.size();
-    }
-
-
-    private static double computeRecall(Map<String, RedactionLogEntry> migratedIds, Map<String, RedactionLogEntry> newIds) {
-
-        return (double) Sets.intersection(newIds.keySet(), migratedIds.keySet()).size() / (double) migratedIds.size();
-    }
-
-
-    private static RedactionLog getOriginalRedactionLog() throws IOException {
-
-        RedactionLog originalRedactionLog;
-        try (var inputStream = new ClassPathResource("files/migration/legacy_redactionlog.json").getInputStream()) {
-            originalRedactionLog = ObjectMapperFactory.create().readValue(inputStream, RedactionLog.class);
-        }
-        return originalRedactionLog;
-    }
-
-}

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/drools/files/management/services/DroolsSyntaxValidationServiceTest.java

@@ -58,12 +58,56 @@ class DroolsSyntaxValidationServiceTest {
     }
 
 
+    @Test
+    @SneakyThrows
+    void testRulesWithRemovedImportsAndChangedName() {
+
+        DroolsSyntaxValidationService droolsSyntaxValidationService = new DroolsSyntaxValidationService(new KieContainerCreationService(rulesClient));
+        var rulesFile = new ClassPathResource("drools/rules.drl");
+
+        String rulesString = new String(rulesFile.getInputStream().readAllBytes());
+        rulesString = rulesString.replaceAll("import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell;", "");
+        rulesString = rulesString.replaceAll("rule \"LDS.0.0: Run local dictionary search\"", "rule \"LDS.0.0: run local dictionary search\"");
+        DroolsSyntaxValidation droolsSyntaxValidation = droolsSyntaxValidationService.testRules(new RuleValidationModel(RuleFileType.ENTITY.name(), rulesString));
+        droolsSyntaxValidation.getDroolsSyntaxErrorMessages().forEach(System.out::println);
+        assertTrue(droolsSyntaxValidation.isCompiled());
+    }
+
+    @Test
+    @SneakyThrows
+    void testRulesWithAddedImports() {
+
+        DroolsSyntaxValidationService droolsSyntaxValidationService = new DroolsSyntaxValidationService(new KieContainerCreationService(rulesClient));
+        var rulesFile = new ClassPathResource("drools/rules.drl");
+
+        String rulesString = new String(rulesFile.getInputStream().readAllBytes());
+        rulesString = rulesString.replaceAll("import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell;", "import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell;\nimport com.iqser.red.service.redaction.v1.server.service.document.EntityComparators;");
+        DroolsSyntaxValidation droolsSyntaxValidation = droolsSyntaxValidationService.testRules(new RuleValidationModel(RuleFileType.ENTITY.name(), rulesString));
+        droolsSyntaxValidation.getDroolsSyntaxErrorMessages().forEach(System.out::println);
+        assertTrue(droolsSyntaxValidation.isCompiled());
+    }
+
+    @Test
+    @SneakyThrows
+    void testRulesWithUnallowedAddedImports() {
+
+        DroolsSyntaxValidationService droolsSyntaxValidationService = new DroolsSyntaxValidationService(new KieContainerCreationService(rulesClient));
+        var rulesFile = new ClassPathResource("drools/rules.drl");
+
+        String rulesString = new String(rulesFile.getInputStream().readAllBytes());
+        rulesString = rulesString.replaceAll("import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell;", "import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell;\nimport com.google.common.collect.Sets;");
+        DroolsSyntaxValidation droolsSyntaxValidation = droolsSyntaxValidationService.testRules(new RuleValidationModel(RuleFileType.ENTITY.name(), rulesString));
+        droolsSyntaxValidation.getDroolsSyntaxErrorMessages().forEach(System.out::println);
+        assertFalse(droolsSyntaxValidation.isCompiled());
+    }
+
+
     @Test
     @SneakyThrows
     void testAllRules() {
 
         DroolsSyntaxValidationService droolsSyntaxValidationService = new DroolsSyntaxValidationService(new KieContainerCreationService(rulesClient));
-        var rulesFile = new ClassPathResource("drools/all_rules.drl");
+        var rulesFile = new ClassPathResource("drools/all_redact_manager_rules.drl");
 
         String rulesString = new String(rulesFile.getInputStream().readAllBytes());
 
@@ -156,7 +200,8 @@ class DroolsSyntaxValidationServiceTest {
         var rulesFile = new ClassPathResource("drools/rules.drl");
 
         String rulesString = new String(rulesFile.getInputStream().readAllBytes());
-        String corruptedRules = rulesString.replaceAll("import com.iqser.red.service.redaction.v1.server.model.document.nodes.ImageType;", "");
+        String corruptedRules = rulesString.replaceAll("import com.iqser.red.service.redaction.v1.server.model.document.nodes.ImageType;",
+                "import com.iqser.red.service.redaction.v1.server.model.document.nodes.ImageType;\nimport com.iqser.red.service.redaction.v1.server.model.document.nodes.SomethingElse;");
 
         DroolsSyntaxValidation droolsSyntaxValidation = droolsSyntaxValidationService.testRules(new RuleValidationModel(RuleFileType.ENTITY.name(), corruptedRules));
         assertFalse(droolsSyntaxValidation.isCompiled());
@@ -216,8 +261,7 @@ class DroolsSyntaxValidationServiceTest {
 
         DroolsSyntaxValidationService droolsSyntaxValidationService = new DroolsSyntaxValidationService(new KieContainerCreationService(rulesClient));
 
-        List<String> ruleFiles = List.of("drools/rules.drl",
-                "drools/all_rules.drl",
+        List<String> ruleFiles = List.of("drools/rules.drl", "drools/all_redact_manager_rules.drl",
                 "drools/documine_flora.drl",
                 "drools/manual_redaction_rules.drl",
                 "drools/acceptance_rules.drl",

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/drools/files/management/services/DroolsUpToDateTest.java

@@ -0,0 +1,53 @@
+package com.iqser.red.service.redaction.v1.server.drools.files.management.services;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.wildfly.common.Assert.assertTrue;
+
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.List;
+
+import org.junit.jupiter.api.Test;
+import org.springframework.core.io.ClassPathResource;
+
+import com.knecon.fforesight.utility.rules.management.factory.RuleFileParser;
+import com.knecon.fforesight.utility.rules.management.models.BasicRule;
+import com.knecon.fforesight.utility.rules.management.models.RuleFileBluePrint;
+import com.knecon.fforesight.utility.rules.management.utils.RuleFileIO;
+
+import lombok.SneakyThrows;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+public class DroolsUpToDateTest {
+
+    @Test
+    @SneakyThrows
+    public void assertAllRuleFilesAreUpToDate() {
+
+        Path droolsPath = new ClassPathResource("drools").getFile().toPath();
+        Files.walk(droolsPath).filter(DroolsUpToDateTest::isEntityRuleFile).forEach(this::validateFile);
+    }
+
+
+    private static boolean isEntityRuleFile(Path droolsFile) {
+
+        String fileName = droolsFile.getFileName().toString();
+        return fileName.endsWith(".drl") && !fileName.endsWith("_components.drl") && !fileName.endsWith("_OLD.drl");
+    }
+
+
+    private void validateFile(Path path) {
+
+        log.info(path.toFile().getAbsolutePath());
+        RuleFileBluePrint allRules = RuleFileParser.buildBluePrintFromAllRuleFiles();
+        RuleFileBluePrint thisRules = RuleFileParser.buildBluePrintFromRulesString(RuleFileIO.getRulesString(path.toFile().getAbsolutePath()));
+        assertTrue(allRules.getAllRuleIdentifiers().containsAll(thisRules.getAllRuleIdentifiers()));
+        for (BasicRule rule : thisRules.getAllRules()) {
+            List<BasicRule> updatedRule = allRules.findRuleByIdentifier(rule.identifier());
+            assert updatedRule.size() == 1;
+            assertEquals(updatedRule.get(0), rule);
+        }
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesEnd2EndTest.java

@@ -17,6 +17,7 @@ import java.util.Set;
 import java.util.UUID;
 
 import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.extension.ExtendWith;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -31,14 +32,15 @@ import org.springframework.context.annotation.Import;
 import org.springframework.context.annotation.Primary;
 import org.springframework.test.context.junit.jupiter.SpringExtension;
 
+import com.iqser.red.commons.jackson.ObjectMapperFactory;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeResult;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Comment;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;
@@ -49,9 +51,9 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Point;
 import com.iqser.red.service.redaction.v1.server.AbstractRedactionIntegrationTest;
 import com.iqser.red.service.redaction.v1.server.Application;
-import com.iqser.red.service.redaction.v1.server.FileSystemBackedStorageService;
 import com.iqser.red.service.redaction.v1.server.annotate.AnnotateRequest;
 import com.iqser.red.service.redaction.v1.server.annotate.AnnotateResponse;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
@@ -63,12 +65,14 @@ import com.iqser.red.service.redaction.v1.server.service.document.EntityCreation
 import com.iqser.red.service.redaction.v1.server.service.document.EntityEnrichmentService;
 import com.iqser.red.storage.commons.StorageAutoConfiguration;
 import com.iqser.red.storage.commons.service.StorageService;
+import com.iqser.red.storage.commons.utils.FileSystemBackedStorageService;
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
 import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingServiceProcessorConfiguration;
 import com.knecon.fforesight.tenantcommons.TenantContext;
 
 import lombok.SneakyThrows;
 
+@Disabled // TODO: only disabled due to OOM in gradle testRunner in GitLab, find a solution!
 @ExtendWith(SpringExtension.class)
 @SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
 @Import(ManualChangesEnd2EndTest.TestConfiguration.class)
@@ -91,7 +95,7 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest {
         @Primary
         public StorageService inmemoryStorage() {
 
-            return new FileSystemBackedStorageService();
+            return new FileSystemBackedStorageService(ObjectMapperFactory.create());
         }
 
     }
@@ -158,7 +162,8 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest {
 
         String idToResize = redactionLog.getEntityLogEntry()
                 .stream()
-                .filter(entry -> entry.getValue().equals(testEntityValue1)).max(Comparator.comparingInt(EntityLogEntry::getStartOffset))
+                .filter(entry -> entry.getValue().equals(testEntityValue1))
+                .max(Comparator.comparingInt(EntityLogEntry::getStartOffset))
                 .get()
                 .getId();
         List<Rectangle> resizedPositions = expandedEntity.getPositionsOnPagePerPage()
@@ -173,8 +178,9 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest {
                 .positions(resizedPositions)
                 .requestDate(OffsetDateTime.now())
                 .status(AnnotationStatus.APPROVED)
+                .updateDictionary(false)
                 .build();
-
+        manualResizeRedaction.setUpdateDictionary(false);
         ManualRedactions manualRedactions = new ManualRedactions();
         manualRedactions.getResizeRedactions().add(manualResizeRedaction);
         request.setManualRedactions(manualRedactions);
@@ -216,7 +222,6 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest {
 
         String manualAddId = UUID.randomUUID().toString();
 
-        Comment comment = Comment.builder().date(OffsetDateTime.now()).user("TEST_USER").text("This is a comment test").build();
         manualRedactions.setIdsToRemove(Set.of(IdRemoval.builder().annotationId("5b940b2cb401ed9f5be6fc24f6e77bcf").fileId("fileId").status(AnnotationStatus.DECLINED).build()));
         manualRedactions.setForceRedactions(Set.of(ManualForceRedaction.builder()
                 .annotationId("675eba69b0c2917de55462c817adaa05")
@@ -225,10 +230,6 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest {
                 .status(AnnotationStatus.APPROVED)
                 .build()));
 
-        manualRedactions.getComments().put("e5be0f1d941bbb92a068e198648d06c4", List.of(comment));
-        manualRedactions.getComments().put("0836727c3508a0b2ea271da69c04cc2f", List.of(comment));
-        manualRedactions.getComments().put(manualAddId, List.of(comment));
-
         ManualRedactionEntry manualRedactionEntry = new ManualRedactionEntry();
         manualRedactionEntry.setAnnotationId(manualAddId);
         manualRedactionEntry.setFileId("fileId");
@@ -334,4 +335,46 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest {
         assertEquals(1, oxfordUniversityPressRecategorized.getManualChanges().size());
     }
 
+
+    @Test
+    @SneakyThrows
+    public void testShrinkingResize() {
+
+        String filePath = "files/new/crafted document.pdf";
+        AnalyzeRequest request = uploadFileToStorage(filePath);
+        analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request);
+        AnalyzeResult result = analyzeService.analyze(request);
+        ManualRedactions manualRedactions = new ManualRedactions();
+        String annotationId = "testAnnotationId";
+
+        manualRedactions.setEntriesToAdd(Set.of(ManualRedactionEntry.builder()
+                .annotationId(annotationId)
+                .requestDate(OffsetDateTime.now())
+                .value("Expand to Hint Clarissa’s Donut ← not added to Dict, should be not annotated Simpson's Tower ← added to Authors-Dict, should be annotated")
+                .positions(List.of(//
+                        new Rectangle(new Point(56.8f, 496.27f), 61.25f, 12.83f, 2), //
+                        new Rectangle(new Point(56.8f, 482.26f), 303.804f, 15.408f, 2), //
+                        new Rectangle(new Point(56.8f, 468.464f), 314.496f, 15.408f, 2))) //
+                .status(AnnotationStatus.APPROVED)
+                .build()));
+        ManualResizeRedaction manualResizeRedaction = ManualResizeRedaction.builder()
+                .annotationId(annotationId)
+                .requestDate(OffsetDateTime.now())
+                .value("Expand to Hint")
+                .positions(List.of(new Rectangle(new Point(56.8f, 496.27f), 61.25f, 12.83f, 2)))
+                .status(AnnotationStatus.APPROVED)
+                .updateDictionary(false)
+                .build();
+        manualRedactions.setResizeRedactions(Set.of(manualResizeRedaction));
+        request.setManualRedactions(manualRedactions);
+
+        analyzeService.reanalyze(request);
+
+        EntityLog entityLog = redactionStorageService.getEntityLog(request.getDossierId(), request.getFileId());
+        EntityLogEntry entityLogEntry = entityLog.getEntityLogEntry().stream().filter(entry -> entry.getId().equals(annotationId)).findFirst().orElseThrow();
+        assertEquals("Expand to Hint", entityLogEntry.getValue());
+        assertEquals(1, entityLogEntry.getPositions().size());
+        assertEquals(ManualRedactionType.RESIZE, entityLogEntry.getManualChanges().get(entityLogEntry.getManualChanges().size() - 1).getManualRedactionType());
+    }
+
 }

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesIntegrationTest.java

@@ -5,99 +5,27 @@ import static org.junit.jupiter.api.Assertions.assertTrue;
 import static org.wildfly.common.Assert.assertFalse;
 
 import java.awt.geom.Rectangle2D;
-import java.io.FileOutputStream;
 import java.time.OffsetDateTime;
 import java.util.Collection;
-import java.util.Comparator;
 import java.util.List;
 import java.util.Set;
 import java.util.stream.Collectors;
 
-import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
-import org.kie.api.KieServices;
-import org.kie.api.builder.KieBuilder;
-import org.kie.api.builder.KieFileSystem;
-import org.kie.api.builder.KieModule;
-import org.kie.api.runtime.KieContainer;
-import org.kie.api.runtime.KieSession;
-import org.kie.internal.io.ResourceFactory;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.beans.factory.annotation.Qualifier;
-import org.springframework.context.annotation.Bean;
-import org.springframework.context.annotation.Configuration;
-import org.springframework.context.annotation.Import;
 
-import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeResult;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
-import com.iqser.red.service.redaction.v1.server.annotate.AnnotateRequest;
-import com.iqser.red.service.redaction.v1.server.annotate.AnnotateResponse;
-import com.iqser.red.service.redaction.v1.server.document.graph.BuildDocumentIntegrationTest;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
-import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.PositionOnPage;
+import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Paragraph;
-import com.iqser.red.service.redaction.v1.server.redaction.utils.OsUtils;
-import com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService;
-import com.iqser.red.service.redaction.v1.server.service.document.EntityEnrichmentService;
-import com.iqser.red.service.redaction.v1.server.service.ManualChangesApplicationService;
-import com.iqser.red.service.redaction.v1.server.utils.exception.NotFoundException;
-import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
-
-import lombok.SneakyThrows;
-
-@Import(ManualChangesIntegrationTest.TestConfiguration.class)
-public class ManualChangesIntegrationTest extends BuildDocumentIntegrationTest {
-
-    private static final String RULES = "drools/manual_redaction_rules.drl";
-
-    @Autowired
-    private EntityEnrichmentService entityEnrichmentService;
-    private EntityCreationService entityCreationService;
-    private KieSession kieSession;
-
-    @Qualifier("kieContainer")
-    @Autowired
-    private KieContainer kieContainer;
-
-    @Configuration
-    @Import(BuildDocumentIntegrationTest.TestConfiguration.class)
-    public static class TestConfiguration {
-
-        @Bean
-        public KieContainer kieContainer() {
-
-            KieServices kieServices = KieServices.Factory.get();
-
-            KieFileSystem kieFileSystem = kieServices.newKieFileSystem();
-            kieFileSystem.write(ResourceFactory.newClassPathResource(RULES));
-            KieBuilder kieBuilder = kieServices.newKieBuilder(kieFileSystem);
-            kieBuilder.buildAll();
-            KieModule kieModule = kieBuilder.getKieModule();
-
-            return kieServices.newKieContainer(kieModule.getReleaseId());
-        }
-
-    }
-
-
-    @BeforeEach
-    public void createServices() {
-
-        kieSession = kieContainer.newKieSession();
-        entityCreationService = new EntityCreationService(entityEnrichmentService, kieSession);
-        ManualChangesApplicationService manualChangesApplicationService = new ManualChangesApplicationService(entityCreationService);
-        kieSession.setGlobal("manualChangesApplicationService", manualChangesApplicationService);
-        kieSession.setGlobal("entityCreationService", entityCreationService);
-    }
+import com.iqser.red.service.redaction.v1.server.rules.RulesIntegrationTest;
 
+public class ManualChangesIntegrationTest extends RulesIntegrationTest {
 
     @Test
     public void manualResizeRedactionTest() {
@@ -117,14 +45,10 @@ public class ManualChangesIntegrationTest extends BuildDocumentIntegrationTest {
                 .positions(toAnnotationRectangles(biggerEntity.getPositionsOnPagePerPage().get(0)))
                 .status(AnnotationStatus.APPROVED)
                 .requestDate(OffsetDateTime.now())
+                .updateDictionary(false)
                 .build();
 
-        
-        kieSession.insert(document);
-        document.streamAllSubNodes().forEach(kieSession::insert);
-        kieSession.insert(manualResizeRedaction);
-        kieSession.fireAllRules();
-        kieSession.dispose();
+        doAnalysis(document, List.of(manualResizeRedaction));
 
         assertEquals(biggerEntity.getTextRange(), entity.getTextRange());
         assertEquals(biggerEntity.getDeepestFullyContainingNode(), entity.getDeepestFullyContainingNode());
@@ -153,12 +77,7 @@ public class ManualChangesIntegrationTest extends BuildDocumentIntegrationTest {
                 .requestDate(OffsetDateTime.now())
                 .build();
 
-        
-        kieSession.insert(manualForceRedaction);
-        kieSession.insert(document);
-        document.streamAllSubNodes().forEach(kieSession::insert);
-        kieSession.fireAllRules();
-        kieSession.dispose();
+        doAnalysis(document, List.of(manualForceRedaction));
 
         assertEquals(Paragraph.class, entity.getDeepestFullyContainingNode().getClass());
         assertFalse(entity.getIntersectingNodes().isEmpty());
@@ -183,12 +102,7 @@ public class ManualChangesIntegrationTest extends BuildDocumentIntegrationTest {
         String initialId = entity.getPositionsOnPagePerPage().get(0).getId();
         IdRemoval idRemoval = IdRemoval.builder().annotationId(initialId).status(AnnotationStatus.APPROVED).requestDate(OffsetDateTime.now()).build();
 
-        
-        kieSession.insert(document);
-        document.streamAllSubNodes().forEach(kieSession::insert);
-        kieSession.insert(idRemoval);
-        kieSession.fireAllRules();
-        kieSession.dispose();
+        doAnalysis(document, List.of(idRemoval));
 
         assertEquals("David Ksenia", entity.getValue());
         assertEquals(initialId, entity.getPositionsOnPagePerPage().get(0).getId());
@@ -213,13 +127,7 @@ public class ManualChangesIntegrationTest extends BuildDocumentIntegrationTest {
                 .requestDate(OffsetDateTime.now())
                 .build();
 
-        
-        kieSession.insert(document);
-        document.streamAllSubNodes().forEach(kieSession::insert);
-        kieSession.insert(idRemoval);
-        kieSession.insert(manualForceRedaction);
-        kieSession.fireAllRules();
-        kieSession.dispose();
+        doAnalysis(document, List.of(manualForceRedaction));
 
         assertEquals(Paragraph.class, entity.getDeepestFullyContainingNode().getClass());
         assertFalse(entity.getIntersectingNodes().isEmpty());
@@ -242,12 +150,7 @@ public class ManualChangesIntegrationTest extends BuildDocumentIntegrationTest {
         String initialId = entity.getPositionsOnPagePerPage().get(0).getId();
         IdRemoval idRemoval = IdRemoval.builder().annotationId(initialId).status(AnnotationStatus.REQUESTED).build();
 
-        
-        kieSession.insert(idRemoval);
-        kieSession.insert(document);
-        document.streamAllSubNodes().forEach(kieSession::insert);
-        kieSession.fireAllRules();
-        kieSession.dispose();
+        doAnalysis(document, List.of(idRemoval));
 
         assertEquals(Paragraph.class, entity.getDeepestFullyContainingNode().getClass());
         assertFalse(entity.getIntersectingNodes().isEmpty());
@@ -287,7 +190,7 @@ public class ManualChangesIntegrationTest extends BuildDocumentIntegrationTest {
 
     private static Rectangle toAnnotationRectangle(Rectangle2D rectangle2D, Integer number) {
 
-        return new Rectangle((float) rectangle2D.getMaxX(), (float) rectangle2D.getMaxY(), (float) rectangle2D.getWidth(), (float) rectangle2D.getHeight(), number);
+        return new Rectangle((float) rectangle2D.getMinX(), (float) rectangle2D.getMinY(), (float) rectangle2D.getWidth(), (float) rectangle2D.getHeight(), number);
     }
 
 }

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualEntityTest.java

@@ -7,6 +7,7 @@ import static org.wildfly.common.Assert.assertFalse;
 import static org.wildfly.common.Assert.assertTrue;
 
 import java.awt.geom.Rectangle2D;
+import java.time.OffsetDateTime;
 import java.util.List;
 import java.util.Set;
 
@@ -19,12 +20,14 @@ import org.springframework.boot.test.mock.mockito.MockBean;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Point;
 import com.iqser.red.service.redaction.v1.server.document.graph.BuildDocumentIntegrationTest;
 import com.iqser.red.service.redaction.v1.server.model.ManualEntity;
 import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryVersion;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
+import com.iqser.red.service.redaction.v1.server.model.document.entity.IEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
 import com.iqser.red.service.redaction.v1.server.service.DictionaryService;
@@ -35,7 +38,7 @@ import com.iqser.red.service.redaction.v1.server.service.document.ManualEntityCr
 
 import lombok.SneakyThrows;
 
-public class ManualEntityCreationServiceTest extends BuildDocumentIntegrationTest {
+public class ManualEntityTest extends BuildDocumentIntegrationTest {
 
     @Autowired
     private EntityEnrichmentService entityEnrichmentService;
@@ -63,39 +66,7 @@ public class ManualEntityCreationServiceTest extends BuildDocumentIntegrationTes
     @SneakyThrows
     public void manualAddRedactionTest() {
 
-        Document document = buildGraph("files/new/VV-919901.pdf");
-        EntityCreationService entityCreationService = new EntityCreationService(entityEnrichmentService);
-
-        List<TextEntity> tempEntities = entityCreationService.byString("To: Syngenta Ltd.", "temp", EntityType.ENTITY, document).toList();
-        assertFalse(tempEntities.isEmpty());
-        var tempEntity = tempEntities.get(0);
-        List<Rectangle> positions = tempEntity.getPositionsOnPagePerPage()
-                .stream()
-                .flatMap(redactionPosition -> redactionPosition.getRectanglePerLine()
-                        .stream()
-                        .map(rectangle2D -> toAnnotationRectangle(rectangle2D, redactionPosition.getPage().getNumber())))
-                .toList();
-
-        ManualRedactionEntry manualRedactionEntry = ManualRedactionEntry.builder()
-                .type("manual")
-                .value(tempEntity.getValue())
-                .reason("reason")
-                .legalBasis("n-a")
-                .section(tempEntity.getDeepestFullyContainingNode().toString())
-                .rectangle(true)
-                .positions(positions)
-                .textAfter("")
-                .textBefore("")
-                .build();
-
-        tempEntity.removeFromGraph();
-        assertTrue(document.getEntities().isEmpty());
-
-        List<ManualEntity> notFoundManualEntities = manualEntityCreationService.createRedactionEntitiesIfFoundAndReturnNotFoundEntries(Set.of(manualRedactionEntry),
-                document,
-                TEST_DOSSIER_TEMPLATE_ID);
-        assertTrue(notFoundManualEntities.isEmpty());
-        assertEquals(1, document.getEntities().size());
+        createFoundManualRedaction();
     }
 
 
@@ -103,6 +74,34 @@ public class ManualEntityCreationServiceTest extends BuildDocumentIntegrationTes
     @SneakyThrows
     public void manualAddRedactionFailingTest() {
 
+        createNotFoundManualRedaction();
+    }
+
+
+    @Test
+    @SneakyThrows
+    public void testFoundManualAddRedactionAndRemovedHasStateRemoved() {
+
+        DocumentAndEntity context = createFoundManualRedaction();
+        IdRemoval removal = IdRemoval.builder().requestDate(OffsetDateTime.now()).build();
+        context.entity().getManualOverwrite().addChange(removal);
+        assertTrue(context.entity().removed());
+    }
+
+
+    @Test
+    @SneakyThrows
+    public void testNotFoundManualAddRedactionAndRemovedHasStateRemoved() {
+
+        DocumentAndEntity context = createNotFoundManualRedaction();
+        IdRemoval removal = IdRemoval.builder().requestDate(OffsetDateTime.now()).build();
+        context.entity().getManualOverwrite().addChange(removal);
+        assertTrue(context.entity().removed());
+    }
+
+
+    private DocumentAndEntity createNotFoundManualRedaction() {
+
         Document document = buildGraph("files/new/VV-919901.pdf");
         // This is important due to PDFTron Web Viewer reordering the content, such that this string is selectable.
         String value = "To: Syngenta Ltd. Jealott’s Hill";
@@ -115,6 +114,7 @@ public class ManualEntityCreationServiceTest extends BuildDocumentIntegrationTes
                 .section("n-a")
                 .rectangle(true)
                 .positions(List.of(new Rectangle(new Point(90, 262), 11, 88, 1), new Rectangle(new Point(90, 247), 11, 131, 1)))
+                .requestDate(OffsetDateTime.now())
                 .textAfter("")
                 .textBefore("")
                 .build();
@@ -136,6 +136,47 @@ public class ManualEntityCreationServiceTest extends BuildDocumentIntegrationTes
         assertEquals(1, redactionLogEntries.size());
         assertEquals(value, redactionLogEntries.get(0).getValue());
         assertEquals(type, redactionLogEntries.get(0).getType());
+        return new DocumentAndEntity(document, notFoundManualEntities.get(0));
+    }
+
+
+    private DocumentAndEntity createFoundManualRedaction() {
+
+        Document document = buildGraph("files/new/VV-919901.pdf");
+        EntityCreationService entityCreationService = new EntityCreationService(entityEnrichmentService);
+
+        List<TextEntity> tempEntities = entityCreationService.byString("To: Syngenta Ltd.", "temp", EntityType.ENTITY, document).toList();
+        assertFalse(tempEntities.isEmpty());
+        var tempEntity = tempEntities.get(0);
+        List<Rectangle> positions = tempEntity.getPositionsOnPagePerPage()
+                .stream()
+                .flatMap(redactionPosition -> redactionPosition.getRectanglePerLine()
+                        .stream()
+                        .map(rectangle2D -> toAnnotationRectangle(rectangle2D, redactionPosition.getPage().getNumber())))
+                .toList();
+
+        ManualRedactionEntry manualRedactionEntry = ManualRedactionEntry.builder()
+                .type("manual")
+                .value(tempEntity.getValue())
+                .reason("reason")
+                .legalBasis("n-a")
+                .section(tempEntity.getDeepestFullyContainingNode().toString())
+                .rectangle(true)
+                .positions(positions)
+                .requestDate(OffsetDateTime.now())
+                .textAfter("")
+                .textBefore("")
+                .build();
+
+        tempEntity.removeFromGraph();
+        assertTrue(document.getEntities().isEmpty());
+
+        List<ManualEntity> notFoundManualEntities = manualEntityCreationService.createRedactionEntitiesIfFoundAndReturnNotFoundEntries(Set.of(manualRedactionEntry),
+                document,
+                TEST_DOSSIER_TEMPLATE_ID);
+        assertTrue(notFoundManualEntities.isEmpty());
+        assertEquals(1, document.getEntities().size());
+        return new DocumentAndEntity(document, document.getEntities().stream().findFirst().get());
     }
 
 
@@ -147,4 +188,9 @@ public class ManualEntityCreationServiceTest extends BuildDocumentIntegrationTes
                 pageNumber);
     }
 
+
+    private record DocumentAndEntity(Document document, IEntity entity) {
+
+    }
+
 }

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/realdata/LiveDataIntegrationTest.java

@@ -37,13 +37,13 @@ import org.springframework.test.context.junit.jupiter.SpringExtension;
 
 import com.fasterxml.jackson.core.type.TypeReference;
 import com.fasterxml.jackson.databind.ObjectMapper;
+import com.iqser.red.commons.jackson.ObjectMapperFactory;
 import com.iqser.red.service.dictionarymerge.commons.DictionaryEntry;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.configuration.Colors;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type;
 import com.iqser.red.service.redaction.v1.server.Application;
-import com.iqser.red.service.redaction.v1.server.FileSystemBackedStorageService;
 import com.iqser.red.service.redaction.v1.server.RedactionServiceSettings;
 import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
 import com.iqser.red.service.redaction.v1.server.client.FileStatusProcessingUpdateClient;
@@ -55,6 +55,7 @@ import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService
 import com.iqser.red.service.redaction.v1.server.utils.ExceptionProvider;
 import com.iqser.red.storage.commons.StorageAutoConfiguration;
 import com.iqser.red.storage.commons.service.StorageService;
+import com.iqser.red.storage.commons.utils.FileSystemBackedStorageService;
 import com.knecon.fforesight.tenantcommons.TenantContext;
 import com.knecon.fforesight.tenantcommons.TenantsClient;
 
@@ -105,9 +106,6 @@ public class LiveDataIntegrationTest {
     @Autowired
     protected RedactionMessageReceiver redactionMessageReceiver;
 
-    @Autowired
-    protected FileSystemBackedStorageService fileSystemBackedStorageService;
-
     @Autowired
     protected RedactionServiceSettings redactionServiceSettings;
 
@@ -125,7 +123,7 @@ public class LiveDataIntegrationTest {
         @Primary
         public StorageService inmemoryStorage() {
 
-            return new FileSystemBackedStorageService();
+            return new FileSystemBackedStorageService(ObjectMapperFactory.create());
         }
 
     }

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/rules/Cbi11Test.java

@@ -0,0 +1,36 @@
+package com.iqser.red.service.redaction.v1.server.rules;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.wildfly.common.Assert.assertFalse;
+
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+
+import org.junit.jupiter.api.Test;
+
+import com.iqser.red.service.redaction.v1.server.model.dictionary.Dictionary;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
+import com.iqser.red.service.redaction.v1.server.utils.Patterns;
+
+public class Cbi11Test extends RulesIntegrationTest {
+
+    @Test
+    public void multiAuthorNameSplittingTest() {
+
+        Document document = buildGraph("files/Minimal Examples/multipleAuthorsInTable.pdf");
+
+        doAnalysis(document, Collections.emptyList());
+
+        List<String> authorNames = document.getEntities().stream().map(Dictionary::splitIntoAuthorNames).flatMap(Collection::stream).toList();
+
+        assertThat(authorNames).containsExactlyInAnyOrder("Cargile", "N.L.", "Ross", "J.A.", "Egli", "Ramsteiner");
+    }
+
+    @Test
+    public void testAndIsNotAnAuthorName() {
+
+        assertFalse(Patterns.AUTHOR_NAME_PATTERN.matcher("and").matches());
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/rules/RulesIntegrationTest.java

@@ -0,0 +1,80 @@
+package com.iqser.red.service.redaction.v1.server.rules;
+
+import java.util.Collection;
+
+import org.junit.jupiter.api.BeforeEach;
+import org.kie.api.KieServices;
+import org.kie.api.builder.KieBuilder;
+import org.kie.api.builder.KieFileSystem;
+import org.kie.api.builder.KieModule;
+import org.kie.api.runtime.KieContainer;
+import org.kie.api.runtime.KieSession;
+import org.kie.internal.io.ResourceFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.context.annotation.Import;
+
+import com.iqser.red.service.redaction.v1.server.document.graph.BuildDocumentIntegrationTest;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
+import com.iqser.red.service.redaction.v1.server.service.ManualChangesApplicationService;
+import com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService;
+import com.iqser.red.service.redaction.v1.server.service.document.EntityEnrichmentService;
+
+public class RulesIntegrationTest extends BuildDocumentIntegrationTest {
+
+    protected static final String RULES = "drools/rules.drl";
+
+    @Autowired
+    protected EntityEnrichmentService entityEnrichmentService;
+    protected EntityCreationService entityCreationService;
+    protected KieSession kieSession;
+
+    @Qualifier("kieContainer")
+    @Autowired
+    private KieContainer kieContainer;
+
+
+    protected void doAnalysis(Document document, Collection<Object> objectToInsert) {
+
+        kieSession.insert(document);
+        document.streamAllSubNodes().forEach(kieSession::insert);
+        objectToInsert.forEach(kieSession::insert);
+        kieSession.fireAllRules();
+        kieSession.dispose();
+    }
+
+
+    @Configuration
+    @Import(BuildDocumentIntegrationTest.TestConfiguration.class)
+    public static class TestConfiguration {
+
+        @Bean
+        public KieContainer kieContainer() {
+
+            KieServices kieServices = KieServices.Factory.get();
+
+            KieFileSystem kieFileSystem = kieServices.newKieFileSystem();
+            kieFileSystem.write(ResourceFactory.newClassPathResource(RULES));
+            KieBuilder kieBuilder = kieServices.newKieBuilder(kieFileSystem);
+            kieBuilder.buildAll();
+            KieModule kieModule = kieBuilder.getKieModule();
+
+            return kieServices.newKieContainer(kieModule.getReleaseId());
+        }
+
+    }
+
+
+    @BeforeEach
+    public void createServices() {
+
+        kieSession = kieContainer.newKieSession();
+        entityCreationService = new EntityCreationService(entityEnrichmentService, kieSession);
+        ManualChangesApplicationService manualChangesApplicationService = new ManualChangesApplicationService(entityCreationService);
+        kieSession.setGlobal("manualChangesApplicationService", manualChangesApplicationService);
+        kieSession.setGlobal("entityCreationService", entityCreationService);
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/service/document/UnprocessedChangesServiceTest.java

@@ -0,0 +1,176 @@
+package com.iqser.red.service.redaction.v1.server.service.document;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.Mockito.when;
+
+import java.util.List;
+import java.util.Optional;
+import java.util.UUID;
+
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
+import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration;
+import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.boot.test.mock.mockito.MockBean;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.ComponentScan;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.context.annotation.FilterType;
+import org.springframework.context.annotation.Import;
+import org.springframework.context.annotation.Primary;
+import org.springframework.test.context.junit.jupiter.SpringExtension;
+
+import com.iqser.red.commons.jackson.ObjectMapperFactory;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
+import com.iqser.red.service.redaction.v1.model.UnprocessedManualEntity;
+import com.iqser.red.service.redaction.v1.server.AbstractRedactionIntegrationTest;
+import com.iqser.red.service.redaction.v1.server.Application;
+import com.iqser.red.service.redaction.v1.server.RedactionIntegrationTest;
+import com.iqser.red.service.redaction.v1.server.service.DictionaryService;
+import com.iqser.red.service.redaction.v1.server.service.UnprocessedChangesService;
+import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
+import com.iqser.red.storage.commons.StorageAutoConfiguration;
+import com.iqser.red.storage.commons.service.StorageService;
+import com.iqser.red.storage.commons.utils.FileSystemBackedStorageService;
+import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
+import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingServiceProcessorConfiguration;
+
+@ExtendWith(SpringExtension.class)
+@SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
+@Import(RedactionIntegrationTest.RedactionIntegrationTestConfiguration.class)
+public class UnprocessedChangesServiceTest extends AbstractRedactionIntegrationTest {
+
+    @Configuration
+    @EnableAutoConfiguration(exclude = {RabbitAutoConfiguration.class})
+    @Import({LayoutParsingServiceProcessorConfiguration.class})
+    @ComponentScan(excludeFilters = {@ComponentScan.Filter(type = FilterType.ASSIGNABLE_TYPE, value = StorageAutoConfiguration.class)})
+    public static class RedactionIntegrationTestConfiguration {
+
+        @Bean
+        @Primary
+        public StorageService inmemoryStorage() {
+
+            return new FileSystemBackedStorageService(ObjectMapperFactory.create());
+        }
+
+    }
+
+    @Autowired
+    RedactionStorageService redactionStorageService;
+
+    @MockBean
+    DictionaryService dictionaryService;
+
+    @Autowired
+    UnprocessedChangesService unprocessedChangesService;
+
+    @Test
+    public void testManualSurroundingText() {
+
+        String pdfFile = "files/new/S4.pdf";
+        when(dictionaryService.getColor(DICTIONARY_AUTHOR, TEST_DOSSIER_TEMPLATE_ID)).thenReturn(new float[]{0f, 0f, 0f});
+        when(dictionaryService.getNotRedactedColor(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(new float[]{0f, 0f, 0f});
+
+        ManualRedactions manualRedactions = new ManualRedactions();
+
+        ManualRedactionEntry manualRedactionEntry = new ManualRedactionEntry();
+        manualRedactionEntry.setAnnotationId(UUID.randomUUID().toString());
+        manualRedactionEntry.setFileId("fileId");
+        manualRedactionEntry.setStatus(AnnotationStatus.APPROVED);
+        manualRedactionEntry.setType("CBI_author");
+        manualRedactionEntry.setValue("rabbits");
+        manualRedactionEntry.setReason("Manual Redaction");
+        manualRedactionEntry.setPositions(List.of(Rectangle.builder().topLeftX(70.944f).topLeftY(670.1595f).width(30.07296f).height(10.048125f).page(1).build()));
+
+        ManualRedactionEntry manualRedactionEntry2 = new ManualRedactionEntry();
+        manualRedactionEntry2.setAnnotationId(UUID.randomUUID().toString());
+        manualRedactionEntry2.setFileId("fileId");
+        manualRedactionEntry2.setStatus(AnnotationStatus.APPROVED);
+        manualRedactionEntry2.setType("CBI_author");
+        manualRedactionEntry2.setValue("rabbits");
+        manualRedactionEntry2.setReason("Manual Redaction");
+        manualRedactionEntry2.setPositions(List.of(Rectangle.builder().topLeftX(470.5204f).topLeftY(746.1195f).width(29.96256f).height(10.048125f).page(1).build()));
+
+        var aoelId = UUID.randomUUID().toString();
+        ManualRedactionEntry manualRedactionEntry3 = new ManualRedactionEntry();
+        manualRedactionEntry3.setAnnotationId(aoelId);
+        manualRedactionEntry3.setFileId("fileId");
+        manualRedactionEntry3.setStatus(AnnotationStatus.APPROVED);
+        manualRedactionEntry3.setType("CBI_author");
+        manualRedactionEntry3.setValue("AOEL");
+        manualRedactionEntry3.setReason("Manual Redaction");
+        manualRedactionEntry3.setPositions(List.of(Rectangle.builder().topLeftX(355.53775f).topLeftY(266.1895f).width(29.32224f).height(10.048125f).page(1).build()));
+
+        var notFoundId = UUID.randomUUID().toString();
+        ManualRedactionEntry manualRedactionEntry4 = new ManualRedactionEntry();
+        manualRedactionEntry4.setAnnotationId(notFoundId);
+        manualRedactionEntry4.setFileId("fileId");
+        manualRedactionEntry4.setStatus(AnnotationStatus.APPROVED);
+        manualRedactionEntry4.setType("CBI_author");
+        manualRedactionEntry4.setValue("Random");
+        manualRedactionEntry4.setReason("Manual Redaction");
+        manualRedactionEntry4.setPositions(List.of(Rectangle.builder().topLeftX(1f).topLeftY(1f).width(1f).height(1f).page(1).build()));
+
+        manualRedactions.getEntriesToAdd().add(manualRedactionEntry);
+        manualRedactions.getEntriesToAdd().add(manualRedactionEntry2);
+        manualRedactions.getEntriesToAdd().add(manualRedactionEntry3);
+        manualRedactions.getEntriesToAdd().add(manualRedactionEntry4);
+
+        AnalyzeRequest request = uploadFileToStorage(pdfFile);
+        request.setManualRedactions(manualRedactions);
+        analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request);
+
+        List<UnprocessedManualEntity> unprocessedManualEntities = unprocessedChangesService.addInformationToUnprocessedManualEntities(TEST_FILE_ID, TEST_DOSSIER_ID, TEST_DOSSIER_TEMPLATE_ID, manualRedactions);
+
+        assertFalse(unprocessedManualEntities.isEmpty());
+        assertEquals(unprocessedManualEntities.size(), 4);
+
+        Optional<UnprocessedManualEntity> optionalUnprocessedManualEntity = unprocessedManualEntities.stream().filter(manualEntity -> manualEntity.getAnnotationId().equals(aoelId)).findFirst();
+        assertTrue(optionalUnprocessedManualEntity.isPresent());
+
+        UnprocessedManualEntity unprocessedManualEntity = optionalUnprocessedManualEntity.get();
+        assertEquals(unprocessedManualEntity.getTextBefore(), "was above the ");
+        assertEquals(unprocessedManualEntity.getTextAfter(), " without PPE (34%");
+        assertEquals(unprocessedManualEntity.getStartOffset(), 2766);
+        assertEquals(unprocessedManualEntity.getEndOffset(), 2770);
+        assertEquals(unprocessedManualEntity.getClosestHeadline(), "2.6.14 Summary of product exposure and risk assessment ");
+        assertEquals(unprocessedManualEntity.getSection(), "[1, 1]: Paragraph: A9396G containing 960 g/L");
+        assertEquals(unprocessedManualEntity.getContainingNodeId().get(0), 1);
+        assertEquals(unprocessedManualEntity.getContainingNodeId().get(1), 1);
+        assertEquals(unprocessedManualEntity.getColor()[0], 0f);
+        assertEquals(unprocessedManualEntity.getColor()[1], 0f);
+        assertEquals(unprocessedManualEntity.getColor()[2], 0f);
+        assertEquals(unprocessedManualEntity.getPositions().get(0).x(), 355.53775f);
+        assertEquals(unprocessedManualEntity.getPositions().get(0).y(), 266.49002f);
+        assertEquals(unprocessedManualEntity.getPositions().get(0).w(), 29.322266f);
+        assertEquals(unprocessedManualEntity.getPositions().get(0).h(), 11.017679f);
+
+        Optional<UnprocessedManualEntity> optionalNotFoundUnprocessedManualEntity = unprocessedManualEntities.stream().filter(manualEntity -> manualEntity.getAnnotationId().equals(notFoundId)).findFirst();
+        assertTrue(optionalNotFoundUnprocessedManualEntity.isPresent());
+
+        UnprocessedManualEntity unprocessedNotFoundManualEntity = optionalNotFoundUnprocessedManualEntity.get();
+        assertEquals(unprocessedNotFoundManualEntity.getTextBefore(), "");
+        assertEquals(unprocessedNotFoundManualEntity.getTextAfter(), "");
+        assertEquals(unprocessedNotFoundManualEntity.getStartOffset(), -1);
+        assertEquals(unprocessedNotFoundManualEntity.getEndOffset(), -1);
+        assertEquals(unprocessedNotFoundManualEntity.getClosestHeadline(), "");
+        assertEquals(unprocessedNotFoundManualEntity.getSection(), "");
+        assertTrue(unprocessedNotFoundManualEntity.getContainingNodeId().isEmpty());
+        assertEquals(unprocessedNotFoundManualEntity.getColor()[0], 0f);
+        assertEquals(unprocessedNotFoundManualEntity.getColor()[1], 0f);
+        assertEquals(unprocessedNotFoundManualEntity.getColor()[2], 0f);
+        assertEquals(unprocessedNotFoundManualEntity.getPositions().get(0).getPageNumber(), 1);
+        assertEquals(unprocessedNotFoundManualEntity.getPositions().get(0).getRectangle()[0], 1f);
+        assertEquals(unprocessedNotFoundManualEntity.getPositions().get(0).getRectangle()[1], 1f);
+        assertEquals(unprocessedNotFoundManualEntity.getPositions().get(0).getRectangle()[2], 1f);
+        assertEquals(unprocessedNotFoundManualEntity.getPositions().get(0).getRectangle()[3], 1f);
+    }
+}

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/utils/LayoutParsingRequestProvider.java

@@ -31,7 +31,6 @@ public class LayoutParsingRequestProvider {
                 .textBlockFileStorageId(textBlockFileStorageId)
                 .positionBlockFileStorageId(positionBlockFileStorageId)
                 .pageFileStorageId(pageFileStorageId)
-                .sectionGridStorageId(sectionGridStorageId)
                 .simplifiedTextStorageId(simplifiedTextStorageId).viewerDocumentStorageId(viewerDocumentStorageId)
                 .build();
     }

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/acceptance_rules.drl

@@ -22,6 +22,7 @@ import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRu
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.*;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Section;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Table;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Paragraph;
@@ -81,25 +82,25 @@ rule "SYN.1.0: Recommend CTL/BL laboratory that start with BL or CTL"
 //------------------------------------ CBI rules ------------------------------------
 
 // Rule unit: CBI.0
-rule "CBI.0.0: Redact CBI Authors (Non Vertebrate Study)"
+rule "CBI.0.0: Redact CBI Authors (non vertebrate Study)"
     when
         not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
         $entity: TextEntity(type == "CBI_author", dictionaryEntry)
     then
-        $entity.apply("CBI.0.0", "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        $entity.redact("CBI.0.0", "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
     end
 
-rule "CBI.0.1: Redact CBI Authors (Vertebrate Study)"
+rule "CBI.0.1: Redact CBI Authors (vertebrate Study)"
     when
         FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
         $entity: TextEntity(type == "CBI_author", dictionaryEntry)
     then
-        $entity.apply("CBI.0.1", "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        $entity.redact("CBI.0.1", "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
     end
 
 
 // Rule unit: CBI.1
-rule "CBI.1.0: Don't redact CBI Address (Non Vertebrate Study)"
+rule "CBI.1.0: Do not redact CBI Address (non vertebrate Study)"
     when
         not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
         $entity: TextEntity(type == "CBI_address", dictionaryEntry)
@@ -107,19 +108,19 @@ rule "CBI.1.0: Don't redact CBI Address (Non Vertebrate Study)"
         $entity.skip("CBI.1.0", "Address found for Non Vertebrate Study");
     end
 
-rule "CBI.1.1: Redact CBI Address (Vertebrate Study)"
+rule "CBI.1.1: Redact CBI Address (vertebrate Study)"
     when
         FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
         $entity: TextEntity(type == "CBI_address", dictionaryEntry)
     then
-        $entity.apply("CBI.1.1", "Address found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        $entity.redact("CBI.1.1", "Address found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
     end
 
 
 // Rule unit: CBI.2
-rule "CBI.2.0: Don't redact genitive CBI_author"
+rule "CBI.2.0: Do not redact genitive CBI Author"
     when
-        $entity: TextEntity(type == "CBI_author", anyMatch(textAfter, "['’’'ʼˈ´`‘′ʻ’']s"), applied())
+        $entity: TextEntity(type == "CBI_author", anyMatch(textAfter, "['’’'ʼˈ´`‘′ʻ’']s"))
     then
         entityCreationService.byTextRange($entity.getTextRange(), "CBI_author", EntityType.FALSE_POSITIVE, document)
         .ifPresent(falsePositive -> falsePositive.skip("CBI.2.0", "Genitive Author found"));
@@ -127,7 +128,7 @@ rule "CBI.2.0: Don't redact genitive CBI_author"
 
 
 // Rule unit: CBI.7
-rule "CBI.7.0: Do not redact Names and Addresses if published information found in section without tables"
+rule "CBI.7.0: Do not redact Names and Addresses if published information found in Section without tables"
     when
         $section: Section(!hasTables(),
                           hasEntitiesOfType("published_information"),
@@ -145,22 +146,17 @@ rule "CBI.7.0: Do not redact Names and Addresses if published information found
 
 rule "CBI.7.1: Do not redact Names and Addresses if published information found in same table row"
     when
-        $table: Table(hasEntitiesOfType("published_information"),
-                     (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
+        $table: Table(hasEntitiesOfType("published_information"), hasEntitiesOfType("CBI_author"))
+        $cellsWithPublishedInformation: TableCell() from $table.streamTableCellsWhichContainType("published_information").toList()
+        $tableCell: TableCell(row == $cellsWithPublishedInformation.row) from $table.streamTableCells().toList()
+        $author: TextEntity(type == "CBI_author", active()) from $tableCell.getEntities()
     then
-        $table.streamEntitiesWhereRowContainsEntitiesOfType(List.of("CBI_author", "CBI_address"))
-            .forEach(redactionEntity -> {
-                redactionEntity.skipWithReferences(
-                        "CBI.7.1",
-                        "Published Information found in row",
-                        $table.getEntitiesOfTypeInSameRow("published_information", redactionEntity)
-                );
-            });
+        $author.skipWithReferences("CBI.7.1", "Published Information found in row", $table.getEntitiesOfTypeInSameRow("published_information", $author));
     end
 
 
 // Rule unit: CBI.9
-rule "CBI.9.0: Redact all Cell's with Header Author(s) as CBI_author (non vertebrate study)"
+rule "CBI.9.0: Redact all cells with Header Author(s) as CBI_author (non vertebrate study)"
     agenda-group "LOCAL_DICTIONARY_ADDS"
     when
         not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
@@ -170,10 +166,10 @@ rule "CBI.9.0: Redact all Cell's with Header Author(s) as CBI_author (non verteb
             .map(tableCell -> entityCreationService.bySemanticNode(tableCell, "CBI_author", EntityType.ENTITY))
             .filter(Optional::isPresent)
             .map(Optional::get)
-            .forEach(redactionEntity -> redactionEntity.apply("CBI.9.0", "Author(s) found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
+            .forEach(redactionEntity -> redactionEntity.redact("CBI.9.0", "Author(s) found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
     end
 
-rule "CBI.9.1: Redact all Cell's with Header Author as CBI_author (non vertebrate study)"
+rule "CBI.9.1: Redact all cells with Header Author as CBI_author (non vertebrate study)"
     agenda-group "LOCAL_DICTIONARY_ADDS"
     when
         not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
@@ -183,12 +179,12 @@ rule "CBI.9.1: Redact all Cell's with Header Author as CBI_author (non vertebrat
             .map(tableCell -> entityCreationService.bySemanticNode(tableCell, "CBI_author", EntityType.ENTITY))
             .filter(Optional::isPresent)
             .map(Optional::get)
-            .forEach(redactionEntity -> redactionEntity.apply("CBI.9.1", "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
+            .forEach(redactionEntity -> redactionEntity.redact("CBI.9.1", "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
     end
 
 
 // Rule unit: CBI.10
-rule "CBI.10.0: Redact all Cell's with Header Author(s) as CBI_author (vertebrate study)"
+rule "CBI.10.0: Redact all cells with Header Author(s) as CBI_author (vertebrate study)"
     agenda-group "LOCAL_DICTIONARY_ADDS"
     when
         FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
@@ -198,10 +194,10 @@ rule "CBI.10.0: Redact all Cell's with Header Author(s) as CBI_author (vertebrat
             .map(tableCell -> entityCreationService.bySemanticNode(tableCell, "CBI_author", EntityType.ENTITY))
             .filter(Optional::isPresent)
             .map(Optional::get)
-            .forEach(redactionEntity -> redactionEntity.apply("CBI.10.0", "Author(s) found", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
+            .forEach(redactionEntity -> redactionEntity.redact("CBI.10.0", "Author(s) found", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
     end
 
-rule "CBI.10.1: Redact all Cell's with Header Author as CBI_author (vertebrate study)"
+rule "CBI.10.1: Redact all cells with Header Author as CBI_author (vertebrate study)"
     agenda-group "LOCAL_DICTIONARY_ADDS"
     when
         FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
@@ -211,7 +207,7 @@ rule "CBI.10.1: Redact all Cell's with Header Author as CBI_author (vertebrate s
             .map(tableCell -> entityCreationService.bySemanticNode(tableCell, "CBI_author", EntityType.ENTITY))
             .filter(Optional::isPresent)
             .map(Optional::get)
-            .forEach(redactionEntity -> redactionEntity.apply("CBI.10.1", "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
+            .forEach(redactionEntity -> redactionEntity.redact("CBI.10.1", "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
     end
 
 
@@ -222,12 +218,12 @@ rule "CBI.11.0: Recommend all CBI_author entities in Table with Vertebrate Study
     when
         $table: Table(hasHeader("Author(s)") && hasHeader("Vertebrate Study Y/N"))
     then
-        $table.getEntitiesOfType("CBI_author").forEach(entity -> dictionary.addMultipleAuthorsAsRecommendation(entity));
+        $table.getEntitiesOfType("CBI_author").stream().filter(IEntity::applied).forEach(entity -> dictionary.addMultipleAuthorsAsRecommendation(entity));
     end
 
 
 // Rule unit: CBI.16
-rule "CBI.16.0: Add CBI_author with \"et al.\" Regex (non vertebrate study)"
+rule "CBI.16.0: Add CBI_author with \"et al.\" RegEx (non vertebrate study)"
     agenda-group "LOCAL_DICTIONARY_ADDS"
     when
         not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
@@ -235,12 +231,12 @@ rule "CBI.16.0: Add CBI_author with \"et al.\" Regex (non vertebrate study)"
     then
         entityCreationService.byRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, 1, $section)
             .forEach(entity -> {
-                entity.apply("CBI.16.0", "Author found by \"et al\" regex", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+                entity.redact("CBI.16.0", "Author found by \"et al\" regex", "Article 39(e)(3) of Regulation (EC) No 178/2002");
                 dictionary.recommendEverywhere(entity);
             });
     end
 
-rule "CBI.16.1: Add CBI_author with \"et al.\" Regex (vertebrate study)"
+rule "CBI.16.1: Add CBI_author with \"et al.\" RegEx (vertebrate study)"
     agenda-group "LOCAL_DICTIONARY_ADDS"
     when
         FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
@@ -248,7 +244,7 @@ rule "CBI.16.1: Add CBI_author with \"et al.\" Regex (vertebrate study)"
     then
         entityCreationService.byRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, 1, $section)
             .forEach(entity -> {
-                entity.apply("CBI.16.1", "Author found by \"et al\" regex", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+                entity.redact("CBI.16.1", "Author found by \"et al\" regex", "Article 39(e)(2) of Regulation (EC) No 178/2002");
                 dictionary.recommendEverywhere(entity);
             });
     end
@@ -294,7 +290,7 @@ rule "CBI.20.1: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJEC
     then
          entityCreationService.betweenStrings("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", EntityType.ENTITY, $section)
          .forEach(laboratoryEntity -> {
-             laboratoryEntity.apply("CBI.20.1", "PERFORMING LABORATORY was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+             laboratoryEntity.redact("CBI.20.1", "PERFORMING LABORATORY was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
              dictionary.recommendEverywhere(laboratoryEntity);
          });
     end
@@ -308,7 +304,7 @@ rule "PII.0.0: Redact all PII (non vertebrate study)"
         not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
         $pii: TextEntity(type == "PII", dictionaryEntry)
     then
-        $pii.apply("PII.0.0", "Personal Information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        $pii.redact("PII.0.0", "Personal Information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
     end
 
 rule "PII.0.1: Redact all PII (vertebrate study)"
@@ -316,7 +312,7 @@ rule "PII.0.1: Redact all PII (vertebrate study)"
         FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
         $pii: TextEntity(type == "PII", dictionaryEntry)
     then
-        $pii.apply("PII.0.1", "Personal Information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        $pii.redact("PII.0.1", "Personal Information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
     end
 
 
@@ -327,7 +323,7 @@ rule "PII.1.0: Redact Emails by RegEx (Non vertebrate study)"
         $section: Section(containsString("@"))
     then
         entityCreationService.byRegex("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", "PII", EntityType.ENTITY, 1, $section)
-            .forEach(emailEntity -> emailEntity.apply("PII.1.0", "Found by Email Regex", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
+            .forEach(emailEntity -> emailEntity.redact("PII.1.0", "Found by Email Regex", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
     end
 
 rule "PII.1.1: Redact Emails by RegEx (vertebrate study)"
@@ -336,7 +332,7 @@ rule "PII.1.1: Redact Emails by RegEx (vertebrate study)"
         $section: Section(containsString("@"))
     then
         entityCreationService.byRegex("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", "PII", EntityType.ENTITY, 1, $section)
-            .forEach(emailEntity -> emailEntity.apply("PII.1.1", "Found by Email Regex", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
+            .forEach(emailEntity -> emailEntity.redact("PII.1.1", "Found by Email Regex", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
     end
 
 
@@ -356,7 +352,7 @@ rule "PII.2.0: Redact Phone and Fax by RegEx (non vertebrate study)"
                               containsString("Fer"))
     then
         entityCreationService.byRegexIgnoreCase("\\b(contact|telephone|phone|ph\\.|fax|tel|ter|mobile|fel|fer)[a-zA-Z\\s]{0,10}[:.\\s]{0,3}([\\+\\d\\(][\\s\\d\\(\\)\\-\\/\\.]{4,100}\\d)\\b", "PII", EntityType.ENTITY, 2, $section)
-            .forEach(contactEntity -> contactEntity.apply("PII.2.0", "Found by Phone and Fax Regex", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
+            .forEach(contactEntity -> contactEntity.redact("PII.2.0", "Found by Phone and Fax Regex", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
     end
 
 rule "PII.2.1: Redact Phone and Fax by RegEx (vertebrate study)"
@@ -374,45 +370,27 @@ rule "PII.2.1: Redact Phone and Fax by RegEx (vertebrate study)"
                               containsString("Fer"))
     then
         entityCreationService.byRegexIgnoreCase("\\b(contact|telephone|phone|ph\\.|fax|tel|ter|mobile|fel|fer)[a-zA-Z\\s]{0,10}[:.\\s]{0,3}([\\+\\d\\(][\\s\\d\\(\\)\\-\\/\\.]{4,100}\\d)\\b", "PII", EntityType.ENTITY, 2, $section)
-            .forEach(contactEntity -> contactEntity.apply("PII.2.1", "Found by Phone and Fax Regex", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
+            .forEach(contactEntity -> contactEntity.redact("PII.2.1", "Found by Phone and Fax Regex", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
     end
 
 
 // Rule unit: PII.9
-rule "PII.9.0: Redact between \"AUTHOR(S)\" and \"COMPLETION DATE\" (non vertebrate study)"
+rule "PII.9.0: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\" (non vertebrate study)"
     when
         not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
-        $section: Section(!hasTables(), containsString("AUTHOR(S):"), containsString("COMPLETION DATE:"), !containsString("STUDY COMPLETION DATE:"))
+        $document: Document(containsStringIgnoreCase("AUTHOR(S)"), containsAnyStringIgnoreCase("COMPLETION DATE", "STUDY COMPLETION DATE"))
     then
-        entityCreationService.betweenStrings("AUTHOR(S):", "COMPLETION DATE:", "PII", EntityType.ENTITY, $section)
-         .forEach(authorEntity -> authorEntity.apply("PII.9.0", "AUTHOR(S) was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
+        entityCreationService.shortestBetweenAnyStringIgnoreCase(List.of("AUTHOR(S)", "AUTHOR(S):"), List.of("COMPLETION DATE", "COMPLETION DATE:", "STUDY COMPLETION DATE", "STUDY COMPLETION DATE:"), "PII", EntityType.ENTITY, $document)
+         .forEach(authorEntity -> authorEntity.redact("PII.9.0", "AUTHOR(S) was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
     end
 
-rule "PII.9.1: Redact between \"AUTHOR(S)\" and \"STUDY COMPLETION DATE\" (non vertebrate study)"
+rule "PII.9.1: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\" (vertebrate study)"
     when
         FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
-        $section: Section(!hasTables(), containsString("AUTHOR(S):"), containsString("COMPLETION DATE:"), !containsString("STUDY COMPLETION DATE:"))
+        $document: Document(containsStringIgnoreCase("AUTHOR(S)"), containsAnyStringIgnoreCase("COMPLETION DATE", "STUDY COMPLETION DATE"))
     then
-        entityCreationService.betweenStrings("AUTHOR(S):", "COMPLETION DATE:", "PII", EntityType.ENTITY, $section)
-         .forEach(authorEntity -> authorEntity.apply("PII.9.1", "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
-    end
-
-rule "PII.9.2: Redact between \"AUTHOR(S)\" and \"COMPLETION DATE\" (non vertebrate study)"
-    when
-        not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
-        $section: Section(!hasTables(), containsString("AUTHOR(S):"), containsString("STUDY COMPLETION DATE:"))
-    then
-        entityCreationService.betweenStrings("AUTHOR(S):", "STUDY COMPLETION DATE:", "PII", EntityType.ENTITY, $section)
-         .forEach(authorEntity -> authorEntity.apply("PII.9.2", "AUTHOR(S) was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
-    end
-
-rule "PII.9.3: Redact between \"AUTHOR(S)\" and \"STUDY COMPLETION DATE\" (vertebrate study)"
-    when
-        FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
-        $section: Section(!hasTables(), containsString("AUTHOR(S):"), containsString("STUDY COMPLETION DATE:"))
-    then
-        entityCreationService.betweenStrings("AUTHOR(S):", "STUDY COMPLETION DATE:", "PII", EntityType.ENTITY, $section)
-         .forEach(authorEntity -> authorEntity.apply("PII.9.3", "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
+        entityCreationService.shortestBetweenAnyStringIgnoreCase(List.of("AUTHOR(S)", "AUTHOR(S):"), List.of("COMPLETION DATE", "COMPLETION DATE:", "STUDY COMPLETION DATE", "STUDY COMPLETION DATE:"), "PII", EntityType.ENTITY, $document)
+        .forEach(authorEntity -> authorEntity.redact("PII.9.1", "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
     end
 
 
@@ -434,25 +412,33 @@ rule "ETC.2.0: Redact signatures (non vertebrate study)"
         not FileAttribute(label == "Vertebrate Study", value == "Yes")
         $signature: Image(imageType == ImageType.SIGNATURE)
     then
-        $signature.apply("ETC.2.0", "Signature Found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        $signature.redact("ETC.2.0", "Signature Found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+    end
+
+rule "ETC.2.1: Redact signatures (vertebrate study)"
+    when
+        FileAttribute(label == "Vertebrate Study", value == "Yes")
+        $signature: Image(imageType == ImageType.SIGNATURE)
+    then
+        $signature.redact("ETC.2.1", "Signature Found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
     end
 
 
 // Rule unit: ETC.3
-rule "ETC.3.0: Redact logos (vertebrate study)"
+rule "ETC.3.0: Skip logos (non vertebrate study)"
     when
         not FileAttribute(label == "Vertebrate Study", value == "Yes")
         $logo: Image(imageType == ImageType.LOGO)
     then
-        $logo.apply("ETC.3.0", "Logo Found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        $logo.skip("ETC.3.0", "Logo Found");
     end
 
-rule "ETC.3.1: Redact logos (non vertebrate study)"
+rule "ETC.3.1: Redact logos (vertebrate study)"
     when
         FileAttribute(label == "Vertebrate Study", value == "Yes")
         $logo: Image(imageType == ImageType.LOGO)
     then
-        $logo.apply("ETC.3.1", "Logo Found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        $logo.redact("ETC.3.1", "Logo Found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
     end
 
 
@@ -470,7 +456,7 @@ rule "ETC.5.0: Ignore dossier_redaction entries if confidentiality is not 'confi
 //------------------------------------ AI rules ------------------------------------
 
 // Rule unit: AI.0
-rule "AI.0.0: add all NER Entities of type CBI_author"
+rule "AI.0.0: Add all NER Entities of type CBI_author"
     salience 999
     when
         nerEntities: NerEntities(hasEntitiesOfType("CBI_author"))
@@ -481,7 +467,7 @@ rule "AI.0.0: add all NER Entities of type CBI_author"
 
 
 // Rule unit: AI.1
-rule "AI.1.0: combine and add NER Entities as CBI_address"
+rule "AI.1.0: Combine and add NER Entities as CBI_address"
     salience 999
     when
         nerEntities: NerEntities(hasEntitiesOfType("ORG") || hasEntitiesOfType("STREET") || hasEntitiesOfType("CITY"))
@@ -490,7 +476,7 @@ rule "AI.1.0: combine and add NER Entities as CBI_address"
     end
 
 
-//------------------------------------ Manual redaction rules ------------------------------------
+//------------------------------------ Manual changes rules ------------------------------------
 
 // Rule unit: MAN.0
 rule "MAN.0.0: Apply manual resize redaction"
@@ -524,7 +510,7 @@ rule "MAN.0.1: Apply manual resize redaction"
 rule "MAN.1.0: Apply id removals that are valid and not in forced redactions to Entity"
     salience 128
     when
-        $idRemoval: IdRemoval($id: annotationId, status == AnnotationStatus.APPROVED)
+        $idRemoval: IdRemoval($id: annotationId, !removeFromDictionary, !removeFromAllDossiers, status == AnnotationStatus.APPROVED)
         $entityToBeRemoved: TextEntity(matchesAnnotationId($id))
     then
         $entityToBeRemoved.getManualOverwrite().addChange($idRemoval);
@@ -635,7 +621,7 @@ rule "MAN.4.1: Apply legal basis change"
 //------------------------------------ Entity merging rules ------------------------------------
 
 // Rule unit: X.0
-rule "X.0.0: remove Entity contained by Entity of same type"
+rule "X.0.0: Remove Entity contained by Entity of same type"
     salience 65
     when
         $larger: TextEntity($type: type, $entityType: entityType, active())
@@ -647,7 +633,7 @@ rule "X.0.0: remove Entity contained by Entity of same type"
 
 
 // Rule unit: X.1
-rule "X.1.0: merge intersecting Entities of same type"
+rule "X.1.0: Merge intersecting Entities of same type"
     salience 64
     when
         $first: TextEntity($type: type, $entityType: entityType, !resized(), active())
@@ -663,7 +649,7 @@ rule "X.1.0: merge intersecting Entities of same type"
 
 
 // Rule unit: X.2
-rule "X.2.0: remove Entity of type ENTITY when contained by FALSE_POSITIVE"
+rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
     salience 64
     when
         $falsePositive: TextEntity($type: type, entityType == EntityType.FALSE_POSITIVE, active())
@@ -676,7 +662,7 @@ rule "X.2.0: remove Entity of type ENTITY when contained by FALSE_POSITIVE"
 
 
 // Rule unit: X.3
-rule "X.3.0: remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"
+rule "X.3.0: Remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"
     salience 64
     when
         $falseRecommendation: TextEntity($type: type, entityType == EntityType.FALSE_RECOMMENDATION, active())
@@ -688,7 +674,7 @@ rule "X.3.0: remove Entity of type RECOMMENDATION when contained by FALSE_RECOMM
 
 
 // Rule unit: X.4
-rule "X.4.0: remove Entity of type RECOMMENDATION when intersected by ENTITY with same type"
+rule "X.4.0: Remove Entity of type RECOMMENDATION when intersected by ENTITY with same type"
     salience 256
     when
         $entity: TextEntity($type: type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
@@ -701,7 +687,7 @@ rule "X.4.0: remove Entity of type RECOMMENDATION when intersected by ENTITY wit
 
 
 // Rule unit: X.5
-rule "X.5.0: remove Entity of type RECOMMENDATION when contained by ENTITY"
+rule "X.5.0: Remove Entity of type RECOMMENDATION when contained by ENTITY"
     salience 256
     when
         $entity: TextEntity((entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
@@ -713,7 +699,7 @@ rule "X.5.0: remove Entity of type RECOMMENDATION when contained by ENTITY"
 
 
 // Rule unit: X.6
-rule "X.6.0: remove Entity of lower rank, when contained by by entity of type ENTITY"
+rule "X.6.0: Remove Entity of lower rank, when contained by by entity of type ENTITY"
     salience 32
     when
         $higherRank: TextEntity($type: type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
@@ -728,7 +714,7 @@ rule "X.6.1: remove Entity of higher rank, when intersected by entity of type EN
     salience 32
     when
         $higherRank: TextEntity($type: type, $value: value, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
-        $lowerRank: TextEntity(intersects($higherRank), type != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active(), $lowerRank.getValue().length() > $value.length())
+        $lowerRank: TextEntity(intersects($higherRank), type != $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active(), $lowerRank.getValue().length() > $value.length())
     then
         $higherRank.getIntersectingNodes().forEach(node -> update(node));
         $higherRank.remove("X.6.1", "remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity");
@@ -739,7 +725,7 @@ rule "X.6.1: remove Entity of higher rank, when intersected by entity of type EN
 //------------------------------------ File attributes rules ------------------------------------
 
 // Rule unit: FA.1
-rule "FA.1.0: remove duplicate FileAttributes"
+rule "FA.1.0: Remove duplicate FileAttributes"
     salience 64
     when
         $fileAttribute: FileAttribute($label: label, $value: value)
@@ -752,7 +738,7 @@ rule "FA.1.0: remove duplicate FileAttributes"
 //------------------------------------ Local dictionary search rules ------------------------------------
 
 // Rule unit: LDS.0
-rule "LDS.0.0: run local dictionary search"
+rule "LDS.0.0: Run local dictionary search"
     agenda-group "LOCAL_DICTIONARY_ADDS"
     salience -999
     when
@@ -760,7 +746,7 @@ rule "LDS.0.0: run local dictionary search"
     then
         entityCreationService.bySearchImplementation($dictionaryModel.getLocalSearch(), $dictionaryModel.getType(), EntityType.RECOMMENDATION, document)
             .forEach(entity ->  {
-                Collection<MatchedRule> matchedRules = $dictionaryModel.getLocalEntriesWithMatchedRules().get(entity.getValue());
-                entity.addMatchedRules(matchedRules);
+                Collection<MatchedRule> matchedRules = $dictionaryModel.getMatchedRulesForLocalDictionaryEntry(entity.getValue());
+                matchedRules.forEach(matchedRule -> entity.addMatchedRule(matchedRule.asSkippedIfApplied()));
             });
     end

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/adama-pilot.drl

@@ -1,916 +0,0 @@
-package drools
-
-import static java.lang.String.format;
-import static com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility.anyMatch;
-import static com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility.exactMatch;
-
-import java.util.List;
-import java.util.LinkedList;
-import java.util.Set;
-import java.util.stream.Collectors;
-import java.util.Collection;
-import java.util.stream.Stream;
-import java.util.Optional;
-
-import com.iqser.red.service.redaction.v1.server.model.document.*;
-import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
-import com.iqser.red.service.redaction.v1.server.model.document.entity.*;
-import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
-import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule;
-import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity
-import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule
-import com.iqser.red.service.redaction.v1.server.model.document.nodes.*;
-import com.iqser.red.service.redaction.v1.server.model.document.nodes.Section;
-import com.iqser.red.service.redaction.v1.server.model.document.nodes.Table;
-import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
-import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
-import com.iqser.red.service.redaction.v1.server.model.document.nodes.Paragraph;
-import com.iqser.red.service.redaction.v1.server.model.document.nodes.Image;
-import com.iqser.red.service.redaction.v1.server.model.document.nodes.ImageType;
-import com.iqser.red.service.redaction.v1.server.model.document.nodes.Page;
-import com.iqser.red.service.redaction.v1.server.model.document.nodes.Headline;
-import com.iqser.red.service.redaction.v1.server.model.document.nodes.SectionIdentifier;
-import com.iqser.red.service.redaction.v1.server.model.document.nodes.Footer;
-import com.iqser.red.service.redaction.v1.server.model.document.nodes.Header;
-import com.iqser.red.service.redaction.v1.server.model.document.nodes.NodeType;
-import com.iqser.red.service.redaction.v1.server.model.document.textblock.*;
-import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
-import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlockCollector;
-import com.iqser.red.service.redaction.v1.server.model.document.textblock.AtomicTextBlock;
-import com.iqser.red.service.redaction.v1.server.model.document.textblock.ConcatenatedTextBlock;
-import com.iqser.red.service.redaction.v1.server.model.NerEntities;
-import com.iqser.red.service.redaction.v1.server.model.dictionary.Dictionary;
-import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryModel;
-import com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService;
-import com.iqser.red.service.redaction.v1.server.service.ManualChangesApplicationService;
-import com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility;
-
-import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus;
-
-global Document document
-global EntityCreationService entityCreationService
-global ManualChangesApplicationService manualChangesApplicationService
-global Dictionary dictionary
-
-//------------------------------------ queries ------------------------------------
-
-query "getFileAttributes"
-    $fileAttribute: FileAttribute()
-    end
-
-//---------------------------------------------------------------------------
-
-rule "H.0.0 retract table of contents page"
-    when
-        $page: Page(getMainBodyTextBlock().getSearchText().contains("........") || (getMainBodyTextBlock().getSearchText().contains("APPENDICES") && getMainBodyTextBlock().getSearchText().contains("TABLES")))
-        $node: SemanticNode(onPage($page.getNumber()), !onPage($page.getNumber() -1), getType() != NodeType.IMAGE)
-    then
-        retract($node);
-    end
-
-
-rule "H.0.0: Ignore Table of Contents"
-    salience 10
-    when
-        $tocHeadline: Headline(containsString("CONTENTS"))
-        $page: Page() from $tocHeadline.getParent().getPages()
-        $node: SemanticNode(this != $tocHeadline, getType() != NodeType.IMAGE, onPage($page.getNumber()), !onPage($page.getNumber() -1))
-    then
-        retract($node);
-    end
-
-//---------------------- CUSTOMER RULES -----------------------------------------------------
-
-rule "DOC.1.0: Adama number"
-    when
-       $paragraph: Paragraph(
-            containsString("R-")
-            && onPage(1)
-        )
-    then
-        entityCreationService
-            .byRegex(
-                "0?0?R\\-[l\\d]{3,5}\\p{Lu}?",
-                "adama_number",
-                EntityType.ENTITY,
-                $paragraph)
-            .findFirst()
-            .ifPresent(entity -> {
-                entity.apply("DOC.1.0", "Adama number found");
-                insert(FileAttribute.builder().label("Adama").value(entity.getValue()).build());
-            });
-    end
-
-rule "DOC.2.0: Study number by keyword"
-    when
-        $studyNumberKeyword: String() from List.of(
-            "BioChem project number:",
-            "DTI Report.",
-            "Final Report N°",
-            "LPT Report No.",
-            "Project identity",
-            "Project No.:",
-            "Protocol No.",
-            "PTRL Report No.",
-            "SLI Report #",
-            "SLI Study #",
-            "Specht Analytical Study Plan",
-            "SPL PROJECT NUMBER:",
-            "Study code:",
-            "Study N°",
-            "Study-No.",
-            "Study No:",
-            "Study number:",
-            "Study Number"
-            )
-        $excludeKeyWords: String() from List.of(
-            "Sponsors study number"
-        )
-        $paragraph: Paragraph(
-            containsStringIgnoreCase($studyNumberKeyword)
-            && !containsStringIgnoreCase($excludeKeyWords)
-            && onPage(1)
-        )
-    then
-        entityCreationService
-            .lineAfterStringIgnoreCase($studyNumberKeyword,"study_number",EntityType.ENTITY,$paragraph)
-            .findFirst()
-            .ifPresent(entity ->
-                {
-                    entity.apply("DOC.2.0", "Study number found");
-                    insert(FileAttribute.builder().label("Study").value(entity.getValue()).build());
-                }
-            );
-    end
-
-rule "DOC.2.1: Study number in header"
-    when
-        $studyNumberKeyword: String() from List.of(
-            "Final Report"
-            )
-        $header: Header(
-            containsStringIgnoreCase($studyNumberKeyword)
-            && onPage(2)
-        )
-    then
-        entityCreationService
-            .lineAfterStringIgnoreCase($studyNumberKeyword,"study_number",EntityType.ENTITY,$header)
-            .findFirst()
-            .ifPresent(entity ->
-                {
-                    entity.apply("DOC.2.1", "Study number found");
-                    insert(FileAttribute.builder().label("Study").value(entity.getValue()).build());
-                }
-            );
-    end
-
-rule "DOC.2.2: Project Number with Section Title"
-    when
-        $title: String() from List.of(
-            "LABORATORY PROJECT ID."
-        )
-        $paragraph: Paragraph(
-            containsStringIgnoreCase($title)
-            && onPage(1)
-        )
-    then
-        entityCreationService.semanticNodeAfterString(
-            $paragraph,
-            "LABORATORY PROJECT ID.",
-            "study_number",
-            EntityType.ENTITY
-            )
-            .ifPresent(
-                entity -> entity.apply("DOC.2.2","Study number by title found.")
-            );
-    end
-
-
-rule "DOC.3.0: Batch Material Number"
-    when
-        $hlKeyword: String() from List.of(
-            "formulation",
-            "formulation:",
-            "test item",
-            "test substance",
-            "test material"
-        )
-        $hlNoKeyword: String() from List.of(
-            "Preparation"
-        )
-        $headline: Headline(
-            containsStringIgnoreCase($hlKeyword)
-            && !containsStringIgnoreCase($hlNoKeyword)
-        )
-        $batchKeyword: String() from List.of(
-            "Batch number",
-            "Batch number:",
-            "Batch number* :",
-            "batch no",
-            "Batch no.",
-            "Lot.n.:"
-            )
-        $section: Section(
-            containsStringIgnoreCase($batchKeyword)
-            && (
-                getHeadline() == $headline
-                || (
-                    containsStringIgnoreCase($hlKeyword)
-                    && !containsStringIgnoreCase($hlNoKeyword)
-                )
-            )
-        )
-    then
-        entityCreationService
-            .lineAfterStringIgnoreCase($batchKeyword,"batch_material_number",EntityType.ENTITY,$section)
-            .forEach(
-                entity -> {
-                    entity.apply("DOC.3.0","Batch number found.");
-                    insert(FileAttribute.builder().label("Batch No").value(entity.getValue()).build());
-                }
-            );
-    end
-
-rule "DOC.4.0: study title as headline"
-    when
-        $hlKeyword: String() from List.of(
-            "Analyitical Method",
-            "Residues",
-            "Acute",
-            "Process",
-            "Application",
-            "Review",
-            "Examination",
-            "Toxicity",
-            "Method Validation",
-            "Explosion",
-            "Sensitisation",
-            "Determination",
-            "Test"
-        )
-        $hlNotKeyword: String() from List.of(
-            "Laboratoire",
-            "Laboratorien"
-        )
-        $species: TextEntity(type=="species")
-        $headline: Headline(
-            onPage(1)
-            && (
-                containsStringIgnoreCase($hlKeyword)
-                || entities contains $species
-            )
-            && !containsStringIgnoreCase($hlNotKeyword)
-        )
-    then
-        entityCreationService.bySemanticNode($headline, "title", EntityType.ENTITY).ifPresent(entity -> {
-            entity.apply("DOC.4.0", "Title found", "n-a");
-            insert(FileAttribute.builder().label("Title").value(entity.getValue()).build());
-        });
-     end
-
-rule "DOC.4.1: study title on cover page between sections"
-    when
-        not TextEntity(type=="title", applied())
-        $page: Page(
-            getNumber() == 1,
-            getMainBodyTextBlock().getSearchText().toLowerCase() (contains "Study Title".toLowerCase())
-            || getMainBodyTextBlock().getSearchText().toLowerCase() (contains "Title".toLowerCase())
-            || getMainBodyTextBlock().getSearchText().toLowerCase() (contains "Report 92 50 12 136".toLowerCase())
-            || getMainBodyTextBlock().getSearchText().toLowerCase() (contains "Final Report".toLowerCase())
-            || getMainBodyTextBlock().getSearchText().toLowerCase() (contains "Final Study Report".toLowerCase()),
-            getMainBodyTextBlock().getSearchText().toLowerCase() (contains "Guideline".toLowerCase())
-            || getMainBodyTextBlock().getSearchText().toLowerCase() (contains "Study Identification".toLowerCase())
-            // too many false positives due to term in header and cover page || getMainBodyTextBlock().getSearchText().toLowerCase() (contains "Final Report".toLowerCase())
-            || getMainBodyTextBlock().getSearchText().toLowerCase() (contains "Data Requirement".toLowerCase())
-            || getMainBodyTextBlock().getSearchText().toLowerCase() (contains "Submitted".toLowerCase())
-            || getMainBodyTextBlock().getSearchText().toLowerCase() (contains "Test Guideline".toLowerCase())
-            || getMainBodyTextBlock().getSearchText().toLowerCase() (contains "Study Director".toLowerCase())
-            || getMainBodyTextBlock().getSearchText().toLowerCase() (contains "Author".toLowerCase())
-            || getMainBodyTextBlock().getSearchText().toLowerCase() (contains "Including:".toLowerCase())
-            || getMainBodyTextBlock().getSearchText().toLowerCase() (contains "Laboratory Investigations".toLowerCase())
-            || getMainBodyTextBlock().getSearchText().toLowerCase() (contains "Test Article".toLowerCase())
-            || getMainBodyTextBlock().getSearchText().toLowerCase() (contains "HLS".toLowerCase())
-            || getMainBodyTextBlock().getSearchText().toLowerCase() (contains "Official Journal".toLowerCase())
-            )
-        then
-            List<String> startStrings = List.of("Study Title", "Study Title:", "Title", "Final Report", "Final Study Report", "Report 92 50 12 136");
-            List<String> stopStrings = List.of("Guideline", "Guidelines", "Study Identification", "Data Requirement", "Submitted", "Test Guideline",
-                                               "Study Director", "Author", "Including:", "Laboratory Investigations", "Test Article", "HLS", "Official Journal");
-            // too many false positives due to term in header and cover page  stopBoundaries.addAll(RedactionSearchUtility.findTextRangesByStringIgnoreCase("Final Report", $page.getMainBodyTextBlock()));
-
-            entityCreationService.shortestBetweenAnyString(startStrings, stopStrings, "title", EntityType.ENTITY, document).forEach(entity -> {
-                entity.apply("DOC.4.1", "Study title found", "n-a");
-                insert(FileAttribute.builder().label("Title").value(entity.getValue()).build());
-            });
-    end
-
-rule "DOC.4.2: Study title by paragraph including species on cover page"
-    when
-        not TextEntity(type=="title", applied())
-        $species: TextEntity(type=="species")
-        $paragraph: Paragraph(
-            onPage(1)
-            && entities contains $species
-        )
-    then
-        entityCreationService.bySemanticNode(
-            $paragraph,
-            "title",
-            EntityType.ENTITY
-        )
-        .ifPresent(entity -> {
-            entity.apply("DOC.4.2","Title with species found on cover page");
-            insert(FileAttribute.builder().label("Title").value(entity.getValue()).build());
-        });
-    end
-
-rule "DOC.5.0: Study Director on cover page in a paragraph on line"
-    when
-        not TextEntity(type=="author", applied())
-        $sectionTitle: String() from List.of(
-            "Study Director/Analyst:",
-            "Study Director:",
-            "Study Director :",
-            "Author:",
-            "Author "
-        )
-        $paragraph: Paragraph(
-            onPage(1)
-            && containsStringIgnoreCase($sectionTitle)
-        )
-    then
-        entityCreationService
-            .lineAfterStringIgnoreCase(
-                $sectionTitle,
-                "author",
-                EntityType.ENTITY,
-                $paragraph
-            )
-            .forEach(entity -> {
-                entity.apply("DOC.5.0","Study Director on cover page found");
-                insert(FileAttribute.builder().label("Author").value(entity.getValue()).build());
-            });
-    end
-
-rule "DOC.5.1: Author/Study Director listing on cover page"
-    when
-        not TextEntity(type=="author", applied())
-        $page: Page(
-            getNumber() == 1
-            && getMainBodyTextBlock().getSearchText().toLowerCase() (
-                contains "Author(s)".toLowerCase()
-                || contains "Authors".toLowerCase()
-                || contains "Author".toLowerCase()
-                || contains "Study Director".toLowerCase()
-                )
-            && getMainBodyTextBlock().getSearchText().toLowerCase() (
-                contains "Test Facility".toLowerCase()
-                || contains "Testing Facility".toLowerCase()
-                || contains "Study Initiation".toLowerCase()
-                || contains "Study Initiated".toLowerCase()
-                || contains "Study completed on".toLowerCase()
-                || contains "Study completion".toLowerCase()
-                || contains "Study amended".toLowerCase()
-                || contains "Report Issue Date".toLowerCase()
-                || contains "Document Date".toLowerCase()
-                || contains "Date".toLowerCase()
-                || contains "Defitrace".toLowerCase() // this should be solved with Organisation Entity as a TextRange
-                )
-            )
-    then
-        List<TextRange> startBoundaries = new LinkedList<>();
-        startBoundaries.addAll(RedactionSearchUtility.findTextRangesByStringIgnoreCase("Author(s)", $page.getMainBodyTextBlock()));
-        startBoundaries.addAll(RedactionSearchUtility.findTextRangesByStringIgnoreCase("Authors", $page.getMainBodyTextBlock()));
-        startBoundaries.addAll(RedactionSearchUtility.findTextRangesByStringIgnoreCase("Author", $page.getMainBodyTextBlock()));
-        startBoundaries.addAll(RedactionSearchUtility.findTextRangesByStringIgnoreCase("Study Director", $page.getMainBodyTextBlock()));
-
-        List<TextRange> stopBoundaries = new LinkedList<>();
-        stopBoundaries.addAll(RedactionSearchUtility.findTextRangesByStringIgnoreCase("Test Facility", $page.getMainBodyTextBlock()));
-        stopBoundaries.addAll(RedactionSearchUtility.findTextRangesByStringIgnoreCase("Testing Facility", $page.getMainBodyTextBlock()));
-        stopBoundaries.addAll(RedactionSearchUtility.findTextRangesByStringIgnoreCase("Study Initiation", $page.getMainBodyTextBlock()));
-        stopBoundaries.addAll(RedactionSearchUtility.findTextRangesByStringIgnoreCase("Study Initiated", $page.getMainBodyTextBlock()));
-        stopBoundaries.addAll(RedactionSearchUtility.findTextRangesByStringIgnoreCase("Study completed on", $page.getMainBodyTextBlock()));
-        stopBoundaries.addAll(RedactionSearchUtility.findTextRangesByStringIgnoreCase("Study completion", $page.getMainBodyTextBlock()));
-        stopBoundaries.addAll(RedactionSearchUtility.findTextRangesByStringIgnoreCase("Study amended", $page.getMainBodyTextBlock()));
-        stopBoundaries.addAll(RedactionSearchUtility.findTextRangesByStringIgnoreCase("Report Issue Date", $page.getMainBodyTextBlock()));
-        stopBoundaries.addAll(RedactionSearchUtility.findTextRangesByStringIgnoreCase("Document Date", $page.getMainBodyTextBlock()));
-        stopBoundaries.addAll(RedactionSearchUtility.findTextRangesByStringIgnoreCase("Date", $page.getMainBodyTextBlock()));
-        stopBoundaries.addAll(RedactionSearchUtility.findTextRangesByStringIgnoreCase("Defitrace", $page.getMainBodyTextBlock()));
-
-        entityCreationService.betweenTextRanges(startBoundaries, stopBoundaries, "author", EntityType.ENTITY, document).forEach(entity -> {
-            entity.apply("DOC.5.1", "Author list found", "n-a");
-            insert(FileAttribute.builder().label("Author").value(entity.getValue()).build());
-        });
-     end
-
-rule "DOC.5.2: Study Director on cover page in a paragraph"
-    when
-        not TextEntity(type=="author", applied())
-        $sectionTitle: String() from List.of(
-            "Study Director"
-        )
-        $node: Paragraph(
-            onPage(1)
-            && containsStringIgnoreCase($sectionTitle)
-        )
-    then
-        entityCreationService
-            .byRegexWithLineBreaksIgnoreCase(
-                "\\n([\\w\\(\\) .]{5,30})\\n",
-                "author",
-                EntityType.ENTITY,
-                1,
-                $node
-            )
-            .filter(
-                entity -> !entity.getValue().toLowerCase().contains($sectionTitle.toLowerCase())
-            )
-            .forEach(entity -> {
-                entity.apply("DOC.5.2","Study Director on cover page found");
-                insert(FileAttribute.builder().label("Author").value(entity.getValue()).build());
-            });
-    end
-
-rule "DOC.5.3: Study Director on cover page in a section"
-    when
-        not TextEntity(type=="author", applied())
-        $page: Page(getNumber() == 1)
-        $sectionTitle: String() from List.of(
-            "Study Director:",
-            "Study Director",
-            "Study Director/Author"
-        )
-        $node: Section(
-            onPage(1)
-            && containsStringIgnoreCase($sectionTitle)
-        )
-    then
-        entityCreationService
-            .byRegexWithLineBreaksIgnoreCase(
-                $sectionTitle+"\\n([\\w\\(\\) .]{5,30})\\n",
-                "author",
-                EntityType.ENTITY,
-                1,
-                $node
-            )
-            .filter(
-                entity -> entity.getPages().contains($page)
-            )
-            .forEach(entity -> {
-                entity.apply("DOC.5.3","Study Director on cover page found");
-                insert(FileAttribute.builder().label("Author").value(entity.getValue()).build());
-            });
-
-        entityCreationService
-            .lineAfterStringIgnoreCase($sectionTitle,"author",EntityType.ENTITY,$node)
-            .forEach(entity -> {
-                entity.apply("DOC.5.2","Study Director on cover page found");
-                insert(FileAttribute.builder().label("Author").value(entity.getValue()).build());
-            });
-    end
-
-rule "DOC.5.4: Author in document section with relevant headline"
-        when
-            not TextEntity(type == "author", applied())
-            $hlKeyword: String() from List.of(
-                "sponsor",
-                "personnel",
-                "staff involved",
-                "study details",
-                "management of study"
-            )
-            $keyword: String() from List.of(
-                "Study Director",
-                "Studv Director",
-                "Study Director:",
-                "Study Director :",
-                "stray Birector:",
-                "Author:",
-                "Author ",
-                "Author(s)"
-            )
-            $section: Section(
-                containsStringIgnoreCase($keyword)
-                && getHeadline().containsStringIgnoreCase($hlKeyword)
-            )
-        then
-            entityCreationService
-                .lineAfterStringIgnoreCase(
-                    $keyword,
-                    "author",
-                    EntityType.ENTITY,
-                    $section
-                )
-                .findFirst()
-                .ifPresent(
-                    entity -> {
-                        entity.apply("DOC.5.4","Author found.");
-                        insert(FileAttribute.builder().label("Author").value(entity.getValue()).build());
-                    }
-                );
-        end
-
-rule "DOC.5.5: Study Director from signature on compliance statement"
-    when
-        not TextEntity(type=="author", applied())
-        $sectionTitle: String() from List.of(
-            "Compliance Statement",
-            "Study Compliance",
-            "Statement of Compliance",
-            "Statement of Study Compliance",
-            "Contributing Scientist",
-            "Compliance with Good Laboratory Practice",
-            "Signatures and Approval"
-        )
-        $role: String() from List.of("Study Director")
-        $node: Section(
-            containsStringIgnoreCase($sectionTitle)
-            && containsStringIgnoreCase($role)
-            && getFirstPage().getNumber() > 4
-        )
-    then
-        entityCreationService
-            .byRegexWithLineBreaksIgnoreCase(
-                "([\\w\\(\\) .,]{5,35})[\\w\\/ ]{0,30}\\n"+$role,
-                "author",
-                EntityType.ENTITY,
-                1,
-                $node
-            )
-            .forEach(entity -> {
-                entity.apply("DOC.5.5","Study Director on compliance page found");
-                insert(FileAttribute.builder().label("Author").value(entity.getValue()).build());
-            });
-    end
-
-rule "DOC.5.6: Study Director from section with relevant headlining responsibility"
-    when
-        not TextEntity(type=="author", applied())
-        $sectionTitle: String() from List.of("Responsible personnel")
-        $role: String() from List.of("Study Director")
-        $node: Section(
-            getHeadline().containsStringIgnoreCase($sectionTitle)
-            && containsStringIgnoreCase($role)
-        )
-        $paragraph: Paragraph(
-            containsStringIgnoreCase($role)
-        )
-    then
-        entityCreationService
-            .lineAfterStringIgnoreCase(
-                $role,
-                "author",
-                EntityType.ENTITY,
-                $paragraph
-            )
-            .forEach(entity -> {
-                entity.apply("DOC.5.6","Study Director on compliance page found");
-                insert(FileAttribute.builder().label("Author").value(entity.getValue()).build());
-            });
-    end
-
-rule "DOC.5.7: Study Director from section with study director in headline"
-    when
-        not TextEntity(type=="author", applied())
-        $sectionTitle: String() from List.of("Study Director")
-        $section: Section(
-            getHeadline().containsStringIgnoreCase($sectionTitle)
-        )
-    then
-        entityCreationService
-            .bySemanticNodeParagraphsOnly(
-                $section,
-                "author",
-                EntityType.ENTITY
-            )
-            .forEach(entity -> {
-                entity.apply("DOC.5.7","Study Director found by headline");
-                insert(FileAttribute.builder().label("Author").value(entity.getValue()).build());
-            });
-    end
-
-rule "DOC.5.9: Remove skipped authors"
-    salience 9999
-    when
-        $author: TextEntity(type=="author", !applied())
-    then
-        $author.removeFromGraph();
-    end
-
-rule "DOC.14.0: GLP Study"
-     when
-        $hlKeywords: String() from List.of(
-            "Good Laboratory Practice",
-            "GLP"
-        )
-        $headline: Headline(
-            containsString($hlKeywords)
-        )
-        $compliance: String() from List.of(
-            "conducted in compliance with",
-            "conducted in accordance with",
-            "conducted and reported in compliance with",
-            "carried out in accordance with",
-            "meets the requirements",
-            "performed in compliance with"
-        )
-        $section: Section(
-            containsStringIgnoreCase($compliance)
-            && getHeadline() == $headline
-        )
-     then
-         entityCreationService.bySemanticNode($headline, "glp_study", EntityType.ENTITY).ifPresent(entity -> {
-             entity.apply("DOC.14.0", "GLP Study found", "n-a");
-             insert(FileAttribute.builder().label("GLP").value("TRUE").build());
-         });
-     end
-
-rule "DOC.14.1: GLP Study"
-     when
-        $compliance: String() from List.of(
-            "conducted in compliance with",
-            "conducted in accordance with",
-            "conducted and reported in compliance with",
-            "carried out in accordance with",
-            "meets the requirements",
-            "performed in compliance with",
-            "conduct"
-        )
-        $principle: String() from List.of(
-            "40 CFR Part 160",
-            "40CFR160",
-            "Good Laboratory Practice",
-            "FIFRA Good Laboratory Practice Standards"
-        )
-        $paragraph: Paragraph(
-            containsStringIgnoreCase($compliance)
-            && containsStringIgnoreCase($principle)
-        )
-     then
-        entityCreationService.betweenStringsIncludeStartAndEndIgnoreCase(
-            $compliance,
-            $principle,
-            "glp_study",
-            EntityType.ENTITY,
-            $paragraph
-            )
-            .forEach(entity -> {
-                entity.apply("DOC.14.1", "GLP Study found", "n-a");
-                insert(FileAttribute.builder().label("GLP").value("TRUE").build());
-            }
-        );
-     end
-
-//------------------------------------ Manual redaction rules ------------------------------------
-
-// Rule unit: MAN.0
-rule "MAN.0.0: Apply manual resize redaction"
-    salience 128
-    when
-        $resizeRedaction: ManualResizeRedaction($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate)
-        not ManualResizeRedaction(annotationId == $id, requestDate.isBefore($requestDate))
-        $entityToBeResized: TextEntity(matchesAnnotationId($id))
-    then
-        manualChangesApplicationService.resizeEntityAndReinsert($entityToBeResized, $resizeRedaction);
-        retract($resizeRedaction);
-        update($entityToBeResized);
-        $entityToBeResized.getIntersectingNodes().forEach(node -> update(node));
-    end
-
-rule "MAN.0.1: Apply manual resize redaction"
-    salience 128
-    when
-        $resizeRedaction: ManualResizeRedaction($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate)
-        not ManualResizeRedaction(annotationId == $id, requestDate.isBefore($requestDate))
-        $imageToBeResized: Image(id == $id)
-    then
-        manualChangesApplicationService.resizeImage($imageToBeResized, $resizeRedaction);
-        retract($resizeRedaction);
-        update($imageToBeResized);
-        update($imageToBeResized.getParent());
-    end
-
-
-// Rule unit: MAN.1
-rule "MAN.1.0: Apply id removals that are valid and not in forced redactions to Entity"
-    salience 128
-    when
-        $idRemoval: IdRemoval($id: annotationId, status == AnnotationStatus.APPROVED)
-        $entityToBeRemoved: TextEntity(matchesAnnotationId($id))
-    then
-        $entityToBeRemoved.getManualOverwrite().addChange($idRemoval);
-        update($entityToBeRemoved);
-        retract($idRemoval);
-        $entityToBeRemoved.getIntersectingNodes().forEach(node -> update(node));
-    end
-
-rule "MAN.1.1: Apply id removals that are valid and not in forced redactions to Image"
-    salience 128
-    when
-        $idRemoval: IdRemoval($id: annotationId, status == AnnotationStatus.APPROVED)
-        $imageEntityToBeRemoved: Image($id == id)
-    then
-        $imageEntityToBeRemoved.getManualOverwrite().addChange($idRemoval);
-        update($imageEntityToBeRemoved);
-        retract($idRemoval);
-        update($imageEntityToBeRemoved.getParent());
-    end
-
-
-// Rule unit: MAN.2
-rule "MAN.2.0: Apply force redaction"
-    salience 128
-    when
-        $force: ManualForceRedaction($id: annotationId, status == AnnotationStatus.APPROVED)
-        $entityToForce: TextEntity(matchesAnnotationId($id))
-    then
-        $entityToForce.getManualOverwrite().addChange($force);
-        update($entityToForce);
-        $entityToForce.getIntersectingNodes().forEach(node -> update(node));
-        retract($force);
-    end
-
-rule "MAN.2.1: Apply force redaction to images"
-    salience 128
-    when
-        $force: ManualForceRedaction($id: annotationId, status == AnnotationStatus.APPROVED)
-        $imageToForce: Image(id == $id)
-    then
-        $imageToForce.getManualOverwrite().addChange($force);
-        update($imageToForce);
-        update($imageToForce.getParent());
-        retract($force);
-    end
-
-
-// Rule unit: MAN.3
-rule "MAN.3.0: Apply entity recategorization"
-    salience 128
-    when
-        $recategorization: ManualRecategorization($id: annotationId, $type: type, status == AnnotationStatus.APPROVED, $requestDate: requestDate)
-        not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))
-        $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type != $type)
-    then
-        $entityToBeRecategorized.getIntersectingNodes().forEach(node -> update(node));
-        manualChangesApplicationService.recategorize($entityToBeRecategorized, $recategorization);
-        retract($recategorization);
-        // Entity is copied and inserted, so the old entity needs to be retracted to avoid duplication.
-        retract($entityToBeRecategorized);
-    end
-
-
-rule "MAN.3.1: Apply entity recategorization of same type"
-    salience 128
-    when
-        $recategorization: ManualRecategorization($id: annotationId, $type: type, status == AnnotationStatus.APPROVED, $requestDate: requestDate)
-        not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))
-        $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type == $type)
-    then
-        $entityToBeRecategorized.getManualOverwrite().addChange($recategorization);
-        retract($recategorization);
-    end
-
-
-rule "MAN.3.2: Apply image recategorization"
-    salience 128
-    when
-        $recategorization: ManualRecategorization($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate)
-        not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))
-        $imageToBeRecategorized: Image($id == id)
-    then
-        manualChangesApplicationService.recategorize($imageToBeRecategorized, $recategorization);
-        update($imageToBeRecategorized);
-        update($imageToBeRecategorized.getParent());
-        retract($recategorization);
-    end
-
-
-// Rule unit: MAN.4
-rule "MAN.4.0: Apply legal basis change"
-    salience 128
-    when
-        $legalbasisChange: ManualLegalBasisChange($id: annotationId, status == AnnotationStatus.APPROVED)
-        $imageToBeRecategorized: Image($id == id)
-    then
-        $imageToBeRecategorized.getManualOverwrite().addChange($legalbasisChange);
-    end
-
-rule "MAN.4.1: Apply legal basis change"
-    salience 128
-    when
-        $legalBasisChange: ManualLegalBasisChange($id: annotationId, status == AnnotationStatus.APPROVED)
-        $entityToBeChanged: TextEntity(matchesAnnotationId($id))
-    then
-        $entityToBeChanged.getManualOverwrite().addChange($legalBasisChange);
-    end
-
-
-//------------------------------------ Entity merging rules ------------------------------------
-
-// Rule unit: X.0
-rule "X.0.0: remove Entity contained by Entity of same type"
-    salience 65
-    when
-        $larger: TextEntity($type: type, $entityType: entityType, active())
-        $contained: TextEntity(containedBy($larger), type == $type, entityType == $entityType, this != $larger, !hasManualChanges(), active())
-    then
-        $contained.remove("X.0.0", "remove Entity contained by Entity of same type");
-        retract($contained);
-    end
-
-
-// Rule unit: X.2
-rule "X.2.0: remove Entity of type ENTITY when contained by FALSE_POSITIVE"
-    salience 64
-    when
-        $falsePositive: TextEntity($type: type, entityType == EntityType.FALSE_POSITIVE, active())
-        $entity: TextEntity(containedBy($falsePositive), type == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges(), active())
-    then
-        $entity.getIntersectingNodes().forEach(node -> update(node));
-        $entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE");
-        retract($entity)
-    end
-
-
-// Rule unit: X.3
-rule "X.3.0: remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"
-    salience 64
-    when
-        $falseRecommendation: TextEntity($type: type, entityType == EntityType.FALSE_RECOMMENDATION, active())
-        $recommendation: TextEntity(containedBy($falseRecommendation), type == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
-    then
-        $recommendation.remove("X.3.0", "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION");
-        retract($recommendation);
-    end
-
-
-// Rule unit: X.4
-rule "X.4.0: remove Entity of type RECOMMENDATION when intersected by ENTITY with same type"
-    salience 256
-    when
-        $entity: TextEntity($type: type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
-        $recommendation: TextEntity(intersects($entity), type == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
-    then
-        $entity.addEngines($recommendation.getEngines());
-        $recommendation.remove("X.4.0", "remove Entity of type RECOMMENDATION when intersected by ENTITY with same type");
-        retract($recommendation);
-    end
-
-
-// Rule unit: X.5
-rule "X.5.0: remove Entity of type RECOMMENDATION when contained by ENTITY"
-    salience 256
-    when
-        $entity: TextEntity((entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
-        $recommendation: TextEntity(containedBy($entity), entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
-    then
-        $recommendation.remove("X.5.0", "remove Entity of type RECOMMENDATION when contained by ENTITY");
-        retract($recommendation);
-    end
-
-
-// Rule unit: X.7
-rule "X.7.0: remove all images"
-    salience 512
-    when
-        $image: Image(imageType != ImageType.OCR, !hasManualChanges())
-    then
-        $image.remove("X.7.0", "remove all images");
-        retract($image);
-    end
-
-
-//------------------------------------ File attributes rules ------------------------------------
-
-// Rule unit: FA.1
-rule "FA.1.0: remove duplicate FileAttributes"
-
-    salience 64
-    when
-        $fileAttribute: FileAttribute($label: label, $value: value)
-        $duplicate: FileAttribute(this != $fileAttribute, label == $label, value == $value)
-    then
-        retract($duplicate);
-    end
-
-
-// Rule unit: LDS.0
-rule "LDS.0.0: run local dictionary search"
-    agenda-group "LOCAL_DICTIONARY_ADDS"
-    salience -999
-    when
-        $dictionaryModel: DictionaryModel(!localEntriesWithMatchedRules.isEmpty()) from dictionary.getDictionaryModels()
-    then
-        entityCreationService.bySearchImplementation($dictionaryModel.getLocalSearch(), $dictionaryModel.getType(), EntityType.RECOMMENDATION, document)
-            .forEach(entity ->  {
-                Collection<MatchedRule> matchedRules = $dictionaryModel.getLocalEntriesWithMatchedRules().get(entity.getValue());
-                entity.addMatchedRules(matchedRules);
-            });
-    end

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora.drl

@@ -22,6 +22,7 @@ import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRu
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.*;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Section;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Table;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Paragraph;
@@ -64,8 +65,10 @@ global Dictionary dictionary
 query "getFileAttributes"
     $fileAttribute: FileAttribute()
     end
-//---------------------------------------------------------------------------
 
+//------------------------------------ Headlines rules ------------------------------------
+
+// Rule unit: H.0
 rule "H.0.0: retract table of contents page"
     when
         $page: Page(getMainBodyTextBlock().getSearchText().contains("........") || (getMainBodyTextBlock().getSearchText().contains("APPENDICES") && getMainBodyTextBlock().getSearchText().contains("TABLES")))
@@ -75,6 +78,7 @@ rule "H.0.0: retract table of contents page"
     end
 
 
+// Rule unit: H.1
 rule "H.1.0: Ignore Table of Contents"
     salience 10
     when
@@ -86,7 +90,7 @@ rule "H.1.0: Ignore Table of Contents"
     end
 
 
-// Rule unit: MAN.0
+// Rule unit: H.2
 rule "H.2.0: Show headlines"
     when
         $headline: Headline()
@@ -95,6 +99,7 @@ rule "H.2.0: Show headlines"
     end
 
 
+// Rule unit: H.3
 rule "H.3.0: Study Type File Attribute"
     when
         not FileAttribute(label == "OECD Number", valueEqualsAnyOf("402","403","404","405","425","429","436","438","439","471","487"))
@@ -111,7 +116,27 @@ rule "H.3.0: Study Type File Attribute"
              .ifPresent(fileAttribute -> insert(fileAttribute));
     end
 
+rule "H.3.1: Study Type File Attribute in Headlines"
+    when
+        not FileAttribute(label == "OECD Number", valueEqualsAnyOf("402","403","404","405","425","429","436","438","439","471","487"))
+        $page: Page($pageNumber:number,
+            getMainBodyTextBlock().getSearchText().contains("DATA REQUIREMENT")
+            || getMainBodyTextBlock().getSearchText().contains("TEST GUIDELINE")
+            || getMainBodyTextBlock().getSearchText().contains("MÉTODO(S) DE REFERÊNCIA(S):"))
+        $headline: Headline(onPage($pageNumber), containsString("OECD") || containsString("EPA")|| containsString("OPPTS"))
+    then
+      Stream.of(RedactionSearchUtility.findTextRangesByRegexIgnoreCase("(?<=OECD)(?:[\\w\\s,\\[\\]\\(\\)\\.]{1,10}|(?:.{5,40}(?:Number |Procedure |Guideline )))(4[\\d]{2})", 1, $headline.getTextBlock()),
+                RedactionSearchUtility.findTextRangesByRegexIgnoreCase("(?<=OECD).{5,40}Method (4[\\d]{2}).{1,65}(\\d{4})\\)", 1, $headline.getTextBlock()),
+                RedactionSearchUtility.findTextRangesByRegexIgnoreCase("(?<=OECD) Guideline (4\\d{2})", 1, $headline.getTextBlock())).flatMap(Collection::stream).findFirst()
+             .map(textRange -> $headline.getTextBlock().subSequence(textRange).toString())
+             .map(value -> FileAttribute.builder().label("OECD Number").value(value).build())
+             .ifPresent(fileAttribute -> insert(fileAttribute));
+    end
 
+
+//------------------------------------ General documine rules ------------------------------------
+
+// Rule unit: DOC.1
 rule "DOC.1.0: Guidelines"
      when
          $section: Section(
@@ -171,7 +196,6 @@ rule "DOC.1.0: Guidelines"
         );
      end
 
-
 rule "DOC.1.2: Guidelines"
      when
          $section: Section(
@@ -197,7 +221,6 @@ rule "DOC.1.2: Guidelines"
          });
      end
 
-
 rule "DOC.1.3: Guidelines"
      when
          $section: Section(
@@ -221,12 +244,65 @@ rule "DOC.1.3: Guidelines"
          )
      then
          $section.getEntitiesOfType(List.of("oecd_guideline", "ec_guideline", "epa_guideline")).forEach(entity -> {
-             entity.removeFromGraph();
+             entity.remove("DOC.1.3", "removed");
              retract(entity);
          });
      end
 
+rule "DOC.1.4: Guideline in Headlines"
+    when
+        $page: Page($pageNumber:number,
+            getMainBodyTextBlock().getSearchText().contains("DATA REQUIREMENT")
+            || getMainBodyTextBlock().getSearchText().contains("TEST GUIDELINE")
+            || getMainBodyTextBlock().getSearchText().contains("MÉTODO(S) DE REFERÊNCIA(S):"))
+        $headline: Headline(onPage($pageNumber), containsString("OECD") || containsString("EPA")|| containsString("OPPTS"))
+    then
+        entityCreationService.byRegex("OECD (No\\.? )?\\d{3}( \\(\\d{4}\\))?", "oecd_guideline", EntityType.ENTITY, $headline).forEach(entity ->
+            entity.apply("DOC.1.4", "OECD Guideline found", "n-a")
+        );
+        entityCreationService.byRegex("OECD[\\s,]{1}(?:.{1,40}.(?>Procedure|Method).{1,20}\\d{3,4}(?>.{1,100}\\d{4}\\))?|\\[.{1,20}.Skin.{1,20}\\]|[\\d\\s,\\(\\)]{7,10}|[\\w\\.\\s]{1,15}[\\d]{3}\\s\\(\\d{4}\\)|.{0,20}[N|n]umber\\s\\d{3}.{0,1}|Test Guideline \\d{3})", "oecd_guideline", EntityType.ENTITY, $headline).forEach(entity ->
+            entity.apply("DOC.1.4", "OECD Guideline found", "n-a")
+        );
+        entityCreationService.byRegex("EPA (OPPTS )?\\d{3}[. ]\\d{4}( \\(\\d{4}\\))?", "epa_guideline", EntityType.ENTITY, $headline).forEach(entity ->
+            entity.apply("DOC.1.4", "EPA Guideline found", "n-a")
+        );
+        entityCreationService.byRegex("EC (Directive )?(No\\.? )?\\d{3,4}\\/\\d{3,4}((,? B(\\.| )\\d{1,2}\\.?)? \\(\\d{4}\\))?", "ec_guideline", EntityType.ENTITY, $headline).forEach(entity ->
+            entity.apply("DOC.1.4", "EC Guideline found", "n-a")
+        );
+        entityCreationService.byRegex("Commission Regulation \\(EC\\) No \\d{3}\\/\\d{4}", "ec_guideline", EntityType.ENTITY, $headline).forEach(entity ->
+            entity.apply("DOC.1.4", "EC Guideline found", "n-a")
+        );
+        entityCreationService.byRegex("OECD Method 4\\d{2}.{5,40}\\(.{5,40}\\d{4}\\)", "oecd_guideline", EntityType.ENTITY, $headline).forEach(entity ->
+            entity.apply("DOC.1.4", "OECD Guideline year found", "n-a")
+        );
+        entityCreationService.byRegex("OPPTS (Guideline Number )?\\d{3}\\.\\d{4}( \\(\\d{4}\\))?", "epa_guideline", EntityType.ENTITY, $headline).forEach(entity ->
+            entity.apply("DOC.1.4", "EPA Guideline found", "n-a")
+        );
+        entityCreationService.byRegex("(?<=OECD)(?:[\\w\\s,\\[\\]\\(\\)\\.]{1,10}|.{5,40}(?:Number |Procedure |Guideline ))(4[\\d]{2})", "oecd_guideline_number", EntityType.ENTITY,1, $headline).forEach(entity ->
+            entity.apply("DOC.1.4", "OECD Guideline no. found", "n-a")
+        );
+        entityCreationService.byRegex("(?<=OECD)(?:[\\w\\s,\\[\\]\\(\\)\\.]{1,10}|.{5,40}(?:Number |Procedure |Guideline ))(4[\\d]{2}),?\\s\\(?(\\d{4})\\)?", "oecd_guideline_year", EntityType.ENTITY,2, $headline).forEach(entity ->
+            entity.apply("DOC.1.4", "OECD Guideline year found", "n-a")
+        );
+        entityCreationService.byRegex("(?<=OECD)[\\w\\s,\\[\\]]{1,10}\\((\\d{4})\\)\\s(4[\\d]{2})", "oecd_guideline_year", EntityType.ENTITY,1, $headline).forEach(entity ->
+            entity.apply("DOC.1.4", "OECD Guideline year found", "n-a")
+        );
+        entityCreationService.byRegex("(?<=OECD).{5,40}Method (4[\\d]{2}).{1,65}(\\d{4})\\)", "oecd_guideline_number", EntityType.ENTITY,1, $headline).forEach(entity ->
+             entity.apply("DOC.1.4", "OECD Guideline number found", "n-a")
+        );
+        entityCreationService.byRegex("(?<=OECD).{5,40}Method (4[\\d]{2}).{1,65}(\\d{4})\\)", "oecd_guideline_year", EntityType.ENTITY,2, $headline).forEach(entity ->
+            entity.apply("DOC.1.4", "OECD Guideline year found", "n-a")
+        );
+        entityCreationService.byRegex("(?<=OECD) Guideline (4\\d{2})", "oecd_guideline_number", EntityType.ENTITY,1, $headline).forEach(entity ->
+            entity.apply("DOC.1.4", "OECD Guideline number found", "n-a")
+        );
+        entityCreationService.byRegex("OECD Guideline 4\\d{2}", "oecd_guideline", EntityType.ENTITY, $headline).forEach(entity ->
+            entity.apply("DOC.1.4", "OECD Guideline found", "n-a")
+        );
+    end
 
+
+// Rule unit: DOC.2
 rule "DOC.2.0: Report number"
     when
        $section: Section(containsString("LABORATORY PROJECT ID") , containsString("Report Number:"))
@@ -237,6 +313,7 @@ rule "DOC.2.0: Report number"
     end
 
 
+// Rule unit: DOC.3
 rule "DOC.3.0: Experimental Starting Date"
     when
         $section: Section(containsString("Experimental I. Starting Date:") || containsString("Experimental II. Starting Date:") || containsStringIgnoreCase("experimental start date") || containsStringIgnoreCase("experimental starting date"))
@@ -257,6 +334,7 @@ rule "DOC.3.0: Experimental Starting Date"
     end
 
 
+// Rule unit: DOC.4
 rule "DOC.4.0: Experimental Completion Date"
     when
         $section: Section(containsStringIgnoreCase("experimental termination date") || containsStringIgnoreCase("experimental completion date"))
@@ -275,20 +353,20 @@ rule "DOC.4.0: Experimental Completion Date"
     end
 
 
- rule "DOC.5.0: Ignore species and strain in irrelevant study types"
+// Rule unit: DOC.5
+rule "DOC.5.0: Ignore species and strain in irrelevant study types"
      salience 1
      when
          FileAttribute(label == "OECD Number", valueEqualsAnyOf("406","428","438","439","471","474","487"))
          $section: Section(hasEntitiesOfType("species") || hasEntitiesOfType("strain"))
      then
          $section.getEntitiesOfType(List.of("species", "strain")).forEach(entity -> {
-             entity.removeFromGraph();
+             entity.remove("DOC.5.0", "removed");
              retract(entity);
          });
      end
 
-
- rule "DOC.5.1: Hide all skipped species and strains except in the relevant sections"
+rule "DOC.5.1: Hide all skipped species and strains except in the relevant sections"
      salience 1
      when
         FileAttribute(label == "OECD Number", valueEqualsAnyOf("402","403","404","405","425","429","436"))
@@ -302,12 +380,11 @@ rule "DOC.4.0: Experimental Completion Date"
         )
      then
          $section.getEntitiesOfType(List.of("species", "strain")).forEach(entity -> {
-             entity.removeFromGraph();
+             entity.remove("DOC.5.1", "removed");
              retract(entity);
          });
      end
 
-
 rule "DOC.5.2: Species"
     when
         FileAttribute(label == "OECD Number", valueEqualsAnyOf("402","403","404","405","425","429","436"))
@@ -319,7 +396,6 @@ rule "DOC.5.2: Species"
         });
     end
 
-
 rule "DOC.5.3: Strain"
     when
         FileAttribute(label == "OECD Number", valueEqualsAnyOf("402","403","404","405","425","429","436"))
@@ -339,20 +415,21 @@ rule "DOC.5.3: Strain"
     end
 
 
+// Rule unit: DOC.6
 rule "DOC.6.0: study title by document structure"
     when
        $table: Table(onPage(1),
                     (containsString("Final Report") || containsString("SPL")),
                      numberOfRows == 1,
                      numberOfCols == 1)
+       $tableCell: TableCell(row == 1, col == 1) from $table.streamTableCells().toList()
+       $paragraph: Paragraph(previousSibling.isPresent(), nextSibling.isPresent()) from $tableCell.streamChildren().toList()
     then
-
-        entityCreationService.bySemanticNode($table.getCell(0, 0).streamChildren().toList().get(1), "title", EntityType.ENTITY).ifPresent(entity -> {
+        entityCreationService.bySemanticNode($paragraph, "title", EntityType.ENTITY).ifPresent(entity -> {
             entity.apply("DOC.6.0", "Study title found", "n-a");
         });
      end
 
-
 rule "DOC.6.1: study title"
     when
         $table: Table(onPage(1), (containsString("Final Report") || containsString("SPL")))
@@ -362,7 +439,6 @@ rule "DOC.6.1: study title"
         });
      end
 
-
 rule "DOC.6.2: study title"
     when
         not Table(onPage(1), (containsString("Final Report") || containsString("SPL")))
@@ -374,7 +450,7 @@ rule "DOC.6.2: study title"
      end
 
 
-
+// Rule unit: DOC.7
 rule "DOC.7.0: Performing Laboratory (Name)"
     when
        $section: Section(containsString("PERFORMING LABORATORY:"))
@@ -384,8 +460,7 @@ rule "DOC.7.0: Performing Laboratory (Name)"
         });
     end
 
-
- rule "DOC.7.1: Performing Laboratory (Country)"
+rule "DOC.7.1: Performing Laboratory (Country)"
      when
          nerEntities: NerEntities(hasEntitiesOfType("COUNTRY"))
          $section: Section(containsString("PERFORMING LABORATORY:"))
@@ -395,27 +470,17 @@ rule "DOC.7.0: Performing Laboratory (Name)"
                      .map(nerEntity -> entityCreationService.byNerEntity(nerEntity, "laboratory_country", EntityType.ENTITY, $section))
                      .forEach(entity -> {
                           entity.apply("DOC.7.1", "Performing Laboratory found", "n-a");
-                          insert(entity);
                      });
      end
 
-
 rule "DOC.7.2: Performing Laboratory (Country & Name) from dict"
     when
-        $section: Section(
-            (hasEntitiesOfType("laboratory_country") || hasEntitiesOfType("laboratory_name"))
-            && (containsString("PERFORMING LABORATORY:") || (containsString("PERFORMING") && containsString("LABORATORY:")))
-        )
+        $section: Section(containsString("PERFORMING LABORATORY:") || (containsString("PERFORMING") && containsString("LABORATORY:")))
+        $countryOrNameFromDictionary: TextEntity(type == "laboratory_country" || type == "laboratory_name", $type: type, isDictionaryEntry()) from $section.getEntities()
     then
-        $section.getEntitiesOfType("laboratory_country").forEach(entity -> {
-            entity.apply("DOC.7.2", "Performing laboratory country dictionary entry found.", "n-a");
-        });
-        $section.getEntitiesOfType("laboratory_name").forEach(entity -> {
-            entity.apply("DOC.7.2", "Performing laboratory name dictionary entry found.", "n-a");
-        });
+        $countryOrNameFromDictionary.apply("DOC.7.2", "Performing " + $type + " dictionary entry found.");
     end
 
-
 rule "DOC.7.3: Performing Laboratory (Country) from dict"
     when
         $section: Section(
@@ -424,12 +489,13 @@ rule "DOC.7.3: Performing Laboratory (Country) from dict"
         )
     then
         $section.getEntitiesOfType(List.of("laboratory_country", "laboratory_name")).forEach(entity -> {
-            entity.removeFromGraph();
+            entity.remove("DOC.7.3", "removed");
             retract(entity);
         });
     end
 
 
+// Rule unit: DOC.8
 rule "DOC.8.0: GLP Study"
     when
         $headline: Headline(containsString("GOOD LABORATORY PRACTICE COMPLIANCE")
@@ -447,6 +513,7 @@ rule "DOC.8.0: GLP Study"
     end
 
 
+// Rule unit: DOC.9
 rule "DOC.9.0: Batch number from CoA"
     when
         $section: Section(
@@ -485,7 +552,6 @@ rule "DOC.9.0: Batch number from CoA"
         });
    end
 
-
 rule "DOC.9.1: Batch number"
     when
         $section: Section(
@@ -518,7 +584,6 @@ rule "DOC.9.1: Batch number"
                       });
    end
 
-
 rule "DOC.9.2: Batch number"
     when
         $section: Section(
@@ -551,8 +616,7 @@ rule "DOC.9.2: Batch number"
    end
 
 
-
-
+// Rule unit: DOC.10
 rule "DOC.10.0: Conclusions - LD50, LC50, Confidence"
     when
         FileAttribute(label == "OECD Number", valueEqualsAnyOf("402","403","425","436"))
@@ -586,6 +650,7 @@ rule "DOC.10.0: Conclusions - LD50, LC50, Confidence"
     end
 
 
+// Rule unit: DOC.11
 rule "DOC.11.0: Guideline Deviation"
     when
         FileAttribute(label == "OECD Number", valueEqualsAnyOf("402","403","404","405","425","429","436","471"))
@@ -611,7 +676,6 @@ rule "DOC.11.0: Guideline Deviation"
         });
     end
 
-
 rule "DOC.11.1: Guideline Deviation in text"
     when
         FileAttribute(label == "OECD Number", valueEqualsAnyOf("402","403","404","405","425","429","436","471"))
@@ -626,6 +690,7 @@ rule "DOC.11.1: Guideline Deviation in text"
     end
 
 
+// Rule unit: DOC.12
 rule "DOC.12.0: Clinical Signs"
     when
         FileAttribute(label == "OECD Number", value == "425")
@@ -636,6 +701,7 @@ rule "DOC.12.0: Clinical Signs"
     end
 
 
+// Rule unit: DOC.13
 rule "DOC.13.0: Dosages"
     when
         FileAttribute(label == "OECD Number", value == "425")
@@ -656,6 +722,7 @@ rule "DOC.13.0: Dosages"
     end
 
 
+// Rule unit: DOC.14
 rule "DOC.14.0: Mortality"
     when
         $headline: Headline(containsString("Mortality") && !containsString("TABLE"))
@@ -666,6 +733,7 @@ rule "DOC.14.0: Mortality"
     end
 
 
+// Rule unit: DOC.15
 rule "DOC.15.0: Study Conclusion"
     when
         FileAttribute(label == "OECD Number", valueEqualsAnyOf("402","403","404","405","425","429","436","471"))
@@ -678,6 +746,7 @@ rule "DOC.15.0: Study Conclusion"
     end
 
 
+// Rule unit: DOC.16
 rule "DOC.16.0: Weight Behavior Changes"
     when
     FileAttribute(label == "OECD Number", value == "402")
@@ -696,6 +765,7 @@ rule "DOC.16.0: Weight Behavior Changes"
     end
 
 
+// Rule unit: DOC.17
 rule "DOC.17.0: Necropsy findings"
     when
     FileAttribute(label == "OECD Number", valueEqualsAnyOf("402","403","436"))
@@ -715,6 +785,7 @@ rule "DOC.17.0: Necropsy findings"
     end
 
 
+// Rule unit: DOC.18
 rule "DOC.18.0: Clinical observations"
     when
         FileAttribute(label == "OECD Number", value == "403")
@@ -735,6 +806,7 @@ rule "DOC.18.0: Clinical observations"
     end
 
 
+// Rule unit: DOC.19
 rule "DOC.19.0: Bodyweight changes"
     when
         FileAttribute(label == "OECD Number", value == "403")
@@ -745,6 +817,7 @@ rule "DOC.19.0: Bodyweight changes"
     end
 
 
+// Rule unit: DOC.20
 rule "DOC.20.0: Study Design"
     when
         FileAttribute(label == "OECD Number", valueEqualsAnyOf("402","404","405","406","428","429","438","439","474","487"))
@@ -757,7 +830,6 @@ rule "DOC.20.0: Study Design"
             .forEach(entity -> entity.apply("DOC.20.0", "Study design section found", "n-a"));
     end
 
-
 rule "DOC.20.1: Study Design"
     when
         Headline(containsStringIgnoreCase("Study Design"), $sectionIdentifier: getSectionIdentifier())
@@ -770,6 +842,7 @@ rule "DOC.20.1: Study Design"
     end
 
 
+// Rule unit: DOC.21
 rule "DOC.21.0: Results and Conclusion (406, 428, 438, 439, 474 & 487)"
     when
         FileAttribute(label == "OECD Number", valueEqualsAnyOf("406","428","438","439","474","487"))
@@ -784,7 +857,6 @@ rule "DOC.21.0: Results and Conclusion (406, 428, 438, 439, 474 & 487)"
             .forEach(entity -> entity.apply("DOC.21.0", "Results and Conclusion found", "n-a"));
     end
 
-
 rule "DOC.21.1: Results and Conclusion (406, 428, 438, 439, 474 & 487)"
     when
         FileAttribute(label == "OECD Number", valueEqualsAnyOf("406","428","438","439","474","487"))
@@ -800,6 +872,7 @@ rule "DOC.21.1: Results and Conclusion (406, 428, 438, 439, 474 & 487)"
     end
 
 
+// Rule unit: DOC.22
 rule "DOC.22.0: Detailing (404 & 405)"
     when
         FileAttribute(label == "OECD Number", valueEqualsAnyOf("404","405"))
@@ -814,6 +887,7 @@ rule "DOC.22.0: Detailing (404 & 405)"
     end
 
 
+// Rule unit: DOC.23
 rule "DOC.23.0: Preliminary Test Results (429)"
     when
         FileAttribute(label == "OECD Number", value == "429")
@@ -827,6 +901,7 @@ rule "DOC.23.0: Preliminary Test Results (429)"
     end
 
 
+// Rule unit: DOC.24
 rule "DOC.24.0: Test Results (429)"
     when
         FileAttribute(label == "OECD Number", value == "429")
@@ -836,7 +911,6 @@ rule "DOC.24.0: Test Results (429)"
         .forEach(entity -> entity.apply("DOC.24.0", "Test Results found", "n-a"));
     end
 
-
 rule "DOC.24.1: Test Results (429)"
     when
         Headline(containsStringIgnoreCase("RESULTS AND DISCUSSION"), $sectionIdentifierResultsAndDiscussion: getSectionIdentifier())
@@ -849,6 +923,7 @@ rule "DOC.24.1: Test Results (429)"
     end
 
 
+// Rule unit: DOC.25
 rule "DOC.25.0: Approach used (429)"
     when
         FileAttribute(label == "OECD Number", value == "429")
@@ -863,6 +938,7 @@ rule "DOC.25.0: Approach used (429)"
     end
 
 
+// Rule unit: DOC.26
 rule "DOC.26.0: Sex"
     when
         FileAttribute(label == "OECD Number", valueEqualsAnyOf("405","429"))
@@ -885,6 +961,7 @@ rule "DOC.26.0: Sex"
     end
 
 
+// Rule unit: DOC.27
 rule "DOC.27.0: Animal Number 405"
     when
         FileAttribute(label == "OECD Number", value == "405")
@@ -910,6 +987,7 @@ rule "DOC.27.0: Animal Number 405"
     end
 
 
+// Rule unit: DOC.28
 rule "DOC.28.0: Animal Number 429"
     when
      FileAttribute(label == "OECD Number", value == "429")
@@ -935,7 +1013,6 @@ rule "DOC.28.0: Animal Number 429"
         });
     end
 
-
 rule "DOC.28.1: No. Of animals - Fallback to appendix tables listing all individual animals for 429"
     when
         $keyword: String() from List.of("Animal Number", "Animal No.", "Animal number")
@@ -948,11 +1025,11 @@ rule "DOC.28.1: No. Of animals - Fallback to appendix tables listing all individ
                 .map(Optional::get)
                 .forEach(entity -> {
                     entity.apply("DOC.28.1", "Animal number found.", "n-a");
-                    insert(entity);
                 });
      end
 
 
+// Rule unit: DOC.29
 rule "DOC.29.0: 4h Exposure"
     when
         FileAttribute(label == "OECD Number", valueEqualsAnyOf("403","436"))
@@ -966,6 +1043,7 @@ rule "DOC.29.0: 4h Exposure"
     end
 
 
+// Rule unit: DOC.30
 rule "DOC.30.0: Dilution of the test substance"
     when
         FileAttribute(label == "OECD Number", value == "404")
@@ -979,6 +1057,7 @@ rule "DOC.30.0: Dilution of the test substance"
     end
 
 
+// Rule unit: DOC.31
 rule "DOC.31.0: Positive Control"
     when
         FileAttribute(label == "OECD Number", value == "429")
@@ -992,6 +1071,7 @@ rule "DOC.31.0: Positive Control"
     end
 
 
+// Rule unit: DOC.32
 rule "DOC.32.0: Mortality Statement"
     when
         FileAttribute(label == "OECD Number", value == "402")
@@ -1002,6 +1082,7 @@ rule "DOC.32.0: Mortality Statement"
     end
 
 
+// Rule unit: DOC.33
 rule "DOC.33.0: Dose Mortality"
     when
         FileAttribute(label == "OECD Number", value == "425")
@@ -1024,7 +1105,6 @@ rule "DOC.33.0: Dose Mortality"
                             .map(Optional::get)
                             .forEach(entity -> {
                                 entity.apply("DOC.33.0", "Dose Mortality Data found.", "n-a");
-                                insert(entity);
                             });
 
         Stream.of($table.streamTableCellsWithHeader("Dose [mg/kg bodyweight]"),
@@ -1040,11 +1120,11 @@ rule "DOC.33.0: Dose Mortality"
                             .map(Optional::get)
                             .forEach(entity -> {
                                 entity.apply("DOC.33.0", "Dose Mortality Data found.", "n-a");
-                                insert(entity);
                             });
     end
 
 
+// Rule unit: DOC.34
 rule "DOC.34.0: Results (Main Study)"
     when
         FileAttribute(label == "OECD Number", value == "429")
@@ -1059,6 +1139,7 @@ rule "DOC.34.0: Results (Main Study)"
     end
 
 
+// Rule unit: DOC.35
 rule "DOC.35.0: Doses (mg/kg bodyweight)"
     when
         FileAttribute(label == "OECD Number", value == "402")
@@ -1070,7 +1151,8 @@ rule "DOC.35.0: Doses (mg/kg bodyweight)"
             .forEach(entity -> entity.apply("DOC.35.0", "Doses per bodyweight information found", "n-a"));
     end
 
-//------------------------------------ Manual redaction rules ------------------------------------
+
+//------------------------------------ Manual changes rules ------------------------------------
 
 // Rule unit: MAN.0
 rule "MAN.0.0: Apply manual resize redaction"
@@ -1104,7 +1186,7 @@ rule "MAN.0.1: Apply manual resize redaction"
 rule "MAN.1.0: Apply id removals that are valid and not in forced redactions to Entity"
     salience 128
     when
-        $idRemoval: IdRemoval($id: annotationId, status == AnnotationStatus.APPROVED)
+        $idRemoval: IdRemoval($id: annotationId, !removeFromDictionary, !removeFromAllDossiers, status == AnnotationStatus.APPROVED)
         $entityToBeRemoved: TextEntity(matchesAnnotationId($id))
     then
         $entityToBeRemoved.getManualOverwrite().addChange($idRemoval);
@@ -1191,6 +1273,14 @@ rule "MAN.3.2: Apply image recategorization"
         retract($recategorization);
     end
 
+rule "MAN.3.3: Apply recategorization entities by default"
+    salience 128
+    when
+        $entity: IEntity(getManualOverwrite().getRecategorized().orElse(false))
+    then
+        $entity.apply("MAN.3.3", "Recategorized entities are applied by default.");
+    end
+
 
 // Rule unit: MAN.4
 rule "MAN.4.0: Apply legal basis change"
@@ -1212,11 +1302,10 @@ rule "MAN.4.1: Apply legal basis change"
     end
 
 
-
 //------------------------------------ Entity merging rules ------------------------------------
 
 // Rule unit: X.0
-rule "X.0.0: remove Entity contained by Entity of same type"
+rule "X.0.0: Remove Entity contained by Entity of same type"
     salience 65
     when
         $larger: TextEntity($type: type, $entityType: entityType, active())
@@ -1228,7 +1317,7 @@ rule "X.0.0: remove Entity contained by Entity of same type"
 
 
 // Rule unit: X.2
-rule "X.2.0: remove Entity of type ENTITY when contained by FALSE_POSITIVE"
+rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
     salience 64
     when
         $falsePositive: TextEntity($type: type, entityType == EntityType.FALSE_POSITIVE, active())
@@ -1241,7 +1330,7 @@ rule "X.2.0: remove Entity of type ENTITY when contained by FALSE_POSITIVE"
 
 
 // Rule unit: X.3
-rule "X.3.0: remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"
+rule "X.3.0: Remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"
     salience 64
     when
         $falseRecommendation: TextEntity($type: type, entityType == EntityType.FALSE_RECOMMENDATION, active())
@@ -1253,7 +1342,7 @@ rule "X.3.0: remove Entity of type RECOMMENDATION when contained by FALSE_RECOMM
 
 
 // Rule unit: X.4
-rule "X.4.0: remove Entity of type RECOMMENDATION when intersected by ENTITY with same type"
+rule "X.4.0: Remove Entity of type RECOMMENDATION when intersected by ENTITY with same type"
     salience 256
     when
         $entity: TextEntity($type: type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
@@ -1266,7 +1355,7 @@ rule "X.4.0: remove Entity of type RECOMMENDATION when intersected by ENTITY wit
 
 
 // Rule unit: X.5
-rule "X.5.0: remove Entity of type RECOMMENDATION when contained by ENTITY"
+rule "X.5.0: Remove Entity of type RECOMMENDATION when contained by ENTITY"
     salience 256
     when
         $entity: TextEntity((entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
@@ -1278,7 +1367,7 @@ rule "X.5.0: remove Entity of type RECOMMENDATION when contained by ENTITY"
 
 
 // Rule unit: X.7
-rule "X.7.0: remove all images"
+rule "X.7.0: Remove all images"
     salience 512
     when
         $image: Image(imageType != ImageType.OCR, !hasManualChanges())
@@ -1291,8 +1380,7 @@ rule "X.7.0: remove all images"
 //------------------------------------ File attributes rules ------------------------------------
 
 // Rule unit: FA.1
-rule "FA.1.0: remove duplicate FileAttributes"
-
+rule "FA.1.0: Remove duplicate FileAttributes"
     salience 64
     when
         $fileAttribute: FileAttribute($label: label, $value: value)
@@ -1302,8 +1390,10 @@ rule "FA.1.0: remove duplicate FileAttributes"
     end
 
 
+//------------------------------------ Local dictionary search rules ------------------------------------
+
 // Rule unit: LDS.0
-rule "LDS.0.0: run local dictionary search"
+rule "LDS.0.0: Run local dictionary search"
     agenda-group "LOCAL_DICTIONARY_ADDS"
     salience -999
     when
@@ -1311,7 +1401,7 @@ rule "LDS.0.0: run local dictionary search"
     then
         entityCreationService.bySearchImplementation($dictionaryModel.getLocalSearch(), $dictionaryModel.getType(), EntityType.RECOMMENDATION, document)
             .forEach(entity ->  {
-                Collection<MatchedRule> matchedRules = $dictionaryModel.getLocalEntriesWithMatchedRules().get(entity.getValue());
-                entity.addMatchedRules(matchedRules);
+                Collection<MatchedRule> matchedRules = $dictionaryModel.getMatchedRulesForLocalDictionaryEntry(entity.getValue());
+                matchedRules.forEach(matchedRule -> entity.addMatchedRule(matchedRule.asSkippedIfApplied()));
             });
     end

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora_components.drl

@@ -217,7 +217,7 @@ rule "GuidelineDeviation.0.0: Guideline deviation as sentences"
 
 rule "Species.0.0: First found species"
     when
-        $oecdNumber: String() from List.of("402", "403", "404", "405", "425", "429", "436", "471")
+        $oecdNumber: String() from List.of("402", "403", "404", "405", "425", "429", "436")
         FileAttribute(label == "OECD Number", value == $oecdNumber)
         $species: List() from collect (Entity(type == "species"))
     then
@@ -313,7 +313,7 @@ rule "Necropsy.2.0: Conducted with 4 hours of exposure as one block"
 
 rule "StudyDesign.0.0: Study design as one block"
     when
-        $oecdNumber: String() from List.of("404", "405", "429", "406", "428", "438", "439", "474", "487")
+        $oecdNumber: String() from List.of("404", "405", "406", "428", "429", "438", "439", "474", "487")
         FileAttribute(label == "OECD Number", value == $oecdNumber)
         $studyDesigns: List() from collect (Entity(type == "study_design"))
     then
@@ -497,15 +497,16 @@ rule "UsedApproach.1.0: Used approach not found and thus 'Individual'"
         componentCreationService.create("UsedApproach.1.0", "What_was_the_approach_used", "Individual", "'Group' when approach used is present, else 'Individual'");
     end
 
-/*
+
 rule "DefaultComponents.999.0: Create components for all unmapped entities."
     salience -999
     when
+        not FileAttribute(label == "OECD Number")
         $allEntities: List(!isEmpty()) from collect (Entity())
     then
         componentCreationService.createComponentsForUnMappedEntities("DefaultComponents.999.0", $allEntities);
     end
-*/
+
 
 //------------------------------------ Component merging rules ------------------------------------
 /*

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/manual_redaction_rules.drl

@@ -22,6 +22,7 @@ import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRu
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.*;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Section;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Table;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Paragraph;
@@ -65,7 +66,7 @@ query "getFileAttributes"
     $fileAttribute: FileAttribute()
     end
 
-//------------------------------------ Manual redaction rules ------------------------------------
+//------------------------------------ Manual changes rules ------------------------------------
 
 // Rule unit: MAN.0
 rule "MAN.0.0: Apply manual resize redaction"
@@ -99,7 +100,7 @@ rule "MAN.0.1: Apply manual resize redaction"
 rule "MAN.1.0: Apply id removals that are valid and not in forced redactions to Entity"
     salience 128
     when
-        $idRemoval: IdRemoval($id: annotationId, status == AnnotationStatus.APPROVED)
+        $idRemoval: IdRemoval($id: annotationId, !removeFromDictionary, !removeFromAllDossiers, status == AnnotationStatus.APPROVED)
         $entityToBeRemoved: TextEntity(matchesAnnotationId($id))
     then
         $entityToBeRemoved.getManualOverwrite().addChange($idRemoval);
@@ -162,7 +163,6 @@ rule "MAN.3.0: Apply entity recategorization"
         retract($entityToBeRecategorized);
     end
 
-
 rule "MAN.3.1: Apply entity recategorization of same type"
     salience 128
     when
@@ -174,7 +174,6 @@ rule "MAN.3.1: Apply entity recategorization of same type"
         retract($recategorization);
     end
 
-
 rule "MAN.3.2: Apply image recategorization"
     salience 128
     when
@@ -207,10 +206,12 @@ rule "MAN.4.1: Apply legal basis change"
     then
         $entityToBeChanged.getManualOverwrite().addChange($legalBasisChange);
     end
+
+
 //------------------------------------ Local dictionary search rules ------------------------------------
 
 // Rule unit: LDS.0
-rule "LDS.0.0: run local dictionary search"
+rule "LDS.0.0: Run local dictionary search"
     agenda-group "LOCAL_DICTIONARY_ADDS"
     salience -999
     when
@@ -218,7 +219,7 @@ rule "LDS.0.0: run local dictionary search"
     then
         entityCreationService.bySearchImplementation($dictionaryModel.getLocalSearch(), $dictionaryModel.getType(), EntityType.RECOMMENDATION, document)
             .forEach(entity ->  {
-                Collection<MatchedRule> matchedRules = $dictionaryModel.getLocalEntriesWithMatchedRules().get(entity.getValue());
-                entity.addMatchedRules(matchedRules);
+                Collection<MatchedRule> matchedRules = $dictionaryModel.getMatchedRulesForLocalDictionaryEntry(entity.getValue());
+                matchedRules.forEach(matchedRule -> entity.addMatchedRule(matchedRule.asSkippedIfApplied()));
             });
     end

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl

@@ -22,6 +22,7 @@ import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRu
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.*;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Section;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Table;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Paragraph;
@@ -83,7 +84,7 @@ rule "SYN.0.0: Redact if CTL/* or BL/* was found (Non Vertebrate Study)"
 //------------------------------------ CBI rules ------------------------------------
 
 // Rule unit: CBI.3
-rule "CBI.3.0: Redacted because Section contains Vertebrate"
+rule "CBI.3.0: Redacted because Section contains a vertebrate"
     when
         $section: Section(!hasTables(), hasEntitiesOfType("vertebrate"), (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
     then
@@ -98,7 +99,7 @@ rule "CBI.3.0: Redacted because Section contains Vertebrate"
             });
     end
 
-rule "CBI.3.1: Redacted because Table Row contains Vertebrate"
+rule "CBI.3.1: Redacted because table row contains a vertebrate"
     when
         $table: Table(hasEntitiesOfType("vertebrate"), (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
     then
@@ -114,7 +115,7 @@ rule "CBI.3.1: Redacted because Table Row contains Vertebrate"
             });
     end
 
-rule "CBI.3.2: Don't redact because Section doesn't contain Vertebrate"
+rule "CBI.3.2: Do not redact because Section does not contain a vertebrate"
     when
         $section: Section(!hasTables(), !hasEntitiesOfType("vertebrate"), (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
     then
@@ -122,7 +123,7 @@ rule "CBI.3.2: Don't redact because Section doesn't contain Vertebrate"
             .forEach(entity -> entity.skip("CBI.3.2", "No vertebrate found"));
     end
 
-rule "CBI.3.3: Dont redact because Table Row doesn't contain Vertebrate"
+rule "CBI.3.3: Do not redact because table row does not contain a vertebrate"
     when
         $table: Table(hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address"))
     then
@@ -133,7 +134,7 @@ rule "CBI.3.3: Dont redact because Table Row doesn't contain Vertebrate"
 
 
 // Rule unit: CBI.4
-rule "CBI.4.0: Dont redact Names and Addresses if no_redaction_indicator is found in Section"
+rule "CBI.4.0: Do not redact Names and Addresses if no_redaction_indicator is found in Section"
     when
         $section: Section(!hasTables(),
                           hasEntitiesOfType("vertebrate"),
@@ -150,7 +151,7 @@ rule "CBI.4.0: Dont redact Names and Addresses if no_redaction_indicator is foun
             });
     end
 
-rule "CBI.4.1: Dont redact Names and Addresses if no_redaction_indicator is found in Table Row"
+rule "CBI.4.1: Do not redact Names and Addresses if no_redaction_indicator is found in table row"
     when
         $table: Table(hasEntitiesOfType("no_redaction_indicator"),
                       hasEntitiesOfType("vertebrate"),
@@ -171,7 +172,7 @@ rule "CBI.4.1: Dont redact Names and Addresses if no_redaction_indicator is foun
 
 
 // Rule unit: CBI.5
-rule "CBI.5.0: Redact Names and Addresses if no_redaction_indicator but also redaction_indicator is found in section"
+rule "CBI.5.0: Redact Names and Addresses if no_redaction_indicator but also redaction_indicator is found in Section"
     when
         $section: Section(!hasTables(),
                           hasEntitiesOfType("redaction_indicator"),
@@ -191,7 +192,7 @@ rule "CBI.5.0: Redact Names and Addresses if no_redaction_indicator but also red
             });
     end
 
-rule "CBI.5.1: Redact Names and Addresses if no_redaction_indicator but also redaction_indicator is found in Table Row"
+rule "CBI.5.1: Redact Names and Addresses if no_redaction_indicator but also redaction_indicator is found in table row"
     when
         $table: Table(hasEntitiesOfType("no_redaction_indicator"),
                       hasEntitiesOfType("redaction_indicator"),
@@ -228,7 +229,7 @@ rule "CBI.8.0: Redacted because Section contains must_redact entity"
             });
     end
 
-rule "CBI.8.1: Redacted because Table Row contains must_redact entity"
+rule "CBI.8.1: Redacted because table row contains must_redact entity"
     when
         $table: Table(hasEntitiesOfType("must_redact"), (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
     then
@@ -246,7 +247,7 @@ rule "CBI.8.1: Redacted because Table Row contains must_redact entity"
 
 
 // Rule unit: CBI.9
-rule "CBI.9.0: Redact all Cell's with Header Author(s) as CBI_author (non vertebrate study)"
+rule "CBI.9.0: Redact all cells with Header Author(s) as CBI_author (non vertebrate study)"
     agenda-group "LOCAL_DICTIONARY_ADDS"
     when
         not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
@@ -256,10 +257,10 @@ rule "CBI.9.0: Redact all Cell's with Header Author(s) as CBI_author (non verteb
             .map(tableCell -> entityCreationService.bySemanticNode(tableCell, "CBI_author", EntityType.ENTITY))
             .filter(Optional::isPresent)
             .map(Optional::get)
-            .forEach(redactionEntity -> redactionEntity.apply("CBI.9.0", "Author(s) found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
+            .forEach(redactionEntity -> redactionEntity.redact("CBI.9.0", "Author(s) found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
     end
 
-rule "CBI.9.1: Redact all Cell's with Header Author as CBI_author (non vertebrate study)"
+rule "CBI.9.1: Redact all cells with Header Author as CBI_author (non vertebrate study)"
     agenda-group "LOCAL_DICTIONARY_ADDS"
     when
         not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
@@ -269,7 +270,7 @@ rule "CBI.9.1: Redact all Cell's with Header Author as CBI_author (non vertebrat
             .map(tableCell -> entityCreationService.bySemanticNode(tableCell, "CBI_author", EntityType.ENTITY))
             .filter(Optional::isPresent)
             .map(Optional::get)
-            .forEach(redactionEntity -> redactionEntity.apply("CBI.9.1", "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
+            .forEach(redactionEntity -> redactionEntity.redact("CBI.9.1", "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
     end
 
 
@@ -280,42 +281,59 @@ rule "CBI.11.0: Recommend all CBI_author entities in Table with Vertebrate Study
     when
         $table: Table(hasHeader("Author(s)") && hasHeader("Vertebrate Study Y/N"))
     then
-        $table.getEntitiesOfType("CBI_author").forEach(entity -> dictionary.addMultipleAuthorsAsRecommendation(entity));
+        $table.getEntitiesOfType("CBI_author").stream().filter(IEntity::applied).forEach(entity -> dictionary.addMultipleAuthorsAsRecommendation(entity));
     end
 
 
 // Rule unit: CBI.12
-rule "CBI.12.0: Add all Cell's with Header Author(s) as CBI_author"
-    salience 1
+rule "CBI.12.0: Redact and recommend TableCell with header 'Author' or 'Author(s)' and header 'Vertebrate study Y/N' with value 'Yes' (non vertebrate study)"
+    agenda-group "LOCAL_DICTIONARY_ADDS"
     when
-        $table: Table(hasHeader("Author(s)") || hasHeader("Author"))
+        not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
+        $table: Table(hasHeader("Author(s)") || hasHeader("Author"), hasHeaderIgnoreCase("Vertebrate Study Y/N"))
+        TableCell(header, containsAnyStringIgnoreCase("Author", "Author(s)"), $authorCol: col) from $table.streamHeaders().toList()
+        TableCell(header, containsStringIgnoreCase("Vertebrate study Y/N"), $vertebrateCol: col) from $table.streamHeaders().toList()
+        TableCell(!header, containsAnyString("Yes", "Y"), $rowWithYes: row) from $table.streamCol($vertebrateCol).toList()
+        $authorCell: TableCell(row == $rowWithYes) from $table.streamCol($authorCol).toList()
     then
-        Stream.concat(
-            $table.streamTableCellsWithHeader("Author(s)"),
-            $table.streamTableCellsWithHeader("Author")
-        )
-            .map(tableCell -> entityCreationService.bySemanticNode(tableCell, "CBI_author", EntityType.ENTITY))
-            .filter(Optional::isPresent)
-            .map(Optional::get)
-            .forEach(redactionEntity -> redactionEntity.skip("CBI.12.0", "Author(s) header found"));
+
+        entityCreationService.bySemanticNode($authorCell, "CBI_author", EntityType.ENTITY)
+          .ifPresent(authorEntity -> {
+              authorEntity.redact("CBI.12.0", "Redacted because it's row belongs to a vertebrate study", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+              dictionary.addMultipleAuthorsAsRecommendation(authorEntity);
+          });
+
     end
 
-rule "CBI.12.1: Dont redact CBI_author, if its row contains a cell with header \"Vertebrate study Y/N\" and value No"
+rule "CBI.12.1: Redact and recommend TableCell with header 'Author' or 'Author(s)' and header 'Vertebrate study Y/N' with value 'Yes' (vertebrate study)"
+    agenda-group "LOCAL_DICTIONARY_ADDS"
     when
-        $table: Table(hasRowWithHeaderAndValue("Vertebrate study Y/N", "N") || hasRowWithHeaderAndValue("Vertebrate study Y/N", "No"))
+        FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
+        $table: Table(hasHeader("Author(s)") || hasHeader("Author"), hasHeaderIgnoreCase("Vertebrate Study Y/N"))
+        TableCell(header, containsAnyStringIgnoreCase("Author", "Author(s)"), $authorCol: col) from $table.streamHeaders().toList()
+        TableCell(header, containsStringIgnoreCase("Vertebrate study Y/N"), $vertebrateCol: col) from $table.streamHeaders().toList()
+        TableCell(!header, containsAnyString("Yes", "Y"), $rowWithYes: row) from $table.streamCol($vertebrateCol).toList()
+        $authorCell: TableCell(row == $rowWithYes) from $table.streamCol($authorCol).toList()
     then
-        $table.streamEntitiesWhereRowHasHeaderAndAnyValue("Vertebrate study Y/N", List.of("N", "No"))
-            .filter(redactionEntity -> redactionEntity.isAnyType(List.of("CBI_author", "CBI_address")))
-            .forEach(authorEntity -> authorEntity.skip("CBI.12.1", "Not redacted because it's row does not belong to a vertebrate study"));
+
+        entityCreationService.bySemanticNode($authorCell, "CBI_author", EntityType.ENTITY)
+          .ifPresent(authorEntity -> {
+              authorEntity.redact("CBI.12.1", "Redacted because it's row belongs to a vertebrate study", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+              dictionary.addMultipleAuthorsAsRecommendation(authorEntity);
+          });
+
     end
 
-rule "CBI.12.2: Redact CBI_author, if its row contains a cell with header \"Vertebrate study Y/N\" and value Yes"
+rule "CBI.12.2: Skip TableCell with header 'Author' or 'Author(s)' and header 'Vertebrate study Y/N' with value 'No'"
     when
-        $table: Table(hasRowWithHeaderAndValue("Vertebrate study Y/N", "Y") || hasRowWithHeaderAndValue("Vertebrate study Y/N", "Yes"))
+        $table: Table(hasHeader("Author(s)") || hasHeader("Author"), hasHeaderIgnoreCase("Vertebrate Study Y/N"))
+        TableCell(header, containsAnyStringIgnoreCase("Author", "Author(s)"), $authorCol: col) from $table.streamHeaders().toList()
+        TableCell(header, containsStringIgnoreCase("Vertebrate study Y/N"), $vertebrateCol: col) from $table.streamHeaders().toList()
+        TableCell(!header, containsAnyString("No", "N"), $rowWithNo: row) from $table.streamCol($vertebrateCol).toList()
+        $authorCell: TableCell(row == $rowWithNo) from $table.streamCol($authorCol).toList()
     then
-        $table.streamEntitiesWhereRowHasHeaderAndAnyValue("Vertebrate study Y/N", List.of("Y", "Yes"))
-            .filter(redactionEntity -> redactionEntity.isAnyType(List.of("CBI_author", "CBI_address")))
-            .forEach(authorEntity -> authorEntity.apply("CBI.12.2", "Redacted because it's row belongs to a vertebrate study", "Reg (EC) No 1107/2009 Art. 63 (2g)"));
+        entityCreationService.bySemanticNode($authorCell, "CBI_author", EntityType.ENTITY)
+          .ifPresent(authorEntity -> authorEntity.skip("CBI.12.2", "Not redacted because it's row does not belong to a vertebrate study"));
     end
 
 
@@ -324,7 +342,7 @@ rule "CBI.14.0: Redact CBI_sponsor entities if preceded by \"batches produced at
     when
         $sponsorEntity: TextEntity(type == "CBI_sponsor", textBefore.contains("batches produced at"))
     then
-        $sponsorEntity.apply("CBI.14.0", "Redacted because it represents a sponsor company", "Reg (EC) No 1107/2009 Art. 63 (2g)");
+        $sponsorEntity.redact("CBI.14.0", "Redacted because it represents a sponsor company", "Reg (EC) No 1107/2009 Art. 63 (2g)");
     end
 
 
@@ -349,7 +367,7 @@ rule "CBI.15.0: Redact row if row contains \"determination of residues\" and liv
             .toList();
 
         $section.getEntitiesOfType(List.of($keyword, $residueKeyword))
-            .forEach(redactionEntity -> redactionEntity.apply("CBI.15.0", "Determination of residues and keyword \"" + $keyword + "\" was found.", "Reg (EC) No 1107/2009 Art. 63 (2g)"));
+            .forEach(redactionEntity -> redactionEntity.redact("CBI.15.0", "Determination of residues and keyword \"" + $keyword + "\" was found.", "Reg (EC) No 1107/2009 Art. 63 (2g)"));
     end
 
 rule "CBI.15.1: Redact CBI_author and CBI_address if row contains \"determination of residues\" and livestock keyword"
@@ -371,12 +389,12 @@ rule "CBI.15.1: Redact CBI_author and CBI_address if row contains \"determinatio
 
         $table.streamEntitiesWhereRowContainsStringsIgnoreCase(List.of($keyword, $residueKeyword))
             .filter(redactionEntity -> redactionEntity.isAnyType(List.of("CBI_author", "CBI_address")))
-            .forEach(redactionEntity -> redactionEntity.apply("CBI.15.1", "Determination of residues and keyword \"" + $keyword + "\" was found.", "Reg (EC) No 1107/2009 Art. 63 (2g)"));
+            .forEach(redactionEntity -> redactionEntity.redact("CBI.15.1", "Determination of residues and keyword \"" + $keyword + "\" was found.", "Reg (EC) No 1107/2009 Art. 63 (2g)"));
     end
 
 
 // Rule unit: CBI.16
-rule "CBI.16.0: Add CBI_author with \"et al.\" Regex (non vertebrate study)"
+rule "CBI.16.0: Add CBI_author with \"et al.\" RegEx (non vertebrate study)"
     agenda-group "LOCAL_DICTIONARY_ADDS"
     when
         not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
@@ -384,12 +402,12 @@ rule "CBI.16.0: Add CBI_author with \"et al.\" Regex (non vertebrate study)"
     then
         entityCreationService.byRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, 1, $section)
             .forEach(entity -> {
-                entity.apply("CBI.16.0", "Author found by \"et al\" regex", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+                entity.redact("CBI.16.0", "Author found by \"et al\" regex", "Article 39(e)(3) of Regulation (EC) No 178/2002");
                 dictionary.recommendEverywhere(entity);
             });
     end
 
-rule "CBI.16.1: Add CBI_author with \"et al.\" Regex (vertebrate study)"
+rule "CBI.16.1: Add CBI_author with \"et al.\" RegEx (vertebrate study)"
     agenda-group "LOCAL_DICTIONARY_ADDS"
     when
         FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
@@ -397,7 +415,7 @@ rule "CBI.16.1: Add CBI_author with \"et al.\" Regex (vertebrate study)"
     then
         entityCreationService.byRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, 1, $section)
             .forEach(entity -> {
-                entity.apply("CBI.16.1", "Author found by \"et al\" regex", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+                entity.redact("CBI.16.1", "Author found by \"et al\" regex", "Article 39(e)(2) of Regulation (EC) No 178/2002");
                 dictionary.recommendEverywhere(entity);
             });
     end
@@ -476,7 +494,7 @@ rule "CBI.20.1: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJEC
     then
          entityCreationService.betweenStrings("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", EntityType.ENTITY, $section)
          .forEach(laboratoryEntity -> {
-             laboratoryEntity.apply("CBI.20.1", "PERFORMING LABORATORY was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+             laboratoryEntity.redact("CBI.20.1", "PERFORMING LABORATORY was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
              dictionary.recommendEverywhere(laboratoryEntity);
          });
     end
@@ -490,7 +508,7 @@ rule "PII.0.0: Redact all PII (non vertebrate study)"
         not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
         $pii: TextEntity(type == "PII", dictionaryEntry)
     then
-        $pii.apply("PII.0.0", "Personal Information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        $pii.redact("PII.0.0", "Personal Information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
     end
 
 rule "PII.0.1: Redact all PII (vertebrate study)"
@@ -498,7 +516,7 @@ rule "PII.0.1: Redact all PII (vertebrate study)"
         FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
         $pii: TextEntity(type == "PII", dictionaryEntry)
     then
-        $pii.apply("PII.0.1", "Personal Information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        $pii.redact("PII.0.1", "Personal Information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
     end
 
 
@@ -509,7 +527,7 @@ rule "PII.1.0: Redact Emails by RegEx (Non vertebrate study)"
         $section: Section(containsString("@"))
     then
         entityCreationService.byRegex("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", "PII", EntityType.ENTITY, 1, $section)
-            .forEach(emailEntity -> emailEntity.apply("PII.1.0", "Found by Email Regex", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
+            .forEach(emailEntity -> emailEntity.redact("PII.1.0", "Found by Email Regex", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
     end
 
 rule "PII.1.1: Redact Emails by RegEx (vertebrate study)"
@@ -518,7 +536,7 @@ rule "PII.1.1: Redact Emails by RegEx (vertebrate study)"
         $section: Section(containsString("@"))
     then
         entityCreationService.byRegex("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", "PII", EntityType.ENTITY, 1, $section)
-            .forEach(emailEntity -> emailEntity.apply("PII.1.1", "Found by Email Regex", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
+            .forEach(emailEntity -> emailEntity.redact("PII.1.1", "Found by Email Regex", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
     end
 
 
@@ -548,12 +566,12 @@ rule "PII.4.0: Redact line after contact information keywords (non vertebrate st
         $section: Section(containsString($contactKeyword))
     then
         entityCreationService.lineAfterString($contactKeyword, "PII", EntityType.ENTITY, $section)
-            .forEach(contactEntity -> contactEntity.apply("PII.4.0", "Found after \"" + $contactKeyword + "\" contact keyword", "Reg (EC) No 1107/2009 Art. 63 (2e)"));
+            .forEach(contactEntity -> contactEntity.redact("PII.4.0", "Found after \"" + $contactKeyword + "\" contact keyword", "Reg (EC) No 1107/2009 Art. 63 (2e)"));
     end
 
-rule "PII.4.1: Redact line after contact information keywords (non vertebrate study)"
+rule "PII.4.1: Redact line after contact information keywords (vertebrate study)"
     when
-        not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
+        FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
         $contactKeyword: String() from List.of("Contact point:",
                                          "Contact:",
                                          "Alternative contact:",
@@ -576,12 +594,12 @@ rule "PII.4.1: Redact line after contact information keywords (non vertebrate st
         $section: Section(containsString($contactKeyword))
     then
         entityCreationService.lineAfterString($contactKeyword, "PII", EntityType.ENTITY, $section)
-            .forEach(contactEntity -> contactEntity.apply("PII.4.1", "Found after \"" + $contactKeyword + "\" contact keyword", "Reg (EC) No 1107/2009 Art. 63 (2e)"));
+            .forEach(contactEntity -> contactEntity.redact("PII.4.1", "Found after \"" + $contactKeyword + "\" contact keyword", "Reg (EC) No 1107/2009 Art. 63 (2e)"));
     end
 
 
 // Rule unit: PII.6
-rule "PII.6.0: redact line between contact keywords (non vertebrate study)"
+rule "PII.6.0: Redact line between contact keywords (non vertebrate study)"
     when
         not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
         $section: Section((containsString("No:") && containsString("Fax")) || (containsString("Contact:") && containsString("Tel")))
@@ -590,10 +608,10 @@ rule "PII.6.0: redact line between contact keywords (non vertebrate study)"
             entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section),
             entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section)
         )
-            .forEach(contactEntity -> contactEntity.apply("PII.6.0", "Found between contact keywords", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
+            .forEach(contactEntity -> contactEntity.redact("PII.6.0", "Found between contact keywords", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
     end
 
-rule "PII.6.1: redact line between contact keywords"
+rule "PII.6.1: Redact line between contact keywords (vertebrate study)"
     when
         FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
         $section: Section((containsString("No:") && containsString("Fax")) || (containsString("Contact:") && containsString("Tel")))
@@ -602,7 +620,7 @@ rule "PII.6.1: redact line between contact keywords"
             entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section),
             entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section)
         )
-            .forEach(contactEntity -> contactEntity.apply("PII.6.1", "Found between contact keywords", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
+            .forEach(contactEntity -> contactEntity.redact("PII.6.1", "Found between contact keywords", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
     end
 
 
@@ -622,10 +640,10 @@ rule "PII.7.0: Redact contact information if applicant is found (non vertebrate
                  entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section),
                  entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section)
              ))
-            .forEach(entity -> entity.apply("PII.7.0", "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
+            .forEach(entity -> entity.redact("PII.7.0", "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
     end
 
-rule "PII.7.1: Redact contact information if applicant is found (non vertebrate study)"
+rule "PII.7.1: Redact contact information if applicant is found (vertebrate study)"
     when
         FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
         $section: Section(getHeadline().containsString("applicant") ||
@@ -640,12 +658,12 @@ rule "PII.7.1: Redact contact information if applicant is found (non vertebrate
                  entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section),
                  entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section)
              ))
-            .forEach(entity -> entity.apply("PII.7.1", "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
+            .forEach(entity -> entity.redact("PII.7.1", "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
     end
 
 
 // Rule unit: PII.8
-rule "PII.8.0: Redact contact information if producer is found"
+rule "PII.8.0: Redact contact information if producer is found (non vertebrate study)"
     when
         not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
         $section: Section(containsStringIgnoreCase("producer of the plant protection") ||
@@ -660,10 +678,10 @@ rule "PII.8.0: Redact contact information if producer is found"
                  entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section),
                  entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section)
              ))
-            .forEach(entity -> entity.apply("PII.8.0", "Producer was found", "Reg (EC) No 1107/2009 Art. 63 (2e)"));
+            .forEach(entity -> entity.redact("PII.8.0", "Producer was found", "Reg (EC) No 1107/2009 Art. 63 (2e)"));
     end
 
-rule "PII.8.1: Redact contact information if producer is found"
+rule "PII.8.1: Redact contact information if producer is found (vertebrate study)"
     when
         FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
         $section: Section(containsStringIgnoreCase("producer of the plant protection") ||
@@ -678,45 +696,27 @@ rule "PII.8.1: Redact contact information if producer is found"
                  entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section),
                  entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section)
              ))
-            .forEach(entity -> entity.apply("PII.8.1", "Producer was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
+            .forEach(entity -> entity.redact("PII.8.1", "Producer was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
     end
 
 
 // Rule unit: PII.9
-rule "PII.9.0: Redact between \"AUTHOR(S)\" and \"COMPLETION DATE\" (non vertebrate study)"
+rule "PII.9.0: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\" (non vertebrate study)"
     when
         not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
-        $section: Section(!hasTables(), containsString("AUTHOR(S):"), containsString("COMPLETION DATE:"), !containsString("STUDY COMPLETION DATE:"))
+        $document: Document(containsStringIgnoreCase("AUTHOR(S)"), containsAnyStringIgnoreCase("COMPLETION DATE", "STUDY COMPLETION DATE"))
     then
-        entityCreationService.betweenStrings("AUTHOR(S):", "COMPLETION DATE:", "PII", EntityType.ENTITY, $section)
-         .forEach(authorEntity -> authorEntity.apply("PII.9.0", "AUTHOR(S) was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
+        entityCreationService.shortestBetweenAnyStringIgnoreCase(List.of("AUTHOR(S)", "AUTHOR(S):"), List.of("COMPLETION DATE", "COMPLETION DATE:", "STUDY COMPLETION DATE", "STUDY COMPLETION DATE:"), "PII", EntityType.ENTITY, $document)
+         .forEach(authorEntity -> authorEntity.redact("PII.9.0", "AUTHOR(S) was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
     end
 
-rule "PII.9.1: Redact between \"AUTHOR(S)\" and \"STUDY COMPLETION DATE\" (non vertebrate study)"
+rule "PII.9.1: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\" (vertebrate study)"
     when
         FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
-        $section: Section(!hasTables(), containsString("AUTHOR(S):"), containsString("COMPLETION DATE:"), !containsString("STUDY COMPLETION DATE:"))
+        $document: Document(containsStringIgnoreCase("AUTHOR(S)"), containsAnyStringIgnoreCase("COMPLETION DATE", "STUDY COMPLETION DATE"))
     then
-        entityCreationService.betweenStrings("AUTHOR(S):", "COMPLETION DATE:", "PII", EntityType.ENTITY, $section)
-         .forEach(authorEntity -> authorEntity.apply("PII.9.1", "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
-    end
-
-rule "PII.9.2: Redact between \"AUTHOR(S)\" and \"COMPLETION DATE\" (non vertebrate study)"
-    when
-        not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
-        $section: Section(!hasTables(), containsString("AUTHOR(S):"), containsString("STUDY COMPLETION DATE:"))
-    then
-        entityCreationService.betweenStrings("AUTHOR(S):", "STUDY COMPLETION DATE:", "PII", EntityType.ENTITY, $section)
-         .forEach(authorEntity -> authorEntity.apply("PII.9.2", "AUTHOR(S) was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
-    end
-
-rule "PII.9.3: Redact between \"AUTHOR(S)\" and \"STUDY COMPLETION DATE\" (vertebrate study)"
-    when
-        FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
-        $section: Section(!hasTables(), containsString("AUTHOR(S):"), containsString("STUDY COMPLETION DATE:"))
-    then
-        entityCreationService.betweenStrings("AUTHOR(S):", "STUDY COMPLETION DATE:", "PII", EntityType.ENTITY, $section)
-         .forEach(authorEntity -> authorEntity.apply("PII.9.3", "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
+        entityCreationService.shortestBetweenAnyStringIgnoreCase(List.of("AUTHOR(S)", "AUTHOR(S):"), List.of("COMPLETION DATE", "COMPLETION DATE:", "STUDY COMPLETION DATE", "STUDY COMPLETION DATE:"), "PII", EntityType.ENTITY, $document)
+        .forEach(authorEntity -> authorEntity.redact("PII.9.1", "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
     end
 
 
@@ -726,7 +726,7 @@ rule "PII.11.0: Redact On behalf of Sequani Ltd.:"
         $section: Section(!hasTables(), containsString("On behalf of Sequani Ltd.: Name Title"))
     then
         entityCreationService.betweenStrings("On behalf of Sequani Ltd.: Name Title", "On behalf of", "PII", EntityType.ENTITY, $section)
-         .forEach(authorEntity -> authorEntity.apply("PII.11.0", "On behalf of Sequani Ltd.: Name Title was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
+         .forEach(authorEntity -> authorEntity.redact("PII.11.0", "On behalf of Sequani Ltd.: Name Title was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
     end
 
 
@@ -748,7 +748,7 @@ rule "ETC.1.0: Redact Purity"
         $section: Section(containsStringIgnoreCase("purity"))
     then
         entityCreationService.byRegex("\\bPurity:\\s*(<?>?\\s*\\d{1,2}(?:\\.\\d{1,2})?\\s*%)", "purity", EntityType.ENTITY, 1, $section)
-            .forEach(entity -> entity.apply("ETC.1.0", "Purity found", "Reg (EC) No 1107/2009 Art. 63 (2a)"));
+            .forEach(entity -> entity.redact("ETC.1.0", "Purity found", "Reg (EC) No 1107/2009 Art. 63 (2a)"));
      end
 
 
@@ -758,7 +758,7 @@ rule "ETC.2.0: Redact signatures (non vertebrate study)"
         not FileAttribute(label == "Vertebrate Study", value == "Yes")
         $signature: Image(imageType == ImageType.SIGNATURE)
     then
-        $signature.apply("ETC.2.0", "Signature Found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        $signature.redact("ETC.2.0", "Signature Found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
     end
 
 rule "ETC.2.1: Redact signatures (vertebrate study)"
@@ -766,25 +766,25 @@ rule "ETC.2.1: Redact signatures (vertebrate study)"
         FileAttribute(label == "Vertebrate Study", value == "Yes")
         $signature: Image(imageType == ImageType.SIGNATURE)
     then
-        $signature.apply("ETC.2.1", "Signature Found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        $signature.redact("ETC.2.1", "Signature Found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
     end
 
 
 // Rule unit: ETC.3
-rule "ETC.3.0: Redact logos (vertebrate study)"
+rule "ETC.3.0: Skip logos (non vertebrate study)"
     when
         not FileAttribute(label == "Vertebrate Study", value == "Yes")
         $logo: Image(imageType == ImageType.LOGO)
     then
-        $logo.apply("ETC.3.0", "Logo Found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        $logo.skip("ETC.3.0", "Logo Found");
     end
 
-rule "ETC.3.1: Redact logos (non vertebrate study)"
+rule "ETC.3.1: Redact logos (vertebrate study)"
     when
         FileAttribute(label == "Vertebrate Study", value == "Yes")
         $logo: Image(imageType == ImageType.LOGO)
     then
-        $logo.apply("ETC.3.1", "Logo Found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        $logo.redact("ETC.3.1", "Logo Found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
     end
 
 
@@ -793,7 +793,7 @@ rule "ETC.4.0: Redact dossier dictionary entries"
     when
         $dossierRedaction: TextEntity(type == "dossier_redaction")
     then
-        $dossierRedaction.apply("ETC.4.0", "Specification of impurity found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        $dossierRedaction.redact("ETC.4.0", "Specification of impurity found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
     end
 
 
@@ -817,14 +817,14 @@ rule "ETC.6.0: Redact CAS Number"
             .map(tableCell -> entityCreationService.bySemanticNode(tableCell, "PII", EntityType.ENTITY))
             .filter(Optional::isPresent)
             .map(Optional::get)
-            .forEach(redactionEntity -> redactionEntity.apply("ETC.6.0", "Sample # found in Header", "Reg (EC) No 1107/2009 Art. 63 (2g)"));
+            .forEach(redactionEntity -> redactionEntity.redact("ETC.6.0", "Sample # found in Header", "Reg (EC) No 1107/2009 Art. 63 (2g)"));
     end
 
 
 // Rule unit: ETC.7
 rule "ETC.7.0: Guidelines FileAttributes"
     when
-        $section: Section(!hasTables(), (containsString("DATA REQUIREMENT(S):") || containsString("TEST GUIDELINE(S):")) && (containsString("OECD") || containsString("EPA") || containsString("OPPTS")))
+        $section: Section(!hasTables(), containsAnyString("DATA REQUIREMENT(S):", "TEST GUIDELINE(S):") && containsAnyString("OECD", "EPA", "OPPTS"))
     then
         RedactionSearchUtility.findTextRangesByRegex("OECD (No\\.? )?\\d{3}( \\(\\d{4}\\))?", $section.getTextBlock()).stream()
             .map(boundary -> $section.getTextBlock().subSequence(boundary).toString())
@@ -839,7 +839,7 @@ rule "ETC.8.0: Redact formulas (vertebrate study)"
         not FileAttribute(label == "Vertebrate Study", value == "Yes")
         $logo: Image(imageType == ImageType.FORMULA)
     then
-        $logo.apply("ETC.8.0", "Logo Found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        $logo.redact("ETC.8.0", "Logo Found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
     end
 
 rule "ETC.8.1: Redact formulas (non vertebrate study)"
@@ -847,14 +847,14 @@ rule "ETC.8.1: Redact formulas (non vertebrate study)"
         FileAttribute(label == "Vertebrate Study", value == "Yes")
         $logo: Image(imageType == ImageType.FORMULA)
     then
-        $logo.apply("ETC.8.1", "Logo Found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        $logo.redact("ETC.8.1", "Logo Found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
     end
 
 
 //------------------------------------ AI rules ------------------------------------
 
 // Rule unit: AI.0
-rule "AI.0.0: add all NER Entities of type CBI_author"
+rule "AI.0.0: Add all NER Entities of type CBI_author"
     salience 999
     when
         nerEntities: NerEntities(hasEntitiesOfType("CBI_author"))
@@ -865,7 +865,7 @@ rule "AI.0.0: add all NER Entities of type CBI_author"
 
 
 // Rule unit: AI.1
-rule "AI.1.0: combine and add NER Entities as CBI_address"
+rule "AI.1.0: Combine and add NER Entities as CBI_address"
     salience 999
     when
         nerEntities: NerEntities(hasEntitiesOfType("ORG") || hasEntitiesOfType("STREET") || hasEntitiesOfType("CITY"))
@@ -874,7 +874,7 @@ rule "AI.1.0: combine and add NER Entities as CBI_address"
     end
 
 
-//------------------------------------ Manual redaction rules ------------------------------------
+//------------------------------------ Manual changes rules ------------------------------------
 
 // Rule unit: MAN.0
 rule "MAN.0.0: Apply manual resize redaction"
@@ -908,7 +908,7 @@ rule "MAN.0.1: Apply manual resize redaction"
 rule "MAN.1.0: Apply id removals that are valid and not in forced redactions to Entity"
     salience 128
     when
-        $idRemoval: IdRemoval($id: annotationId, status == AnnotationStatus.APPROVED)
+        $idRemoval: IdRemoval($id: annotationId, !removeFromDictionary, !removeFromAllDossiers, status == AnnotationStatus.APPROVED)
         $entityToBeRemoved: TextEntity(matchesAnnotationId($id))
     then
         $entityToBeRemoved.getManualOverwrite().addChange($idRemoval);
@@ -1019,7 +1019,7 @@ rule "MAN.4.1: Apply legal basis change"
 //------------------------------------ Entity merging rules ------------------------------------
 
 // Rule unit: X.0
-rule "X.0.0: remove Entity contained by Entity of same type"
+rule "X.0.0: Remove Entity contained by Entity of same type"
     salience 65
     when
         $larger: TextEntity($type: type, $entityType: entityType, active())
@@ -1031,7 +1031,7 @@ rule "X.0.0: remove Entity contained by Entity of same type"
 
 
 // Rule unit: X.1
-rule "X.1.0: merge intersecting Entities of same type"
+rule "X.1.0: Merge intersecting Entities of same type"
     salience 64
     when
         $first: TextEntity($type: type, $entityType: entityType, !resized(), active())
@@ -1047,7 +1047,7 @@ rule "X.1.0: merge intersecting Entities of same type"
 
 
 // Rule unit: X.2
-rule "X.2.0: remove Entity of type ENTITY when contained by FALSE_POSITIVE"
+rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
     salience 64
     when
         $falsePositive: TextEntity($type: type, entityType == EntityType.FALSE_POSITIVE, active())
@@ -1060,7 +1060,7 @@ rule "X.2.0: remove Entity of type ENTITY when contained by FALSE_POSITIVE"
 
 
 // Rule unit: X.3
-rule "X.3.0: remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"
+rule "X.3.0: Remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"
     salience 64
     when
         $falseRecommendation: TextEntity($type: type, entityType == EntityType.FALSE_RECOMMENDATION, active())
@@ -1072,7 +1072,7 @@ rule "X.3.0: remove Entity of type RECOMMENDATION when contained by FALSE_RECOMM
 
 
 // Rule unit: X.4
-rule "X.4.0: remove Entity of type RECOMMENDATION when intersected by ENTITY with same type"
+rule "X.4.0: Remove Entity of type RECOMMENDATION when intersected by ENTITY with same type"
     salience 256
     when
         $entity: TextEntity($type: type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
@@ -1085,7 +1085,7 @@ rule "X.4.0: remove Entity of type RECOMMENDATION when intersected by ENTITY wit
 
 
 // Rule unit: X.5
-rule "X.5.0: remove Entity of type RECOMMENDATION when contained by ENTITY"
+rule "X.5.0: Remove Entity of type RECOMMENDATION when contained by ENTITY"
     salience 256
     when
         $entity: TextEntity((entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
@@ -1097,7 +1097,7 @@ rule "X.5.0: remove Entity of type RECOMMENDATION when contained by ENTITY"
 
 
 // Rule unit: X.6
-rule "X.6.0: remove Entity of lower rank, when contained by entity of type ENTITY"
+rule "X.6.0: Remove Entity of lower rank, when contained by by entity of type ENTITY"
     salience 32
     when
         $higherRank: TextEntity($type: type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
@@ -1112,7 +1112,7 @@ rule "X.6.1: remove Entity of higher rank, when intersected by entity of type EN
     salience 32
     when
         $higherRank: TextEntity($type: type, $value: value, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
-        $lowerRank: TextEntity(intersects($higherRank), type != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active(), $lowerRank.getValue().length() > $value.length())
+        $lowerRank: TextEntity(intersects($higherRank), type != $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active(), $lowerRank.getValue().length() > $value.length())
     then
         $higherRank.getIntersectingNodes().forEach(node -> update(node));
         $higherRank.remove("X.6.1", "remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity");
@@ -1123,7 +1123,7 @@ rule "X.6.1: remove Entity of higher rank, when intersected by entity of type EN
 //------------------------------------ File attributes rules ------------------------------------
 
 // Rule unit: FA.1
-rule "FA.1.0: remove duplicate FileAttributes"
+rule "FA.1.0: Remove duplicate FileAttributes"
     salience 64
     when
         $fileAttribute: FileAttribute($label: label, $value: value)
@@ -1136,7 +1136,7 @@ rule "FA.1.0: remove duplicate FileAttributes"
 //------------------------------------ Local dictionary search rules ------------------------------------
 
 // Rule unit: LDS.0
-rule "LDS.0.0: run local dictionary search"
+rule "LDS.0.0: Run local dictionary search"
     agenda-group "LOCAL_DICTIONARY_ADDS"
     salience -999
     when
@@ -1144,7 +1144,7 @@ rule "LDS.0.0: run local dictionary search"
     then
         entityCreationService.bySearchImplementation($dictionaryModel.getLocalSearch(), $dictionaryModel.getType(), EntityType.RECOMMENDATION, document)
             .forEach(entity ->  {
-                Collection<MatchedRule> matchedRules = $dictionaryModel.getLocalEntriesWithMatchedRules().get(entity.getValue());
-                entity.addMatchedRules(matchedRules);
+                Collection<MatchedRule> matchedRules = $dictionaryModel.getMatchedRulesForLocalDictionaryEntry(entity.getValue());
+                matchedRules.forEach(matchedRule -> entity.addMatchedRule(matchedRule.asSkippedIfApplied()));
             });
     end

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_v2.drl

@@ -22,6 +22,7 @@ import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRu
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.*;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Section;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Table;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Paragraph;
@@ -59,16 +60,40 @@ global EntityCreationService entityCreationService
 global ManualChangesApplicationService manualChangesApplicationService
 global Dictionary dictionary
 
-// --------------------------------------- queries -------------------------------------------------------------------
+//------------------------------------ queries ------------------------------------
 
 query "getFileAttributes"
     $fileAttribute: FileAttribute()
     end
 
-// --------------------------------------- NER Entities rules -------------------------------------------------------------------
+//------------------------------------ CBI rules ------------------------------------
+
+// Rule unit: CBI.0
+rule "CBI.0.0: Redact CBI Authors (non vertebrate Study)"
+    when
+        not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
+        $entity: TextEntity(type == "CBI_author", dictionaryEntry)
+    then
+        $entity.redact("CBI.0.0", "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+    end
+
+
+//------------------------------------ PII rules ------------------------------------
+
+// Rule unit: PII.0
+rule "PII.0.0: Redact all PII (non vertebrate study)"
+    when
+        not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
+        $pii: TextEntity(type == "PII", dictionaryEntry)
+    then
+        $pii.redact("PII.0.0", "Personal Information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+    end
+
+
+//------------------------------------ AI rules ------------------------------------
 
 // Rule unit: AI.0
-rule "AI.0.0: add all NER Entities of type CBI_author"
+rule "AI.0.0: Add all NER Entities of type CBI_author"
     salience 999
     when
         nerEntities: NerEntities(hasEntitiesOfType("CBI_author"))
@@ -77,28 +102,8 @@ rule "AI.0.0: add all NER Entities of type CBI_author"
             .forEach(nerEntity -> entityCreationService.byNerEntity(nerEntity, EntityType.RECOMMENDATION, document));
     end
 
-// --------------------------------------- CBI rules -------------------------------------------------------------------
 
-rule "CBI.0.0: Always redact CBI_author"
-
-    when
-        $cbiAuthor: TextEntity(type == "CBI_author", entityType == EntityType.ENTITY)
-    then
-        $cbiAuthor.apply("CBI.0.0", "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
-    end
-
-// --------------------------------------- PII rules -------------------------------------------------------------------
-
-rule "PII.0.0: Always redact PII"
-
-    when
-        $cbiAuthor: TextEntity(type == "PII", entityType == EntityType.ENTITY)
-    then
-        $cbiAuthor.apply("PII.0.0", "PII found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
-    end
-
-
-//------------------------------------ Manual redaction rules ------------------------------------
+//------------------------------------ Manual changes rules ------------------------------------
 
 // Rule unit: MAN.0
 rule "MAN.0.0: Apply manual resize redaction"
@@ -132,7 +137,7 @@ rule "MAN.0.1: Apply manual resize redaction"
 rule "MAN.1.0: Apply id removals that are valid and not in forced redactions to Entity"
     salience 128
     when
-        $idRemoval: IdRemoval($id: annotationId, status == AnnotationStatus.APPROVED)
+        $idRemoval: IdRemoval($id: annotationId, !removeFromDictionary, !removeFromAllDossiers, status == AnnotationStatus.APPROVED)
         $entityToBeRemoved: TextEntity(matchesAnnotationId($id))
     then
         $entityToBeRemoved.getManualOverwrite().addChange($idRemoval);
@@ -156,7 +161,6 @@ rule "MAN.1.1: Apply id removals that are valid and not in forced redactions to
 
 // Rule unit: MAN.2
 rule "MAN.2.0: Apply force redaction"
-    no-loop true
     salience 128
     when
         $force: ManualForceRedaction($id: annotationId, status == AnnotationStatus.APPROVED)
@@ -165,10 +169,10 @@ rule "MAN.2.0: Apply force redaction"
         $entityToForce.getManualOverwrite().addChange($force);
         update($entityToForce);
         $entityToForce.getIntersectingNodes().forEach(node -> update(node));
+        retract($force);
     end
 
 rule "MAN.2.1: Apply force redaction to images"
-    no-loop true
     salience 128
     when
         $force: ManualForceRedaction($id: annotationId, status == AnnotationStatus.APPROVED)
@@ -177,6 +181,7 @@ rule "MAN.2.1: Apply force redaction to images"
         $imageToForce.getManualOverwrite().addChange($force);
         update($imageToForce);
         update($imageToForce.getParent());
+        retract($force);
     end
 
 
@@ -184,9 +189,9 @@ rule "MAN.2.1: Apply force redaction to images"
 rule "MAN.3.0: Apply entity recategorization"
     salience 128
     when
-        $recategorization: ManualRecategorization($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate)
+        $recategorization: ManualRecategorization($id: annotationId, $type: type, status == AnnotationStatus.APPROVED, $requestDate: requestDate)
         not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))
-        $entityToBeRecategorized: TextEntity(matchesAnnotationId($id))
+        $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type != $type)
     then
         $entityToBeRecategorized.getIntersectingNodes().forEach(node -> update(node));
         manualChangesApplicationService.recategorize($entityToBeRecategorized, $recategorization);
@@ -195,16 +200,14 @@ rule "MAN.3.0: Apply entity recategorization"
         retract($entityToBeRecategorized);
     end
 
-rule "MAN.3.1: Apply image recategorization"
+rule "MAN.3.1: Apply entity recategorization of same type"
     salience 128
     when
-        $recategorization: ManualRecategorization($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate)
+        $recategorization: ManualRecategorization($id: annotationId, $type: type, status == AnnotationStatus.APPROVED, $requestDate: requestDate)
         not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))
-        $imageToBeRecategorized: Image($id == id)
+        $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type == $type)
     then
-        manualChangesApplicationService.recategorize($imageToBeRecategorized, $recategorization);
-        update($imageToBeRecategorized);
-        update($imageToBeRecategorized.getParent());
+        $entityToBeRecategorized.getManualOverwrite().addChange($recategorization);
         retract($recategorization);
     end
 
@@ -232,7 +235,7 @@ rule "MAN.4.1: Apply legal basis change"
 //------------------------------------ Entity merging rules ------------------------------------
 
 // Rule unit: X.0
-rule "X.0.0: remove Entity contained by Entity of same type"
+rule "X.0.0: Remove Entity contained by Entity of same type"
     salience 65
     when
         $larger: TextEntity($type: type, $entityType: entityType, active())
@@ -244,7 +247,7 @@ rule "X.0.0: remove Entity contained by Entity of same type"
 
 
 // Rule unit: X.1
-rule "X.1.0: merge intersecting Entities of same type"
+rule "X.1.0: Merge intersecting Entities of same type"
     salience 64
     when
         $first: TextEntity($type: type, $entityType: entityType, !resized(), active())
@@ -260,7 +263,7 @@ rule "X.1.0: merge intersecting Entities of same type"
 
 
 // Rule unit: X.2
-rule "X.2.0: remove Entity of type ENTITY when contained by FALSE_POSITIVE"
+rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
     salience 64
     when
         $falsePositive: TextEntity($type: type, entityType == EntityType.FALSE_POSITIVE, active())
@@ -273,7 +276,7 @@ rule "X.2.0: remove Entity of type ENTITY when contained by FALSE_POSITIVE"
 
 
 // Rule unit: X.3
-rule "X.3.0: remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"
+rule "X.3.0: Remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"
     salience 64
     when
         $falseRecommendation: TextEntity($type: type, entityType == EntityType.FALSE_RECOMMENDATION, active())
@@ -285,7 +288,7 @@ rule "X.3.0: remove Entity of type RECOMMENDATION when contained by FALSE_RECOMM
 
 
 // Rule unit: X.4
-rule "X.4.0: remove Entity of type RECOMMENDATION when intersected by ENTITY with same type"
+rule "X.4.0: Remove Entity of type RECOMMENDATION when intersected by ENTITY with same type"
     salience 256
     when
         $entity: TextEntity($type: type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
@@ -298,7 +301,7 @@ rule "X.4.0: remove Entity of type RECOMMENDATION when intersected by ENTITY wit
 
 
 // Rule unit: X.5
-rule "X.5.0: remove Entity of type RECOMMENDATION when contained by ENTITY"
+rule "X.5.0: Remove Entity of type RECOMMENDATION when contained by ENTITY"
     salience 256
     when
         $entity: TextEntity((entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
@@ -310,7 +313,7 @@ rule "X.5.0: remove Entity of type RECOMMENDATION when contained by ENTITY"
 
 
 // Rule unit: X.6
-rule "X.6.0: remove Entity of lower rank, when contained by by entity of type ENTITY"
+rule "X.6.0: Remove Entity of lower rank, when contained by by entity of type ENTITY"
     salience 32
     when
         $higherRank: TextEntity($type: type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
@@ -325,7 +328,7 @@ rule "X.6.1: remove Entity of higher rank, when intersected by entity of type EN
     salience 32
     when
         $higherRank: TextEntity($type: type, $value: value, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
-        $lowerRank: TextEntity(intersects($higherRank), type != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active(), $lowerRank.getValue().length() > $value.length())
+        $lowerRank: TextEntity(intersects($higherRank), type != $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active(), $lowerRank.getValue().length() > $value.length())
     then
         $higherRank.getIntersectingNodes().forEach(node -> update(node));
         $higherRank.remove("X.6.1", "remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity");
@@ -336,7 +339,7 @@ rule "X.6.1: remove Entity of higher rank, when intersected by entity of type EN
 //------------------------------------ File attributes rules ------------------------------------
 
 // Rule unit: FA.1
-rule "FA.1.0: remove duplicate FileAttributes"
+rule "FA.1.0: Remove duplicate FileAttributes"
     salience 64
     when
         $fileAttribute: FileAttribute($label: label, $value: value)
@@ -349,7 +352,7 @@ rule "FA.1.0: remove duplicate FileAttributes"
 //------------------------------------ Local dictionary search rules ------------------------------------
 
 // Rule unit: LDS.0
-rule "LDS.0.0: run local dictionary search"
+rule "LDS.0.0: Run local dictionary search"
     agenda-group "LOCAL_DICTIONARY_ADDS"
     salience -999
     when
@@ -357,7 +360,7 @@ rule "LDS.0.0: run local dictionary search"
     then
         entityCreationService.bySearchImplementation($dictionaryModel.getLocalSearch(), $dictionaryModel.getType(), EntityType.RECOMMENDATION, document)
             .forEach(entity ->  {
-                Collection<MatchedRule> matchedRules = $dictionaryModel.getLocalEntriesWithMatchedRules().get(entity.getValue());
-                entity.addMatchedRules(matchedRules);
+                Collection<MatchedRule> matchedRules = $dictionaryModel.getMatchedRulesForLocalDictionaryEntry(entity.getValue());
+                matchedRules.forEach(matchedRule -> entity.addMatchedRule(matchedRule.asSkippedIfApplied()));
             });
     end

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/table_demo.drl

@@ -0,0 +1,487 @@
+package drools
+
+import static java.lang.String.format;
+import static com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility.anyMatch;
+import static com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility.exactMatch;
+
+import java.util.List;
+import java.util.LinkedList;
+import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.Collection;
+import java.util.stream.Stream;
+import java.util.Optional;
+
+import com.iqser.red.service.redaction.v1.server.model.document.*;
+import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
+import com.iqser.red.service.redaction.v1.server.model.document.entity.*;
+import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
+import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule;
+import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity
+import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.*;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.Section;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.Table;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.Paragraph;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.Image;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.ImageType;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.Page;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.Headline;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.SectionIdentifier;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.Footer;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.Header;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.NodeType;
+import com.iqser.red.service.redaction.v1.server.model.document.textblock.*;
+import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
+import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlockCollector;
+import com.iqser.red.service.redaction.v1.server.model.document.textblock.AtomicTextBlock;
+import com.iqser.red.service.redaction.v1.server.model.document.textblock.ConcatenatedTextBlock;
+import com.iqser.red.service.redaction.v1.server.model.NerEntities;
+import com.iqser.red.service.redaction.v1.server.model.dictionary.Dictionary;
+import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryModel;
+import com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService;
+import com.iqser.red.service.redaction.v1.server.service.ManualChangesApplicationService;
+import com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility;
+
+import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus;
+
+global Document document
+global EntityCreationService entityCreationService
+global ManualChangesApplicationService manualChangesApplicationService
+global Dictionary dictionary
+
+//------------------------------------ queries ------------------------------------
+
+query "getFileAttributes"
+    $fileAttribute: FileAttribute()
+    end
+
+//------------------------------------ Table extraction rules ------------------------------------
+
+// Rule unit: TAB.0
+rule "TAB.0.0: Study Type File Attribute"
+    when
+        not FileAttribute(label == "OECD Number", valueEqualsAnyOf("402","403","404","405","425","429","436","438","439","471","487"))
+        $section: Section(containsAnyString("DATA REQUIREMENT", "TEST GUIDELINE", "MÉTODO(S) DE REFERÊNCIA(S):")
+            && containsAnyString("OECD", "EPA", "OPPTS"))
+    then
+        RedactionSearchUtility.findTextRangesByRegexIgnoreCase("(?<=OECD)(?:[\\w\\s,\\[\\]\\(\\)\\.]{1,10}|(?:.{5,40}(?:Number |Procedure |Guideline )))(4[\\d]{2})", 1 ,$section.getTextBlock()).stream()
+            .map(boundary -> $section.getTextBlock().subSequence(boundary).toString())
+            .map(value -> FileAttribute.builder().label("OECD Number").value(value).build())
+            .forEach(fileAttribute -> insert(fileAttribute));
+        RedactionSearchUtility.findTextRangesByRegexIgnoreCase("(?<=OECD).{5,40}Method (4[\\d]{2}).{1,65}(\\d{4})\\)", 1, $section.getTextBlock()).stream()
+            .map(boundary -> $section.getTextBlock().subSequence(boundary).toString())
+            .map(value -> FileAttribute.builder().label("OECD Number").value(value).build())
+            .forEach(fileAttribute -> insert(fileAttribute));
+    end
+
+rule "TAB.0.1: Guidelines"
+    when
+        $section: Section(containsAnyString("DATA REQUIREMENT", "TEST GUIDELINE", "MÉTODO(S) DE REFERÊNCIA(S):") && containsAnyString("OECD", "EPA", "OPPTS"))
+    then
+        entityCreationService.byRegex("(?<=OECD)(?:[\\w\\s,\\[\\]\\(\\)\\.]{1,10}|.{5,40}(?:Number |Procedure |Guideline ))(4[\\d]{2})", "oecd_guideline_number", EntityType.ENTITY, 1, $section)
+            .forEach(guideline -> guideline.apply("TAB.0.1", "OECD Guideline no. found"));
+        entityCreationService.byRegex("(?<=OECD)(?:[\\w\\s,\\[\\]\\(\\)\\.]{1,10}|.{5,40}(?:Number |Procedure |Guideline ))(4[\\d]{2}),?\\s\\(?(\\d{4})\\)?", "oecd_guideline_year", EntityType.ENTITY, 2, $section)
+            .forEach(guideline -> guideline.apply("TAB.0.1", "OECD Guideline year found"));
+        entityCreationService.byRegex("(?<=OECD)[\\w\\s,\\[\\]]{1,10}\\((\\d{4})\\)\\s(4[\\d]{2})", "oecd_guideline_year", EntityType.ENTITY, 1, $section)
+            .forEach(guideline -> guideline.apply("TAB.0.1", "OECD Guideline year found"));
+        entityCreationService.byRegex("(?<=OECD).{5,40}Method (4[\\d]{2}).{1,65}(\\d{4})\\)", "oecd_guideline_number", EntityType.ENTITY, 1, $section)
+            .forEach(guideline -> guideline.apply("TAB.0.1", "OECD Guideline number found"));
+        entityCreationService.byRegex("(?<=OECD).{5,40}Method (4[\\d]{2}).{1,65}(\\d{4})\\)", "oecd_guideline_year", EntityType.ENTITY, 2, $section)
+            .forEach(guideline -> guideline.apply("TAB.0.1", "OECD Guideline year found"));
+    end
+
+
+// Rule unit: TAB.1
+rule "TAB.1.0: Full Table extraction (Guideline Deviation)"
+    when
+        FileAttribute(label == "OECD Number", valueEqualsAnyOf("425"))
+        $section: Section(getHeadline().containsString("Full Table"))
+        $table: Table() from $section.getParent().streamAllSubNodesOfType(NodeType.TABLE).toList()
+        $tableCell: TableCell(!header) from $table.streamTableCells().toList()
+    then
+        entityCreationService.bySemanticNode($tableCell, "full_table_row", EntityType.ENTITY)
+            .ifPresent(entity -> entity.apply("TAB.1.0", "full table extracted"));
+    end
+
+
+// Rule unit: TAB.2
+rule "TAB.2.0: Individual row extraction (Clinical Signs)"
+    when
+        FileAttribute(label == "OECD Number", valueEqualsAnyOf("425"))
+        $section: Section(getHeadline().containsString("Individual Rows Extraction"))
+        $table: Table(hasHeader("Animal No."), (hasRowWithHeaderAndAnyValue("Animal No.", List.of("120-2", "120-5")))) from $section.streamChildren().toList()
+        TableCell($row: row, containsAnyString("120-2", "120-5")) from $table.streamTableCellsWithHeader("Animal No.").toList()
+        $tableCell: TableCell($row == row) from $table.streamTableCells().toList()
+    then
+        entityCreationService.bySemanticNode($tableCell, "clinical_signs", EntityType.ENTITY)
+            .ifPresent(entity -> entity.apply("TAB.2.0", "Individual row based on animal number"));
+    end
+
+
+// Rule unit: TAB.3
+rule "TAB.3.0: Individual column extraction (Strain)"
+    when
+        FileAttribute(label == "OECD Number", valueEqualsAnyOf("425"))
+        $section: Section(getHeadline().containsString("Individual Column"))
+        $table: Table(hasHeader("Sex")) from $section.streamChildren().toList()
+    then
+        $table.streamTableCellsWithHeader("Sex")
+            .map(tableCell -> entityCreationService.bySemanticNode(tableCell, "dosages", EntityType.ENTITY))
+            .filter(Optional::isPresent)
+            .map(Optional::get)
+            .forEach(redactionEntity -> redactionEntity.apply("TAB.3.0", "Individual column based on column header"));
+    end
+
+
+// Rule unit: TAB.4
+rule "TAB.4.0: Combined Columns Extraction - Sex and Dosage"
+    when
+        FileAttribute(label == "OECD Number", valueEqualsAnyOf("425"))
+        $section: Section(getHeadline().containsString("Combined Columns"))
+        $table: Table(hasHeader("Dosage (mg/kg bw)")) from $section.getParent().streamAllSubNodesOfType(NodeType.TABLE).toList()
+    then
+        $table.streamTableCellsWithHeader("Dosage (mg/kg bw)")
+            .map(tableCell -> entityCreationService.bySemanticNode(tableCell, "dose_mortality_dose", EntityType.ENTITY))
+            .filter(Optional::isPresent)
+            .map(Optional::get)
+            .forEach(redactionEntity -> redactionEntity.apply("TAB.4.0", "Dose Mortality dose found."));
+    end
+
+rule "TAB.4.1: Combined Columns Extraction - Sex and Mortality"
+    when
+        FileAttribute(label == "OECD Number", valueEqualsAnyOf("425"))
+        $section: Section(getHeadline().containsString("Combined Columns"))
+        $table: Table(hasHeader("Mortality")) from $section.getParent().streamAllSubNodesOfType(NodeType.TABLE).toList()
+    then
+        $table.streamTableCellsWithHeader("Mortality")
+            .map(tableCell -> entityCreationService.bySemanticNode(tableCell, "dose_mortality", EntityType.ENTITY))
+            .filter(Optional::isPresent)
+            .map(Optional::get)
+            .forEach(redactionEntity -> redactionEntity.apply("TAB.4.1", "Dose Mortality found."));
+    end
+
+
+// Rule unit: TAB.5
+rule "TAB.5.0: Targeted cell extraction"
+    when
+        FileAttribute(label == "OECD Number", valueEqualsAnyOf("425"))
+        $section: Section(getHeadline().containsString("Value Extraction"))
+        $table: Table(hasHeader("Mortality"), hasRowWithHeaderAndAnyValue("Sex", List.of("male", "Male")), hasRowWithHeaderAndValue("Mortality", "Survived")) from $section.streamChildren().toList()
+        TableCell(containsWordIgnoreCase("Male"), $row: row) from $table.streamTableCellsWithHeader("Sex").toList()
+        TableCell($row == row, containsStringIgnoreCase("Survived")) from $table.streamTableCellsWithHeader("Mortality").toList()
+        $dosageCell: TableCell($row == row) from $table.streamTableCellsWithHeader("Dosage").toList()
+    then
+        entityCreationService.bySemanticNode($dosageCell,"doses_mg_kg_bw", EntityType.ENTITY)
+            .ifPresent(entity -> entity.apply("TAB.5.0", "Dosage found in row with survived male"));
+    end
+
+
+// Rule unit: TAB.6
+rule "TAB.6.0: Targeted cell extraction (Experimental Stop date)"
+    when
+        $section: Section(getHeadline().containsString("Advanced Table Extraction"), containsAllStrings("female", "Female", "Survived", "Group 2"))
+        $table: Table(hasHeader("Group 2")) from $section.streamChildren().toList()
+        TableCell(containsWordIgnoreCase("Female"), $row: row) from $table.streamTableCellsWithHeader("Group 2").toList()
+        TableCell($row == row, containsStringIgnoreCase("Survived")) from $table.streamTableCellsWithHeader("Group 2").toList()
+        $femaleSurvived: TableCell($row == row) from $table.streamTableCellsWithHeader("Group 2").toList()
+    then
+        entityCreationService.bySemanticNode($femaleSurvived, "experiment_female_survived", EntityType.ENTITY)
+            .ifPresent(entity -> entity.apply("TAB.6.0", "Female in group to experimental start date"));
+    end
+
+
+// Rule unit: TAB.7
+rule "TAB.7.0: Indicator (Species)"
+    when
+        FileAttribute(label == "OECD Number", valueEqualsAnyOf("425"))
+        $section: Section(getHeadline().containsString("Entity-Based"))
+        $table: Table() from $section.streamAllSubNodesOfType(NodeType.TABLE).toList()
+        TableCell(isHeader(), containsString("Title"), $col: col) from $table.streamTableCells().toList()
+        TableCell(hasEntitiesOfType("vertebrates"), $row: row) from $table.streamTableCells().toList()
+        $cell: TableCell($col == col, $row == row) from $table.streamTableCells().toList()
+    then
+        entityCreationService.bySemanticNode($cell, "study_design", EntityType.ENTITY)
+            .ifPresent(redactionEntity -> redactionEntity.apply("TAB.7.0", "Vertebrate study found"));
+    end
+
+
+//------------------------------------ Manual changes rules ------------------------------------
+
+// Rule unit: MAN.0
+rule "MAN.0.0: Apply manual resize redaction"
+    salience 128
+    when
+        $resizeRedaction: ManualResizeRedaction($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate)
+        not ManualResizeRedaction(annotationId == $id, requestDate.isBefore($requestDate))
+        $entityToBeResized: TextEntity(matchesAnnotationId($id))
+    then
+        manualChangesApplicationService.resizeEntityAndReinsert($entityToBeResized, $resizeRedaction);
+        retract($resizeRedaction);
+        update($entityToBeResized);
+        $entityToBeResized.getIntersectingNodes().forEach(node -> update(node));
+    end
+
+rule "MAN.0.1: Apply manual resize redaction"
+    salience 128
+    when
+        $resizeRedaction: ManualResizeRedaction($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate)
+        not ManualResizeRedaction(annotationId == $id, requestDate.isBefore($requestDate))
+        $imageToBeResized: Image(id == $id)
+    then
+        manualChangesApplicationService.resizeImage($imageToBeResized, $resizeRedaction);
+        retract($resizeRedaction);
+        update($imageToBeResized);
+        update($imageToBeResized.getParent());
+    end
+
+
+// Rule unit: MAN.1
+rule "MAN.1.0: Apply id removals that are valid and not in forced redactions to Entity"
+    salience 128
+    when
+        $idRemoval: IdRemoval($id: annotationId, !removeFromDictionary, !removeFromAllDossiers, status == AnnotationStatus.APPROVED)
+        $entityToBeRemoved: TextEntity(matchesAnnotationId($id))
+    then
+        $entityToBeRemoved.getManualOverwrite().addChange($idRemoval);
+        update($entityToBeRemoved);
+        retract($idRemoval);
+        $entityToBeRemoved.getIntersectingNodes().forEach(node -> update(node));
+    end
+
+rule "MAN.1.1: Apply id removals that are valid and not in forced redactions to Image"
+    salience 128
+    when
+        $idRemoval: IdRemoval($id: annotationId, status == AnnotationStatus.APPROVED)
+        $imageEntityToBeRemoved: Image($id == id)
+    then
+        $imageEntityToBeRemoved.getManualOverwrite().addChange($idRemoval);
+        update($imageEntityToBeRemoved);
+        retract($idRemoval);
+        update($imageEntityToBeRemoved.getParent());
+    end
+
+
+// Rule unit: MAN.2
+rule "MAN.2.0: Apply force redaction"
+    salience 128
+    when
+        $force: ManualForceRedaction($id: annotationId, status == AnnotationStatus.APPROVED)
+        $entityToForce: TextEntity(matchesAnnotationId($id))
+    then
+        $entityToForce.getManualOverwrite().addChange($force);
+        update($entityToForce);
+        $entityToForce.getIntersectingNodes().forEach(node -> update(node));
+        retract($force);
+    end
+
+rule "MAN.2.1: Apply force redaction to images"
+    salience 128
+    when
+        $force: ManualForceRedaction($id: annotationId, status == AnnotationStatus.APPROVED)
+        $imageToForce: Image(id == $id)
+    then
+        $imageToForce.getManualOverwrite().addChange($force);
+        update($imageToForce);
+        update($imageToForce.getParent());
+        retract($force);
+    end
+
+
+// Rule unit: MAN.3
+rule "MAN.3.0: Apply entity recategorization"
+    salience 128
+    when
+        $recategorization: ManualRecategorization($id: annotationId, $type: type, status == AnnotationStatus.APPROVED, $requestDate: requestDate)
+        not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))
+        $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type != $type)
+    then
+        $entityToBeRecategorized.getIntersectingNodes().forEach(node -> update(node));
+        manualChangesApplicationService.recategorize($entityToBeRecategorized, $recategorization);
+        retract($recategorization);
+        // Entity is copied and inserted, so the old entity needs to be retracted to avoid duplication.
+        retract($entityToBeRecategorized);
+    end
+
+rule "MAN.3.1: Apply entity recategorization of same type"
+    salience 128
+    when
+        $recategorization: ManualRecategorization($id: annotationId, $type: type, status == AnnotationStatus.APPROVED, $requestDate: requestDate)
+        not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))
+        $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type == $type)
+    then
+        $entityToBeRecategorized.getManualOverwrite().addChange($recategorization);
+        retract($recategorization);
+    end
+
+rule "MAN.3.2: Apply image recategorization"
+    salience 128
+    when
+        $recategorization: ManualRecategorization($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate)
+        not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))
+        $imageToBeRecategorized: Image($id == id)
+    then
+        manualChangesApplicationService.recategorize($imageToBeRecategorized, $recategorization);
+        update($imageToBeRecategorized);
+        update($imageToBeRecategorized.getParent());
+        retract($recategorization);
+    end
+
+
+// Rule unit: MAN.4
+rule "MAN.4.0: Apply legal basis change"
+    salience 128
+    when
+        $legalbasisChange: ManualLegalBasisChange($id: annotationId, status == AnnotationStatus.APPROVED)
+        $imageToBeRecategorized: Image($id == id)
+    then
+        $imageToBeRecategorized.getManualOverwrite().addChange($legalbasisChange);
+    end
+
+rule "MAN.4.1: Apply legal basis change"
+    salience 128
+    when
+        $legalBasisChange: ManualLegalBasisChange($id: annotationId, status == AnnotationStatus.APPROVED)
+        $entityToBeChanged: TextEntity(matchesAnnotationId($id))
+    then
+        $entityToBeChanged.getManualOverwrite().addChange($legalBasisChange);
+    end
+
+
+//------------------------------------ Entity merging rules ------------------------------------
+
+// Rule unit: X.0
+rule "X.0.0: Remove Entity contained by Entity of same type"
+    salience 65
+    when
+        $larger: TextEntity($type: type, $entityType: entityType, active())
+        $contained: TextEntity(containedBy($larger), type == $type, entityType == $entityType, this != $larger, !hasManualChanges(), active())
+    then
+        $contained.remove("X.0.0", "remove Entity contained by Entity of same type");
+        retract($contained);
+    end
+
+
+// Rule unit: X.2
+rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
+    salience 64
+    when
+        $falsePositive: TextEntity($type: type, entityType == EntityType.FALSE_POSITIVE, active())
+        $entity: TextEntity(containedBy($falsePositive), type == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges(), active())
+    then
+        $entity.getIntersectingNodes().forEach(node -> update(node));
+        $entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE");
+        retract($entity)
+    end
+
+
+// Rule unit: X.3
+rule "X.3.0: Remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"
+    salience 64
+    when
+        $falseRecommendation: TextEntity($type: type, entityType == EntityType.FALSE_RECOMMENDATION, active())
+        $recommendation: TextEntity(containedBy($falseRecommendation), type == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
+    then
+        $recommendation.remove("X.3.0", "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION");
+        retract($recommendation);
+    end
+
+
+// Rule unit: X.4
+rule "X.4.0: Remove Entity of type RECOMMENDATION when intersected by ENTITY with same type"
+    salience 256
+    when
+        $entity: TextEntity($type: type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
+        $recommendation: TextEntity(intersects($entity), type == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
+    then
+        $entity.addEngines($recommendation.getEngines());
+        $recommendation.remove("X.4.0", "remove Entity of type RECOMMENDATION when intersected by ENTITY with same type");
+        retract($recommendation);
+    end
+
+
+// Rule unit: X.5
+rule "X.5.0: Remove Entity of type RECOMMENDATION when contained by ENTITY"
+    salience 256
+    when
+        $entity: TextEntity((entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
+        $recommendation: TextEntity(containedBy($entity), entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
+    then
+        $recommendation.remove("X.5.0", "remove Entity of type RECOMMENDATION when contained by ENTITY");
+        retract($recommendation);
+    end
+
+
+// Rule unit: X.6
+rule "X.6.0: Remove Entity of lower rank, when contained by by entity of type ENTITY"
+    salience 32
+    when
+        $higherRank: TextEntity($type: type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
+        $lowerRank: TextEntity(containedBy($higherRank), type != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active())
+    then
+        $lowerRank.getIntersectingNodes().forEach(node -> update(node));
+        $lowerRank.remove("X.6.0", "remove Entity of lower rank, when contained by entity of type ENTITY");
+        retract($lowerRank);
+    end
+
+rule "X.6.1: remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity"
+    salience 32
+    when
+        $higherRank: TextEntity($type: type, $value: value, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
+        $lowerRank: TextEntity(intersects($higherRank), type != $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active(), $lowerRank.getValue().length() > $value.length())
+    then
+        $higherRank.getIntersectingNodes().forEach(node -> update(node));
+        $higherRank.remove("X.6.1", "remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity");
+        retract($higherRank);
+    end
+
+
+// Rule unit: X.7
+rule "X.7.0: Remove all images"
+    salience 512
+    when
+        $image: Image(imageType != ImageType.OCR, !hasManualChanges())
+    then
+        $image.remove("X.7.0", "remove all images");
+        retract($image);
+    end
+
+
+//------------------------------------ File attributes rules ------------------------------------
+
+// Rule unit: FA.1
+rule "FA.1.0: Remove duplicate FileAttributes"
+    salience 64
+    when
+        $fileAttribute: FileAttribute($label: label, $value: value)
+        $duplicate: FileAttribute(this != $fileAttribute, label == $label, value == $value)
+    then
+        retract($duplicate);
+    end
+
+
+//------------------------------------ Local dictionary search rules ------------------------------------
+
+// Rule unit: LDS.0
+rule "LDS.0.0: Run local dictionary search"
+    agenda-group "LOCAL_DICTIONARY_ADDS"
+    salience -999
+    when
+        $dictionaryModel: DictionaryModel(!localEntriesWithMatchedRules.isEmpty()) from dictionary.getDictionaryModels()
+    then
+        entityCreationService.bySearchImplementation($dictionaryModel.getLocalSearch(), $dictionaryModel.getType(), EntityType.RECOMMENDATION, document)
+            .forEach(entity ->  {
+                Collection<MatchedRule> matchedRules = $dictionaryModel.getMatchedRulesForLocalDictionaryEntry(entity.getValue());
+                matchedRules.forEach(matchedRule -> entity.addMatchedRule(matchedRule.asSkippedIfApplied()));
+            });
+    end

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/table_demo_components.drl

@@ -0,0 +1,126 @@
+package drools
+
+import static java.lang.String.format;
+import static com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility.anyMatch;
+import static com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility.exactMatch;
+
+import java.util.List;
+import java.util.LinkedList;
+import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.Collection;
+import java.util.stream.Stream;
+import java.util.Optional;
+
+import com.iqser.red.service.redaction.v1.server.model.component.Component;
+import com.iqser.red.service.redaction.v1.server.model.component.Entity;
+import com.iqser.red.service.redaction.v1.server.service.document.ComponentCreationService;
+
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Change;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryType;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualChange;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position;
+
+import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;
+
+global ComponentCreationService componentCreationService
+
+//------------------------------------ queries ------------------------------------
+
+query "getFileAttributes"
+    $fileAttribute: FileAttribute()
+    end
+
+query "getComponents"
+    $component: Component()
+    end
+
+//------------------------------------ Guideline mapping object ------------------------------------
+
+declare GuidelineMapping
+    number: String
+    year: String
+    guideline: String
+ end
+
+//------------------------------------ Demo Components rules ------------------------------------
+
+rule "FullTable.1.1: All rows of matching table"
+    when
+        $tableRowValues: List() from collect (Entity(type == "full_table_row"))
+    then
+        componentCreationService.joiningFromSameTableRow("FullTable.1.1", "1.1 Full Table", $tableRowValues);
+    end
+
+
+rule "IndivRowExtr.1.2: Individual rows of table"
+    when
+        $clinicalSigns: List() from collect (Entity(type == "clinical_signs"))
+    then
+        componentCreationService.joiningFromSameTableRow("IndivRowExtr.1.2", "1.2 Individual Rows", $clinicalSigns);
+    end
+
+
+rule "IndivColExtr.1.3: Individual column of table"
+    when
+        $tableColValues: List() from collect (Entity(type == "dosages"))
+    then
+        componentCreationService.joiningFromSameTableRow("IndivColExtr.1.3", "1.3 Individual Column", $tableColValues);
+    end
+
+
+rule "CombColExtr.2.1: Combined Columns Extraction"
+    when
+        $doseMortalities: List() from collect (Entity(type == "dose_mortality" || type == "dose_mortality_dose"))
+    then
+        componentCreationService.joiningFromSameTableRow("DoseMortality.0.0", "2.1 Combined Columns", $doseMortalities);
+    end
+
+
+rule "ValueExtrRef.2.2: Cells containing dose for survived males"
+    when
+        $tableValues: List() from collect (Entity(type == "doses_mg_kg_bw"))
+    then
+        componentCreationService.joining("ValueExtrRef.2.2", "2.2 Individual Table Values", $tableValues);
+    end
+
+
+rule "AdvTableExtr.2.3: Cells containing dose for survived males"
+    when
+        $tableValues: List() from collect (Entity(type == "experiment_female_survived"))
+    then
+        componentCreationService.rowValueCount("AdvTableExtr.2.3", "2.3 Advanced Table Values", $tableValues);
+    end
+
+
+rule "EntityBasedExtr.2.4: Cells containing dose for survived males"
+    when
+        $tableValues: List() from collect (Entity(type == "study_design"))
+    then
+        componentCreationService.joiningFromSameTableRow("EntityBasedExtr.2.4", "2.4 Entity-Based Values", $tableValues);
+    end
+
+
+rule "DefaultComponents.999.0: Create components for all unmapped entities."
+    salience -999
+    when
+        not FileAttribute(label == "OECD Number")
+        $allEntities: List(!isEmpty()) from collect (Entity())
+    then
+        componentCreationService.createComponentsForUnMappedEntities("DefaultComponents.999.0", $allEntities);
+    end
+
+
+//------------------------------------ Component merging rules ------------------------------------
+/*
+rule "X.0.0: merge duplicate component references"
+    when
+        $first: Component()
+        $duplicate: Component(this != $first, name == $first.name, value == $first.value)
+    then
+        $first.getReferences().addAll($duplicate.getReferences());
+        retract($duplicate);
+    end
+*/

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/test_components.drl

@@ -0,0 +1,55 @@
+package drools
+
+import static java.lang.String.format;
+import static com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility.anyMatch;
+import static com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility.exactMatch;
+
+import java.util.List;
+import java.util.LinkedList;
+import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.Collection;
+import java.util.stream.Stream;
+import java.util.Optional;
+
+import com.iqser.red.service.redaction.v1.server.model.component.Component;
+import com.iqser.red.service.redaction.v1.server.model.component.Entity;
+import com.iqser.red.service.redaction.v1.server.service.document.ComponentCreationService;
+
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Change;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryType;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualChange;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position;
+
+import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;
+
+global ComponentCreationService componentCreationService
+
+//------------------------------------ queries ------------------------------------
+
+query "getFileAttributes"
+    $fileAttribute: FileAttribute()
+    end
+
+query "getComponents"
+    $component: Component()
+    end
+
+//------------------------------------ Guideline mapping object ------------------------------------
+
+declare GuidelineMapping
+    number: String
+    year: String
+    guideline: String
+ end
+
+//------------------------------------ Test Components rules ------------------------------------
+
+rule "AdvTableExtr.2.3: Cells containing dose for survived males"
+    when
+        $tableValues: List() from collect (Entity(type == "test"))
+    then
+        componentCreationService.rowValueCount("AdvTableExtr.2.3", "2.3 Advanced Table Values", $tableValues);
+    end

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/test_rules.drl

@@ -0,0 +1,403 @@
+package drools
+
+import static java.lang.String.format;
+import static com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility.anyMatch;
+import static com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility.exactMatch;
+
+import java.util.List;
+import java.util.LinkedList;
+import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.Collection;
+import java.util.stream.Stream;
+import java.util.Optional;
+
+import com.iqser.red.service.redaction.v1.server.model.document.*;
+import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
+import com.iqser.red.service.redaction.v1.server.model.document.entity.*;
+import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
+import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule;
+import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity
+import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.*;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.Section;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.Table;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.Paragraph;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.Image;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.ImageType;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.Page;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.Headline;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.SectionIdentifier;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.Footer;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.Header;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.NodeType;
+import com.iqser.red.service.redaction.v1.server.model.document.textblock.*;
+import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
+import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlockCollector;
+import com.iqser.red.service.redaction.v1.server.model.document.textblock.AtomicTextBlock;
+import com.iqser.red.service.redaction.v1.server.model.document.textblock.ConcatenatedTextBlock;
+import com.iqser.red.service.redaction.v1.server.model.NerEntities;
+import com.iqser.red.service.redaction.v1.server.model.dictionary.Dictionary;
+import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryModel;
+import com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService;
+import com.iqser.red.service.redaction.v1.server.service.ManualChangesApplicationService;
+import com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility;
+
+import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus;
+
+global Document document
+global EntityCreationService entityCreationService
+global ManualChangesApplicationService manualChangesApplicationService
+global Dictionary dictionary
+
+//------------------------------------ queries ------------------------------------
+
+query "getFileAttributes"
+    $fileAttribute: FileAttribute()
+    end
+
+//------------------------------------ Table extraction rules ------------------------------------
+
+// Rule unit: TAB.0
+rule "TAB.0.0: Study Type File Attribute"
+    when
+        not FileAttribute(label == "OECD Number", valueEqualsAnyOf("402","403","404","405","425","429","436","438","439","471","487"))
+        $section: Section(containsAnyString("DATA REQUIREMENT", "TEST GUIDELINE", "MÉTODO(S) DE REFERÊNCIA(S):")
+            && containsAnyString("OECD", "EPA", "OPPTS"))
+    then
+        RedactionSearchUtility.findTextRangesByRegexIgnoreCase("(?<=OECD)(?:[\\w\\s,\\[\\]\\(\\)\\.]{1,10}|(?:.{5,40}(?:Number |Procedure |Guideline )))(4[\\d]{2})", 1 ,$section.getTextBlock()).stream()
+            .map(boundary -> $section.getTextBlock().subSequence(boundary).toString())
+            .map(value -> FileAttribute.builder().label("OECD Number").value(value).build())
+            .forEach(fileAttribute -> insert(fileAttribute));
+        RedactionSearchUtility.findTextRangesByRegexIgnoreCase("(?<=OECD).{5,40}Method (4[\\d]{2}).{1,65}(\\d{4})\\)", 1, $section.getTextBlock()).stream()
+            .map(boundary -> $section.getTextBlock().subSequence(boundary).toString())
+            .map(value -> FileAttribute.builder().label("OECD Number").value(value).build())
+            .forEach(fileAttribute -> insert(fileAttribute));
+    end
+
+rule "TAB.0.1: Guidelines"
+    when
+        $section: Section(containsAnyString("DATA REQUIREMENT", "TEST GUIDELINE", "MÉTODO(S) DE REFERÊNCIA(S):") && containsAnyString("OECD", "EPA", "OPPTS"))
+    then
+        entityCreationService.byRegex("(?<=OECD)(?:[\\w\\s,\\[\\]\\(\\)\\.]{1,10}|.{5,40}(?:Number |Procedure |Guideline ))(4[\\d]{2})", "oecd_guideline_number", EntityType.ENTITY, 1, $section)
+            .forEach(guideline -> guideline.apply("TAB.0.1", "OECD Guideline no. found"));
+        entityCreationService.byRegex("(?<=OECD)(?:[\\w\\s,\\[\\]\\(\\)\\.]{1,10}|.{5,40}(?:Number |Procedure |Guideline ))(4[\\d]{2}),?\\s\\(?(\\d{4})\\)?", "oecd_guideline_year", EntityType.ENTITY, 2, $section)
+            .forEach(guideline -> guideline.apply("TAB.0.1", "OECD Guideline year found"));
+        entityCreationService.byRegex("(?<=OECD)[\\w\\s,\\[\\]]{1,10}\\((\\d{4})\\)\\s(4[\\d]{2})", "oecd_guideline_year", EntityType.ENTITY, 1, $section)
+            .forEach(guideline -> guideline.apply("TAB.0.1", "OECD Guideline year found"));
+        entityCreationService.byRegex("(?<=OECD).{5,40}Method (4[\\d]{2}).{1,65}(\\d{4})\\)", "oecd_guideline_number", EntityType.ENTITY, 1, $section)
+            .forEach(guideline -> guideline.apply("TAB.0.1", "OECD Guideline number found"));
+        entityCreationService.byRegex("(?<=OECD).{5,40}Method (4[\\d]{2}).{1,65}(\\d{4})\\)", "oecd_guideline_year", EntityType.ENTITY, 2, $section)
+            .forEach(guideline -> guideline.apply("TAB.0.1", "OECD Guideline year found"));
+    end
+
+
+// Rule unit: TAB.6
+rule "TAB.6.0: Targeted cell extraction (Experimental Stop date)"
+    when
+        $section: Section(getHeadline().containsString("Advanced Table Extraction"), containsAllStrings("female", "Female", "Survived", "Group 2"))
+        $table: Table(hasHeader("Group 2")) from $section.streamChildren().toList()
+        TableCell(containsWordIgnoreCase("Female"), $row: row) from $table.streamTableCellsWithHeader("Group 2").toList()
+        TableCell($row == row, containsStringIgnoreCase("Survived")) from $table.streamTableCellsWithHeader("Group 2").toList()
+        $femaleSurvived: TableCell($row == row) from $table.streamTableCellsWithHeader("Group 2").toList()
+    then
+        entityCreationService.bySemanticNode($femaleSurvived, "experiment_female_survived", EntityType.ENTITY)
+            .ifPresent(entity -> entity.apply("TAB.6.0", "Female in group to experimental start date"));
+    end
+
+
+//------------------------------------ Manual changes rules ------------------------------------
+
+// Rule unit: MAN.0
+rule "MAN.0.0: Apply manual resize redaction"
+    salience 128
+    when
+        $resizeRedaction: ManualResizeRedaction($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate)
+        not ManualResizeRedaction(annotationId == $id, requestDate.isBefore($requestDate))
+        $entityToBeResized: TextEntity(matchesAnnotationId($id))
+    then
+        manualChangesApplicationService.resizeEntityAndReinsert($entityToBeResized, $resizeRedaction);
+        retract($resizeRedaction);
+        update($entityToBeResized);
+        $entityToBeResized.getIntersectingNodes().forEach(node -> update(node));
+    end
+
+rule "MAN.0.1: Apply manual resize redaction"
+    salience 128
+    when
+        $resizeRedaction: ManualResizeRedaction($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate)
+        not ManualResizeRedaction(annotationId == $id, requestDate.isBefore($requestDate))
+        $imageToBeResized: Image(id == $id)
+    then
+        manualChangesApplicationService.resizeImage($imageToBeResized, $resizeRedaction);
+        retract($resizeRedaction);
+        update($imageToBeResized);
+        update($imageToBeResized.getParent());
+    end
+
+
+// Rule unit: MAN.1
+rule "MAN.1.0: Apply id removals that are valid and not in forced redactions to Entity"
+    salience 128
+    when
+        $idRemoval: IdRemoval($id: annotationId, !removeFromDictionary, !removeFromAllDossiers, status == AnnotationStatus.APPROVED)
+        $entityToBeRemoved: TextEntity(matchesAnnotationId($id))
+    then
+        $entityToBeRemoved.getManualOverwrite().addChange($idRemoval);
+        update($entityToBeRemoved);
+        retract($idRemoval);
+        $entityToBeRemoved.getIntersectingNodes().forEach(node -> update(node));
+    end
+
+rule "MAN.1.1: Apply id removals that are valid and not in forced redactions to Image"
+    salience 128
+    when
+        $idRemoval: IdRemoval($id: annotationId, status == AnnotationStatus.APPROVED)
+        $imageEntityToBeRemoved: Image($id == id)
+    then
+        $imageEntityToBeRemoved.getManualOverwrite().addChange($idRemoval);
+        update($imageEntityToBeRemoved);
+        retract($idRemoval);
+        update($imageEntityToBeRemoved.getParent());
+    end
+
+
+// Rule unit: MAN.2
+rule "MAN.2.0: Apply force redaction"
+    salience 128
+    when
+        $force: ManualForceRedaction($id: annotationId, status == AnnotationStatus.APPROVED)
+        $entityToForce: TextEntity(matchesAnnotationId($id))
+    then
+        $entityToForce.getManualOverwrite().addChange($force);
+        update($entityToForce);
+        $entityToForce.getIntersectingNodes().forEach(node -> update(node));
+        retract($force);
+    end
+
+rule "MAN.2.1: Apply force redaction to images"
+    salience 128
+    when
+        $force: ManualForceRedaction($id: annotationId, status == AnnotationStatus.APPROVED)
+        $imageToForce: Image(id == $id)
+    then
+        $imageToForce.getManualOverwrite().addChange($force);
+        update($imageToForce);
+        update($imageToForce.getParent());
+        retract($force);
+    end
+
+
+// Rule unit: MAN.3
+rule "MAN.3.0: Apply entity recategorization"
+    salience 128
+    when
+        $recategorization: ManualRecategorization($id: annotationId, $type: type, status == AnnotationStatus.APPROVED, $requestDate: requestDate)
+        not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))
+        $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type != $type)
+    then
+        $entityToBeRecategorized.getIntersectingNodes().forEach(node -> update(node));
+        manualChangesApplicationService.recategorize($entityToBeRecategorized, $recategorization);
+        retract($recategorization);
+        // Entity is copied and inserted, so the old entity needs to be retracted to avoid duplication.
+        retract($entityToBeRecategorized);
+    end
+
+rule "MAN.3.1: Apply entity recategorization of same type"
+    salience 128
+    when
+        $recategorization: ManualRecategorization($id: annotationId, $type: type, status == AnnotationStatus.APPROVED, $requestDate: requestDate)
+        not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))
+        $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type == $type)
+    then
+        $entityToBeRecategorized.getManualOverwrite().addChange($recategorization);
+        retract($recategorization);
+    end
+
+rule "MAN.3.2: Apply image recategorization"
+    salience 128
+    when
+        $recategorization: ManualRecategorization($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate)
+        not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))
+        $imageToBeRecategorized: Image($id == id)
+    then
+        manualChangesApplicationService.recategorize($imageToBeRecategorized, $recategorization);
+        update($imageToBeRecategorized);
+        update($imageToBeRecategorized.getParent());
+        retract($recategorization);
+    end
+
+
+// Rule unit: MAN.4
+rule "MAN.4.0: Apply legal basis change"
+    salience 128
+    when
+        $legalbasisChange: ManualLegalBasisChange($id: annotationId, status == AnnotationStatus.APPROVED)
+        $imageToBeRecategorized: Image($id == id)
+    then
+        $imageToBeRecategorized.getManualOverwrite().addChange($legalbasisChange);
+    end
+
+rule "MAN.4.1: Apply legal basis change"
+    salience 128
+    when
+        $legalBasisChange: ManualLegalBasisChange($id: annotationId, status == AnnotationStatus.APPROVED)
+        $entityToBeChanged: TextEntity(matchesAnnotationId($id))
+    then
+        $entityToBeChanged.getManualOverwrite().addChange($legalBasisChange);
+    end
+
+
+//------------------------------------ Entity merging rules ------------------------------------
+
+// Rule unit: X.0
+rule "X.0.0: Remove Entity contained by Entity of same type"
+    salience 65
+    when
+        $larger: TextEntity($type: type, $entityType: entityType, active())
+        $contained: TextEntity(containedBy($larger), type == $type, entityType == $entityType, this != $larger, !hasManualChanges(), active())
+    then
+        $contained.remove("X.0.0", "remove Entity contained by Entity of same type");
+        retract($contained);
+    end
+
+
+// Rule unit: X.1
+rule "X.1.0: Merge intersecting Entities of same type"
+    salience 64
+    when
+        $first: TextEntity($type: type, $entityType: entityType, !resized(), active())
+        $second: TextEntity(intersects($first), type == $type, entityType == $entityType, this != $first, !hasManualChanges(), active())
+    then
+        TextEntity mergedEntity = entityCreationService.mergeEntitiesOfSameType(List.of($first, $second), $type, $entityType, document);
+        $first.remove("X.1.0", "merge intersecting Entities of same type");
+        $second.remove("X.1.0", "merge intersecting Entities of same type");
+        retract($first);
+        retract($second);
+        mergedEntity.getIntersectingNodes().forEach(node -> update(node));
+    end
+
+
+// Rule unit: X.2
+rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
+    salience 64
+    when
+        $falsePositive: TextEntity($type: type, entityType == EntityType.FALSE_POSITIVE, active())
+        $entity: TextEntity(containedBy($falsePositive), type == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges(), active())
+    then
+        $entity.getIntersectingNodes().forEach(node -> update(node));
+        $entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE");
+        retract($entity)
+    end
+
+
+// Rule unit: X.3
+rule "X.3.0: Remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"
+    salience 64
+    when
+        $falseRecommendation: TextEntity($type: type, entityType == EntityType.FALSE_RECOMMENDATION, active())
+        $recommendation: TextEntity(containedBy($falseRecommendation), type == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
+    then
+        $recommendation.remove("X.3.0", "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION");
+        retract($recommendation);
+    end
+
+
+// Rule unit: X.4
+rule "X.4.0: Remove Entity of type RECOMMENDATION when intersected by ENTITY with same type"
+    salience 256
+    when
+        $entity: TextEntity($type: type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
+        $recommendation: TextEntity(intersects($entity), type == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
+    then
+        $entity.addEngines($recommendation.getEngines());
+        $recommendation.remove("X.4.0", "remove Entity of type RECOMMENDATION when intersected by ENTITY with same type");
+        retract($recommendation);
+    end
+
+
+// Rule unit: X.5
+rule "X.5.0: Remove Entity of type RECOMMENDATION when contained by ENTITY"
+    salience 256
+    when
+        $entity: TextEntity((entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
+        $recommendation: TextEntity(containedBy($entity), entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
+    then
+        $recommendation.remove("X.5.0", "remove Entity of type RECOMMENDATION when contained by ENTITY");
+        retract($recommendation);
+    end
+
+
+// Rule unit: X.6
+rule "X.6.0: Remove Entity of lower rank, when contained by by entity of type ENTITY"
+    salience 32
+    when
+        $higherRank: TextEntity($type: type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
+        $lowerRank: TextEntity(containedBy($higherRank), type != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active())
+    then
+        $lowerRank.getIntersectingNodes().forEach(node -> update(node));
+        $lowerRank.remove("X.6.0", "remove Entity of lower rank, when contained by entity of type ENTITY");
+        retract($lowerRank);
+    end
+
+rule "X.6.1: remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity"
+    salience 32
+    when
+        $higherRank: TextEntity($type: type, $value: value, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
+        $lowerRank: TextEntity(intersects($higherRank), type != $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active(), $lowerRank.getValue().length() > $value.length())
+    then
+        $higherRank.getIntersectingNodes().forEach(node -> update(node));
+        $higherRank.remove("X.6.1", "remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity");
+        retract($higherRank);
+    end
+
+
+// Rule unit: X.7
+rule "X.7.0: Remove all images"
+    salience 512
+    when
+        $image: Image(imageType != ImageType.OCR, !hasManualChanges())
+    then
+        $image.remove("X.7.0", "remove all images");
+        retract($image);
+    end
+
+
+//------------------------------------ File attributes rules ------------------------------------
+
+// Rule unit: FA.1
+rule "FA.1.0: Remove duplicate FileAttributes"
+    salience 64
+    when
+        $fileAttribute: FileAttribute($label: label, $value: value)
+        $duplicate: FileAttribute(this != $fileAttribute, label == $label, value == $value)
+    then
+        retract($duplicate);
+    end
+
+
+//------------------------------------ Local dictionary search rules ------------------------------------
+
+// Rule unit: LDS.0
+rule "LDS.0.0: Run local dictionary search"
+    agenda-group "LOCAL_DICTIONARY_ADDS"
+    salience -999
+    when
+        $dictionaryModel: DictionaryModel(!localEntriesWithMatchedRules.isEmpty()) from dictionary.getDictionaryModels()
+    then
+        entityCreationService.bySearchImplementation($dictionaryModel.getLocalSearch(), $dictionaryModel.getType(), EntityType.RECOMMENDATION, document)
+            .forEach(entity ->  {
+                Collection<MatchedRule> matchedRules = $dictionaryModel.getMatchedRulesForLocalDictionaryEntry(entity.getValue());
+                matchedRules.forEach(matchedRule -> entity.addMatchedRule(matchedRule.asSkippedIfApplied()));
+            });
+    end

redaction-service-v1/redaction-service-server-v1/src/test/resources/files/migration/def8f960580f088b975ba806dfae1f87.IMAGE_INFO.json

@@ -0,0 +1,891 @@
+{
+  "dossierId": "12778ddd-0d87-4127-90f4-a15802e8f967",
+  "fileId": "4a15b5c89a4310eaef6243fb6f1f4f6f",
+  "targetFileExtension": "ORIGIN.pdf.gz",
+  "responseFileExtension": "IMAGE_INFO.json.gz",
+  "X-TENANT-ID": "documine",
+  "data": [
+    {
+      "classification": {
+        "label": "signature",
+        "probabilities": {
+          "signature": 0.7805,
+          "other": 0.1362,
+          "logo": 0.0629,
+          "formula": 0.0204
+        }
+      },
+      "representation": "F8F3FFCFCC30F0CFCFFCF37FF",
+      "position": {
+        "x1": 0,
+        "x2": 595,
+        "y1": 0,
+        "y2": 842,
+        "pageNumber": 1
+      },
+      "geometry": {
+        "width": 595,
+        "height": 842
+      },
+      "alpha": false,
+      "filters": {
+        "geometry": {
+          "imageSize": {
+            "quotient": 1.0,
+            "tooLarge": true,
+            "tooSmall": false
+          },
+          "imageFormat": {
+            "quotient": 0.7067,
+            "tooTall": false,
+            "tooWide": false
+          }
+        },
+        "probability": {
+          "unconfident": false
+        },
+        "allPassed": false
+      }
+    },
+    {
+      "classification": {
+        "label": "signature",
+        "probabilities": {
+          "signature": 0.9955,
+          "other": 0.0023,
+          "logo": 0.0022,
+          "formula": 0.0
+        }
+      },
+      "representation": "97FFFFFFFF10608F7E18107EF",
+      "position": {
+        "x1": 0,
+        "x2": 595,
+        "y1": 0,
+        "y2": 842,
+        "pageNumber": 2
+      },
+      "geometry": {
+        "width": 595,
+        "height": 842
+      },
+      "alpha": false,
+      "filters": {
+        "geometry": {
+          "imageSize": {
+            "quotient": 1.0,
+            "tooLarge": true,
+            "tooSmall": false
+          },
+          "imageFormat": {
+            "quotient": 0.7067,
+            "tooTall": false,
+            "tooWide": false
+          }
+        },
+        "probability": {
+          "unconfident": false
+        },
+        "allPassed": false
+      }
+    },
+    {
+      "classification": {
+        "label": "signature",
+        "probabilities": {
+          "signature": 0.9998,
+          "logo": 0.0001,
+          "other": 0.0001,
+          "formula": 0.0
+        }
+      },
+      "representation": "FFFFF10EFFF3E0810608307FF",
+      "position": {
+        "x1": 0,
+        "x2": 595,
+        "y1": 0,
+        "y2": 842,
+        "pageNumber": 3
+      },
+      "geometry": {
+        "width": 595,
+        "height": 842
+      },
+      "alpha": false,
+      "filters": {
+        "geometry": {
+          "imageSize": {
+            "quotient": 1.0,
+            "tooLarge": true,
+            "tooSmall": false
+          },
+          "imageFormat": {
+            "quotient": 0.7067,
+            "tooTall": false,
+            "tooWide": false
+          }
+        },
+        "probability": {
+          "unconfident": false
+        },
+        "allPassed": false
+      }
+    },
+    {
+      "classification": {
+        "label": "signature",
+        "probabilities": {
+          "signature": 0.9998,
+          "other": 0.0001,
+          "formula": 0.0,
+          "logo": 0.0
+        }
+      },
+      "representation": "93EFF76E08F0E88F1E187C7EF",
+      "position": {
+        "x1": 0,
+        "x2": 595,
+        "y1": 0,
+        "y2": 842,
+        "pageNumber": 4
+      },
+      "geometry": {
+        "width": 595,
+        "height": 842
+      },
+      "alpha": false,
+      "filters": {
+        "geometry": {
+          "imageSize": {
+            "quotient": 1.0,
+            "tooLarge": true,
+            "tooSmall": false
+          },
+          "imageFormat": {
+            "quotient": 0.7067,
+            "tooTall": false,
+            "tooWide": false
+          }
+        },
+        "probability": {
+          "unconfident": false
+        },
+        "allPassed": false
+      }
+    },
+    {
+      "classification": {
+        "label": "signature",
+        "probabilities": {
+          "signature": 0.9997,
+          "logo": 0.0001,
+          "other": 0.0001,
+          "formula": 0.0
+        }
+      },
+      "representation": "FFFFF33EFC3060818608787FF",
+      "position": {
+        "x1": 0,
+        "x2": 595,
+        "y1": 0,
+        "y2": 842,
+        "pageNumber": 5
+      },
+      "geometry": {
+        "width": 595,
+        "height": 842
+      },
+      "alpha": false,
+      "filters": {
+        "geometry": {
+          "imageSize": {
+            "quotient": 1.0,
+            "tooLarge": true,
+            "tooSmall": false
+          },
+          "imageFormat": {
+            "quotient": 0.7067,
+            "tooTall": false,
+            "tooWide": false
+          }
+        },
+        "probability": {
+          "unconfident": false
+        },
+        "allPassed": false
+      }
+    },
+    {
+      "classification": {
+        "label": "signature",
+        "probabilities": {
+          "signature": 0.9085,
+          "logo": 0.0453,
+          "other": 0.0413,
+          "formula": 0.005
+        }
+      },
+      "representation": "FFF3F506F81060CF1E18FC7FF",
+      "position": {
+        "x1": 0,
+        "x2": 595,
+        "y1": 0,
+        "y2": 842,
+        "pageNumber": 6
+      },
+      "geometry": {
+        "width": 595,
+        "height": 842
+      },
+      "alpha": false,
+      "filters": {
+        "geometry": {
+          "imageSize": {
+            "quotient": 1.0,
+            "tooLarge": true,
+            "tooSmall": false
+          },
+          "imageFormat": {
+            "quotient": 0.7067,
+            "tooTall": false,
+            "tooWide": false
+          }
+        },
+        "probability": {
+          "unconfident": false
+        },
+        "allPassed": false
+      }
+    },
+    {
+      "classification": {
+        "label": "signature",
+        "probabilities": {
+          "signature": 0.7596,
+          "other": 0.1218,
+          "logo": 0.1073,
+          "formula": 0.0113
+        }
+      },
+      "representation": "FFFFFFF70C1070C10F3C10FFF",
+      "position": {
+        "x1": 0,
+        "x2": 595,
+        "y1": 0,
+        "y2": 842,
+        "pageNumber": 7
+      },
+      "geometry": {
+        "width": 595,
+        "height": 842
+      },
+      "alpha": false,
+      "filters": {
+        "geometry": {
+          "imageSize": {
+            "quotient": 1.0,
+            "tooLarge": true,
+            "tooSmall": false
+          },
+          "imageFormat": {
+            "quotient": 0.7067,
+            "tooTall": false,
+            "tooWide": false
+          }
+        },
+        "probability": {
+          "unconfident": false
+        },
+        "allPassed": false
+      }
+    },
+    {
+      "classification": {
+        "label": "signature",
+        "probabilities": {
+          "signature": 0.4095,
+          "other": 0.2126,
+          "formula": 0.2008,
+          "logo": 0.1771
+        }
+      },
+      "representation": "FF70C1070C1070CFD70C10FFF",
+      "position": {
+        "x1": 0,
+        "x2": 595,
+        "y1": 0,
+        "y2": 842,
+        "pageNumber": 8
+      },
+      "geometry": {
+        "width": 595,
+        "height": 842
+      },
+      "alpha": false,
+      "filters": {
+        "geometry": {
+          "imageSize": {
+            "quotient": 1.0,
+            "tooLarge": true,
+            "tooSmall": false
+          },
+          "imageFormat": {
+            "quotient": 0.7067,
+            "tooTall": false,
+            "tooWide": false
+          }
+        },
+        "probability": {
+          "unconfident": true
+        },
+        "allPassed": false
+      }
+    },
+    {
+      "classification": {
+        "label": "signature",
+        "probabilities": {
+          "signature": 0.9282,
+          "logo": 0.0402,
+          "other": 0.0305,
+          "formula": 0.0011
+        }
+      },
+      "representation": "F4F1CF170CF170C1070CF7FFF",
+      "position": {
+        "x1": 0,
+        "x2": 595,
+        "y1": 0,
+        "y2": 842,
+        "pageNumber": 9
+      },
+      "geometry": {
+        "width": 595,
+        "height": 842
+      },
+      "alpha": false,
+      "filters": {
+        "geometry": {
+          "imageSize": {
+            "quotient": 1.0,
+            "tooLarge": true,
+            "tooSmall": false
+          },
+          "imageFormat": {
+            "quotient": 0.7067,
+            "tooTall": false,
+            "tooWide": false
+          }
+        },
+        "probability": {
+          "unconfident": false
+        },
+        "allPassed": false
+      }
+    },
+    {
+      "classification": {
+        "label": "signature",
+        "probabilities": {
+          "signature": 0.9699,
+          "logo": 0.02,
+          "other": 0.0097,
+          "formula": 0.0004
+        }
+      },
+      "representation": "FF70FF4E1D3070D1070CF0FFF",
+      "position": {
+        "x1": 0,
+        "x2": 595,
+        "y1": 0,
+        "y2": 842,
+        "pageNumber": 10
+      },
+      "geometry": {
+        "width": 595,
+        "height": 842
+      },
+      "alpha": false,
+      "filters": {
+        "geometry": {
+          "imageSize": {
+            "quotient": 1.0,
+            "tooLarge": true,
+            "tooSmall": false
+          },
+          "imageFormat": {
+            "quotient": 0.7067,
+            "tooTall": false,
+            "tooWide": false
+          }
+        },
+        "probability": {
+          "unconfident": false
+        },
+        "allPassed": false
+      }
+    },
+    {
+      "classification": {
+        "label": "other",
+        "probabilities": {
+          "other": 0.9181,
+          "signature": 0.0432,
+          "formula": 0.0355,
+          "logo": 0.0032
+        }
+      },
+      "representation": "F3F1F7DF1F10F0C1070CF3FFF",
+      "position": {
+        "x1": 0,
+        "x2": 595,
+        "y1": 0,
+        "y2": 842,
+        "pageNumber": 11
+      },
+      "geometry": {
+        "width": 595,
+        "height": 842
+      },
+      "alpha": false,
+      "filters": {
+        "geometry": {
+          "imageSize": {
+            "quotient": 1.0,
+            "tooLarge": true,
+            "tooSmall": false
+          },
+          "imageFormat": {
+            "quotient": 0.7067,
+            "tooTall": false,
+            "tooWide": false
+          }
+        },
+        "probability": {
+          "unconfident": false
+        },
+        "allPassed": false
+      }
+    },
+    {
+      "classification": {
+        "label": "signature",
+        "probabilities": {
+          "signature": 0.663,
+          "other": 0.137,
+          "logo": 0.1327,
+          "formula": 0.0674
+        }
+      },
+      "representation": "B770C10F7C1070C10F0C97FFF",
+      "position": {
+        "x1": 0,
+        "x2": 595,
+        "y1": 0,
+        "y2": 842,
+        "pageNumber": 12
+      },
+      "geometry": {
+        "width": 595,
+        "height": 842
+      },
+      "alpha": false,
+      "filters": {
+        "geometry": {
+          "imageSize": {
+            "quotient": 1.0,
+            "tooLarge": true,
+            "tooSmall": false
+          },
+          "imageFormat": {
+            "quotient": 0.7067,
+            "tooTall": false,
+            "tooWide": false
+          }
+        },
+        "probability": {
+          "unconfident": false
+        },
+        "allPassed": false
+      }
+    },
+    {
+      "classification": {
+        "label": "signature",
+        "probabilities": {
+          "signature": 0.5023,
+          "other": 0.2343,
+          "logo": 0.2255,
+          "formula": 0.0379
+        }
+      },
+      "representation": "B370C10FFC1070CF3F1CF0FFF",
+      "position": {
+        "x1": 0,
+        "x2": 595,
+        "y1": 0,
+        "y2": 842,
+        "pageNumber": 13
+      },
+      "geometry": {
+        "width": 595,
+        "height": 842
+      },
+      "alpha": false,
+      "filters": {
+        "geometry": {
+          "imageSize": {
+            "quotient": 1.0,
+            "tooLarge": true,
+            "tooSmall": false
+          },
+          "imageFormat": {
+            "quotient": 0.7067,
+            "tooTall": false,
+            "tooWide": false
+          }
+        },
+        "probability": {
+          "unconfident": false
+        },
+        "allPassed": false
+      }
+    },
+    {
+      "classification": {
+        "label": "signature",
+        "probabilities": {
+          "signature": 0.9999,
+          "other": 0.0001,
+          "formula": 0.0,
+          "logo": 0.0
+        }
+      },
+      "representation": "97FFFFFFFFFFFF810608907EF",
+      "position": {
+        "x1": 0,
+        "x2": 595,
+        "y1": 0,
+        "y2": 842,
+        "pageNumber": 14
+      },
+      "geometry": {
+        "width": 595,
+        "height": 842
+      },
+      "alpha": false,
+      "filters": {
+        "geometry": {
+          "imageSize": {
+            "quotient": 1.0,
+            "tooLarge": true,
+            "tooSmall": false
+          },
+          "imageFormat": {
+            "quotient": 0.7067,
+            "tooTall": false,
+            "tooWide": false
+          }
+        },
+        "probability": {
+          "unconfident": false
+        },
+        "allPassed": false
+      }
+    },
+    {
+      "classification": {
+        "label": "signature",
+        "probabilities": {
+          "signature": 0.9099,
+          "logo": 0.0548,
+          "other": 0.0339,
+          "formula": 0.0014
+        }
+      },
+      "representation": "FFFFF10F7DDD70C1070CF77FF",
+      "position": {
+        "x1": 0,
+        "x2": 595,
+        "y1": 0,
+        "y2": 842,
+        "pageNumber": 15
+      },
+      "geometry": {
+        "width": 595,
+        "height": 842
+      },
+      "alpha": false,
+      "filters": {
+        "geometry": {
+          "imageSize": {
+            "quotient": 1.0,
+            "tooLarge": true,
+            "tooSmall": false
+          },
+          "imageFormat": {
+            "quotient": 0.7067,
+            "tooTall": false,
+            "tooWide": false
+          }
+        },
+        "probability": {
+          "unconfident": false
+        },
+        "allPassed": false
+      }
+    },
+    {
+      "classification": {
+        "label": "signature",
+        "probabilities": {
+          "signature": 0.6538,
+          "other": 0.1607,
+          "logo": 0.1489,
+          "formula": 0.0366
+        }
+      },
+      "representation": "FF70CF370CFE70C1070CB3FFF",
+      "position": {
+        "x1": 0,
+        "x2": 595,
+        "y1": 0,
+        "y2": 842,
+        "pageNumber": 16
+      },
+      "geometry": {
+        "width": 595,
+        "height": 842
+      },
+      "alpha": false,
+      "filters": {
+        "geometry": {
+          "imageSize": {
+            "quotient": 1.0,
+            "tooLarge": true,
+            "tooSmall": false
+          },
+          "imageFormat": {
+            "quotient": 0.7067,
+            "tooTall": false,
+            "tooWide": false
+          }
+        },
+        "probability": {
+          "unconfident": false
+        },
+        "allPassed": false
+      }
+    },
+    {
+      "classification": {
+        "label": "signature",
+        "probabilities": {
+          "signature": 0.9637,
+          "logo": 0.0179,
+          "other": 0.0156,
+          "formula": 0.0028
+        }
+      },
+      "representation": "FF70CF170C10F0C1070CF7FFF",
+      "position": {
+        "x1": 0,
+        "x2": 595,
+        "y1": 0,
+        "y2": 842,
+        "pageNumber": 17
+      },
+      "geometry": {
+        "width": 595,
+        "height": 842
+      },
+      "alpha": false,
+      "filters": {
+        "geometry": {
+          "imageSize": {
+            "quotient": 1.0,
+            "tooLarge": true,
+            "tooSmall": false
+          },
+          "imageFormat": {
+            "quotient": 0.7067,
+            "tooTall": false,
+            "tooWide": false
+          }
+        },
+        "probability": {
+          "unconfident": false
+        },
+        "allPassed": false
+      }
+    },
+    {
+      "classification": {
+        "label": "other",
+        "probabilities": {
+          "other": 1.0,
+          "formula": 0.0,
+          "logo": 0.0,
+          "signature": 0.0
+        }
+      },
+      "representation": "FFFFF7070E1870E18608106FF",
+      "position": {
+        "x1": 0,
+        "x2": 595,
+        "y1": 0,
+        "y2": 842,
+        "pageNumber": 18
+      },
+      "geometry": {
+        "width": 595,
+        "height": 842
+      },
+      "alpha": false,
+      "filters": {
+        "geometry": {
+          "imageSize": {
+            "quotient": 1.0,
+            "tooLarge": true,
+            "tooSmall": false
+          },
+          "imageFormat": {
+            "quotient": 0.7067,
+            "tooTall": false,
+            "tooWide": false
+          }
+        },
+        "probability": {
+          "unconfident": false
+        },
+        "allPassed": false
+      }
+    },
+    {
+      "classification": {
+        "label": "other",
+        "probabilities": {
+          "other": 1.0,
+          "formula": 0.0,
+          "logo": 0.0,
+          "signature": 0.0
+        }
+      },
+      "representation": "9B70E1870E1870E1870810EFF",
+      "position": {
+        "x1": 0,
+        "x2": 595,
+        "y1": 0,
+        "y2": 842,
+        "pageNumber": 19
+      },
+      "geometry": {
+        "width": 595,
+        "height": 842
+      },
+      "alpha": false,
+      "filters": {
+        "geometry": {
+          "imageSize": {
+            "quotient": 1.0,
+            "tooLarge": true,
+            "tooSmall": false
+          },
+          "imageFormat": {
+            "quotient": 0.7067,
+            "tooTall": false,
+            "tooWide": false
+          }
+        },
+        "probability": {
+          "unconfident": false
+        },
+        "allPassed": false
+      }
+    },
+    {
+      "classification": {
+        "label": "signature",
+        "probabilities": {
+          "signature": 0.3619,
+          "other": 0.2659,
+          "logo": 0.2169,
+          "formula": 0.1552
+        }
+      },
+      "representation": "FFFFF30FE910ECF1E708106FF",
+      "position": {
+        "x1": 0,
+        "x2": 595,
+        "y1": 0,
+        "y2": 842,
+        "pageNumber": 20
+      },
+      "geometry": {
+        "width": 595,
+        "height": 842
+      },
+      "alpha": false,
+      "filters": {
+        "geometry": {
+          "imageSize": {
+            "quotient": 1.0,
+            "tooLarge": true,
+            "tooSmall": false
+          },
+          "imageFormat": {
+            "quotient": 0.7067,
+            "tooTall": false,
+            "tooWide": false
+          }
+        },
+        "probability": {
+          "unconfident": true
+        },
+        "allPassed": false
+      }
+    },
+    {
+      "classification": {
+        "label": "signature",
+        "probabilities": {
+          "signature": 0.9984,
+          "other": 0.001,
+          "logo": 0.0006,
+          "formula": 0.0
+        }
+      },
+      "representation": "B7E9F70F3C73F1C78F3F706EF",
+      "position": {
+        "x1": 0,
+        "x2": 595,
+        "y1": 0,
+        "y2": 842,
+        "pageNumber": 21
+      },
+      "geometry": {
+        "width": 595,
+        "height": 842
+      },
+      "alpha": false,
+      "filters": {
+        "geometry": {
+          "imageSize": {
+            "quotient": 1.0,
+            "tooLarge": true,
+            "tooSmall": false
+          },
+          "imageFormat": {
+            "quotient": 0.7067,
+            "tooTall": false,
+            "tooWide": false
+          }
+        },
+        "probability": {
+          "unconfident": false
+        },
+        "allPassed": false
+      }
+    }
+  ]
+}