Merge branch 'hotfix' into 'master'

hotfix: filter entities with empty value in redactionLog and entityLog

See merge request redactmanager/redaction-service!215

.gitlab-ci.yml

@@ -12,7 +12,7 @@ deploy:
   script:
     - echo "Building with gradle version ${BUILDVERSION}"
     - gradle -Pversion=${BUILDVERSION} publish
-    - gradle bootBuildImage --cleanCache --publishImage -PbuildbootDockerHostNetwork=true -Pversion=${BUILDVERSION}
+    - gradle bootBuildImage --publishImage -PbuildbootDockerHostNetwork=true -Pversion=${BUILDVERSION}
     - echo "BUILDVERSION=$BUILDVERSION" >> version.env
   artifacts:
     reports:

redaction-service-v1/redaction-service-api-v1/build.gradle.kts

@@ -7,7 +7,7 @@ description = "redaction-service-api-v1"
 
 dependencies {
     implementation("org.springframework:spring-web:6.0.12")
-    implementation("com.iqser.red.service:persistence-service-internal-api-v1:2.207.0")
+    implementation("com.iqser.red.service:persistence-service-internal-api-v1:2.240.0")
 }
 
 publishing {

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/MigrationRequest.java

@@ -0,0 +1,17 @@
+package com.iqser.red.service.redaction.v1.model;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
+public class MigrationRequest {
+
+    String dossierId;
+    String fileId;
+
+}

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/MigrationResponse.java

@@ -0,0 +1,17 @@
+package com.iqser.red.service.redaction.v1.model;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
+public class MigrationResponse {
+
+    String dossierId;
+    String fileId;
+
+}

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/QueueNames.java

@@ -0,0 +1,14 @@
+package com.iqser.red.service.redaction.v1.model;
+
+public class QueueNames {
+
+    public static final String REDACTION_QUEUE = "redactionQueue";
+    public static final String REDACTION_DQL = "redactionDQL";
+
+    public static final String REDACTION_PRIORITY_QUEUE = "redactionPriorityQueue";
+
+    public static final String MIGRATION_QUEUE = "migrationQueue";
+    public static final String MIGRATION_DLQ = "migrationDLQ";
+    public static final String MIGRATION_RESPONSE_QUEUE = "migrationResponseQueue";
+
+}

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/UnprocessedManualEntity.java

@@ -1,7 +1,10 @@
 package com.iqser.red.service.redaction.v1.model;
 
+import java.util.ArrayList;
 import java.util.List;
 
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position;
+
 import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;
@@ -22,6 +25,9 @@ public class UnprocessedManualEntity {
     private int endOffset;
     private String closestHeadline;
     private String section;
+    private String legalBasis;
     private float[] color;
     private List<Integer> containingNodeId;
+    @Builder.Default
+    private List<Position> positions = new ArrayList<>();
 }

redaction-service-v1/redaction-service-server-v1/build.gradle.kts

@@ -12,11 +12,11 @@ plugins {
 description = "redaction-service-server-v1"
 
 
-val layoutParserVersion = "0.74.0"
+val layoutParserVersion = "0.75.0"
 val jacksonVersion = "2.15.2"
 val droolsVersion = "9.44.0.Final"
 val pdfBoxVersion = "3.0.0"
-val persistenceServiceVersion = "2.223.0"
+val persistenceServiceVersion = "2.242.0"
 
 configurations {
     all {
@@ -51,6 +51,7 @@ dependencies {
     implementation("org.springframework.cloud:spring-cloud-starter-openfeign:4.0.4")
     implementation("org.springframework.boot:spring-boot-starter-amqp:3.1.4")
 
+    testImplementation(project(":rules-management"))
     testImplementation("org.apache.pdfbox:pdfbox:${pdfBoxVersion}")
     testImplementation("org.apache.pdfbox:pdfbox-tools:${pdfBoxVersion}")
 
@@ -75,7 +76,7 @@ tasks.named<BootBuildImage>("bootBuildImage") {
     environment.put("BPE_DELIM_JAVA_TOOL_OPTIONS", " ")
     environment.put("BPE_APPEND_JAVA_TOOL_OPTIONS", "-Dfile.encoding=UTF-8")
 
-    imageName.set("nexus.knecon.com:5001/red/${project.name}:${project.version}")
+    imageName.set("nexus.knecon.com:5001/red/${project.name}")// must build image with same name always, otherwise the builder will not know which image to use as cache. DO NOT CHANGE!
     if (project.hasProperty("buildbootDockerHostNetwork")) {
         network.set("host")
     }
@@ -91,5 +92,8 @@ tasks.named<BootBuildImage>("bootBuildImage") {
             email.set(providers.gradleProperty("mavenEmail").getOrNull())
             url.set("https://nexus.knecon.com:5001/")
         }
+
+        val dockerTag = "nexus.knecon.com:5001/red/${project.name}:${project.version}"
+        tags.set(listOf(dockerTag))
     }
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/ManualEntity.java

@@ -4,10 +4,9 @@ import java.util.List;
 import java.util.PriorityQueue;
 
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
 import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
-import com.iqser.red.service.redaction.v1.server.model.document.entity.IEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
+import com.iqser.red.service.redaction.v1.server.model.document.entity.IEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.ManualChangeOverwrite;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule;
 
@@ -44,39 +43,6 @@ public class ManualEntity implements IEntity {
     ManualChangeOverwrite manualOverwrite = new ManualChangeOverwrite();
 
 
-    public static ManualEntity fromRedactionLogEntry(RedactionLogEntry redactionLogEntry) {
-
-        String ruleIdentifier = redactionLogEntry.getType() + "." + redactionLogEntry.getMatchedRule() + ".0";
-        List<RectangleWithPage> rectangleWithPages = redactionLogEntry.getPositions().stream().map(RectangleWithPage::fromRedactionLogRectangle).toList();
-        return ManualEntity.builder()
-                .id(redactionLogEntry.getId())
-                .value(redactionLogEntry.getValue())
-                .entityPosition(rectangleWithPages)
-                .ruleIdentifier(ruleIdentifier)
-                .reason(redactionLogEntry.getReason())
-                .legalBasis(redactionLogEntry.getLegalBasis())
-                .type(redactionLogEntry.getType())
-                .section(redactionLogEntry.getSection()).entityType(getEntityType(redactionLogEntry))
-                .applied(redactionLogEntry.isRedacted())
-                .isDictionaryEntry(redactionLogEntry.isDictionaryEntry())
-                .isDossierDictionaryEntry(redactionLogEntry.isDossierDictionaryEntry())
-                .rectangle(redactionLogEntry.isRectangle())
-                .build();
-    }
-
-
-    private static EntityType getEntityType(RedactionLogEntry redactionLogEntry) {
-
-        if (redactionLogEntry.isRecommendation()) {
-            return EntityType.RECOMMENDATION;
-        }
-        if (redactionLogEntry.isHint()) {
-            return EntityType.HINT;
-        }
-        return EntityType.ENTITY;
-    }
-
-
     public static ManualEntity fromManualRedactionEntry(ManualRedactionEntry manualRedactionEntry, boolean hint) {
 
         List<RectangleWithPage> rectangleWithPages = manualRedactionEntry.getPositions().stream().map(RectangleWithPage::fromAnnotationRectangle).toList();
@@ -90,7 +56,8 @@ public class ManualEntity implements IEntity {
                 .reason(manualRedactionEntry.getReason())
                 .legalBasis(manualRedactionEntry.getLegalBasis())
                 .type(manualRedactionEntry.getType())
-                .section(manualRedactionEntry.getSection()).entityType(hint ? EntityType.HINT : EntityType.ENTITY)
+                .section(manualRedactionEntry.getSection())
+                .entityType(hint ? EntityType.HINT : EntityType.ENTITY)
                 .applied(true)
                 .isDictionaryEntry(false)
                 .isDossierDictionaryEntry(false)

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/MigratedEntityLog.java

@@ -0,0 +1,20 @@
+package com.iqser.red.service.redaction.v1.server.model;
+
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.migration.MigratedIds;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
+public class MigratedEntityLog {
+
+    MigratedIds migratedIds;
+    EntityLog entityLog;
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/MigrationEntity.java

@@ -0,0 +1,315 @@
+package com.iqser.red.service.redaction.v1.server.model;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ChangeType;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryType;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Change;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ManualChange;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ManualRedactionType;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
+import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
+import com.iqser.red.service.redaction.v1.server.model.document.entity.IEntity;
+import com.iqser.red.service.redaction.v1.server.model.document.entity.PositionOnPage;
+import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.Image;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.ImageType;
+
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.RequiredArgsConstructor;
+
+@Data
+@AllArgsConstructor
+@RequiredArgsConstructor
+public final class MigrationEntity {
+
+    private final ManualEntity manualEntity;
+    private final RedactionLogEntry redactionLogEntry;
+    private IEntity migratedEntity;
+    private String oldId;
+    private String newId;
+
+
+    public static MigrationEntity fromRedactionLogEntry(RedactionLogEntry redactionLogEntry) {
+
+        return new MigrationEntity(createManualEntity(redactionLogEntry), redactionLogEntry);
+    }
+
+
+    public static ManualEntity createManualEntity(RedactionLogEntry redactionLogEntry) {
+
+        String ruleIdentifier = "OLD." + redactionLogEntry.getMatchedRule() + ".0";
+        List<RectangleWithPage> rectangleWithPages = redactionLogEntry.getPositions().stream().map(RectangleWithPage::fromRedactionLogRectangle).toList();
+
+        return ManualEntity.builder()
+                .id(redactionLogEntry.getId())
+                .value(redactionLogEntry.getValue())
+                .entityPosition(rectangleWithPages)
+                .ruleIdentifier(ruleIdentifier)
+                .reason(redactionLogEntry.getReason())
+                .legalBasis(redactionLogEntry.getLegalBasis())
+                .type(redactionLogEntry.getType())
+                .section(redactionLogEntry.getSection())
+                .entityType(getEntityType(redactionLogEntry))
+                .applied(redactionLogEntry.isRedacted())
+                .isDictionaryEntry(redactionLogEntry.isDictionaryEntry())
+                .isDossierDictionaryEntry(redactionLogEntry.isDossierDictionaryEntry())
+                .rectangle(redactionLogEntry.isRectangle())
+                .build();
+    }
+
+
+    private static EntityType getEntityType(RedactionLogEntry redactionLogEntry) {
+
+        if (redactionLogEntry.isRecommendation()) {
+            return EntityType.RECOMMENDATION;
+        }
+        if (redactionLogEntry.isHint()) {
+            return EntityType.HINT;
+        }
+        return EntityType.ENTITY;
+    }
+
+
+    private static com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Change toEntityLogChanges(Change change) {
+
+        return new com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Change(change.getAnalysisNumber(),
+                toEntityLogType(change.getType()),
+                change.getDateTime());
+    }
+
+
+    private static EntryType getEntryType(EntityType entityType) {
+
+        return switch (entityType) {
+            case ENTITY -> EntryType.ENTITY;
+            case HINT -> EntryType.HINT;
+            case FALSE_POSITIVE -> EntryType.FALSE_POSITIVE;
+            case RECOMMENDATION -> EntryType.RECOMMENDATION;
+            case FALSE_RECOMMENDATION -> EntryType.FALSE_RECOMMENDATION;
+        };
+    }
+
+
+    private static com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualChange toEntityLogManualChanges(ManualChange manualChange) {
+
+        return new com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualChange(toManualRedactionType(manualChange.getManualRedactionType()),
+                manualChange.getProcessedDate(),
+                manualChange.getRequestedDate(),
+                manualChange.getUserId(),
+                manualChange.getPropertyChanges());
+    }
+
+
+    private static ChangeType toEntityLogType(com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ChangeType type) {
+
+        return switch (type) {
+            case ADDED -> ChangeType.ADDED;
+            case REMOVED -> ChangeType.REMOVED;
+            case CHANGED -> ChangeType.CHANGED;
+        };
+    }
+
+
+    private static com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType toManualRedactionType(ManualRedactionType manualRedactionType) {
+
+        return switch (manualRedactionType) {
+            case ADD_LOCALLY -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.ADD_LOCALLY;
+            case ADD_TO_DICTIONARY -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.ADD_TO_DICTIONARY;
+            case REMOVE_LOCALLY -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.REMOVE_LOCALLY;
+            case REMOVE_FROM_DICTIONARY -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.REMOVE_FROM_DICTIONARY;
+            case FORCE_REDACT -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.FORCE_REDACT;
+            case FORCE_HINT -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.FORCE_HINT;
+            case RECATEGORIZE -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.RECATEGORIZE;
+            case LEGAL_BASIS_CHANGE -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.LEGAL_BASIS_CHANGE;
+            case RESIZE -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.RESIZE;
+        };
+    }
+
+
+    public EntityLogEntry toEntityLogEntry(Map<String, String> oldToNewIdMapping) {
+
+        EntityLogEntry entityLogEntry;
+        if (migratedEntity instanceof Image image) {
+            entityLogEntry = createEntityLogEntry(image);
+        } else if (migratedEntity instanceof TextEntity textEntity) {
+            entityLogEntry = createEntityLogEntry(textEntity);
+        } else if (migratedEntity instanceof ManualEntity entity) {
+            entityLogEntry = createEntityLogEntry(entity);
+        } else {
+            throw new UnsupportedOperationException("Unknown subclass " + migratedEntity.getClass());
+        }
+        entityLogEntry.setChanges(redactionLogEntry.getChanges().stream().map(MigrationEntity::toEntityLogChanges).toList());
+        entityLogEntry.setManualChanges(redactionLogEntry.getManualChanges()
+                .stream()
+                .filter(manualChange -> manualChange.getAnnotationStatus().equals(AnnotationStatus.APPROVED))
+                .map(MigrationEntity::toEntityLogManualChanges)
+                .toList());
+        entityLogEntry.setColor(redactionLogEntry.getColor());
+        entityLogEntry.setReference(migrateSetOfIds(redactionLogEntry.getReference(), oldToNewIdMapping));
+        entityLogEntry.setImportedRedactionIntersections(migrateSetOfIds(redactionLogEntry.getImportedRedactionIntersections(), oldToNewIdMapping));
+        entityLogEntry.setEngines(getMigratedEngines(redactionLogEntry));
+        if (redactionLogEntry.getLegalBasis() != null) {
+            entityLogEntry.setLegalBasis(redactionLogEntry.getLegalBasis());
+        }
+
+        return entityLogEntry;
+    }
+
+
+    private static Set<com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine> getMigratedEngines(RedactionLogEntry entry) {
+
+        if (entry.getEngines() == null) {
+            return Collections.emptySet();
+        }
+        return entry.getEngines().stream().map(MigrationEntity::toEntityLogEngine).collect(Collectors.toSet());
+    }
+
+
+    private Set<String> migrateSetOfIds(Set<String> ids, Map<String, String> oldToNewIdMapping) {
+
+        if (ids == null) {
+            return Collections.emptySet();
+        }
+        return ids.stream().map(oldToNewIdMapping::get).collect(Collectors.toSet());
+    }
+
+
+    private static com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine toEntityLogEngine(Engine engine) {
+
+        return switch (engine) {
+            case DICTIONARY -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine.DICTIONARY;
+            case NER -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine.NER;
+            case RULE -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine.RULE;
+        };
+    }
+
+
+    public EntityLogEntry createEntityLogEntry(Image image) {
+
+        String imageType = image.getImageType().equals(ImageType.OTHER) ? "image" : image.getImageType().toString().toLowerCase(Locale.ENGLISH);
+        return EntityLogEntry.builder()
+                .id(image.getId())
+                .value(image.value())
+                .type(imageType)
+                .reason(image.buildReasonWithManualChangeDescriptions())
+                .legalBasis(image.legalBasis())
+                .matchedRule(image.getMatchedRule().getRuleIdentifier().toString())
+                .dictionaryEntry(false)
+                .positions(List.of(new Position(image.getPosition(), image.getPage().getNumber())))
+                .containingNodeId(image.getTreeId())
+                .closestHeadline(image.getHeadline().getTextBlock().getSearchText())
+                .section(image.getManualOverwrite().getSection().orElse(image.getParent().toString()))
+                .imageHasTransparency(image.isTransparent())
+                .state(buildEntryState(image))
+                .entryType(redactionLogEntry.isHint() ? EntryType.IMAGE_HINT : EntryType.IMAGE)
+                .build();
+
+    }
+
+
+    public EntityLogEntry createEntityLogEntry(ManualEntity manualEntity) {
+
+        String type = manualEntity.getManualOverwrite().getType().orElse(manualEntity.getType());
+        return EntityLogEntry.builder()
+                .id(manualEntity.getId())
+                .reason(manualEntity.buildReasonWithManualChangeDescriptions())
+                .legalBasis(manualEntity.legalBasis())
+                .value(manualEntity.value())
+                .type(type)
+                .state(buildEntryState(manualEntity))
+                .entryType(buildEntryType(manualEntity))
+                .section(manualEntity.getManualOverwrite().getSection().orElse(manualEntity.getSection()))
+                .containingNodeId(Collections.emptyList())
+                .closestHeadline("")
+                .matchedRule(manualEntity.getMatchedRule().getRuleIdentifier().toString())
+                .dictionaryEntry(manualEntity.isDictionaryEntry())
+                .dossierDictionaryEntry(manualEntity.isDossierDictionaryEntry())
+                .textAfter("")
+                .textBefore("")
+                .startOffset(-1)
+                .endOffset(-1)
+                .positions(manualEntity.getManualOverwrite()
+                        .getPositions()
+                        .orElse(manualEntity.getEntityPosition())
+                        .stream()
+                        .map(entityPosition -> new Position(entityPosition.rectangle2D(), entityPosition.pageNumber()))
+                        .toList())
+                .engines(Collections.emptySet())
+                .build();
+    }
+
+
+    public EntityLogEntry createEntityLogEntry(TextEntity entity) {
+
+        assert entity.getPositionsOnPagePerPage().size() == 1;
+        PositionOnPage positionOnPage = entity.getPositionsOnPagePerPage().get(0);
+        List<Position> rectanglesPerLine = positionOnPage.getRectanglePerLine()
+                .stream()
+                .map(rectangle2D -> new Position(rectangle2D, positionOnPage.getPage().getNumber()))
+                .toList();
+        return EntityLogEntry.builder()
+                .id(positionOnPage.getId())
+                .positions(rectanglesPerLine)
+                .reason(entity.buildReasonWithManualChangeDescriptions())
+                .legalBasis(entity.legalBasis())
+                .value(entity.getManualOverwrite().getValue().orElse(entity.getMatchedRule().isWriteValueWithLineBreaks() ? entity.getValueWithLineBreaks() : entity.getValue()))
+                .type(entity.getType())
+                .section(entity.getManualOverwrite().getSection().orElse(entity.getDeepestFullyContainingNode().toString()))
+                .containingNodeId(entity.getDeepestFullyContainingNode().getTreeId())
+                .closestHeadline(entity.getDeepestFullyContainingNode().getHeadline().getTextBlock().getSearchText())
+                .matchedRule(entity.getMatchedRule().getRuleIdentifier().toString())
+                .dictionaryEntry(entity.isDictionaryEntry())
+                .textAfter(entity.getTextAfter())
+                .textBefore(entity.getTextBefore())
+                .startOffset(entity.getTextRange().start())
+                .endOffset(entity.getTextRange().end())
+                .dossierDictionaryEntry(entity.isDossierDictionaryEntry())
+                .engines(entity.getEngines() != null ? entity.getEngines() : Collections.emptySet())
+                .state(buildEntryState(entity))
+                .entryType(buildEntryType(entity))
+                .build();
+    }
+
+
+    private EntryState buildEntryState(IEntity entity) {
+
+        if (entity.applied() && entity.active()) {
+            return EntryState.APPLIED;
+        } else if (entity.skipped() && entity.active()) {
+            return EntryState.SKIPPED;
+        } else if (entity.ignored()) {
+            return EntryState.IGNORED;
+        } else {
+            return EntryState.REMOVED;
+        }
+    }
+
+
+    private EntryType buildEntryType(IEntity entity) {
+
+        if (entity instanceof TextEntity textEntity) {
+            return getEntryType(textEntity.getEntityType());
+        } else if (entity instanceof ManualEntity manualEntity) {
+            if (manualEntity.isRectangle()) {
+                return EntryType.AREA;
+            }
+            return getEntryType(manualEntity.getEntityType());
+        } else if (entity instanceof Image) {
+            return EntryType.IMAGE;
+        }
+        throw new UnsupportedOperationException(String.format("Entity subclass %s is not implemented!", entity.getClass()));
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/component/Entity.java

@@ -80,7 +80,9 @@ public class Entity {
                 .legalBasis(e.getLegalBasis())
                 .imported(e.isImported())
                 .section(e.getSection())
-                .color(e.getColor()).positions(e.getPositions()).containingNode(document.getDocumentTree().getEntryById(e.getContainingNodeId()).getNode())
+                .color(e.getColor())
+                .positions(e.getPositions())
+                .containingNode(document.getDocumentTree().getEntryById(e.getContainingNodeId()).getNode())
                 .textBefore(e.getTextBefore())
                 .textAfter(e.getTextAfter())
                 .startOffset(e.getStartOffset())

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/Dictionary.java

@@ -2,23 +2,23 @@ package com.iqser.red.service.redaction.v1.server.model.dictionary;
 
 import static java.lang.String.format;
 
+import java.util.Arrays;
 import java.util.Collection;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
+import java.util.Locale;
 import java.util.Map;
 import java.util.Set;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
 import org.apache.commons.lang3.StringUtils;
 
-import com.iqser.red.service.redaction.v1.server.utils.exception.NotFoundException;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
 import com.iqser.red.service.redaction.v1.server.utils.Patterns;
+import com.iqser.red.service.redaction.v1.server.utils.exception.NotFoundException;
 
 import lombok.Data;
 import lombok.Getter;
@@ -107,11 +107,21 @@ public class Dictionary {
         if (StringUtils.isEmpty(value)) {
             throw new IllegalArgumentException(format("%s is not a valid dictionary entry", value));
         }
+        boolean isCaseInsensitive = localAccessMap.get(type).isCaseInsensitive();
         Set<MatchedRule> matchedRulesSet = new HashSet<>(matchedRules);
-        localAccessMap.get(type).getLocalEntriesWithMatchedRules().merge(value.trim(), matchedRulesSet, (set1, set2) -> Stream.concat(set1.stream(), set2.stream()).collect(Collectors.toSet()));
+
+        String cleanedValue = value;
+        if (isCaseInsensitive) {
+            cleanedValue = cleanedValue.toLowerCase(Locale.US);
+        }
+        localAccessMap.get(type)
+                .getLocalEntriesWithMatchedRules()
+                .merge(cleanedValue.trim(), matchedRulesSet, (set1, set2) -> Stream.concat(set1.stream(), set2.stream()).collect(Collectors.toSet()));
         if (alsoAddLastname) {
-            String lastname = value.split(" ")[0];
-            localAccessMap.get(type).getLocalEntriesWithMatchedRules().merge(lastname, matchedRulesSet, (set1, set2) -> Stream.concat(set1.stream(), set2.stream()).collect(Collectors.toSet()));
+            String lastname = cleanedValue.split(" ")[0];
+            localAccessMap.get(type)
+                    .getLocalEntriesWithMatchedRules()
+                    .merge(lastname, matchedRulesSet, (set1, set2) -> Stream.concat(set1.stream(), set2.stream()).collect(Collectors.toSet()));
         }
     }
 
@@ -130,16 +140,20 @@ public class Dictionary {
 
     public void addMultipleAuthorsAsRecommendation(TextEntity textEntity) {
 
-        String cleanedWord = textEntity.getValue().replaceAll(",", " ").replaceAll("  ", " ").trim() + " ";
-        Pattern pattern = Patterns.AUTHOR_TABLE_SPLITTER;
-        Matcher matcher = pattern.matcher(cleanedWord);
+        splitIntoAuthorNames(textEntity).forEach(authorName -> addLocalDictionaryEntry(textEntity.getType(), authorName, textEntity.getMatchedRuleList(), true));
 
-        while (matcher.find()) {
-            String match = matcher.group().trim();
-            if (match.length() >= 3) {
-                addLocalDictionaryEntry(textEntity.getType(), match, textEntity.getMatchedRuleList(), true);
-            }
+    }
+
+
+    public static List<String> splitIntoAuthorNames(TextEntity textEntity) {
+
+        List<String> splitAuthorNames;
+        if (textEntity.getValue().contains(",")) {
+            splitAuthorNames = Arrays.asList(textEntity.getValue().split(","));
+        } else {
+            splitAuthorNames = Arrays.asList(textEntity.getValueWithLineBreaks().split("\n"));
         }
+        return splitAuthorNames.stream().map(String::trim).filter(authorName -> Patterns.AUTHOR_NAME_PATTERN.matcher(authorName).matches()).toList();
     }
 
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/DictionaryModel.java

@@ -2,6 +2,8 @@ package com.iqser.red.service.redaction.v1.server.model.dictionary;
 
 import java.io.Serializable;
 import java.util.HashMap;
+import java.util.List;
+import java.util.Locale;
 import java.util.Set;
 import java.util.stream.Collectors;
 
@@ -109,4 +111,11 @@ public class DictionaryModel implements Serializable {
         return falseRecommendationsSearch;
     }
 
+    public Set<MatchedRule> getMatchedRulesForLocalDictionaryEntry(String value) {
+        var cleanedValue = isCaseInsensitive() ? value.toLowerCase(Locale.US) : value;
+
+        return localEntriesWithMatchedRules.get(cleanedValue);
+
+    }
+
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/ManualChangeOverwrite.java

@@ -1,6 +1,5 @@
 package com.iqser.red.service.redaction.v1.server.model.document.entity;
 
-import java.awt.geom.Rectangle2D;
 import java.util.Collections;
 import java.util.Comparator;
 import java.util.LinkedList;
@@ -8,6 +7,7 @@ import java.util.List;
 import java.util.Map;
 import java.util.Optional;
 
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.BaseAnnotation;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction;
@@ -83,7 +83,18 @@ public class ManualChangeOverwrite {
 
             if (manualChange instanceof IdRemoval) {
                 applied = false;
-                ignored = true;
+                if (sortedManualChanges.get(0) instanceof ManualRedactionEntry) { // set to removed if it's a Manual Redaction
+                    removed = true;
+                } else {
+                    ignored = true;
+                }
+            }
+
+            if(manualChange instanceof ManualRedactionEntry manualRedactionEntry){
+                removed = false;
+                ignored = false;
+                applied = true;
+                legalBasis = manualRedactionEntry.getLegalBasis();
             }
 
             if (manualChange instanceof ManualForceRedaction manualForceRedaction) {
@@ -210,10 +221,4 @@ public class ManualChangeOverwrite {
         return positions == null ? Optional.empty() : Optional.of(positions);
     }
 
-
-    public static Rectangle2D toRectangle2D(com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle rect) {
-
-        return new Rectangle2D.Double(rect.getTopLeftX() - rect.getWidth(), rect.getTopLeftY() - rect.getHeight(), rect.getWidth(), rect.getHeight());
-    }
-
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/MatchedRule.java

@@ -31,11 +31,11 @@ public final class MatchedRule implements Comparable<MatchedRule> {
     String reason = "";
     @Builder.Default
     String legalBasis = "";
-    boolean applied;
     boolean writeValueWithLineBreaks;
+    boolean applied;
     boolean removed;
     boolean ignored;
-    boolean resized;
+
     @Builder.Default
     Set<TextEntity> references = Collections.emptySet();
 
@@ -46,6 +46,20 @@ public final class MatchedRule implements Comparable<MatchedRule> {
     }
 
 
+    public MatchedRule asSkippedIfApplied() {
+
+        if (!this.isApplied()) {
+            return this;
+        }
+        return MatchedRule.builder().ruleIdentifier(getRuleIdentifier())
+                .writeValueWithLineBreaks(this.isWriteValueWithLineBreaks())
+                .legalBasis(this.getLegalBasis())
+                .reason(this.getReason())
+                .references(this.getReferences())
+                .build();
+    }
+
+
     @Override
     public int compareTo(MatchedRule matchedRule) {
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/SemanticNode.java

@@ -86,6 +86,12 @@ public interface SemanticNode {
     }
 
 
+    /**
+     * Checks if the given page number exists in the list of pages.
+     *
+     * @param pageNumber the page number to be checked
+     * @return true if the page number exists, otherwise false
+     */
     default boolean onPage(int pageNumber) {
 
         return getPages().stream().anyMatch(page -> page.getNumber() == pageNumber);
@@ -425,6 +431,71 @@ public interface SemanticNode {
     }
 
 
+    /**
+     * Checks whether this SemanticNode contains exactly the provided String as a word.
+     * @param word - String which the TextBlock might contain
+     * @return true, if this node's TextBlock contains string
+     */
+    default boolean containsWord(String word) {
+
+        return getTextBlock().getWords().stream().anyMatch(s -> s.equals(word));
+    }
+
+
+    /**
+     * Checks whether this SemanticNode contains exactly the provided String as a word ignoring case.
+     * @param word - String which the TextBlock might contain
+     * @return true, if this node's TextBlock contains string
+     */
+    default boolean containsWordIgnoreCase(String word) {
+
+        return getTextBlock().getWords().stream().map(String::toLowerCase).anyMatch(s -> s.equals(word.toLowerCase(Locale.ENGLISH)));
+    }
+
+
+    /**
+     * Checks whether this SemanticNode contains any of the provided Strings as a word.
+     * @param words - A List of Strings which the TextBlock might contain
+     * @return true, if this node's TextBlock contains any of the provided strings
+     */
+    default boolean containsAnyWord(String... words) {
+
+        return Arrays.stream(words).anyMatch(word -> getTextBlock().getWords().stream().anyMatch(word::equals));
+    }
+
+
+    /**
+     * Checks whether this SemanticNode contains any of the provided Strings as a word ignoring case.
+     * @param words - A List of Strings which the TextBlock might contain
+     * @return true, if this node's TextBlock contains any of the provided strings
+     */
+    default boolean containsAnyWordIgnoreCase(String... words) {
+
+        return Arrays.stream(words).map(String::toLowerCase).anyMatch(word -> getTextBlock().getWords().stream().map(String::toLowerCase).anyMatch(word::equals));
+    }
+
+
+    /**
+     * Checks whether this SemanticNode contains all the provided Strings as word.
+     * @param words - A List of Strings which the TextBlock might contain
+     * @return true, if this node's TextBlock contains all the provided strings
+     */
+    default boolean containsAllWords(String... words) {
+
+        return Arrays.stream(words).allMatch(word -> getTextBlock().getWords().stream().anyMatch(word::equals));
+    }
+
+
+    /**
+     * Checks whether this SemanticNode contains all the provided Strings as word ignoring case.
+     * @param words - A List of Strings which the TextBlock might contain
+     * @return true, if this node's TextBlock contains all the provided strings
+     */
+    default boolean containsAllWordsIgnoreCase(String... words) {
+
+        return Arrays.stream(words).map(String::toLowerCase).allMatch(word -> getTextBlock().getWords().stream().map(String::toLowerCase).anyMatch(word::equals));
+    }
+
     /**
      * Checks whether this SemanticNode matches the provided regex pattern.
      *

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Table.java

@@ -172,6 +172,18 @@ public class Table implements SemanticNode {
         return streamChildrenOfType(NodeType.TABLE_CELL).map(node -> (TableCell) node);
     }
 
+    /**
+     * Streams all TableCells that contain at least one entity of a given type in this Table row-wise.
+     *
+     * @param type The type of the entity
+     * @return Stream of filtered TableCells
+     */
+    public Stream<TableCell> streamTableCellsWhichContainType(String type) {
+
+        return streamTableCells()
+                .filter(tableCell -> tableCell.getEntities().stream().filter(TextEntity::active).anyMatch(entity -> entity.getType().equals(type)));
+    }
+
 
     /**
      * Streams all TableCells in this Table which have the provided header row-wise.

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/textblock/AtomicTextBlock.java

@@ -3,11 +3,14 @@ package com.iqser.red.service.redaction.v1.server.model.document.textblock;
 import static java.lang.String.format;
 
 import java.awt.geom.Rectangle2D;
+import java.text.BreakIterator;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
+import java.util.Locale;
 import java.util.Map;
 import java.util.Set;
 import java.util.stream.Collectors;
@@ -40,6 +43,7 @@ public class AtomicTextBlock implements TextBlock {
     //string coordinates
     TextRange textRange;
     String searchText;
+    List<String> words;
     List<Integer> lineBreaks;
 
     //position coordinates
@@ -114,6 +118,20 @@ public class AtomicTextBlock implements TextBlock {
         return new TextRange(lineBreaks.get(lineNumber - 1) + textRange.start(), lineBreaks.get(lineNumber) + textRange.start());
     }
 
+    public List<String> getWords() {
+
+        if (words == null) {
+            words = new ArrayList<>();
+            BreakIterator iterator = BreakIterator.getWordInstance(Locale.ENGLISH);
+            iterator.setText(searchText);
+            int start = iterator.first();
+            for (int end = iterator.next(); end != BreakIterator.DONE; start = end, end = iterator.next()) {
+                words.add(searchText.substring(start, end));
+            }
+        }
+        return words;
+    }
+
 
     @Override
     public List<AtomicTextBlock> getAtomicTextBlocks() {

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/textblock/ConcatenatedTextBlock.java

@@ -7,6 +7,7 @@ import java.util.Collections;
 import java.util.HashMap;
 import java.util.LinkedList;
 import java.util.List;
+import java.util.Collection;
 import java.util.Map;
 import java.util.stream.Stream;
 
@@ -86,6 +87,13 @@ public class ConcatenatedTextBlock implements TextBlock {
     }
 
 
+    @Override
+    public List<String> getWords() {
+
+        return atomicTextBlocks.stream().map(AtomicTextBlock::getWords).flatMap(Collection::stream).toList();
+    }
+
+
     @Override
     public int numberOfLines() {
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/textblock/TextBlock.java

@@ -18,6 +18,7 @@ public interface TextBlock extends CharSequence {
 
     String getSearchText();
 
+    List<String> getWords();
 
     List<AtomicTextBlock> getAtomicTextBlocks();
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/drools/BasicRule.java

@@ -1,5 +1,6 @@
 package com.iqser.red.service.redaction.v1.server.model.drools;
 
+import org.drools.drl.ast.descr.AttributeDescr;
 import org.drools.drl.ast.descr.RuleDescr;
 
 import lombok.AccessLevel;
@@ -18,6 +19,7 @@ public class BasicRule {
     RuleIdentifier identifier;
     String name;
     String code;
+    String agendaGroup;
     int line;
 
 
@@ -26,7 +28,8 @@ public class BasicRule {
         RuleIdentifier identifier = RuleIdentifier.fromName(rule.getName());
         String nameWithoutIdentifier = rule.getName().replace(identifier + ":", "");
         String code = rulesString.substring(rule.getStartCharacter(), rule.getEndCharacter());
-        return new BasicRule(identifier, nameWithoutIdentifier, code, rule.getLine());
+        String agendaGroup = rule.getAttributes().getOrDefault("agenda-group", new AttributeDescr("agenda-group", "DEFAULT")).getValue();
+        return new BasicRule(identifier, nameWithoutIdentifier, code, agendaGroup, rule.getLine());
     }
 
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/drools/RuleFileBluePrint.java

@@ -1,10 +1,12 @@
 package com.iqser.red.service.redaction.v1.server.model.drools;
 
+import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
 import java.util.Objects;
 import java.util.Optional;
+import java.util.Set;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
@@ -39,6 +41,12 @@ public final class RuleFileBluePrint {
     }
 
 
+    public Set<String> getImportSplitByKeyword() {
+
+        return Arrays.stream(imports.replaceAll("\n", "").split("import")).map(String::trim).collect(Collectors.toSet());
+    }
+
+
     public List<BasicRule> findRulesByIdentifier(RuleIdentifier ruleIdentifier) {
 
         if (Objects.isNull(ruleIdentifier.unit())) {

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/queue/MessageReceiver.java

@@ -1,13 +1,11 @@
 package com.iqser.red.service.redaction.v1.server.queue;
 
-import static com.iqser.red.service.redaction.v1.server.queue.MessagingConfiguration.REDACTION_QUEUE;
+import static com.iqser.red.service.redaction.v1.model.QueueNames.REDACTION_QUEUE;
 
 import org.springframework.amqp.core.Message;
 import org.springframework.amqp.rabbit.annotation.RabbitHandler;
 import org.springframework.amqp.rabbit.annotation.RabbitListener;
 
-import com.fasterxml.jackson.core.JsonProcessingException;
-
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/queue/MessagingConfiguration.java

@@ -1,6 +1,10 @@
 package com.iqser.red.service.redaction.v1.server.queue;
 
-import lombok.RequiredArgsConstructor;
+import static com.iqser.red.service.redaction.v1.model.QueueNames.MIGRATION_DLQ;
+import static com.iqser.red.service.redaction.v1.model.QueueNames.MIGRATION_QUEUE;
+import static com.iqser.red.service.redaction.v1.model.QueueNames.REDACTION_DQL;
+import static com.iqser.red.service.redaction.v1.model.QueueNames.REDACTION_PRIORITY_QUEUE;
+import static com.iqser.red.service.redaction.v1.model.QueueNames.REDACTION_QUEUE;
 
 import org.springframework.amqp.core.Queue;
 import org.springframework.amqp.core.QueueBuilder;
@@ -8,15 +12,12 @@ import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;
 
+import lombok.RequiredArgsConstructor;
+
 @Configuration
 @RequiredArgsConstructor
 public class MessagingConfiguration {
 
-    public static final String REDACTION_QUEUE = "redactionQueue";
-    public static final String REDACTION_DQL = "redactionDQL";
-
-    public static final String REDACTION_PRIORITY_QUEUE = "redactionPriorityQueue";
-
     public static final String X_ERROR_INFO_HEADER = "x-error-message";
     public static final String X_ERROR_INFO_TIMESTAMP_HEADER = "x-error-message-timestamp";
 
@@ -37,6 +38,20 @@ public class MessagingConfiguration {
     }
 
 
+    @Bean
+    public Queue migrationQueue() {
+
+        return QueueBuilder.durable(MIGRATION_QUEUE).withArgument("x-dead-letter-exchange", "").withArgument("x-dead-letter-routing-key", MIGRATION_DLQ).maxPriority(2).build();
+    }
+
+
+    @Bean
+    public Queue migrationDLQ() {
+
+        return QueueBuilder.durable(MIGRATION_DLQ).build();
+    }
+
+
     @Bean
     public Queue redactionQueue() {
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/queue/MigrationMessageReceiver.java

@@ -0,0 +1,66 @@
+package com.iqser.red.service.redaction.v1.server.queue;
+
+import static com.iqser.red.service.redaction.v1.model.QueueNames.MIGRATION_QUEUE;
+import static com.iqser.red.service.redaction.v1.model.QueueNames.MIGRATION_RESPONSE_QUEUE;
+
+import org.springframework.amqp.core.Message;
+import org.springframework.amqp.rabbit.annotation.RabbitHandler;
+import org.springframework.amqp.rabbit.annotation.RabbitListener;
+import org.springframework.amqp.rabbit.core.RabbitTemplate;
+import org.springframework.stereotype.Service;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog;
+import com.iqser.red.service.redaction.v1.model.MigrationRequest;
+import com.iqser.red.service.redaction.v1.model.MigrationResponse;
+import com.iqser.red.service.redaction.v1.server.model.MigratedEntityLog;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
+import com.iqser.red.service.redaction.v1.server.service.RedactionLogToEntityLogMigrationService;
+import com.iqser.red.service.redaction.v1.server.service.document.DocumentGraphMapper;
+import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
+
+import lombok.AccessLevel;
+import lombok.RequiredArgsConstructor;
+import lombok.SneakyThrows;
+import lombok.experimental.FieldDefaults;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+@Service
+@RequiredArgsConstructor
+@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
+public class MigrationMessageReceiver {
+
+    ObjectMapper objectMapper;
+    RedactionLogToEntityLogMigrationService redactionLogToEntityLogMigrationService;
+    RedactionStorageService redactionStorageService;
+    RabbitTemplate rabbitTemplate;
+
+
+    @SneakyThrows
+    @RabbitHandler
+    @RabbitListener(queues = MIGRATION_QUEUE)
+    public void receiveMigrationRequest(Message message) {
+
+        MigrationRequest migrationRequest = objectMapper.readValue(message.getBody(), MigrationRequest.class);
+
+        RedactionLog redactionLog = redactionStorageService.getRedactionLog(migrationRequest.getDossierId(), migrationRequest.getFileId());
+        Document document = DocumentGraphMapper.toDocumentGraph(redactionStorageService.getDocumentData(migrationRequest.getDossierId(), migrationRequest.getFileId()));
+
+        MigratedEntityLog migratedEntityLog = redactionLogToEntityLogMigrationService.migrate(redactionLog, document);
+
+        redactionStorageService.storeObject(migrationRequest.getDossierId(), migrationRequest.getFileId(), FileType.ENTITY_LOG, migratedEntityLog.getEntityLog());
+        redactionStorageService.storeObject(migrationRequest.getDossierId(), migrationRequest.getFileId(), FileType.MIGRATED_IDS, migratedEntityLog.getMigratedIds());
+
+        sendFinished(MigrationResponse.builder().dossierId(migrationRequest.getDossierId()).fileId(migrationRequest.getFileId()).build());
+    }
+
+
+    @SneakyThrows
+    public void sendFinished(MigrationResponse migrationResponse) {
+
+        rabbitTemplate.convertAndSend(MIGRATION_RESPONSE_QUEUE, migrationResponse);
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/queue/PriorityMessageReceiver.java

@@ -1,13 +1,11 @@
 package com.iqser.red.service.redaction.v1.server.queue;
 
-import static com.iqser.red.service.redaction.v1.server.queue.MessagingConfiguration.REDACTION_PRIORITY_QUEUE;
+import static com.iqser.red.service.redaction.v1.model.QueueNames.REDACTION_PRIORITY_QUEUE;
 
 import org.springframework.amqp.core.Message;
 import org.springframework.amqp.rabbit.annotation.RabbitHandler;
 import org.springframework.amqp.rabbit.annotation.RabbitListener;
 
-import com.fasterxml.jackson.core.JsonProcessingException;
-
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/queue/RedactionMessageReceiver.java

@@ -1,8 +1,8 @@
 package com.iqser.red.service.redaction.v1.server.queue;
 
-import static com.iqser.red.service.redaction.v1.server.queue.MessagingConfiguration.REDACTION_DQL;
-import static com.iqser.red.service.redaction.v1.server.queue.MessagingConfiguration.REDACTION_PRIORITY_QUEUE;
-import static com.iqser.red.service.redaction.v1.server.queue.MessagingConfiguration.REDACTION_QUEUE;
+import static com.iqser.red.service.redaction.v1.model.QueueNames.REDACTION_DQL;
+import static com.iqser.red.service.redaction.v1.model.QueueNames.REDACTION_PRIORITY_QUEUE;
+import static com.iqser.red.service.redaction.v1.model.QueueNames.REDACTION_QUEUE;
 import static com.iqser.red.service.redaction.v1.server.queue.MessagingConfiguration.X_ERROR_INFO_HEADER;
 import static com.iqser.red.service.redaction.v1.server.queue.MessagingConfiguration.X_ERROR_INFO_TIMESTAMP_HEADER;
 import static java.lang.String.format;
@@ -23,8 +23,8 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeResu
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileErrorInfo;
 import com.iqser.red.service.redaction.v1.server.client.FileStatusProcessingUpdateClient;
 import com.iqser.red.service.redaction.v1.server.client.RulesClient;
-import com.iqser.red.service.redaction.v1.server.utils.exception.DroolsTimeoutException;
 import com.iqser.red.service.redaction.v1.server.service.AnalyzeService;
+import com.iqser.red.service.redaction.v1.server.utils.exception.DroolsTimeoutException;
 
 import lombok.RequiredArgsConstructor;
 import lombok.SneakyThrows;
@@ -128,7 +128,7 @@ public class RedactionMessageReceiver {
 
     @RabbitHandler
     @RabbitListener(queues = REDACTION_DQL)
-    public void receiveAnalyzeRequestDQL(Message in) throws IOException {
+    public void receiveAnalyzeRequestDLQ(Message in) throws IOException {
 
         var analyzeRequest = objectMapper.readValue(in.getBody(), AnalyzeRequest.class);
         log.info("Failed to process analyze request: {}", analyzeRequest);

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalyzeService.java

@@ -18,7 +18,6 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileTyp
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentLog;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogChanges;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Comment;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.legalbasis.LegalBasis;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog;
@@ -112,7 +111,10 @@ public class AnalyzeService {
                     startTime,
                     kieContainerCreationService.getLatestKieContainer(analyzeRequest.getDossierTemplateId(), RuleFileType.COMPONENT),
                     entityLogCreatorService.updateVersionsAndReturnChanges(previousEntityLog,
-                            dictionaryIncrement.getDictionaryVersion(), analyzeRequest.getDossierTemplateId(), false), document,
+                            dictionaryIncrement.getDictionaryVersion(),
+                            analyzeRequest.getDossierTemplateId(),
+                            false),
+                    document,
                     previousRedactionLog,
                     document.getNumberOfPages(),
                     dictionaryIncrement.getDictionaryVersion(),
@@ -153,7 +155,10 @@ public class AnalyzeService {
                 dictionary.getVersion());
 
         return finalizeAnalysis(analyzeRequest,
-                startTime, kieContainerCreationService.getLatestKieContainer(analyzeRequest.getDossierTemplateId(), RuleFileType.COMPONENT), entityLogChanges, document,
+                startTime,
+                kieContainerCreationService.getLatestKieContainer(analyzeRequest.getDossierTemplateId(), RuleFileType.COMPONENT),
+                entityLogChanges,
+                document,
                 redactionLog,
                 document.getNumberOfPages(),
                 dictionaryIncrement.getDictionaryVersion(),
@@ -205,7 +210,11 @@ public class AnalyzeService {
                 dictionary.getVersion(),
                 kieWrapperEntityRules.rulesVersion());
 
-        return finalizeAnalysis(analyzeRequest, startTime, kieWrapperComponentRules, new EntityLogChanges(entityLog, false), document,
+        return finalizeAnalysis(analyzeRequest,
+                startTime,
+                kieWrapperComponentRules,
+                new EntityLogChanges(entityLog, false),
+                document,
                 redactionLog,
                 document.getNumberOfPages(),
                 dictionary.getVersion(),
@@ -223,8 +232,7 @@ public class AnalyzeService {
 
         List<RedactionLogEntry> newRedactionLogEntries = redactionLogCreatorService.createRedactionLog(document,
                 analyzeRequest.getDossierTemplateId(),
-                notFoundManualRedactionEntries,
-                getComments(analyzeRequest));
+                notFoundManualRedactionEntries);
 
         var importedRedactionFilteredEntries = importedRedactionService.processImportedRedactions(analyzeRequest.getDossierTemplateId(),
                 analyzeRequest.getDossierId(),
@@ -234,12 +242,18 @@ public class AnalyzeService {
 
         previousRedactionLog.getRedactionLogEntry()
                 .removeIf(entry -> sectionsToReanalyseIds.contains(entry.getSectionNumber()) && !entry.getType().equals(ImportedRedactionService.IMPORTED_REDACTION_TYPE));
+
         previousRedactionLog.getRedactionLogEntry().addAll(importedRedactionFilteredEntries);
+
         return previousRedactionLog;
     }
 
 
-    private AnalyzeResult finalizeAnalysis(AnalyzeRequest analyzeRequest, long startTime, KieWrapper kieWrapperComponentRules, EntityLogChanges entityLogChanges, Document document,
+    private AnalyzeResult finalizeAnalysis(AnalyzeRequest analyzeRequest,
+                                           long startTime,
+                                           KieWrapper kieWrapperComponentRules,
+                                           EntityLogChanges entityLogChanges,
+                                           Document document,
                                            RedactionLog redactionLog,
                                            int numberOfPages,
                                            DictionaryVersion dictionaryVersion,
@@ -282,7 +296,9 @@ public class AnalyzeService {
     }
 
 
-    private void computeComponentsWhenRulesArePresent(AnalyzeRequest analyzeRequest, KieWrapper kieWrapperComponentRules, Document document,
+    private void computeComponentsWhenRulesArePresent(AnalyzeRequest analyzeRequest,
+                                                      KieWrapper kieWrapperComponentRules,
+                                                      Document document,
                                                       Set<FileAttribute> addedFileAttributes,
                                                       EntityLogChanges entityLogChanges,
                                                       DictionaryVersion dictionaryVersion) {
@@ -291,7 +307,9 @@ public class AnalyzeService {
             return;
         }
 
-        List<Component> components = componentDroolsExecutionService.executeRules(kieWrapperComponentRules.container(), entityLogChanges.getEntityLog(), document,
+        List<Component> components = componentDroolsExecutionService.executeRules(kieWrapperComponentRules.container(),
+                entityLogChanges.getEntityLog(),
+                document,
                 addedFileAttributes.stream().toList());
         log.info("Finished component rule execution for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
 
@@ -376,8 +394,7 @@ public class AnalyzeService {
 
         List<RedactionLogEntry> redactionLogEntries = redactionLogCreatorService.createRedactionLog(document,
                 analyzeRequest.getDossierTemplateId(),
-                notFoundManualRedactionEntries,
-                getComments(analyzeRequest));
+                notFoundManualRedactionEntries);
 
         List<LegalBasis> legalBasis = legalBasisClient.getLegalBasisMapping(analyzeRequest.getDossierTemplateId());
         RedactionLog redactionLog = new RedactionLog(redactionServiceSettings.getAnalysisVersion(),
@@ -399,18 +416,6 @@ public class AnalyzeService {
     }
 
 
-    private static Map<String, List<Comment>> getComments(AnalyzeRequest analyzeRequest) {
-
-        if (analyzeRequest.getManualRedactions() == null) {
-            return Collections.emptyMap();
-        }
-        if (analyzeRequest.getManualRedactions().getComments() == null) {
-            return Collections.emptyMap();
-        }
-        return analyzeRequest.getManualRedactions().getComments();
-    }
-
-
     public List<RedactionLogLegalBasis> toRedactionLogLegalBasis(List<LegalBasis> legalBasis) {
 
         return legalBasis.stream().map(l -> new RedactionLogLegalBasis(l.getName(), l.getDescription(), l.getReason())).collect(Collectors.toList());

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/DictionaryService.java

@@ -47,7 +47,7 @@ import lombok.extern.slf4j.Slf4j;
 @RequiredArgsConstructor
 public class DictionaryService {
 
-    private static final String DEFAULT_COLOR = "#cccccc";
+    public static final String DEFAULT_COLOR = "#cccccc";
     private final DictionaryClient dictionaryClient;
 
     private final RedactionServiceSettings settings;

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java

@@ -1,5 +1,7 @@
 package com.iqser.red.service.redaction.v1.server.service;
 
+import static com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryType.IMAGE;
+
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashSet;
@@ -58,13 +60,11 @@ public class EntityLogCreatorService {
     }
 
 
-    public EntityLog createInitialEntityLog(AnalyzeRequest analyzeRequest,
-                                            Document document,
-                                            List<ManualEntity> notFoundManualRedactionEntries,
+    public EntityLog createInitialEntityLog(AnalyzeRequest analyzeRequest, Document document, List<ManualEntity> notFoundManualEntities,
                                             DictionaryVersion dictionaryVersion,
                                             long rulesVersion) {
 
-        List<EntityLogEntry> entityLogEntries = createEntityLogEntries(document, analyzeRequest.getDossierTemplateId(), notFoundManualRedactionEntries);
+        List<EntityLogEntry> entityLogEntries = createEntityLogEntries(document, analyzeRequest.getDossierTemplateId(), notFoundManualEntities);
 
         List<LegalBasis> legalBasis = legalBasisClient.getLegalBasisMapping(analyzeRequest.getDossierTemplateId());
         EntityLog entityLog = new EntityLog(redactionServiceSettings.getAnalysisVersion(),
@@ -121,10 +121,11 @@ public class EntityLogCreatorService {
 
         List<EntityLogEntry> newEntityLogEntries = createEntityLogEntries(document, analyzeRequest.getDossierTemplateId(), notFoundManualRedactionEntries);
         Set<String> newEntityIds = newEntityLogEntries.stream().map(EntityLogEntry::getId).collect(Collectors.toSet());
+
         List<EntityLogEntry> previousEntriesFromReAnalyzedSections = previousEntityLog.getEntityLogEntry()
                 .stream()
-                .filter(entry -> (newEntityIds.contains(entry.getId()) || sectionsToReanalyseIds.contains(entry.getContainingNodeId().get(0))) && !entry.getType()
-                        .equals(ImportedRedactionService.IMPORTED_REDACTION_TYPE))
+                .filter(entry -> !entry.getType()
+                        .equals(ImportedRedactionService.IMPORTED_REDACTION_TYPE) && (newEntityIds.contains(entry.getId()) || entry.getContainingNodeId().isEmpty() || sectionsToReanalyseIds.contains(entry.getContainingNodeId().get(0))))
                 .toList();
         previousEntityLog.getEntityLogEntry().removeAll(previousEntriesFromReAnalyzedSections);
 
@@ -150,6 +151,7 @@ public class EntityLogCreatorService {
         Set<String> processedIds = new HashSet<>();
         document.getEntities()
                 .stream()
+                .filter(entity -> !entity.getValue().isEmpty())
                 .filter(EntityLogCreatorService::notFalsePositiveOrFalseRecommendation)
                 .filter(entity -> !entity.removed())
                 .forEach(entityNode -> entries.addAll(toEntityLogEntries(entityNode, dossierTemplateId, processedIds)));
@@ -185,7 +187,7 @@ public class EntityLogCreatorService {
     }
 
 
-    public EntityLogEntry createEntityLogEntry(Image image, String dossierTemplateId) {
+    private EntityLogEntry createEntityLogEntry(Image image, String dossierTemplateId) {
 
         String imageType = image.getImageType().equals(ImageType.OTHER) ? "image" : image.getImageType().toString().toLowerCase(Locale.ENGLISH);
         boolean isHint = dictionaryService.isHint(imageType, dossierTemplateId);
@@ -205,8 +207,7 @@ public class EntityLogCreatorService {
                 .section(image.getManualOverwrite().getSection().orElse(image.getParent().toString()))
                 .imageHasTransparency(image.isTransparent())
                 .manualChanges(manualChangeFactory.toManualChangeList(image.getManualOverwrite().getManualChangeLog(), isHint))
-                .state(buildEntryState(image))
-                .entryType(buildEntryType(image))
+                .state(buildEntryState(image)).entryType(isHint ? EntryType.IMAGE_HINT : EntryType.IMAGE)
                 .build();
 
     }
@@ -290,6 +291,7 @@ public class EntityLogCreatorService {
             return dictionaryService.getNotRedactedColor(dossierTemplateId);
         }
         return dictionaryService.getColor(type, dossierTemplateId);
+
     }
 
 
@@ -312,12 +314,10 @@ public class EntityLogCreatorService {
         if (entity instanceof TextEntity textEntity) {
             return getEntryType(textEntity.getEntityType());
         } else if (entity instanceof ManualEntity manualEntity) {
-            if (((ManualEntity) entity).isRectangle()) {
+            if (manualEntity.isRectangle()) {
                 return EntryType.AREA;
             }
             return getEntryType(manualEntity.getEntityType());
-        } else if (entity instanceof Image) {
-            return EntryType.IMAGE;
         }
         throw new UnsupportedOperationException(String.format("Entity subclass %s is not implemented!", entity.getClass()));
     }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/ImportedRedactionService.java

@@ -1,19 +1,24 @@
 package com.iqser.red.service.redaction.v1.server.service;
 
+import java.util.ArrayList;
+import java.util.Collections;
 import java.util.HashSet;
 import java.util.List;
+import java.util.stream.Collectors;
 
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedaction;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedactions;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Point;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
 import org.springframework.stereotype.Service;
 
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryType;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Point;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.imported.ImportedRedaction;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.imported.ImportedRedactions;
+
+
 import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
 
 import io.micrometer.core.annotation.Timed;
@@ -55,22 +60,21 @@ public class ImportedRedactionService {
     @Timed("redactmanager_processImportedRedactions")
     public List<EntityLogEntry> processImportedEntities(String dossierTemplateId,
                                                         String dossierId,
-                                                        String fileId,
-                                                        List<EntityLogEntry> redactionLogEntries,
+                                                        String fileId, List<EntityLogEntry> entityLogEntries,
                                                         boolean addImportedRedactions) {
 
         ImportedRedactions importedRedactions = redactionStorageService.getImportedRedactions(dossierId, fileId);
         if (importedRedactions == null) {
-            return redactionLogEntries;
+            return entityLogEntries;
         }
 
-        redactionLogEntries.forEach(redactionLogEntry -> addIntersections(redactionLogEntry, importedRedactions));
+        entityLogEntries.forEach(redactionLogEntry -> addIntersections(redactionLogEntry, importedRedactions));
 
         if (addImportedRedactions) {
-            return addImportedRedactionsEntityLogEntries(dossierTemplateId, redactionLogEntries, importedRedactions);
+            return addImportedRedactionsEntityLogEntries(dossierTemplateId, entityLogEntries, importedRedactions);
         }
 
-        return redactionLogEntries;
+        return entityLogEntries;
     }
 
 
@@ -86,7 +90,7 @@ public class ImportedRedactionService {
                         .type(IMPORTED_REDACTION_TYPE)
                         .imported(true)
                         .redacted(true)
-                        .positions(importedRedaction.getPositions())
+                        .positions(importedRedaction.getPositions().stream().map(this::normalize).collect(Collectors.toList()))
                         .color(getColor(IMPORTED_REDACTION_TYPE, dossierTemplateId))
                         .build();
 
@@ -107,8 +111,10 @@ public class ImportedRedactionService {
                         .type(IMPORTED_REDACTION_TYPE).entryType(EntryType.ENTITY)
                         .imported(true)
                         .state(EntryState.APPLIED)
-                        .positions(toPositions(importedRedaction.getPositions()))
+                        .positions(importedRedaction.getPositions())
                         .color(getColor(IMPORTED_REDACTION_TYPE, dossierTemplateId))
+                        .containingNodeId(Collections.emptyList())
+                        .value("")
                         .build();
 
                 entityLogEntries.add(redactionLogEntry);
@@ -139,7 +145,7 @@ public class ImportedRedactionService {
             if (importedRedactions.getImportedRedactions().containsKey(rectangle.getPage())) {
                 var importedRedactionsOnPage = importedRedactions.getImportedRedactions().get(rectangle.getPage());
                 for (ImportedRedaction importedRedaction : importedRedactionsOnPage) {
-                    for (Rectangle importedRedactionPosition : importedRedaction.getPositions()) {
+                    for (Position importedRedactionPosition : importedRedaction.getPositions()) {
                         if (rectOverlap(normalizedRectangle, normalize(importedRedactionPosition))) {
                             if (redactionLogEntry.getImportedRedactionIntersections() == null) {
                                 redactionLogEntry.setImportedRedactionIntersections(new HashSet<>());
@@ -160,7 +166,7 @@ public class ImportedRedactionService {
             if (importedRedactions.getImportedRedactions().containsKey(rectangle.getPageNumber())) {
                 var importedRedactionsOnPage = importedRedactions.getImportedRedactions().get(rectangle.getPageNumber());
                 for (ImportedRedaction importedRedaction : importedRedactionsOnPage) {
-                    for (Rectangle importedRedactionPosition : importedRedaction.getPositions()) {
+                    for (Position importedRedactionPosition : importedRedaction.getPositions()) {
                         if (rectOverlap(normalizedRectangle, normalize(importedRedactionPosition))) {
                             if (redactionLogEntry.getImportedRedactionIntersections() == null) {
                                 redactionLogEntry.setImportedRedactionIntersections(new HashSet<>());

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RedactionChangeLogService.java

@@ -13,6 +13,7 @@ import org.springframework.stereotype.Service;
 
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Change;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ChangeType;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ManualRedactionType;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogChanges;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
@@ -102,7 +103,7 @@ public class RedactionChangeLogService {
 
         Set<RedactionLogEntry> currentExistingEntries = currentRedactionLog.getRedactionLogEntry()
                 .stream()
-                .filter(entry -> entry.getChanges().isEmpty() || !entry.lastChangeIsRemoved())
+                .filter(entry -> (entry.getChanges().isEmpty() || !entry.lastChangeIsRemoved()) && !isLastManualChangeRemove(entry))
                 .collect(Collectors.toSet());
 
         previouslyExistingEntries.forEach(currentExistingEntries::remove);
@@ -113,4 +114,12 @@ public class RedactionChangeLogService {
         return addedIds;
     }
 
+
+    private static boolean isLastManualChangeRemove(RedactionLogEntry redactionLogEntry){
+        if(redactionLogEntry.getManualChanges() == null || redactionLogEntry.getManualChanges().isEmpty()){
+            return false;
+        }
+        return redactionLogEntry.getManualChanges().get(redactionLogEntry.getManualChanges().size() -1).getManualRedactionType() == ManualRedactionType.REMOVE_LOCALLY;
+    }
+
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RedactionLogCreatorService.java

@@ -6,7 +6,6 @@ import java.util.Collections;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Locale;
-import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
 import java.util.stream.Collectors;
@@ -15,13 +14,11 @@ import org.springframework.beans.BeanUtils;
 import org.springframework.stereotype.Service;
 
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Comment;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ManualChange;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ManualRedactionType;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Point;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogComment;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
 import com.iqser.red.service.redaction.v1.server.model.ManualEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
@@ -46,18 +43,18 @@ public class RedactionLogCreatorService {
     private final ManualChangeFactory manualChangeFactory;
 
 
-    public List<RedactionLogEntry> createRedactionLog(Document document,
-                                                      String dossierTemplateId,
-                                                      List<ManualEntity> notFoundManualRedactionEntries,
-                                                      Map<String, List<Comment>> comments) {
+    public List<RedactionLogEntry> createRedactionLog(Document document, String dossierTemplateId, List<ManualEntity> notFoundManualRedactionEntries) {
 
         List<RedactionLogEntry> entries = new ArrayList<>();
         Set<String> processIds = new HashSet<>();
-        document.getEntities().stream().filter(RedactionLogCreatorService::notFalsePositiveOrFalseRecommendation)
+        document.getEntities()
+                .stream()
+                .filter(entity -> !entity.getValue().isEmpty())
+                .filter(RedactionLogCreatorService::notFalsePositiveOrFalseRecommendation)
                 .filter(IEntity::active)
-                .forEach(entityNode -> entries.addAll(toRedactionLogEntries(entityNode, dossierTemplateId, comments, processIds)));
-        document.streamAllImages().filter(image -> !image.removed()).forEach(imageNode -> entries.add(createRedactionLogEntry(imageNode, dossierTemplateId, comments)));
-        notFoundManualRedactionEntries.forEach(entityIdentifier -> entries.add(createRedactionLogEntry(entityIdentifier, dossierTemplateId, comments)));
+                .forEach(entityNode -> entries.addAll(toRedactionLogEntries(entityNode, dossierTemplateId, processIds)));
+        document.streamAllImages().filter(image -> !image.removed()).forEach(imageNode -> entries.add(createRedactionLogEntry(imageNode, dossierTemplateId)));
+        notFoundManualRedactionEntries.forEach(entityIdentifier -> entries.add(createRedactionLogEntry(entityIdentifier, dossierTemplateId)));
         return entries;
     }
 
@@ -68,7 +65,7 @@ public class RedactionLogCreatorService {
     }
 
 
-    private List<RedactionLogEntry> toRedactionLogEntries(TextEntity textEntity, String dossierTemplateId, Map<String, List<Comment>> comments, Set<String> processedIds) {
+    private List<RedactionLogEntry> toRedactionLogEntries(TextEntity textEntity, String dossierTemplateId, Set<String> processedIds) {
 
         List<RedactionLogEntry> redactionLogEntities = new ArrayList<>();
 
@@ -82,7 +79,6 @@ public class RedactionLogCreatorService {
 
             RedactionLogEntry redactionLogEntry = createRedactionLogEntry(textEntity, dossierTemplateId);
             redactionLogEntry.setId(positionOnPage.getId());
-            redactionLogEntry.setComments(buildRedactionLogComments(comments, positionOnPage.getId()));
 
             List<Rectangle> rectanglesPerLine = positionOnPage.getRectanglePerLine()
                     .stream()
@@ -97,44 +93,14 @@ public class RedactionLogCreatorService {
     }
 
 
-    private List<RedactionLogComment> buildRedactionLogComments(Map<String, List<Comment>> commentsPerId, String id) {
-
-        if (!commentsPerId.containsKey(id)) {
-            return Collections.emptyList();
-        }
-        List<Comment> comments = commentsPerId.get(id);
-        if (comments == null || comments.isEmpty()) {
-            return Collections.emptyList();
-        }
-        return toRedactionLogComments(comments);
-    }
-
-
-    private List<RedactionLogComment> toRedactionLogComments(List<Comment> comments) {
-
-        return comments.stream().map(this::toRedactionLogComment).toList();
-    }
-
-
-    private RedactionLogComment toRedactionLogComment(Comment comment) {
-
-        return new RedactionLogComment(comment.getId(),
-                comment.getUser(),
-                comment.getText(),
-                comment.getAnnotationId(),
-                comment.getFileId(),
-                comment.getDate(),
-                comment.getSoftDeletedTime());
-    }
-
-
     private RedactionLogEntry createRedactionLogEntry(TextEntity entity, String dossierTemplateId) {
 
         Set<String> referenceIds = new HashSet<>();
         entity.references().stream().filter(TextEntity::active).forEach(ref -> ref.getPositionsOnPagePerPage().forEach(pos -> referenceIds.add(pos.getId())));
         int sectionNumber = entity.getDeepestFullyContainingNode().getTreeId().isEmpty() ? 0 : entity.getDeepestFullyContainingNode().getTreeId().get(0);
         boolean isHint = isHint(entity.getEntityType());
-        return RedactionLogEntry.builder().color(getColor(entity.getType(), dossierTemplateId, entity.applied(), isHint))
+        return RedactionLogEntry.builder()
+                .color(getColor(entity.getType(), dossierTemplateId, entity.applied(), isHint))
                 .reason(entity.buildReasonWithManualChangeDescriptions())
                 .legalBasis(entity.legalBasis())
                 .value(entity.getManualOverwrite().getValue().orElse(entity.getMatchedRule().isWriteValueWithLineBreaks() ? entity.getValueWithLineBreaks() : entity.getValue()))
@@ -220,11 +186,13 @@ public class RedactionLogCreatorService {
     }
 
 
-    public RedactionLogEntry createRedactionLogEntry(Image image, String dossierTemplateId, Map<String, List<Comment>> comments) {
+    public RedactionLogEntry createRedactionLogEntry(Image image, String dossierTemplateId) {
 
         String imageType = image.getImageType().equals(ImageType.OTHER) ? "image" : image.getImageType().toString().toLowerCase(Locale.ENGLISH);
         boolean isHint = dictionaryService.isHint(imageType, dossierTemplateId);
-        return RedactionLogEntry.builder().id(image.getId()).color(getColor(imageType, dossierTemplateId, image.applied(), isHint))
+        return RedactionLogEntry.builder()
+                .id(image.getId())
+                .color(getColor(imageType, dossierTemplateId, image.applied(), isHint))
                 .isImage(true)
                 .value(image.value())
                 .type(imageType)
@@ -240,7 +208,6 @@ public class RedactionLogCreatorService {
                 .section(image.getManualOverwrite().getSection().orElse(image.getParent().toString()))
                 .imageHasTransparency(image.isTransparent())
                 .manualChanges(mapManualChanges(image.getManualOverwrite(), isHint))
-                .comments(buildRedactionLogComments(comments, image.getId()))
                 .build();
 
     }
@@ -252,7 +219,7 @@ public class RedactionLogCreatorService {
     }
 
 
-    public RedactionLogEntry createRedactionLogEntry(ManualEntity manualEntity, String dossierTemplateId, Map<String, List<Comment>> comments) {
+    public RedactionLogEntry createRedactionLogEntry(ManualEntity manualEntity, String dossierTemplateId) {
 
         String type = manualEntity.getManualOverwrite().getType().orElse(manualEntity.getType());
         boolean isHint = isHint(manualEntity.getEntityType());
@@ -276,13 +243,16 @@ public class RedactionLogCreatorService {
                 .textAfter("")
                 .textBefore("")
                 .startOffset(-1)
-                .endOffset(-1).positions(manualEntity.getManualOverwrite().getPositions().orElse(manualEntity.getEntityPosition())
+                .endOffset(-1)
+                .positions(manualEntity.getManualOverwrite()
+                        .getPositions()
+                        .orElse(manualEntity.getEntityPosition())
                         .stream()
-                        .map(entityPosition -> toRedactionLogRectangle(entityPosition.rectangle2D(), entityPosition.pageNumber())).toList())
+                        .map(entityPosition -> toRedactionLogRectangle(entityPosition.rectangle2D(), entityPosition.pageNumber()))
+                        .toList())
                 .engines(Collections.emptySet())
                 .reference(Collections.emptySet())
                 .manualChanges(mapManualChanges(manualEntity.getManualOverwrite(), isHint))
-                .comments(buildRedactionLogComments(comments, manualEntity.getId()))
                 .build();
     }
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RedactionLogToEntityLogMigrationService.java

@@ -0,0 +1,187 @@
+package com.iqser.red.service.redaction.v1.server.service;
+
+import java.util.Collection;
+import java.util.Comparator;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import org.springframework.stereotype.Service;
+
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogLegalBasis;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.migration.MigratedIds;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogLegalBasis;
+import com.iqser.red.service.redaction.v1.server.model.ManualEntity;
+import com.iqser.red.service.redaction.v1.server.model.MigratedEntityLog;
+import com.iqser.red.service.redaction.v1.server.model.MigrationEntity;
+import com.iqser.red.service.redaction.v1.server.model.RectangleWithPage;
+import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
+import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.Image;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
+import com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService;
+import com.iqser.red.service.redaction.v1.server.service.document.EntityEnrichmentService;
+import com.iqser.red.service.redaction.v1.server.service.document.EntityFindingUtility;
+import com.iqser.red.service.redaction.v1.server.utils.MigratedIdsCollector;
+
+import lombok.AccessLevel;
+import lombok.RequiredArgsConstructor;
+import lombok.experimental.FieldDefaults;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+@Service
+@RequiredArgsConstructor
+@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
+//TODO: remove this, once the migration is done
+public class RedactionLogToEntityLogMigrationService {
+
+    private static final double MATCH_THRESHOLD = 10;
+    EntityFindingUtility entityFindingUtility;
+    EntityEnrichmentService entityEnrichmentService;
+
+
+    public MigratedEntityLog migrate(RedactionLog redactionLog, Document document) {
+
+        List<MigrationEntity> entitiesToMigrate = calculateMigrationEntitiesFromRedactionLog(redactionLog, document);
+        MigratedIds migratedIds = entitiesToMigrate.stream().collect(new MigratedIdsCollector());
+
+        EntityLog entityLog = new EntityLog();
+        entityLog.setAnalysisNumber(redactionLog.getAnalysisNumber());
+        entityLog.setRulesVersion(redactionLog.getRulesVersion());
+        entityLog.setDictionaryVersion(redactionLog.getDictionaryVersion());
+        entityLog.setDossierDictionaryVersion(redactionLog.getDossierDictionaryVersion());
+        entityLog.setLegalBasisVersion(redactionLog.getLegalBasisVersion());
+        entityLog.setAnalysisVersion(redactionLog.getAnalysisVersion());
+        entityLog.setLegalBasis(redactionLog.getLegalBasis().stream().map(RedactionLogToEntityLogMigrationService::toEntityLogLegalBasis).toList());
+
+        Map<String, String> oldToNewIDMapping = migratedIds.buildOldToNewMapping();
+        entityLog.setEntityLogEntry(entitiesToMigrate.stream().map(migrationEntity -> migrationEntity.toEntityLogEntry(oldToNewIDMapping)).toList());
+
+        if (redactionLog.getRedactionLogEntry().size() != entityLog.getEntityLogEntry().size()) {
+            String message = String.format("Not all entities have been found during the migration redactionLog has %d entries and new entityLog %d",
+                    redactionLog.getRedactionLogEntry().size(),
+                    entityLog.getEntityLogEntry().size());
+            log.error(message);
+            throw new AssertionError(message);
+        }
+
+        return new MigratedEntityLog(migratedIds, entityLog);
+    }
+
+
+    private List<MigrationEntity> calculateMigrationEntitiesFromRedactionLog(RedactionLog redactionLog, Document document) {
+
+        List<MigrationEntity> images = getImageBasedMigrationEntities(redactionLog, document);
+        List<MigrationEntity> textMigrationEntities = getTextBasedMigrationEntities(redactionLog, document);
+        return Stream.of(textMigrationEntities.stream(), images.stream()).flatMap(Function.identity()).toList();
+    }
+
+
+    private static EntityLogLegalBasis toEntityLogLegalBasis(RedactionLogLegalBasis redactionLogLegalBasis) {
+
+        return new EntityLogLegalBasis(redactionLogLegalBasis.getName(), redactionLogLegalBasis.getDescription(), redactionLogLegalBasis.getReason());
+    }
+
+
+    private List<MigrationEntity> getImageBasedMigrationEntities(RedactionLog redactionLog, Document document) {
+
+        List<Image> images = document.streamAllImages().collect(Collectors.toList());
+        List<RedactionLogEntry> redactionLogImages = redactionLog.getRedactionLogEntry().stream().filter(RedactionLogEntry::isImage).toList();
+        List<MigrationEntity> migrationEntities = new LinkedList<>();
+        for (RedactionLogEntry redactionLogImage : redactionLogImages) {
+            List<RectangleWithPage> imagePositions = redactionLogImage.getPositions().stream().map(RectangleWithPage::fromRedactionLogRectangle).toList();
+            assert imagePositions.size() == 1;
+            Image closestImage = images.stream()
+                    .filter(image -> image.onPage(redactionLogImage.getPositions().get(0).getPage()))
+                    .min(Comparator.comparingDouble(image -> entityFindingUtility.calculateDistance(image.getPosition(), imagePositions.get(0).rectangle2D())))
+                    .orElseThrow(() -> new RuntimeException("Image from redaction log not found: " + redactionLogImage));
+
+            double minDistance = entityFindingUtility.calculateDistance(closestImage.getPosition(), imagePositions.get(0).rectangle2D());
+            if (minDistance > MATCH_THRESHOLD) {
+                throw new RuntimeException(String.format("Closest image has a distance of %.2f which is higher than the allowed %.2f", minDistance, MATCH_THRESHOLD));
+            }
+
+            images.remove(closestImage);
+
+            String ruleIdentifier = "OLDIMG." + redactionLogImage.getMatchedRule() + ".0";
+            if (redactionLogImage.lastChangeIsRemoved()) {
+                closestImage.remove(ruleIdentifier, redactionLogImage.getReason());
+            } else if (redactionLogImage.isRedacted()) {
+                closestImage.apply(ruleIdentifier, redactionLogImage.getReason(), redactionLogImage.getLegalBasis());
+            } else {
+                closestImage.skip(ruleIdentifier, redactionLogImage.getReason());
+            }
+            migrationEntities.add(new MigrationEntity(null, redactionLogImage, closestImage, redactionLogImage.getId(), closestImage.getId()));
+        }
+        return migrationEntities;
+    }
+
+
+    private List<MigrationEntity> getTextBasedMigrationEntities(RedactionLog redactionLog, Document document) {
+
+        List<MigrationEntity> entitiesToMigrate = redactionLog.getRedactionLogEntry()
+                .stream()
+                .filter(redactionLogEntry -> !redactionLogEntry.isImage())
+                .map(MigrationEntity::fromRedactionLogEntry)
+                .peek(migrationEntity -> {
+                    if (migrationEntity.getRedactionLogEntry().lastChangeIsRemoved()) {
+                        migrationEntity.getManualEntity().remove(migrationEntity.getManualEntity().getRuleIdentifier(), migrationEntity.getRedactionLogEntry().getReason());
+                    } else if (migrationEntity.getManualEntity().isApplied()) {
+                        migrationEntity.getManualEntity()
+                                .apply(migrationEntity.getManualEntity().getRuleIdentifier(),
+                                        migrationEntity.getManualEntity().getReason(),
+                                        migrationEntity.getManualEntity().getLegalBasis());
+                    } else {
+                        migrationEntity.getManualEntity().skip(migrationEntity.getManualEntity().getRuleIdentifier(), migrationEntity.getManualEntity().getReason());
+                    }
+                })
+                .toList();
+
+        Map<String, List<TextEntity>> tempEntitiesByValue = entityFindingUtility.findAllPossibleEntitiesAndGroupByValue(document,
+                entitiesToMigrate.stream().map(MigrationEntity::getManualEntity).toList());
+
+        for (MigrationEntity migrationEntity : entitiesToMigrate) {
+            Optional<TextEntity> optionalTextEntity = entityFindingUtility.findClosestEntityAndReturnEmptyIfNotFound(migrationEntity.getManualEntity(),
+                    tempEntitiesByValue,
+                    MATCH_THRESHOLD);
+
+            if (optionalTextEntity.isEmpty()) {
+                migrationEntity.setMigratedEntity(migrationEntity.getManualEntity());
+                migrationEntity.setOldId(migrationEntity.getManualEntity().getId());
+                migrationEntity.setNewId(migrationEntity.getManualEntity().getId());
+                continue;
+            }
+
+            TextEntity entity = createCorrectEntity(migrationEntity.getManualEntity(), document, optionalTextEntity.get().getTextRange());
+            migrationEntity.setMigratedEntity(entity);
+            migrationEntity.setOldId(migrationEntity.getManualEntity().getId());
+            migrationEntity.setNewId(entity.getPositionsOnPagePerPage().get(0).getId()); // Can only be on one page, since redactionLogEntries can only be on one page
+        }
+
+        tempEntitiesByValue.values().stream().flatMap(Collection::stream).forEach(TextEntity::removeFromGraph);
+        return entitiesToMigrate;
+    }
+
+
+    private TextEntity createCorrectEntity(ManualEntity manualEntity, SemanticNode node, TextRange closestTextRange) {
+
+        EntityCreationService entityCreationService = new EntityCreationService(entityEnrichmentService);
+        TextEntity correctEntity = entityCreationService.forceByTextRange(closestTextRange, manualEntity.getType(), manualEntity.getEntityType(), node);
+
+        correctEntity.addMatchedRules(manualEntity.getMatchedRuleList());
+        correctEntity.setDictionaryEntry(manualEntity.isDictionaryEntry());
+        correctEntity.setDossierDictionaryEntry(manualEntity.isDossierDictionaryEntry());
+        correctEntity.getManualOverwrite().addChanges(manualEntity.getManualOverwrite().getManualChangeLog());
+        return correctEntity;
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/UnprocessedChangesService.java

@@ -2,12 +2,14 @@ package com.iqser.red.service.redaction.v1.server.service;
 
 import java.util.ArrayList;
 import java.util.Collections;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
 import java.util.stream.Collectors;
 
 import org.springframework.stereotype.Service;
 
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
 import com.iqser.red.service.redaction.v1.model.UnprocessedManualEntity;
@@ -33,6 +35,10 @@ public class UnprocessedChangesService {
 
     public List<UnprocessedManualEntity> addInformationToUnprocessedManualEntities(String fileId, String dossierId, String dossierTemplateId, ManualRedactions manualRedactions) {
 
+        // FIXME, because of the color if you start redaction-service a no processing is performed, a nullpointer will be throw when getting color.
+        // The entire concept needs to be rethought, because it is bullshit.
+        dictionaryService.updateDictionary(dossierTemplateId, dossierId);
+
         Document document = DocumentGraphMapper.toDocumentGraph(redactionStorageService.getDocumentData(dossierId, fileId));
         List<UnprocessedManualEntity> unprocessedManualEntities = new ArrayList<>();
         Set<String> annotationIds = manualRedactions.getEntriesToAdd().stream().map(ManualRedactionEntry::getAnnotationId).collect(Collectors.toSet());
@@ -40,17 +46,29 @@ public class UnprocessedChangesService {
         List<ManualEntity> notFoundManualEntities = manualEntityCreationService.toTextEntity(manualEntitiesConverter(manualRedactions), document);
 
         document.getEntities().forEach(textEntity -> {
-            unprocessedManualEntities.add(UnprocessedManualEntity.builder()
-                    .annotationId(annotationIds.stream().filter(textEntity::matchesAnnotationId).findFirst().orElse(""))
-                    .closestHeadline(textEntity.getDeepestFullyContainingNode().getHeadline().getTextBlock().getSearchText())
-                    .color(getColor(textEntity.getType(), dossierTemplateId, textEntity.applied(), textEntity.getEntityType().equals(EntityType.HINT)))
-                    .startOffset(textEntity.getTextRange().start())
-                    .endOffset(textEntity.getTextRange().end())
-                    .containingNodeId(textEntity.getDeepestFullyContainingNode().getTreeId())
-                    .textBefore(textEntity.getTextBefore())
-                    .textAfter(textEntity.getTextAfter())
-                    .section(textEntity.getManualOverwrite().getSection().orElse(textEntity.getDeepestFullyContainingNode().toString()))
-                    .build());
+            Set<String> processedIds = new HashSet<>();
+            for (var positionsOnPerPage : textEntity.getPositionsOnPagePerPage()) {
+                if (processedIds.contains(positionsOnPerPage.getId())) {
+                    continue;
+                }
+                processedIds.add(positionsOnPerPage.getId());
+                List<Position> positions = positionsOnPerPage.getRectanglePerLine().stream().map(rectangle2D -> new Position(rectangle2D, positionsOnPerPage.getPage().getNumber()))
+                        .toList();
+                unprocessedManualEntities.add(UnprocessedManualEntity.builder()
+                        .annotationId(annotationIds.stream().filter(textEntity::matchesAnnotationId).findFirst().orElse(""))
+                        .closestHeadline(textEntity.getDeepestFullyContainingNode().getHeadline().getTextBlock().getSearchText())
+                        .color(getColor(textEntity.getType(), dossierTemplateId, textEntity.applied(), textEntity.getEntityType().equals(EntityType.HINT)))
+                        .startOffset(textEntity.getTextRange().start())
+                        .endOffset(textEntity.getTextRange().end())
+                        .containingNodeId(textEntity.getDeepestFullyContainingNode().getTreeId())
+                        .textBefore(textEntity.getTextBefore())
+                        .textAfter(textEntity.getTextAfter())
+                        .section(textEntity.getManualOverwrite().getSection().orElse(textEntity.getDeepestFullyContainingNode().toString()))
+                        .positions(positions)
+                        .legalBasis(textEntity.legalBasis())
+                        .build());
+            }
+
         });
 
         notFoundManualEntities.forEach(manualEntity -> {
@@ -64,6 +82,12 @@ public class UnprocessedChangesService {
                     .textAfter("")
                     .textBefore("")
                     .section("")
+                    .positions(manualEntity.getManualOverwrite()
+                            .getPositions()
+                            .orElse(manualEntity.getEntityPosition())
+                            .stream()
+                            .map(entityPosition -> new Position(entityPosition.rectangle2D(), entityPosition.pageNumber()))
+                            .toList())
                     .build());
         });
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/ComponentCreationService.java

@@ -19,6 +19,7 @@ import org.kie.api.runtime.KieSession;
 
 import com.iqser.red.service.redaction.v1.server.model.component.Component;
 import com.iqser.red.service.redaction.v1.server.model.component.Entity;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.Paragraph;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Table;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell;
@@ -102,7 +103,11 @@ public class ComponentCreationService {
 
         referencedEntities.addAll(references);
 
-        kieSession.insert(Component.builder().matchedRule(RuleIdentifier.fromString(ruleIdentifier)).name(name).value(value).valueDescription(valueDescription)
+        kieSession.insert(Component.builder()
+                .matchedRule(RuleIdentifier.fromString(ruleIdentifier))
+                .name(name)
+                .value(value)
+                .valueDescription(valueDescription)
                 .references(new LinkedList<>(references))
                 .build());
     }
@@ -293,6 +298,30 @@ public class ComponentCreationService {
     }
 
 
+    /**
+     * Counts the distinct rows where any of the provided entities occur and creates a component for each distinct table with its respective row count.
+     *
+     * @param ruleIdentifier the identifier of the rule
+     * @param name           the name of the record
+     * @param entities       the collection of entities to count row values from
+     */
+    public void rowValueCount(String ruleIdentifier, String name, Collection<Entity> entities) {
+
+        entities.stream().collect(Collectors.groupingBy(this::getFirstTable)).forEach((optionalTable, groupedEntities) -> {
+
+            if (optionalTable.isEmpty()) {
+                return;
+            }
+
+            long count = groupedEntities.stream()
+                    .collect(Collectors.groupingBy(entity -> getFirstTableCell(entity).map(TableCell::getRow).orElse(-1)))
+                    .size();
+
+            create(ruleIdentifier, name, String.valueOf(count), "Count rows with values in the entity references in same table", entities);
+        });
+    }
+
+
     /**
      * Creates a component for each sentence in the collection of entities.
      *
@@ -354,7 +383,9 @@ public class ComponentCreationService {
      */
     public void createComponentsForUnMappedEntities(String ruleIdentifier, Collection<Entity> entities) {
 
-        entities.stream().filter(entity -> !referencedEntities.contains(entity)).sorted(EntityComparators.first())
+        entities.stream()
+                .filter(entity -> !referencedEntities.contains(entity))
+                .sorted(EntityComparators.first())
                 .forEach(entity -> create(ruleIdentifier, entity.getType(), entity.getValue(), "Unmapped Entity", List.of(entity)));
     }
 
@@ -421,10 +452,6 @@ public class ComponentCreationService {
                 groupedEntities.forEach(entity -> create(ruleIdentifier, name, entity.getValue(), valueDescription, entity));
             }
 
-            groupedEntities.stream()
-                    .filter(entity -> !(entity.getContainingNode() instanceof TableCell))
-                    .forEach(entity -> create(ruleIdentifier, name, entity.getValue(), valueDescription, entity));
-
             groupedEntities.stream()
                     .filter(entity -> entity.getContainingNode() instanceof TableCell)
                     .collect(Collectors.groupingBy(entity -> ((TableCell) entity.getContainingNode()).getRow()))
@@ -434,7 +461,7 @@ public class ComponentCreationService {
                     .map(Map.Entry::getValue)
                     .forEach(entitiesInSameRow -> create(ruleIdentifier,
                             name,
-                            entitiesInSameRow.stream().sorted(Comparator.comparing(Entity::getType).reversed()).map(Entity::getValue).collect(Collectors.joining(", ")),
+                            entitiesInSameRow.stream().sorted(EntityComparators.first()).map(Entity::getValue).collect(Collectors.joining(", ")),
                             valueDescription,
                             entitiesInSameRow));
         });
@@ -455,6 +482,20 @@ public class ComponentCreationService {
     }
 
 
+    private Optional<TableCell> getFirstTableCell(Entity entity) {
+
+        SemanticNode node = entity.getContainingNode();
+        while (!(node instanceof TableCell)) {
+            if (!node.hasParent()) {
+                return Optional.empty();
+            }
+            node = node.getParent();
+        }
+
+        return Optional.of((TableCell) node);
+    }
+
+
     /**
      * Creates a new component with the given rule identifier, name, value, and value description.
      * If the component is part of a table, it also takes a list of entities that belong to the same table row.
@@ -479,7 +520,9 @@ public class ComponentCreationService {
      */
     public void create(String ruleIdentifier, String name, String value) {
 
-        kieSession.insert(Component.builder().matchedRule(RuleIdentifier.fromString(ruleIdentifier)).name(name)
+        kieSession.insert(Component.builder()
+                .matchedRule(RuleIdentifier.fromString(ruleIdentifier))
+                .name(name)
                 .value(value)
                 .valueDescription("")
                 .references(Collections.emptyList())

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityCreationService.java

@@ -58,6 +58,8 @@ public class EntityCreationService {
 
     public Stream<TextEntity> betweenStrings(String start, String stop, String type, EntityType entityType, SemanticNode node) {
 
+        checkIfBothStartAndEndAreEmpty(start, stop);
+
         List<TextRange> startTextRanges = RedactionSearchUtility.findTextRangesByString(start, node.getTextBlock());
         List<TextRange> stopTextRanges = RedactionSearchUtility.findTextRangesByString(stop, node.getTextBlock());
 
@@ -67,6 +69,8 @@ public class EntityCreationService {
 
     public Stream<TextEntity> betweenStringsIgnoreCase(String start, String stop, String type, EntityType entityType, SemanticNode node) {
 
+        checkIfBothStartAndEndAreEmpty(start, stop);
+
         List<TextRange> startBoundaries = RedactionSearchUtility.findTextRangesByStringIgnoreCase(start, node.getTextBlock());
         List<TextRange> stopBoundaries = RedactionSearchUtility.findTextRangesByStringIgnoreCase(stop, node.getTextBlock());
 
@@ -76,6 +80,8 @@ public class EntityCreationService {
 
     public Stream<TextEntity> betweenStringsIncludeStart(String start, String stop, String type, EntityType entityType, SemanticNode node) {
 
+        checkIfBothStartAndEndAreEmpty(start, stop);
+
         List<TextRange> startBoundaries = RedactionSearchUtility.findTextRangesByString(start, node.getTextBlock());
         List<TextRange> stopBoundaries = RedactionSearchUtility.findTextRangesByString(stop, node.getTextBlock());
 
@@ -90,6 +96,8 @@ public class EntityCreationService {
 
     public Stream<TextEntity> betweenStringsIncludeStartIgnoreCase(String start, String stop, String type, EntityType entityType, SemanticNode node) {
 
+        checkIfBothStartAndEndAreEmpty(start, stop);
+
         List<TextRange> startBoundaries = RedactionSearchUtility.findTextRangesByStringIgnoreCase(start, node.getTextBlock());
         List<TextRange> stopBoundaries = RedactionSearchUtility.findTextRangesByStringIgnoreCase(stop, node.getTextBlock());
 
@@ -104,6 +112,8 @@ public class EntityCreationService {
 
     public Stream<TextEntity> betweenStringsIncludeEnd(String start, String stop, String type, EntityType entityType, SemanticNode node) {
 
+        checkIfBothStartAndEndAreEmpty(start, stop);
+
         List<TextRange> startBoundaries = RedactionSearchUtility.findTextRangesByString(start, node.getTextBlock());
         List<TextRange> stopBoundaries = RedactionSearchUtility.findTextRangesByString(stop, node.getTextBlock());
 
@@ -118,6 +128,8 @@ public class EntityCreationService {
 
     public Stream<TextEntity> betweenStringsIncludeEndIgnoreCase(String start, String stop, String type, EntityType entityType, SemanticNode node) {
 
+        checkIfBothStartAndEndAreEmpty(start, stop);
+
         List<TextRange> startBoundaries = RedactionSearchUtility.findTextRangesByStringIgnoreCase(start, node.getTextBlock());
         List<TextRange> stopBoundaries = RedactionSearchUtility.findTextRangesByStringIgnoreCase(stop, node.getTextBlock());
 
@@ -132,6 +144,8 @@ public class EntityCreationService {
 
     public Stream<TextEntity> betweenStringsIncludeStartAndEnd(String start, String stop, String type, EntityType entityType, SemanticNode node) {
 
+        checkIfBothStartAndEndAreEmpty(start, stop);
+
         List<TextRange> startBoundaries = RedactionSearchUtility.findTextRangesByString(start, node.getTextBlock());
         List<TextRange> stopBoundaries = RedactionSearchUtility.findTextRangesByString(stop, node.getTextBlock());
 
@@ -150,6 +164,8 @@ public class EntityCreationService {
 
     public Stream<TextEntity> betweenStringsIncludeStartAndEndIgnoreCase(String start, String stop, String type, EntityType entityType, SemanticNode node) {
 
+        checkIfBothStartAndEndAreEmpty(start, stop);
+
         List<TextRange> startBoundaries = RedactionSearchUtility.findTextRangesByStringIgnoreCase(start, node.getTextBlock());
         List<TextRange> stopBoundaries = RedactionSearchUtility.findTextRangesByStringIgnoreCase(stop, node.getTextBlock());
 
@@ -168,6 +184,8 @@ public class EntityCreationService {
 
     public Stream<TextEntity> shortestBetweenAnyString(List<String> starts, List<String> stops, String type, EntityType entityType, SemanticNode node) {
 
+        checkIfBothStartAndEndAreEmpty(starts, stops);
+
         List<TextRange> startTextRanges = RedactionSearchUtility.findTextRangesByList(starts, node.getTextBlock());
         List<TextRange> stopTextRanges = RedactionSearchUtility.findTextRangesByList(stops, node.getTextBlock());
 
@@ -177,6 +195,8 @@ public class EntityCreationService {
 
     public Stream<TextEntity> shortestBetweenAnyStringIgnoreCase(List<String> starts, List<String> stops, String type, EntityType entityType, SemanticNode node) {
 
+        checkIfBothStartAndEndAreEmpty(starts, stops);
+
         List<TextRange> startTextRanges = RedactionSearchUtility.findTextRangesByListIgnoreCase(starts, node.getTextBlock());
         List<TextRange> stopTextRanges = RedactionSearchUtility.findTextRangesByListIgnoreCase(stops, node.getTextBlock());
 
@@ -185,6 +205,8 @@ public class EntityCreationService {
 
     public Stream<TextEntity> shortestBetweenAnyStringIgnoreCase(List<String> starts, List<String> stops, String type, EntityType entityType, SemanticNode node, int limit) {
 
+        checkIfBothStartAndEndAreEmpty(starts, stops);
+
         List<TextRange> startTextRanges = RedactionSearchUtility.findTextRangesByListIgnoreCase(starts, node.getTextBlock());
         List<TextRange> stopTextRanges = RedactionSearchUtility.findTextRangesByListIgnoreCase(stops, node.getTextBlock());
 
@@ -219,9 +241,6 @@ public class EntityCreationService {
 
     public Stream<TextEntity> betweenTextRanges(List<TextRange> startBoundaries, List<TextRange> stopBoundaries, String type, EntityType entityType, SemanticNode node, int limit) {
 
-        if (startBoundaries.isEmpty() || stopBoundaries.isEmpty()) {
-            return Stream.empty();
-        }
         List<TextRange> entityBoundaries = findNonOverlappingBoundariesBetweenBoundariesWithMinimalDistances(startBoundaries, stopBoundaries);
         return entityBoundaries.stream()
                 .filter(range -> (limit == 0 || range.length() <= limit))
@@ -263,6 +282,18 @@ public class EntityCreationService {
     }
 
 
+    private void checkIfBothStartAndEndAreEmpty(String start, String end) {
+        checkIfBothStartAndEndAreEmpty(List.of(start), List.of(end));
+    }
+
+
+    private <T> void checkIfBothStartAndEndAreEmpty(List<T> start, List<T> end) {
+        if ((start == null || start.isEmpty()) && (end == null || end.isEmpty())) {
+            throw new IllegalArgumentException("Start and end values are empty!");
+        }
+    }
+
+
     public Stream<TextEntity> bySearchImplementation(SearchImplementation searchImplementation, String type, EntityType entityType, SemanticNode node) {
 
         return searchImplementation.getBoundaries(node.getTextBlock(), node.getTextRange())

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityFindingUtility.java

@@ -0,0 +1,164 @@
+package com.iqser.red.service.redaction.v1.server.service.document;
+
+import static java.lang.String.format;
+import static java.util.stream.Collectors.groupingBy;
+
+import java.awt.geom.Rectangle2D;
+import java.util.Collection;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+
+import com.iqser.red.service.redaction.v1.server.model.ManualEntity;
+import com.iqser.red.service.redaction.v1.server.model.RectangleWithPage;
+import com.iqser.red.service.redaction.v1.server.model.dictionary.SearchImplementation;
+import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
+import com.iqser.red.service.redaction.v1.server.model.document.entity.PositionOnPage;
+import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.Page;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
+import com.iqser.red.service.redaction.v1.server.service.DictionaryService;
+
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+@Service
+public class EntityFindingUtility {
+
+    EntityCreationService entityCreationService;
+    DictionaryService dictionaryService;
+
+
+    @Autowired
+    public EntityFindingUtility(EntityEnrichmentService entityEnrichmentService, DictionaryService dictionaryService) {
+
+        entityCreationService = new EntityCreationService(entityEnrichmentService);
+        this.dictionaryService = dictionaryService;
+    }
+
+
+    public Optional<TextEntity> findClosestEntityAndReturnEmptyIfNotFound(ManualEntity identifier, Map<String, List<TextEntity>> entitiesWithSameValue, double matchThreshold) {
+
+        List<TextEntity> possibleEntities = entitiesWithSameValue.get(identifier.getValue().toLowerCase(Locale.ENGLISH));
+
+        if (entityIdentifierValueNotFound(possibleEntities)) {
+            log.warn("Entity could not be created with identifier: {}, due to the value {} not being found anywhere.", identifier, identifier.getValue());
+            return Optional.empty();
+        }
+
+        Optional<TextEntity> optionalClosestEntity = possibleEntities.stream()
+                .filter(entity -> pagesMatch(entity, identifier.getEntityPosition()))
+                .min(Comparator.comparingDouble(entity -> calculateMinDistance(identifier.getEntityPosition(), entity)));
+
+        if (optionalClosestEntity.isEmpty()) {
+            log.warn("No Entity with value {} found on page {}", identifier.getValue(), identifier.getEntityPosition());
+            return Optional.empty();
+        }
+
+        TextEntity closestEntity = optionalClosestEntity.get();
+        double distance = calculateMinDistance(identifier.getEntityPosition(), closestEntity);
+        if (distance > matchThreshold) {
+            log.warn("For entity {} on page {} with positions {} distance to closest found entity is {} and therefore higher than the threshold of {}",
+                    identifier.getValue(),
+                    identifier.getEntityPosition().get(0).pageNumber(),
+                    identifier.getEntityPosition().stream().map(RectangleWithPage::rectangle2D).toList(),
+                    distance,
+                    matchThreshold);
+            return Optional.empty();
+        }
+
+        return Optional.of(closestEntity);
+    }
+
+
+    private static boolean entityIdentifierValueNotFound(List<TextEntity> possibleEntities) {
+
+        return possibleEntities == null || possibleEntities.isEmpty();
+    }
+
+
+    private static boolean pagesMatch(TextEntity entity, List<RectangleWithPage> originalPositions) {
+
+        Set<Integer> entityPageNumbers = entity.getPositionsOnPagePerPage().stream().map(PositionOnPage::getPage).map(Page::getNumber).collect(Collectors.toSet());
+        Set<Integer> originalPageNumbers = originalPositions.stream().map(RectangleWithPage::pageNumber).collect(Collectors.toSet());
+        return entityPageNumbers.containsAll(originalPageNumbers);
+    }
+
+
+    private double calculateMinDistance(List<RectangleWithPage> originalPositions, TextEntity entity) {
+
+        if (originalPositions.size() != countRectangles(entity)) {
+            return Double.MAX_VALUE;
+        }
+        return originalPositions.stream()
+                .mapToDouble(rectangleWithPage -> calculateMinDistancePerRectangle(entity, rectangleWithPage.pageNumber(), rectangleWithPage.rectangle2D()))
+                .sum();
+    }
+
+
+    private static long countRectangles(TextEntity entity) {
+
+        return entity.getPositionsOnPagePerPage().stream().mapToLong(redactionPosition -> redactionPosition.getRectanglePerLine().size()).sum();
+    }
+
+
+    private double calculateMinDistancePerRectangle(TextEntity entity, int pageNumber, Rectangle2D originalRectangle) {
+
+        return entity.getPositionsOnPagePerPage()
+                .stream()
+                .filter(redactionPosition -> redactionPosition.getPage().getNumber() == pageNumber)
+                .map(PositionOnPage::getRectanglePerLine)
+                .flatMap(Collection::stream)
+                .mapToDouble(rectangle -> calculateDistance(rectangle, originalRectangle))
+                .min()
+                .orElse(Double.MAX_VALUE);
+    }
+
+
+    public double calculateDistance(Rectangle2D rectangle1, Rectangle2D rectangle2) {
+
+        // mirrored coordinates safe comparison
+        double minX1 = Math.min(rectangle1.getMinX(), rectangle1.getMaxX());
+        double minY1 = Math.min(rectangle1.getMinY(), rectangle1.getMaxY());
+        double maxX1 = Math.max(rectangle1.getMinX(), rectangle1.getMaxX());
+        double maxY1 = Math.max(rectangle1.getMinY(), rectangle1.getMaxY());
+
+        double minX2 = Math.min(rectangle2.getMinX(), rectangle2.getMaxX());
+        double minY2 = Math.min(rectangle2.getMinY(), rectangle2.getMaxY());
+        double maxX2 = Math.max(rectangle2.getMinX(), rectangle2.getMaxX());
+        double maxY2 = Math.max(rectangle2.getMinY(), rectangle2.getMaxY());
+
+        return Math.abs(minX1 - minX2) //
+                + Math.abs(minY1 - minY2) //
+                + Math.abs(maxX1 - maxX2) //
+                + Math.abs(maxY1 - maxY2);
+    }
+
+
+    public Map<String, List<TextEntity>> findAllPossibleEntitiesAndGroupByValue(SemanticNode node, List<ManualEntity> manualEntities) {
+
+        Set<Integer> pageNumbers = manualEntities.stream().flatMap(entry -> entry.getEntityPosition().stream().map(RectangleWithPage::pageNumber)).collect(Collectors.toSet());
+        Set<String> entryValues = manualEntities.stream().map(ManualEntity::getValue).map(String::toLowerCase).collect(Collectors.toSet());
+
+        if (!pageNumbers.stream().allMatch(node::onPage)) {
+            throw new IllegalArgumentException(format("SemanticNode \"%s\" does not contain these pages %s, it has pages: %s",
+                    node,
+                    pageNumbers.stream().filter(pageNumber -> !node.onPage(pageNumber)).toList(),
+                    node.getPages()));
+        }
+        SearchImplementation searchImplementation = new SearchImplementation(entryValues, true);
+
+        return searchImplementation.getBoundaries(node.getTextBlock(), node.getTextRange())
+                .stream()
+                .map(boundary -> entityCreationService.forceByTextRange(boundary, "temp", EntityType.ENTITY, node))
+                .collect(groupingBy(entity -> entity.getValue().toLowerCase(Locale.ROOT)));
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/ManualEntityCreationService.java

@@ -1,33 +1,22 @@
 package com.iqser.red.service.redaction.v1.server.service.document;
 
-import static java.lang.String.format;
-import static java.util.stream.Collectors.groupingBy;
-
-import java.awt.geom.Rectangle2D;
 import java.util.ArrayList;
 import java.util.Collection;
-import java.util.Comparator;
 import java.util.LinkedList;
 import java.util.List;
-import java.util.Locale;
 import java.util.Map;
 import java.util.Optional;
 import java.util.Set;
-import java.util.stream.Collectors;
 
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
 
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog;
 import com.iqser.red.service.redaction.v1.server.model.ManualEntity;
-import com.iqser.red.service.redaction.v1.server.model.RectangleWithPage;
-import com.iqser.red.service.redaction.v1.server.model.dictionary.SearchImplementation;
 import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.PositionOnPage;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
-import com.iqser.red.service.redaction.v1.server.model.document.nodes.Page;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
 import com.iqser.red.service.redaction.v1.server.service.DictionaryService;
 
@@ -41,41 +30,47 @@ import lombok.extern.slf4j.Slf4j;
 public class ManualEntityCreationService {
 
     static double MATCH_THRESHOLD = 5; // Is compared to the sum of distances in pdf coordinates for each corner of the bounding box of the entities
+    EntityFindingUtility entityFindingUtility;
     EntityCreationService entityCreationService;
     DictionaryService dictionaryService;
 
 
     @Autowired
-    public ManualEntityCreationService(EntityEnrichmentService entityEnrichmentService, DictionaryService dictionaryService) {
+    public ManualEntityCreationService(EntityEnrichmentService entityEnrichmentService, DictionaryService dictionaryService, EntityFindingUtility entityFindingUtility) {
 
+        this.entityFindingUtility = entityFindingUtility;
         entityCreationService = new EntityCreationService(entityEnrichmentService);
         this.dictionaryService = dictionaryService;
     }
 
 
-    public List<ManualEntity> toTextEntity(RedactionLog redactionLog, SemanticNode node) {
+    public List<ManualEntity> createRedactionEntitiesIfFoundAndReturnNotFoundEntries(Set<ManualRedactionEntry> manualRedactionEntries,
+                                                                                     SemanticNode node,
+                                                                                     String dossierTemplateId) {
+
+        List<ManualEntity> manualEntities = manualRedactionEntries.stream()
+                .filter(manualRedactionEntry -> !(manualRedactionEntry.isAddToDictionary() || manualRedactionEntry.isAddToDossierDictionary()))
+                .map(manualRedactionEntry -> ManualEntity.fromManualRedactionEntry(manualRedactionEntry,
+                        dictionaryService.isHint(manualRedactionEntry.getType(), dossierTemplateId))).peek(manualEntity -> {
+                    if (manualEntity.getEntityType().equals(EntityType.HINT)) {
+                        manualEntity.skip("MAN.5.1", "manual hint is skipped by default");
+                    } else {
+                        manualEntity.apply("MAN.5.0", "manual entries are applied by default", manualEntity.getLegalBasis());
+                    }
+                })
+                .toList();
 
-        List<ManualEntity> manualEntities = redactionLog.getRedactionLogEntry().stream().map(ManualEntity::fromRedactionLogEntry).peek(manualEntity -> {
-            if (manualEntity.isApplied()) {
-                manualEntity.apply(manualEntity.getRuleIdentifier(), manualEntity.getReason(), manualEntity.getLegalBasis());
-            } else {
-                manualEntity.skip(manualEntity.getRuleIdentifier(), manualEntity.getReason());
-            }
-        }).toList();
         return toTextEntity(manualEntities, node);
     }
 
 
     public List<ManualEntity> toTextEntity(List<ManualEntity> manualEntities, SemanticNode node) {
 
-        Set<Integer> pageNumbers = manualEntities.stream().flatMap(entry -> entry.getEntityPosition().stream().map(RectangleWithPage::pageNumber)).collect(Collectors.toSet());
-        Set<String> entryValues = manualEntities.stream().map(ManualEntity::getValue).map(String::toLowerCase).collect(Collectors.toSet());
-
-        Map<String, List<TextEntity>> tempEntitiesByValue = findAllPossibleEntitiesAndGroupByValue(node, pageNumbers, entryValues);
+        Map<String, List<TextEntity>> tempEntitiesByValue = entityFindingUtility.findAllPossibleEntitiesAndGroupByValue(node, manualEntities);
 
         List<ManualEntity> notFoundManualEntities = new LinkedList<>();
         for (ManualEntity manualEntity : manualEntities) {
-            Optional<TextEntity> optionalRedactionEntity = findClosestEntityAndReturnEmptyIfNotFound(manualEntity, tempEntitiesByValue);
+            Optional<TextEntity> optionalRedactionEntity = entityFindingUtility.findClosestEntityAndReturnEmptyIfNotFound(manualEntity, tempEntitiesByValue, MATCH_THRESHOLD);
             if (optionalRedactionEntity.isEmpty()) {
                 notFoundManualEntities.add(manualEntity);
                 continue;
@@ -87,27 +82,12 @@ public class ManualEntityCreationService {
     }
 
 
-    public List<ManualEntity> createRedactionEntitiesIfFoundAndReturnNotFoundEntries(Set<ManualRedactionEntry> manualRedactionEntries,
-                                                                                     SemanticNode node,
-                                                                                     String dossierTemplateId) {
-
-        List<ManualEntity> manualEntities = manualRedactionEntries.stream()
-                .filter(manualRedactionEntry -> !(manualRedactionEntry.isAddToDictionary() || manualRedactionEntry.isAddToDossierDictionary()))
-                .map(manualRedactionEntry -> ManualEntity.fromManualRedactionEntry(manualRedactionEntry,
-                        dictionaryService.isHint(manualRedactionEntry.getType(), dossierTemplateId)))
-                .peek(manualEntity -> manualEntity.apply("MAN.5.0", "manual entries are applied by default", manualEntity.getLegalBasis()))
-                .toList();
-
-        return toTextEntity(manualEntities, node);
-    }
-
-
     /**
      * Deletes the temp Entity and creates a RedactionEntity with correct values, based on the given parameters.
      *
-     * @param manualEntity The entity identifier for the RedactionEntity.
+     * @param manualEntity     The entity identifier for the RedactionEntity.
      * @param node             The SemanticNode associated with the RedactionEntity.
-     * @param closestTextRange  The closest Boundary to the RedactionEntity.
+     * @param closestTextRange The closest Boundary to the RedactionEntity.
      */
     private void createCorrectEntity(ManualEntity manualEntity, SemanticNode node, TextRange closestTextRange) {
 
@@ -118,7 +98,6 @@ public class ManualEntityCreationService {
         correctEntity.setDossierDictionaryEntry(manualEntity.isDossierDictionaryEntry());
         correctEntity.getManualOverwrite().addChanges(manualEntity.getManualOverwrite().getManualChangeLog());
 
-
         // AnnotationIds must match the IDs in the add requests, or comments break. Maybe think about migrating IDs on the fly!
         List<PositionOnPage> redactionPositionsWithIdOfManualOnPage = new ArrayList<>(correctEntity.getPositionsOnPagePerPage().size());
         for (PositionOnPage positionOnPage : correctEntity.getPositionsOnPagePerPage()) {
@@ -127,123 +106,4 @@ public class ManualEntityCreationService {
         correctEntity.setPositionsOnPagePerPage(redactionPositionsWithIdOfManualOnPage);
     }
 
-
-    private Optional<TextEntity> findClosestEntityAndReturnEmptyIfNotFound(ManualEntity identifier, Map<String, List<TextEntity>> entitiesWithSameValue) {
-
-        List<TextEntity> possibleEntities = entitiesWithSameValue.get(identifier.getValue().toLowerCase(Locale.ENGLISH));
-
-        if (entityIdentifierValueNotFound(possibleEntities)) {
-            log.warn("Entity could not be created with identifier: {}, due to the value {} not being found anywhere.", identifier, identifier.getValue());
-            return Optional.empty();
-        }
-
-        Optional<TextEntity> optionalClosestEntity = possibleEntities.stream()
-                .filter(entity -> pagesMatch(entity, identifier.getEntityPosition()))
-                .min(Comparator.comparingDouble(entity -> calculateMinDistance(identifier.getEntityPosition(), entity)));
-
-        if (optionalClosestEntity.isEmpty()) {
-            log.warn("No Entity with value {} found on page {}", identifier.getValue(), identifier.getEntityPosition());
-            return Optional.empty();
-        }
-
-        TextEntity closestEntity = optionalClosestEntity.get();
-        double distance = calculateMinDistance(identifier.getEntityPosition(), closestEntity);
-        if (distance > MATCH_THRESHOLD) {
-            log.warn(format("Distance to closest found entity is %.2f and therefore higher than the threshold of %.2f for \n%s \n%s",
-                    distance,
-                    MATCH_THRESHOLD,
-                    identifier.getEntityPosition(),
-                    closestEntity.getPositionsOnPagePerPage()));
-            return Optional.empty();
-        }
-
-        return Optional.of(closestEntity);
-    }
-
-
-    private static boolean entityIdentifierValueNotFound(List<TextEntity> possibleEntities) {
-
-        return possibleEntities == null || possibleEntities.isEmpty();
-    }
-
-
-    private Map<String, List<TextEntity>> findAllPossibleEntitiesAndGroupByValue(SemanticNode node, Set<Integer> pageNumbers, Set<String> entryValues) {
-
-        if (!pageNumbers.stream().allMatch(node::onPage)) {
-            throw new IllegalArgumentException(format("SemanticNode \"%s\" does not contain these pages %s, it has pages: %s",
-                    node,
-                    pageNumbers.stream().filter(pageNumber -> !node.onPage(pageNumber)).toList(),
-                    node.getPages()));
-        }
-        SearchImplementation searchImplementation = new SearchImplementation(entryValues, true);
-
-        return searchImplementation.getBoundaries(node.getTextBlock(), node.getTextRange())
-                .stream().map(boundary -> entityCreationService.forceByTextRange(boundary, "temp", EntityType.ENTITY, node))
-                .collect(groupingBy(entity -> entity.getValue().toLowerCase(Locale.ROOT)));
-    }
-
-
-    private static boolean allValuesFound(Map<String, List<TextEntity>> entitiesByValue, Set<String> entryValues) {
-
-        return entitiesByValue.keySet().equals(entryValues);
-    }
-
-
-    private static boolean pagesMatch(TextEntity entity, List<RectangleWithPage> originalPositions) {
-
-        Set<Integer> entityPageNumbers = entity.getPositionsOnPagePerPage().stream().map(PositionOnPage::getPage).map(Page::getNumber).collect(Collectors.toSet());
-        Set<Integer> originalPageNumbers = originalPositions.stream().map(RectangleWithPage::pageNumber).collect(Collectors.toSet());
-        return entityPageNumbers.containsAll(originalPageNumbers);
-    }
-
-
-    private double calculateMinDistance(List<RectangleWithPage> originalPositions, TextEntity entity) {
-
-        if (originalPositions.size() != countRectangles(entity)) {
-            return Double.MAX_VALUE;
-        }
-        return originalPositions.stream()
-                .mapToDouble(rectangleWithPage -> calculateMinDistancePerRectangle(entity, rectangleWithPage.pageNumber(), rectangleWithPage.rectangle2D()))
-                .sum();
-    }
-
-
-    private static long countRectangles(TextEntity entity) {
-
-        return entity.getPositionsOnPagePerPage().stream().mapToLong(redactionPosition -> redactionPosition.getRectanglePerLine().size()).sum();
-    }
-
-
-    private double calculateMinDistancePerRectangle(TextEntity entity, int pageNumber, Rectangle2D originalRectangle) {
-
-        return entity.getPositionsOnPagePerPage()
-                .stream()
-                .filter(redactionPosition -> redactionPosition.getPage().getNumber() == pageNumber)
-                .map(PositionOnPage::getRectanglePerLine)
-                .flatMap(Collection::stream)
-                .mapToDouble(rectangle -> calculateDistance(rectangle, originalRectangle))
-                .min()
-                .orElse(Double.MAX_VALUE);
-    }
-
-
-    private double calculateDistance(Rectangle2D rectangle, Rectangle2D rectangle2D) {
-
-        // mirrored coordinates safe comparison
-        double minX1 = Math.min(rectangle.getMinX(), rectangle.getMaxX());
-        double minY1 = Math.min(rectangle.getMinY(), rectangle.getMaxY());
-        double maxX1 = Math.max(rectangle.getMinX(), rectangle.getMaxX());
-        double maxY1 = Math.max(rectangle.getMinY(), rectangle.getMaxY());
-
-        double minX2 = Math.min(rectangle2D.getMinX(), rectangle2D.getMaxX());
-        double minY2 = Math.min(rectangle2D.getMinY(), rectangle2D.getMaxY());
-        double maxX2 = Math.max(rectangle2D.getMinX(), rectangle2D.getMaxX());
-        double maxY2 = Math.max(rectangle2D.getMinY(), rectangle2D.getMaxY());
-
-        return Math.abs(minX1 - minX2) //
-                + Math.abs(minY1 - minY2) //
-                + Math.abs(maxX1 - maxX2) //
-                + Math.abs(maxY1 - maxY2);
-    }
-
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/DroolsSyntaxValidationService.java

@@ -1,18 +1,21 @@
 package com.iqser.red.service.redaction.v1.server.service.drools;
 
 import java.util.List;
+import java.util.Set;
+import java.util.regex.Pattern;
 import java.util.stream.Collectors;
 
+import org.drools.drl.parser.DroolsParserException;
 import org.kie.api.builder.KieBuilder;
 import org.kie.api.builder.Message;
 import org.springframework.stereotype.Service;
 
+import com.google.common.collect.Sets;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType;
 import com.iqser.red.service.redaction.v1.model.DroolsSyntaxErrorMessage;
 import com.iqser.red.service.redaction.v1.model.DroolsSyntaxValidation;
 import com.iqser.red.service.redaction.v1.model.RuleValidationModel;
 import com.iqser.red.service.redaction.v1.server.model.drools.BasicQuery;
-import com.iqser.red.service.redaction.v1.server.model.drools.BasicRule;
 import com.iqser.red.service.redaction.v1.server.model.drools.RuleFileBluePrint;
 import com.iqser.red.service.redaction.v1.server.storage.RuleManagementResources;
 
@@ -22,6 +25,8 @@ import lombok.SneakyThrows;
 @Service
 @RequiredArgsConstructor
 public class DroolsSyntaxValidationService {
+    
+    private static final Pattern allowedImportsPattern = Pattern.compile("^(?:import\\s+static\\s+)?(?:import\\s+)?(?:com\\.knecon\\.fforesight|com\\.iqser\\.red)\\..*;$");
 
     private final KieContainerCreationService kieContainerCreationService;
 
@@ -29,14 +34,20 @@ public class DroolsSyntaxValidationService {
     @SneakyThrows
     public DroolsSyntaxValidation testRules(RuleValidationModel rules) {
 
-        DroolsSyntaxValidation customDroolsSyntaxValidation = buildCustomDroolsSyntaxValidation(rules.getRulesString(), RuleFileType.valueOf(rules.getRuleFileType()));
+        DroolsSyntaxValidation customDroolsSyntaxValidation;
+        try {
+            customDroolsSyntaxValidation = buildCustomDroolsSyntaxValidation(rules.getRulesString(), RuleFileType.valueOf(rules.getRuleFileType()));
+        } catch (DroolsParserException e) {
+            // this means the parser could not parse the file at all. In this case use drools compiler only as it will return useful error messages.
+            customDroolsSyntaxValidation = new DroolsSyntaxValidation();
+        }
         DroolsSyntaxValidation droolsCompilerSyntaxValidation = buildDroolsCompilerSyntaxValidation(rules);
         droolsCompilerSyntaxValidation.getDroolsSyntaxErrorMessages().addAll(customDroolsSyntaxValidation.getDroolsSyntaxErrorMessages());
         return droolsCompilerSyntaxValidation;
     }
 
 
-    private DroolsSyntaxValidation buildCustomDroolsSyntaxValidation(String ruleString, RuleFileType ruleFileType) {
+    private DroolsSyntaxValidation buildCustomDroolsSyntaxValidation(String ruleString, RuleFileType ruleFileType) throws DroolsParserException {
 
         RuleFileBluePrint ruleFileBluePrint = RuleFileParser.buildBluePrintFromRulesString(ruleString);
         DroolsSyntaxValidation customSyntaxValidation = ruleFileBluePrint.getDroolsSyntaxValidation();
@@ -46,7 +57,7 @@ public class DroolsSyntaxValidationService {
             case COMPONENT -> RuleFileParser.buildBluePrintFromRulesString(RuleManagementResources.getBaseComponentRuleFileString());
         };
 
-        if (!ruleFileBluePrint.getImports().equals(baseRuleFileBluePrint.getImports())) {
+        if (!importsAreValid(baseRuleFileBluePrint, ruleFileBluePrint)) {
             customSyntaxValidation.getDroolsSyntaxErrorMessages()
                     .add(DroolsSyntaxErrorMessage.builder()
                             .line(ruleFileBluePrint.getImportLine())
@@ -56,8 +67,10 @@ public class DroolsSyntaxValidationService {
         }
         if (!ruleFileBluePrint.getGlobals().equals(baseRuleFileBluePrint.getGlobals())) {
             customSyntaxValidation.getDroolsSyntaxErrorMessages()
-                    .add(DroolsSyntaxErrorMessage.builder().line(ruleFileBluePrint.getGlobalsLine())
-                            .column(0).message(String.format("Changing the globals is not allowed! Must be: %n%s", baseRuleFileBluePrint.getGlobals()))
+                    .add(DroolsSyntaxErrorMessage.builder()
+                            .line(ruleFileBluePrint.getGlobalsLine())
+                            .column(0)
+                            .message(String.format("Changing the globals is not allowed! Must be: %n%s", baseRuleFileBluePrint.getGlobals()))
                             .build());
         }
         baseRuleFileBluePrint.getQueries().forEach(basicQuery -> {
@@ -70,23 +83,33 @@ public class DroolsSyntaxValidationService {
                                 .build());
             }
         });
-        baseRuleFileBluePrint.streamAllRules().forEach(basicRule -> {
-            if (!validateRuleIsPresent(basicRule, ruleFileBluePrint)) {
-                customSyntaxValidation.getDroolsSyntaxErrorMessages()
-                        .add(DroolsSyntaxErrorMessage.builder()
-                                .line(basicRule.getLine())
-                                .column(0)
-                                .message(String.format("Changing or removing the rule %s is not allowed! Must be: %n%s", basicRule.getName(), basicRule.getCode()))
+        if (ruleFileType.equals(RuleFileType.ENTITY)) {
+            String requiredAgendaGroup = "LOCAL_DICTIONARY_ADDS";
+            if (!validateAgendaGroupIsPresent(ruleFileBluePrint, requiredAgendaGroup)) {
+                customSyntaxValidation.getDroolsSyntaxErrorMessages().add(DroolsSyntaxErrorMessage.builder().line(0)
+                                .column(0).message(String.format("At least one rule with Agenda-Group '%s' required!", requiredAgendaGroup))
                                 .build());
             }
-        });
+        }
         return customSyntaxValidation;
     }
 
 
-    private static boolean validateRuleIsPresent(BasicRule basicRule, RuleFileBluePrint ruleFileBluePrint) {
+    private boolean validateAgendaGroupIsPresent(RuleFileBluePrint ruleFileBluePrint, String agendaGroupName) {
 
-        return ruleFileBluePrint.findRulesByIdentifier(basicRule.getIdentifier()).stream().anyMatch(otherRule -> otherRule.getCode().equals(basicRule.getCode()));
+        return ruleFileBluePrint.streamAllRules().anyMatch(basicRule -> basicRule.getAgendaGroup().equals(agendaGroupName));
+    }
+
+
+    private boolean importsAreValid(RuleFileBluePrint baseRuleFileBluePrint, RuleFileBluePrint ruleFileBluePrint) {
+
+        // imports may shrink, but not add anything new!
+        Set<String> baseImports = baseRuleFileBluePrint.getImportSplitByKeyword();
+        Set<String> imports = ruleFileBluePrint.getImportSplitByKeyword();
+
+        Set<String> additionalImports = Sets.difference(imports, baseImports);
+
+        return additionalImports.stream().allMatch(additionalImport -> allowedImportsPattern.matcher(additionalImport).matches());
     }
 
 
@@ -101,7 +124,9 @@ public class DroolsSyntaxValidationService {
         var versionId = System.currentTimeMillis();
         var testRules = "test-rules";
         KieBuilder kieBuilder = kieContainerCreationService.registerNewKieContainerVersion(testRules,
-                versionId, rules.getRulesString(), RuleFileType.valueOf(rules.getRuleFileType()));
+                versionId,
+                rules.getRulesString(),
+                RuleFileType.valueOf(rules.getRuleFileType()));
         return buildDroolsCompilerSyntaxValidation(kieBuilder);
     }
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/RedactionStorageService.java

@@ -3,6 +3,9 @@ package com.iqser.red.service.redaction.v1.server.storage;
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.InputStream;
+import java.util.stream.Collectors;
+
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedactions;
 
 import org.springframework.stereotype.Service;
 
@@ -10,11 +13,9 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.imported.ImportedRedactions;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.SectionGrid;
 import com.iqser.red.service.redaction.v1.server.client.model.NerEntitiesModel;
-import com.iqser.red.service.redaction.v1.server.utils.exception.NotFoundException;
 import com.iqser.red.service.redaction.v1.server.model.document.DocumentData;
+import com.iqser.red.service.redaction.v1.server.utils.exception.NotFoundException;
 import com.iqser.red.storage.commons.exception.StorageObjectDoesNotExist;
 import com.iqser.red.storage.commons.service.StorageService;
 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPage;
@@ -91,7 +92,11 @@ public class RedactionStorageService {
     public RedactionLog getRedactionLog(String dossierId, String fileId) {
 
         try {
-            return storageService.readJSONObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.REDACTION_LOG), RedactionLog.class);
+            RedactionLog redactionLog = storageService.readJSONObject(TenantContext.getTenantId(),
+                    StorageIdUtils.getStorageId(dossierId, fileId, FileType.REDACTION_LOG),
+                    RedactionLog.class);
+            redactionLog.setRedactionLogEntry(redactionLog.getRedactionLogEntry().stream().filter(entry -> !entry.getValue().isEmpty()).collect(Collectors.toList()));
+            return redactionLog;
         } catch (StorageObjectDoesNotExist e) {
             log.debug("RedactionLog not available.");
             return null;
@@ -99,13 +104,16 @@ public class RedactionStorageService {
 
     }
 
+
     @Timed("redactmanager_getRedactionLog")
     public EntityLog getEntityLog(String dossierId, String fileId) {
 
         try {
-            return storageService.readJSONObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.ENTITY_LOG), EntityLog.class);
+            EntityLog entityLog = storageService.readJSONObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.ENTITY_LOG), EntityLog.class);
+            entityLog.setEntityLogEntry(entityLog.getEntityLogEntry().stream().filter(entry -> !entry.getValue().isEmpty()).collect(Collectors.toList()));
+            return entityLog;
         } catch (StorageObjectDoesNotExist e) {
-            log.debug("RedactionLog not available.");
+            log.debug("EntityLog not available.");
             return null;
         }
 
@@ -117,10 +125,18 @@ public class RedactionStorageService {
 
         try {
             return DocumentData.builder()
-                    .documentStructure(storageService.readJSONObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_STRUCTURE), DocumentStructure.class))
-                    .documentTextData(storageService.readJSONObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_TEXT), DocumentTextData[].class))
-                    .documentPositionData(storageService.readJSONObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_POSITION), DocumentPositionData[].class))
-                    .documentPages(storageService.readJSONObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_PAGES), DocumentPage[].class))
+                    .documentStructure(storageService.readJSONObject(TenantContext.getTenantId(),
+                            StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_STRUCTURE),
+                            DocumentStructure.class))
+                    .documentTextData(storageService.readJSONObject(TenantContext.getTenantId(),
+                            StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_TEXT),
+                            DocumentTextData[].class))
+                    .documentPositionData(storageService.readJSONObject(TenantContext.getTenantId(),
+                            StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_POSITION),
+                            DocumentPositionData[].class))
+                    .documentPages(storageService.readJSONObject(TenantContext.getTenantId(),
+                            StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_PAGES),
+                            DocumentPage[].class))
                     .build();
         } catch (StorageObjectDoesNotExist e) {
             log.debug("DocumentData not available.");
@@ -140,17 +156,6 @@ public class RedactionStorageService {
     }
 
 
-    @Timed("redactmanager_getSectionGrid")
-    public SectionGrid getSectionGrid(String dossierId, String fileId) {
-
-        try {
-            return storageService.readJSONObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.SECTION_GRID), SectionGrid.class);
-        } catch (StorageObjectDoesNotExist e) {
-            throw new NotFoundException("Section Grid is not available.");
-        }
-    }
-
-
     public ComponentLog getComponentLog(String dossierId, String fileId) {
 
         try {

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/utils/MigratedIdsCollector.java

@@ -0,0 +1,54 @@
+package com.iqser.red.service.redaction.v1.server.utils;
+
+import java.util.LinkedList;
+import java.util.Set;
+import java.util.function.BiConsumer;
+import java.util.function.BinaryOperator;
+import java.util.function.Function;
+import java.util.function.Supplier;
+import java.util.stream.Collector;
+
+import com.google.common.base.Functions;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.migration.MigratedIds;
+import com.iqser.red.service.redaction.v1.server.model.MigrationEntity;
+
+public class MigratedIdsCollector implements Collector<MigrationEntity, MigratedIds, MigratedIds> {
+
+    @Override
+    public Supplier<MigratedIds> supplier() {
+
+        return () -> new MigratedIds(new LinkedList<>());
+    }
+
+
+    @Override
+    public BiConsumer<MigratedIds, MigrationEntity> accumulator() {
+
+        return (migratedIds, migrationEntity) -> migratedIds.addMapping(migrationEntity.getOldId(), migrationEntity.getNewId());
+    }
+
+
+    @Override
+    public BinaryOperator<MigratedIds> combiner() {
+
+        return (migratedIds, migratedIds2) -> {
+            migratedIds.getMappings().addAll(migratedIds2.getMappings());
+            return migratedIds;
+        };
+    }
+
+
+    @Override
+    public Function<MigratedIds, MigratedIds> finisher() {
+
+        return Functions.identity();
+    }
+
+
+    @Override
+    public Set<Characteristics> characteristics() {
+
+        return Set.of(Characteristics.IDENTITY_FINISH, Characteristics.CONCURRENT, Characteristics.UNORDERED);
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/utils/Patterns.java

@@ -11,8 +11,7 @@ public final class Patterns {
 
     public static final Map<String, Pattern> patternCache = new HashMap<>();
 
-    public static final Pattern AUTHOR_TABLE_SPLITTER = Pattern.compile(
-            "(((((di)|(van)) )|[A-Z]’)?[A-ZÄÖÜ][\\wäöüéèê]{2,500}( ?[A-ZÄÖÜ]{1,2}\\.){1,3})|(((((di)|(van)) )|[A-Z]’)?[A-ZÄÖÜ][\\wäöüéèê]{2,500}( ?[A-ZÄÖÜ]{1,2} ){1,3})");
+    public static final Pattern AUTHOR_NAME_PATTERN = Pattern.compile("^(?!\\w*[.]$)\\p{Lu}\\p{L}*[.\\p{L}\\s]*$");
 
 
     public Pattern getCompiledPattern(String pattern, boolean caseInsensitive) {

redaction-service-v1/redaction-service-server-v1/src/main/resources/application-dev.yaml

@@ -1,5 +1,5 @@
 server:
-  port: 8083
+  port: 8077
 
 persistence-service.url: "http://localhost:8085"
 tenant-user-management-service.url: "http://localhost:8091/internal"

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AbstractRedactionIntegrationTest.java

@@ -1,5 +1,28 @@
 package com.iqser.red.service.redaction.v1.server;
 
+import static org.mockito.Mockito.when;
+
+import java.io.File;
+import java.io.InputStream;
+import java.net.URL;
+import java.nio.file.Files;
+import java.time.OffsetDateTime;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import org.junit.jupiter.api.AfterEach;
+import org.mockito.stubbing.Answer;
+import org.springframework.amqp.rabbit.core.RabbitTemplate;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.test.mock.mockito.MockBean;
+import org.springframework.core.io.ClassPathResource;
+
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.iqser.red.service.dictionarymerge.commons.DictionaryEntry;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
@@ -12,10 +35,10 @@ import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient;
 import com.iqser.red.service.redaction.v1.server.client.RulesClient;
 import com.iqser.red.service.redaction.v1.server.controller.RedactionController;
 import com.iqser.red.service.redaction.v1.server.service.AnalyzeService;
-import com.iqser.red.service.redaction.v1.server.utils.ResourceLoader;
-import com.iqser.red.service.redaction.v1.server.utils.TextNormalizationUtilities;
 import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
 import com.iqser.red.service.redaction.v1.server.utils.LayoutParsingRequestProvider;
+import com.iqser.red.service.redaction.v1.server.utils.ResourceLoader;
+import com.iqser.red.service.redaction.v1.server.utils.TextNormalizationUtilities;
 import com.iqser.red.storage.commons.service.StorageService;
 import com.iqser.red.storage.commons.utils.FileSystemBackedStorageService;
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingFinishedEvent;
@@ -23,23 +46,8 @@ import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsi
 import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingPipeline;
 import com.knecon.fforesight.tenantcommons.TenantContext;
 import com.knecon.fforesight.tenantcommons.TenantsClient;
+
 import lombok.SneakyThrows;
-import org.junit.jupiter.api.AfterEach;
-import org.mockito.stubbing.Answer;
-import org.springframework.amqp.rabbit.core.RabbitTemplate;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.boot.test.mock.mockito.MockBean;
-import org.springframework.core.io.ClassPathResource;
-
-import java.io.File;
-import java.io.InputStream;
-import java.net.URL;
-import java.nio.file.Files;
-import java.time.OffsetDateTime;
-import java.util.*;
-import java.util.stream.Collectors;
-
-import static org.mockito.Mockito.when;
 
 public abstract class AbstractRedactionIntegrationTest {
 
@@ -432,7 +440,6 @@ public abstract class AbstractRedactionIntegrationTest {
     @SneakyThrows
     protected LayoutParsingFinishedEvent analyzeDocumentStructure(LayoutParsingType layoutParsingType, AnalyzeRequest request) {
 
-
         return layoutParsingPipeline.parseLayoutAndSaveFilesToStorage(LayoutParsingRequestProvider.build(layoutParsingType, request));
     }
 

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AnalysisTest.java

@@ -0,0 +1,122 @@
+package com.iqser.red.service.redaction.v1.server;
+
+import static org.mockito.Mockito.when;
+
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.util.List;
+
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Disabled;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
+import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration;
+import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.ComponentScan;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.context.annotation.FilterType;
+import org.springframework.context.annotation.Import;
+import org.springframework.context.annotation.Primary;
+import org.springframework.test.context.junit.jupiter.SpringExtension;
+
+import com.iqser.red.commons.jackson.ObjectMapperFactory;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeResult;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type;
+import com.iqser.red.service.redaction.v1.server.annotate.AnnotateRequest;
+import com.iqser.red.service.redaction.v1.server.annotate.AnnotateResponse;
+import com.iqser.red.service.redaction.v1.server.redaction.utils.OsUtils;
+import com.iqser.red.storage.commons.StorageAutoConfiguration;
+import com.iqser.red.storage.commons.service.StorageService;
+import com.iqser.red.storage.commons.utils.FileSystemBackedStorageService;
+import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
+import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingServiceProcessorConfiguration;
+import com.knecon.fforesight.tenantcommons.TenantContext;
+
+@ExtendWith(SpringExtension.class)
+@SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT, properties = {"application.type=DocuMine"})
+@Import(AnalysisTest.RedactionIntegrationTestConfiguration.class)
+public class AnalysisTest extends AbstractRedactionIntegrationTest {
+
+    private static final String RULES = loadFromClassPath("drools/test_rules.drl");
+    private static final String COMPONENT_RULES = loadFromClassPath("drools/test_components.drl");
+
+
+    @Test
+    @Disabled
+    public void analyzeTableDemoFile() throws IOException {
+
+        AnalyzeRequest request = uploadFileToStorage("files/Metolachlor/S-Metolachlor_RAR_01_Volume_1_2018-09-06.pdf");
+
+        System.out.println("Start Full integration test");
+        analyzeDocumentStructure(LayoutParsingType.DOCUMINE, request);
+        System.out.println("Finished structure analysis");
+        AnalyzeResult result = analyzeService.analyze(request);
+        System.out.println("Finished analysis");
+        var entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
+        var componentLog = redactionStorageService.getComponentLog(TEST_DOSSIER_ID, TEST_FILE_ID);
+        AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder().dossierId(TEST_DOSSIER_ID).fileId(TEST_FILE_ID).build());
+
+        String outputFileName = OsUtils.getTemporaryDirectory() + "/Documine.pdf";
+
+        try (FileOutputStream fileOutputStream = new FileOutputStream(outputFileName)) {
+            fileOutputStream.write(annotateResponse.getDocument());
+        }
+    }
+
+
+    @Configuration
+    @EnableAutoConfiguration(exclude = {RabbitAutoConfiguration.class})
+    @Import(LayoutParsingServiceProcessorConfiguration.class)
+    @ComponentScan(excludeFilters = {@ComponentScan.Filter(type = FilterType.ASSIGNABLE_TYPE, value = StorageAutoConfiguration.class)})
+    static class RedactionIntegrationTestConfiguration {
+
+        @Bean
+        @Primary
+        public StorageService inmemoryStorage() {
+
+            return new FileSystemBackedStorageService(ObjectMapperFactory.create());
+        }
+
+    }
+
+
+    @BeforeEach
+    public void stubClients() {
+
+        TenantContext.setTenantId("documine");
+
+        when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.ENTITY)).thenReturn(System.currentTimeMillis());
+        when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.ENTITY)).thenReturn(JSONPrimitive.of(RULES));
+        when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.COMPONENT)).thenReturn(System.currentTimeMillis());
+        when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.COMPONENT)).thenReturn(JSONPrimitive.of(COMPONENT_RULES));
+
+        loadDictionaryForTest();
+        loadTypeForTest();
+        loadNerForTest();
+        when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L);
+        when(dictionaryClient.getAllTypesForDossierTemplate(TEST_DOSSIER_TEMPLATE_ID, false)).thenReturn(getTypeResponse());
+
+        when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L);
+        when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, false)).thenReturn(List.of(Type.builder()
+                .id(DOSSIER_REDACTIONS_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID)
+                .type(DOSSIER_REDACTIONS_INDICATOR)
+                .dossierTemplateId(TEST_DOSSIER_ID)
+                .hexColor("#ffe187")
+                .isHint(hintTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
+                .isCaseInsensitive(caseInSensitiveMap.get(DOSSIER_REDACTIONS_INDICATOR))
+                .isRecommendation(recommendationTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
+                .rank(rankTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
+                .build()));
+
+        mockDictionaryCalls(null);
+
+        when(dictionaryClient.getColors(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(colors);
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DocumineFloraTest.java

@@ -6,6 +6,8 @@ import java.io.FileOutputStream;
 import java.io.IOException;
 import java.util.List;
 
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
+import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.Test;
@@ -19,6 +21,7 @@ import org.springframework.context.annotation.Configuration;
 import org.springframework.context.annotation.FilterType;
 import org.springframework.context.annotation.Import;
 import org.springframework.context.annotation.Primary;
+import org.springframework.core.io.ClassPathResource;
 import org.springframework.test.context.junit.jupiter.SpringExtension;
 
 import com.iqser.red.commons.jackson.ObjectMapperFactory;
@@ -50,7 +53,11 @@ public class DocumineFloraTest extends AbstractRedactionIntegrationTest {
 //    @Disabled
     public void titleExtraction() throws IOException {
 
-        AnalyzeRequest request = uploadFileToStorage("files/Documine/Flora/402Study-ocred.pdf");
+        AnalyzeRequest request = uploadFileToStorage("files/ImportedRedactions/18 Chlorothalonil RAR 08 Volume 3CA B 6a Oct 2017.pdf");
+
+        ClassPathResource importedRedactionClasspathResource = new ClassPathResource("files/ImportedRedactions/18 Chlorothalonil RAR 08 Volume 3CA B 6a Oct 2017.IMPORTED_REDACTIONS.json");
+        storageService.storeObject(TenantContext.getTenantId(), RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.IMPORTED_REDACTIONS), importedRedactionClasspathResource.getInputStream());
+
 
 //        AnalyzeRequest request = prepareStorage("files/Documine/Flora/ProblemDocs/SOLICITA_VICTRATO-GOLD-II_Item 21_Mutacao_Genica (1).pdf",
 //                "files/Documine/Flora/ProblemDocs/SOLICITA_VICTRATO-GOLD-II_Item 21_Mutacao_Genica (1).TABLES.json");

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/MigrationIntegrationTest.java

@@ -0,0 +1,189 @@
+package com.iqser.red.service.redaction.v1.server;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.nio.file.Path;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import org.junit.jupiter.api.Disabled;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.context.annotation.Import;
+import org.springframework.core.io.ClassPathResource;
+import org.springframework.test.context.junit.jupiter.SpringExtension;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.migration.MigratedIds;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
+import com.iqser.red.service.redaction.v1.server.annotate.AnnotateRequest;
+import com.iqser.red.service.redaction.v1.server.annotate.AnnotateResponse;
+import com.iqser.red.service.redaction.v1.server.document.graph.BuildDocumentIntegrationTest;
+import com.iqser.red.service.redaction.v1.server.model.MigratedEntityLog;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
+import com.iqser.red.service.redaction.v1.server.redaction.utils.OsUtils;
+import com.iqser.red.service.redaction.v1.server.service.RedactionLogToEntityLogMigrationService;
+
+import lombok.SneakyThrows;
+
+@Disabled
+@ExtendWith(SpringExtension.class)
+@SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
+@Import(MigrationIntegrationTest.TestConfiguration.class)
+public class MigrationIntegrationTest extends BuildDocumentIntegrationTest {
+
+    @Autowired
+    RedactionLogToEntityLogMigrationService redactionLogToEntityLogMigrationService;
+
+    @Autowired
+    ObjectMapper mapper;
+
+
+    @Test
+    @SneakyThrows
+    public void testSave() {
+
+        MigratedIds ids = new MigratedIds(new LinkedList<>());
+        ids.addMapping("123", "321");
+        ids.addMapping("123", "321");
+        ids.addMapping("123", "321");
+        ids.addMapping("123", "321");
+        ids.addMapping("123", "321");
+
+        mapper.writeValue(new FileOutputStream("/tmp/testIds.json"), ids);
+        var ids2 = mapper.readValue(new FileInputStream("/tmp/testIds.json"), MigratedIds.class);
+        assert ids2.getMappings().size() == 5;
+    }
+
+    @Test
+    @SneakyThrows
+    public void testMigration() {
+
+        String fileName = "files/migration/def8f960580f088b975ba806dfae1f87.ORIGIN.pdf";
+        String imageFileName = "files/migration/def8f960580f088b975ba806dfae1f87.IMAGE_INFO.json";
+        String tableFileName = "files/migration/def8f960580f088b975ba806dfae1f87.TABLES.json";
+
+        Document document = buildGraph(fileName, imageFileName, tableFileName);
+        RedactionLog redactionLog;
+        try (var in = new ClassPathResource("files/migration/def8f960580f088b975ba806dfae1f87.REDACTION_LOG.json").getInputStream()) {
+            redactionLog = mapper.readValue(in, RedactionLog.class);
+        }
+        MigratedEntityLog migratedEntityLog = redactionLogToEntityLogMigrationService.migrate(redactionLog, document);
+
+        redactionStorageService.storeObject(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.ENTITY_LOG, migratedEntityLog.getEntityLog());
+        assertEquals(redactionLog.getRedactionLogEntry().size(), migratedEntityLog.getEntityLog().getEntityLogEntry().size());
+        assertEquals(redactionLog.getRedactionLogEntry().size(), migratedEntityLog.getMigratedIds().getMappings().size());
+        EntityLog entityLog = migratedEntityLog.getEntityLog();
+        assertEquals(redactionLog.getAnalysisNumber(), entityLog.getAnalysisNumber());
+        assertEquals(redactionLog.getAnalysisVersion(), entityLog.getAnalysisVersion());
+        assertEquals(redactionLog.getDictionaryVersion(), entityLog.getDictionaryVersion());
+        assertEquals(redactionLog.getDossierDictionaryVersion(), entityLog.getDossierDictionaryVersion());
+        assertEquals(redactionLog.getLegalBasisVersion(), entityLog.getLegalBasisVersion());
+        assertEquals(redactionLog.getRulesVersion(), entityLog.getRulesVersion());
+        assertEquals(redactionLog.getLegalBasis().size(), entityLog.getLegalBasis().size());
+
+        Map<String, String> migratedIds = migratedEntityLog.getMigratedIds().buildOldToNewMapping();
+        migratedIds.forEach((oldId, newId) -> assertEntryIsEqual(oldId, newId, redactionLog, entityLog, migratedIds));
+
+        AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder().dossierId(TEST_DOSSIER_ID).fileId(TEST_FILE_ID).build());
+        File outputFile = Path.of(OsUtils.getTemporaryDirectory()).resolve(Path.of(fileName.replaceAll(".pdf", "_MIGRATED.pdf")).getFileName()).toFile();
+
+        try (FileOutputStream fileOutputStream = new FileOutputStream(outputFile)) {
+            fileOutputStream.write(annotateResponse.getDocument());
+        }
+    }
+
+
+    private void assertEntryIsEqual(String oldId, String newId, RedactionLog redactionLog, EntityLog entityLog, Map<String, String> oldToNewMapping) {
+
+        RedactionLogEntry redactionLogEntry = redactionLog.getRedactionLogEntry().stream().filter(entry -> entry.getId().equals(oldId)).findAny().orElseThrow();
+        EntityLogEntry entityLogEntry = entityLog.getEntityLogEntry().stream().filter(entry -> entry.getId().equals(newId)).findAny().orElseThrow();
+
+        if (!redactionLogEntry.isImage()) {
+            assertEquals(redactionLogEntry.getValue().toLowerCase(Locale.ENGLISH), entityLogEntry.getValue().toLowerCase(Locale.ENGLISH));
+        }
+        assertEquals(redactionLogEntry.getChanges().size(), entityLogEntry.getChanges().size());
+        assertEquals(redactionLogEntry.getManualChanges().size(), entityLogEntry.getManualChanges().size());
+        assertEquals(redactionLogEntry.getPositions().size(), entityLogEntry.getPositions().size());
+        assertTrue(positionsAlmostEqual(redactionLogEntry.getPositions(), entityLogEntry.getPositions()));
+        assertEquals(redactionLogEntry.getColor(), entityLogEntry.getColor());
+        assertEqualsNullSafe(redactionLogEntry.getLegalBasis(), entityLogEntry.getLegalBasis());
+        assertEqualsNullSafe(redactionLogEntry.getReason(), entityLogEntry.getReason());
+        assertReferencesEqual(redactionLogEntry.getReference(), entityLogEntry.getReference(), oldToNewMapping);
+        assertEquals(redactionLogEntry.isDictionaryEntry(), entityLogEntry.isDictionaryEntry());
+        assertEquals(redactionLogEntry.isDossierDictionaryEntry(), entityLogEntry.isDossierDictionaryEntry());
+        assertEquals(redactionLogEntry.getEngines().stream().map(Enum::name).collect(Collectors.toSet()),
+                entityLogEntry.getEngines().stream().map(Enum::name).collect(Collectors.toSet()));
+    }
+
+
+    private boolean positionsAlmostEqual(List<Rectangle> positions1, List<Position> positions2) {
+
+        double tolerance = 2;
+        for (int i = 0; i < positions1.size(); i++) {
+            Rectangle p1 = positions1.get(0);
+            Position p2 = positions2.get(0);
+            if (p1.getPage() != p2.getPageNumber()) {
+                return false;
+            }
+            if (Math.abs(p1.getHeight() - p2.getRectangle()[3]) > tolerance) {
+                return false;
+            }
+            if (Math.abs(p1.getWidth() - p2.getRectangle()[2]) > tolerance) {
+                return false;
+            }
+            if (Math.abs(p1.getTopLeft().getX() - p2.getRectangle()[0]) > tolerance) {
+                return false;
+            }
+            if (Math.abs(p1.getTopLeft().getY() - p2.getRectangle()[1]) > tolerance) {
+                return false;
+            }
+        }
+        return true;
+    }
+
+
+    private void assertEqualsNullSafe(String string1, String string2) {
+
+        if (Objects.isNull(string1) && Objects.isNull(string2)) {
+            assertEquals(string1, string2);
+        } else if (Objects.isNull(string1)) {
+            assertEquals("", string2);
+        } else if (Objects.isNull(string2)) {
+            assertEquals("", string1);
+        } else {
+            assertEquals(string1, string2);
+        }
+
+    }
+
+
+    private void assertReferencesEqual(Set<String> reference, Set<String> reference1, Map<String, String> oldToNewMapping) {
+
+        if (reference == null) {
+            assertTrue(reference1.isEmpty());
+            return;
+        }
+
+        assertEquals(reference.stream().map(oldToNewMapping::get).collect(Collectors.toSet()), reference1);
+
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionAcceptanceTest.java

@@ -1,5 +1,6 @@
 package com.iqser.red.service.redaction.v1.server;
 
+import static org.assertj.core.api.Assertions.assertThat;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.mockito.Mockito.when;
 
@@ -103,6 +104,25 @@ public class RedactionAcceptanceTest extends AbstractRedactionIntegrationTest {
     }
 
 
+    @Test
+    public void authorNameSplittingTest() {
+
+        AnalyzeRequest request = uploadFileToStorage("files/Minimal Examples/tableWithAuthors.pdf");
+
+        analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request);
+        AnalyzeResult result = analyzeService.analyze(request);
+
+        var entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
+
+        var recommendations = entityLog.getEntityLogEntry()
+                .stream()
+                .filter(entityLogEntry -> entityLogEntry.getEntryType().equals(EntryType.RECOMMENDATION))
+                .map(EntityLogEntry::getValue)
+                .toList();
+
+        assertThat(recommendations).containsExactlyInAnyOrder("Michael N.", "Funnarie B.", "Feuer A.");
+    }
+
     @Test
     public void acceptanceTests() throws IOException {
 
@@ -148,7 +168,8 @@ public class RedactionAcceptanceTest extends AbstractRedactionIntegrationTest {
         return redactionLog.getEntityLogEntry()
                 .stream()
                 .filter(entry -> entry.getType().equals(type))
-                .filter(entry -> entry.getValue().equals(value)).filter(entry -> entry.getContainingNodeId().get(0).equals(sectionNumber.get(0)));
+                .filter(entry -> entry.getValue().equals(value))
+                .filter(entry -> entry.getContainingNodeId().get(0).equals(sectionNumber.get(0)));
     }
 
 

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java

@@ -50,7 +50,6 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryType;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Comment;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;
@@ -214,17 +213,16 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
 //        }
 
         AnalyzeRequest request = uploadFileToStorage("files/new/crafted document.pdf");
-        ManualResizeRedaction manualResizeRedaction = ManualResizeRedaction.builder()
-                .annotationId("c6be5277f5ee60dc3d83527798b7fe02")
-                .value("Dr. Alan")
-                .positions(List.of(new Rectangle(236.8f, 182.90005f, 40.584f, 12.642f, 7)))
-                .status(AnnotationStatus.APPROVED)
-                .requestDate(OffsetDateTime.now())
-                .build();
-        manualResizeRedaction.setUpdateDictionary(false);
-
-        ManualRedactions manualRedactions = ManualRedactions.builder().resizeRedactions(Set.of())
 
+        ManualRedactions manualRedactions = ManualRedactions.builder()
+                .resizeRedactions(Set.of(ManualResizeRedaction.builder()
+                        .annotationId("c6be5277f5ee60dc3d83527798b7fe02")
+                        .value("Dr. Alan")
+                        .positions(List.of(new Rectangle(236.8f, 182.90005f, 40.584f, 12.642f, 7)))
+                        .status(AnnotationStatus.APPROVED)
+                        .requestDate(OffsetDateTime.now())
+                        .updateDictionary(false)
+                        .build()))
                 .build();
 
         request.setManualRedactions(manualRedactions);
@@ -265,10 +263,38 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
     }
 
 
+    @Test
+    public void importedRedactionsTest() throws IOException {
+
+        ClassPathResource importedRedactionClasspathResource = new ClassPathResource(
+                "files/ImportedRedactions/18 Chlorothalonil RAR 08 Volume 3CA B 6a Oct 2017.IMPORTED_REDACTIONS.json");
+        storageService.storeObject(TenantContext.getTenantId(),
+                RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.IMPORTED_REDACTIONS),
+                importedRedactionClasspathResource.getInputStream());
+
+        AnalyzeRequest request = uploadFileToStorage("files/ImportedRedactions/18 Chlorothalonil RAR 08 Volume 3CA B 6a Oct 2017.pdf");
+        System.out.println("Start Full integration test");
+        analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request);
+        System.out.println("Finished structure analysis");
+        AnalyzeResult result = analyzeService.analyze(request);
+        System.out.println("Finished analysis");
+        var entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
+
+        AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder().dossierId(TEST_DOSSIER_ID).fileId(TEST_FILE_ID).build());
+
+        String outputFileName = OsUtils.getTemporaryDirectory() + "/Annotated.pdf";
+
+        try (FileOutputStream fileOutputStream = new FileOutputStream(outputFileName)) {
+            fileOutputStream.write(annotateResponse.getDocument());
+        }
+
+    }
+
+
     @Test
     public void titleExtraction() throws IOException {
 
-        AnalyzeRequest request = uploadFileToStorage("files/Metolachlor/S-Metolachlor_RAR_02_Volume_2_2018-09-06.pdf");
+        AnalyzeRequest request = uploadFileToStorage("files/migration/def8f960580f088b975ba806dfae1f87.ORIGIN.pdf");
         System.out.println("Start Full integration test");
         analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request);
         System.out.println("Finished structure analysis");
@@ -573,7 +599,9 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
             fileOutputStream.write(objectMapper.writeValueAsBytes(redactionStorageService.getDocumentData(TEST_DOSSIER_ID, TEST_FILE_ID)));
         }
 
-        List<String> valuesInDocument = entityLog.getEntityLogEntry().stream().filter(e -> !e.getEntryType().equals(EntryType.IMAGE))
+        List<String> valuesInDocument = entityLog.getEntityLogEntry()
+                .stream()
+                .filter(e -> !e.getEntryType().equals(EntryType.IMAGE))
                 .map(redactionLogEntry -> new TextRange(redactionLogEntry.getStartOffset(), redactionLogEntry.getEndOffset()))
                 .map(boundary -> documentGraph.getTextBlock().subSequence(boundary).toString())
                 .toList();
@@ -900,9 +928,9 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
                 .value("Bera P")
                 .positions(List.of(new Rectangle(382.12485f, 235.94246f, 8.768621f, -21.632504f, 1)))
                 .textBefore("")
+                .updateDictionary(false)
                 .textAfter("")
                 .build();
-        manualResizeRedaction.setUpdateDictionary(false);
         manualRedactions.getResizeRedactions().add(manualResizeRedaction);
 
         request.setManualRedactions(manualRedactions);
@@ -942,7 +970,11 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
 
         redactionLog.getEntityLogEntry().forEach(entry -> {
             if (!entry.getEntryType().equals(EntryType.HINT)) {
-                assertThat(entry.getReason()).isEqualTo("Not redacted because it's row does not belong to a vertebrate study");
+                if (entry.getType().equals("CBI_author")) {
+                    assertThat(entry.getReason()).isEqualTo("Not redacted because it's row does not belong to a vertebrate study");
+                } else if (entry.getType().equals("CBI_address")) {
+                    assertThat(entry.getReason()).isEqualTo("No vertebrate found");
+                }
             }
         });
     }
@@ -980,7 +1012,6 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
 
         String manualAddId = UUID.randomUUID().toString();
 
-        Comment comment = Comment.builder().date(OffsetDateTime.now()).user("TEST_USER").text("This is a comment test").build();
         manualRedactions.setIdsToRemove(Set.of(IdRemoval.builder()
                 .annotationId("5b940b2cb401ed9f5be6fc24f6e77bcf")
                 .fileId("fileId")
@@ -997,10 +1028,6 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
                 .processedDate(OffsetDateTime.now())
                 .build()));
 
-        manualRedactions.getComments().put("e5be0f1d941bbb92a068e198648d06c4", List.of(comment));
-        manualRedactions.getComments().put("0836727c3508a0b2ea271da69c04cc2f", List.of(comment));
-        manualRedactions.getComments().put(manualAddId, List.of(comment));
-
         ManualRedactionEntry manualRedactionEntry = new ManualRedactionEntry();
         manualRedactionEntry.setAnnotationId(manualAddId);
         manualRedactionEntry.setFileId("fileId");
@@ -1047,6 +1074,7 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
                         Rectangle.builder().topLeftX(307.89517f).topLeftY(327.567f).width(7.648041f).height(61.670967f).page(1).build(),
                         Rectangle.builder().topLeftX(316.99985f).topLeftY(327.567f).width(7.648041f).height(38.104286f).page(1).build()))
                 .status(AnnotationStatus.APPROVED)
+                .updateDictionary(false)
                 .build()));
 
         analyzeService.reanalyze(request);
@@ -1088,6 +1116,7 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
 
 
     @Test
+    @Disabled
     public void testImportedRedactions() throws IOException {
 
         String outputFileName = OsUtils.getTemporaryDirectory() + "/ImportedRedactions.pdf";

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/annotate/AnnotationService.java

@@ -29,7 +29,6 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryType;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
-import com.iqser.red.service.redaction.v1.server.service.DictionaryService;
 import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
 import com.iqser.red.service.redaction.v1.server.utils.RectangleTransformations;
 
@@ -41,11 +40,10 @@ import lombok.SneakyThrows;
 public class AnnotationService {
 
     private final RedactionStorageService redactionStorageService;
-    private final DictionaryService dictionaryService;
 
 
     /**
-     * This method draws a PDF Text Markup Annotation for each RedactionLogEntry and the section grid.
+     * This method draws a PDF Text Markup Annotation for each EntityLogEntry and the Viewer document.
      * A Text Markup Annotation has two main functionalities.
      * First, it highlights text with a color and second, when hovered over in a PDF Viewer it displays a popup message.
      * Where the Popup message appears is defined by the Rectangle provided to the Annotation.
@@ -70,7 +68,6 @@ public class AnnotationService {
         try (PDDocument pdDocument = Loader.loadPDF(storedObjectFile)) {
             pdDocument.setAllSecurityToBeRemoved(true);
 
-            dictionaryService.updateDictionary(annotateRequest.getDossierTemplateId(), annotateRequest.getDossierId());
             annotate(pdDocument, entityLog);
 
             PDDocumentCatalog catalog = pdDocument.getDocumentCatalog();

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/BuildDocumentIntegrationTest.java

@@ -46,6 +46,7 @@ public class BuildDocumentIntegrationTest extends AbstractRedactionIntegrationTe
 
     }
 
+
     @SneakyThrows
     protected Document buildGraph(String filename) {
 
@@ -65,6 +66,20 @@ public class BuildDocumentIntegrationTest extends AbstractRedactionIntegrationTe
     }
 
 
+    @SneakyThrows
+    protected Document buildGraph(String filename, String imageFileName, String tableFileName) {
+
+        String fileNameWithPdf = filename;
+        if (!filename.endsWith(".pdf")) {
+            fileNameWithPdf = filename + ".pdf";
+        }
+        AnalyzeRequest request = prepareStorage(fileNameWithPdf, tableFileName, imageFileName);
+
+        analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request);
+        return DocumentGraphMapper.toDocumentGraph(redactionStorageService.getDocumentData(request.getDossierId(), request.getFileId()));
+    }
+
+
     @SneakyThrows
     protected Document buildGraphNoImages(String filename) {
 

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/MigrationPocTest.java

@@ -1,162 +0,0 @@
-package com.iqser.red.service.redaction.v1.server.document.graph;
-
-import static java.util.stream.Collectors.toMap;
-import static org.junit.jupiter.api.Assertions.assertTrue;
-import static org.mockito.Mockito.when;
-
-import java.io.IOException;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-import java.util.stream.Stream;
-
-import org.drools.io.ClassPathResource;
-import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.Disabled;
-import org.junit.jupiter.api.Test;
-import org.springframework.beans.factory.annotation.Autowired;
-
-import com.google.common.base.Functions;
-import com.google.common.collect.Sets;
-import com.iqser.red.commons.jackson.ObjectMapperFactory;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeResult;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
-import com.iqser.red.service.redaction.v1.server.model.ManualEntity;
-import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
-import com.iqser.red.service.redaction.v1.server.service.RedactionLogCreatorService;
-import com.iqser.red.service.redaction.v1.server.service.document.DocumentGraphMapper;
-import com.iqser.red.service.redaction.v1.server.service.document.ManualEntityCreationService;
-import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
-import com.knecon.fforesight.tenantcommons.TenantContext;
-
-import lombok.SneakyThrows;
-
-public class MigrationPocTest extends BuildDocumentIntegrationTest {
-
-    private static final String RULES = loadFromClassPath("drools/rules.drl");
-
-    @Autowired
-    private ManualEntityCreationService redactionLogAdapter;
-    @Autowired
-    private RedactionLogCreatorService redactionLogCreatorService;
-
-
-    @BeforeEach
-    public void stubClients() {
-
-        TenantContext.setTenantId("redaction");
-        when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.ENTITY)).thenReturn(System.currentTimeMillis());
-        when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.ENTITY)).thenReturn(JSONPrimitive.of(RULES));
-        when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.COMPONENT)).thenReturn(-1L);
-
-        loadDictionaryForTest();
-        loadTypeForTest();
-        loadNerForTest();
-        when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L);
-        when(dictionaryClient.getAllTypesForDossierTemplate(TEST_DOSSIER_TEMPLATE_ID, false)).thenReturn(getTypeResponse());
-
-        when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L);
-        when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, false)).thenReturn(List.of(Type.builder()
-                .id(DOSSIER_REDACTIONS_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID)
-                .type(DOSSIER_REDACTIONS_INDICATOR)
-                .dossierTemplateId(TEST_DOSSIER_ID)
-                .hexColor("#ffe187")
-                .isHint(hintTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
-                .isCaseInsensitive(caseInSensitiveMap.get(DOSSIER_REDACTIONS_INDICATOR))
-                .isRecommendation(recommendationTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
-                .rank(rankTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
-                .build()));
-
-        mockDictionaryCalls(null);
-
-        when(dictionaryClient.getColors(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(colors);
-    }
-
-
-    @Test
-    @Disabled // Enable if you fix the TODO in EntityCreationService
-    @SneakyThrows
-    public void testMigration() {
-
-        AnalyzeRequest request = uploadFileToStorage("files/new/crafted document.pdf");
-        analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request);
-        AnalyzeResult result = analyzeService.analyze(request);
-        var newRedactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID);
-
-        var originalRedactionLog = getOriginalRedactionLog();
-
-        // IMPORTANT: always use the graph which is mapped from the DocumentData, since rounding errors occur during storage.
-        Document document = DocumentGraphMapper.toDocumentGraph(redactionStorageService.getDocumentData(request.getDossierId(), request.getFileId()));
-        List<ManualEntity> notFoundManualRedactionEntries = redactionLogAdapter.toTextEntity(originalRedactionLog, document);
-
-        var migratedRedactionLogEntries = redactionLogCreatorService.createRedactionLog(document, TEST_DOSSIER_TEMPLATE_ID, notFoundManualRedactionEntries, Collections.emptyMap());
-
-        Map<String, RedactionLogEntry> migratedIds = migratedRedactionLogEntries.stream().collect(toMap(RedactionLogEntry::getId, Functions.identity()));
-        Map<String, RedactionLogEntry> newIds = newRedactionLog.getRedactionLogEntry().stream().collect(toMap(RedactionLogEntry::getId, Functions.identity()));
-
-        logPrecision(migratedIds, newIds);
-        logRecall(migratedIds, newIds);
-    }
-
-
-    private static void logPrecision(Map<String, RedactionLogEntry> migratedIds, Map<String, RedactionLogEntry> newIds) {
-
-        var precision = computePrecision(migratedIds, newIds);
-        System.out.printf("precision %.2f\n", precision);
-        System.out.println("New Entries");
-        getAddedEntries(migratedIds, newIds).forEach(System.out::println);
-        assertTrue(precision >= 0.85);
-        System.out.println();
-    }
-
-
-    private static void logRecall(Map<String, RedactionLogEntry> migratedIds, Map<String, RedactionLogEntry> newIds) {
-
-        var recall = computeRecall(migratedIds, newIds);
-        System.out.printf("recall %.2f\n", recall);
-        System.out.println("Missing entries");
-        getMissingEntries(migratedIds, newIds).forEach(System.out::println);
-        assertTrue(recall >= 0.85);
-        System.out.println();
-    }
-
-
-    private static Stream<RedactionLogEntry> getMissingEntries(Map<String, RedactionLogEntry> migratedIds, Map<String, RedactionLogEntry> newIds) {
-
-        return migratedIds.entrySet().stream().filter(entry -> !newIds.containsKey(entry.getKey())).map(Map.Entry::getValue);
-    }
-
-
-    private static Stream<RedactionLogEntry> getAddedEntries(Map<String, RedactionLogEntry> migratedIds, Map<String, RedactionLogEntry> newIds) {
-
-        return newIds.entrySet().stream().filter(entry -> !migratedIds.containsKey(entry.getKey())).map(Map.Entry::getValue);
-    }
-
-
-    private static double computePrecision(Map<String, RedactionLogEntry> migratedIds, Map<String, RedactionLogEntry> newIds) {
-
-        return (double) Sets.intersection(newIds.keySet(), migratedIds.keySet()).size() / (double) newIds.size();
-    }
-
-
-    private static double computeRecall(Map<String, RedactionLogEntry> migratedIds, Map<String, RedactionLogEntry> newIds) {
-
-        return (double) Sets.intersection(newIds.keySet(), migratedIds.keySet()).size() / (double) migratedIds.size();
-    }
-
-
-    private static RedactionLog getOriginalRedactionLog() throws IOException {
-
-        RedactionLog originalRedactionLog;
-        try (var inputStream = new ClassPathResource("files/migration/legacy_redactionlog.json").getInputStream()) {
-            originalRedactionLog = ObjectMapperFactory.create().readValue(inputStream, RedactionLog.class);
-        }
-        return originalRedactionLog;
-    }
-
-}

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/drools/files/management/services/DroolsSyntaxValidationServiceTest.java

@@ -58,6 +58,50 @@ class DroolsSyntaxValidationServiceTest {
     }
 
 
+    @Test
+    @SneakyThrows
+    void testRulesWithRemovedImportsAndChangedName() {
+
+        DroolsSyntaxValidationService droolsSyntaxValidationService = new DroolsSyntaxValidationService(new KieContainerCreationService(rulesClient));
+        var rulesFile = new ClassPathResource("drools/rules.drl");
+
+        String rulesString = new String(rulesFile.getInputStream().readAllBytes());
+        rulesString = rulesString.replaceAll("import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell;", "");
+        rulesString = rulesString.replaceAll("rule \"LDS.0.0: Run local dictionary search\"", "rule \"LDS.0.0: run local dictionary search\"");
+        DroolsSyntaxValidation droolsSyntaxValidation = droolsSyntaxValidationService.testRules(new RuleValidationModel(RuleFileType.ENTITY.name(), rulesString));
+        droolsSyntaxValidation.getDroolsSyntaxErrorMessages().forEach(System.out::println);
+        assertTrue(droolsSyntaxValidation.isCompiled());
+    }
+
+    @Test
+    @SneakyThrows
+    void testRulesWithAddedImports() {
+
+        DroolsSyntaxValidationService droolsSyntaxValidationService = new DroolsSyntaxValidationService(new KieContainerCreationService(rulesClient));
+        var rulesFile = new ClassPathResource("drools/rules.drl");
+
+        String rulesString = new String(rulesFile.getInputStream().readAllBytes());
+        rulesString = rulesString.replaceAll("import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell;", "import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell;\nimport com.iqser.red.service.redaction.v1.server.service.document.EntityComparators;");
+        DroolsSyntaxValidation droolsSyntaxValidation = droolsSyntaxValidationService.testRules(new RuleValidationModel(RuleFileType.ENTITY.name(), rulesString));
+        droolsSyntaxValidation.getDroolsSyntaxErrorMessages().forEach(System.out::println);
+        assertTrue(droolsSyntaxValidation.isCompiled());
+    }
+
+    @Test
+    @SneakyThrows
+    void testRulesWithUnallowedAddedImports() {
+
+        DroolsSyntaxValidationService droolsSyntaxValidationService = new DroolsSyntaxValidationService(new KieContainerCreationService(rulesClient));
+        var rulesFile = new ClassPathResource("drools/rules.drl");
+
+        String rulesString = new String(rulesFile.getInputStream().readAllBytes());
+        rulesString = rulesString.replaceAll("import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell;", "import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell;\nimport com.google.common.collect.Sets;");
+        DroolsSyntaxValidation droolsSyntaxValidation = droolsSyntaxValidationService.testRules(new RuleValidationModel(RuleFileType.ENTITY.name(), rulesString));
+        droolsSyntaxValidation.getDroolsSyntaxErrorMessages().forEach(System.out::println);
+        assertFalse(droolsSyntaxValidation.isCompiled());
+    }
+
+
     @Test
     @SneakyThrows
     void testAllRules() {
@@ -156,7 +200,8 @@ class DroolsSyntaxValidationServiceTest {
         var rulesFile = new ClassPathResource("drools/rules.drl");
 
         String rulesString = new String(rulesFile.getInputStream().readAllBytes());
-        String corruptedRules = rulesString.replaceAll("import com.iqser.red.service.redaction.v1.server.model.document.nodes.ImageType;", "");
+        String corruptedRules = rulesString.replaceAll("import com.iqser.red.service.redaction.v1.server.model.document.nodes.ImageType;",
+                "import com.iqser.red.service.redaction.v1.server.model.document.nodes.ImageType;\nimport com.iqser.red.service.redaction.v1.server.model.document.nodes.SomethingElse;");
 
         DroolsSyntaxValidation droolsSyntaxValidation = droolsSyntaxValidationService.testRules(new RuleValidationModel(RuleFileType.ENTITY.name(), corruptedRules));
         assertFalse(droolsSyntaxValidation.isCompiled());

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/drools/files/management/services/DroolsUpToDateTest.java

@@ -0,0 +1,53 @@
+package com.iqser.red.service.redaction.v1.server.drools.files.management.services;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.wildfly.common.Assert.assertTrue;
+
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.List;
+
+import org.junit.jupiter.api.Test;
+import org.springframework.core.io.ClassPathResource;
+
+import com.knecon.fforesight.utility.rules.management.factory.RuleFileParser;
+import com.knecon.fforesight.utility.rules.management.models.BasicRule;
+import com.knecon.fforesight.utility.rules.management.models.RuleFileBluePrint;
+import com.knecon.fforesight.utility.rules.management.utils.RuleFileIO;
+
+import lombok.SneakyThrows;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+public class DroolsUpToDateTest {
+
+    @Test
+    @SneakyThrows
+    public void assertAllRuleFilesAreUpToDate() {
+
+        Path droolsPath = new ClassPathResource("drools").getFile().toPath();
+        Files.walk(droolsPath).filter(DroolsUpToDateTest::isEntityRuleFile).forEach(this::validateFile);
+    }
+
+
+    private static boolean isEntityRuleFile(Path droolsFile) {
+
+        String fileName = droolsFile.getFileName().toString();
+        return fileName.endsWith(".drl") && !fileName.endsWith("_components.drl") && !fileName.endsWith("_OLD.drl");
+    }
+
+
+    private void validateFile(Path path) {
+
+        log.info(path.toFile().getAbsolutePath());
+        RuleFileBluePrint allRules = RuleFileParser.buildBluePrintFromAllRuleFiles();
+        RuleFileBluePrint thisRules = RuleFileParser.buildBluePrintFromRulesString(RuleFileIO.getRulesString(path.toFile().getAbsolutePath()));
+        assertTrue(allRules.getAllRuleIdentifiers().containsAll(thisRules.getAllRuleIdentifiers()));
+        for (BasicRule rule : thisRules.getAllRules()) {
+            List<BasicRule> updatedRule = allRules.findRuleByIdentifier(rule.identifier());
+            assert updatedRule.size() == 1;
+            assertEquals(updatedRule.get(0), rule);
+        }
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesEnd2EndTest.java

@@ -17,6 +17,7 @@ import java.util.Set;
 import java.util.UUID;
 
 import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.extension.ExtendWith;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -40,7 +41,6 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Comment;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;
@@ -72,6 +72,7 @@ import com.knecon.fforesight.tenantcommons.TenantContext;
 
 import lombok.SneakyThrows;
 
+@Disabled // TODO: only disabled due to OOM in gradle testRunner in GitLab, find a solution!
 @ExtendWith(SpringExtension.class)
 @SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
 @Import(ManualChangesEnd2EndTest.TestConfiguration.class)
@@ -160,7 +161,9 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest {
         TextEntity expandedEntity = entityCreationService.byString(expandedEntityKeyword, "PII", EntityType.ENTITY, document).findFirst().get();
 
         String idToResize = redactionLog.getEntityLogEntry()
-                .stream().filter(entry -> entry.getValue().equals(testEntityValue1)).max(Comparator.comparingInt(EntityLogEntry::getStartOffset))
+                .stream()
+                .filter(entry -> entry.getValue().equals(testEntityValue1))
+                .max(Comparator.comparingInt(EntityLogEntry::getStartOffset))
                 .get()
                 .getId();
         List<Rectangle> resizedPositions = expandedEntity.getPositionsOnPagePerPage()
@@ -175,6 +178,7 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest {
                 .positions(resizedPositions)
                 .requestDate(OffsetDateTime.now())
                 .status(AnnotationStatus.APPROVED)
+                .updateDictionary(false)
                 .build();
         manualResizeRedaction.setUpdateDictionary(false);
         ManualRedactions manualRedactions = new ManualRedactions();
@@ -218,7 +222,6 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest {
 
         String manualAddId = UUID.randomUUID().toString();
 
-        Comment comment = Comment.builder().date(OffsetDateTime.now()).user("TEST_USER").text("This is a comment test").build();
         manualRedactions.setIdsToRemove(Set.of(IdRemoval.builder().annotationId("5b940b2cb401ed9f5be6fc24f6e77bcf").fileId("fileId").status(AnnotationStatus.DECLINED).build()));
         manualRedactions.setForceRedactions(Set.of(ManualForceRedaction.builder()
                 .annotationId("675eba69b0c2917de55462c817adaa05")
@@ -227,10 +230,6 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest {
                 .status(AnnotationStatus.APPROVED)
                 .build()));
 
-        manualRedactions.getComments().put("e5be0f1d941bbb92a068e198648d06c4", List.of(comment));
-        manualRedactions.getComments().put("0836727c3508a0b2ea271da69c04cc2f", List.of(comment));
-        manualRedactions.getComments().put(manualAddId, List.of(comment));
-
         ManualRedactionEntry manualRedactionEntry = new ManualRedactionEntry();
         manualRedactionEntry.setAnnotationId(manualAddId);
         manualRedactionEntry.setFileId("fileId");
@@ -362,9 +361,10 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest {
                 .annotationId(annotationId)
                 .requestDate(OffsetDateTime.now())
                 .value("Expand to Hint")
-                .positions(List.of(new Rectangle(new Point(56.8f, 496.27f), 61.25f, 12.83f, 2))).status(AnnotationStatus.APPROVED)
+                .positions(List.of(new Rectangle(new Point(56.8f, 496.27f), 61.25f, 12.83f, 2)))
+                .status(AnnotationStatus.APPROVED)
+                .updateDictionary(false)
                 .build();
-        manualResizeRedaction.setUpdateDictionary(false);
         manualRedactions.setResizeRedactions(Set.of(manualResizeRedaction));
         request.setManualRedactions(manualRedactions);
 

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesIntegrationTest.java

@@ -5,99 +5,27 @@ import static org.junit.jupiter.api.Assertions.assertTrue;
 import static org.wildfly.common.Assert.assertFalse;
 
 import java.awt.geom.Rectangle2D;
-import java.io.FileOutputStream;
 import java.time.OffsetDateTime;
 import java.util.Collection;
-import java.util.Comparator;
 import java.util.List;
 import java.util.Set;
 import java.util.stream.Collectors;
 
-import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
-import org.kie.api.KieServices;
-import org.kie.api.builder.KieBuilder;
-import org.kie.api.builder.KieFileSystem;
-import org.kie.api.builder.KieModule;
-import org.kie.api.runtime.KieContainer;
-import org.kie.api.runtime.KieSession;
-import org.kie.internal.io.ResourceFactory;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.beans.factory.annotation.Qualifier;
-import org.springframework.context.annotation.Bean;
-import org.springframework.context.annotation.Configuration;
-import org.springframework.context.annotation.Import;
 
-import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeResult;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
-import com.iqser.red.service.redaction.v1.server.annotate.AnnotateRequest;
-import com.iqser.red.service.redaction.v1.server.annotate.AnnotateResponse;
-import com.iqser.red.service.redaction.v1.server.document.graph.BuildDocumentIntegrationTest;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
-import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.PositionOnPage;
+import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Paragraph;
-import com.iqser.red.service.redaction.v1.server.redaction.utils.OsUtils;
-import com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService;
-import com.iqser.red.service.redaction.v1.server.service.document.EntityEnrichmentService;
-import com.iqser.red.service.redaction.v1.server.service.ManualChangesApplicationService;
-import com.iqser.red.service.redaction.v1.server.utils.exception.NotFoundException;
-import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
-
-import lombok.SneakyThrows;
-
-@Import(ManualChangesIntegrationTest.TestConfiguration.class)
-public class ManualChangesIntegrationTest extends BuildDocumentIntegrationTest {
-
-    private static final String RULES = "drools/manual_redaction_rules.drl";
-
-    @Autowired
-    private EntityEnrichmentService entityEnrichmentService;
-    private EntityCreationService entityCreationService;
-    private KieSession kieSession;
-
-    @Qualifier("kieContainer")
-    @Autowired
-    private KieContainer kieContainer;
-
-    @Configuration
-    @Import(BuildDocumentIntegrationTest.TestConfiguration.class)
-    public static class TestConfiguration {
-
-        @Bean
-        public KieContainer kieContainer() {
-
-            KieServices kieServices = KieServices.Factory.get();
-
-            KieFileSystem kieFileSystem = kieServices.newKieFileSystem();
-            kieFileSystem.write(ResourceFactory.newClassPathResource(RULES));
-            KieBuilder kieBuilder = kieServices.newKieBuilder(kieFileSystem);
-            kieBuilder.buildAll();
-            KieModule kieModule = kieBuilder.getKieModule();
-
-            return kieServices.newKieContainer(kieModule.getReleaseId());
-        }
-
-    }
-
-
-    @BeforeEach
-    public void createServices() {
-
-        kieSession = kieContainer.newKieSession();
-        entityCreationService = new EntityCreationService(entityEnrichmentService, kieSession);
-        ManualChangesApplicationService manualChangesApplicationService = new ManualChangesApplicationService(entityCreationService);
-        kieSession.setGlobal("manualChangesApplicationService", manualChangesApplicationService);
-        kieSession.setGlobal("entityCreationService", entityCreationService);
-    }
+import com.iqser.red.service.redaction.v1.server.rules.RulesIntegrationTest;
 
+public class ManualChangesIntegrationTest extends RulesIntegrationTest {
 
     @Test
     public void manualResizeRedactionTest() {
@@ -117,14 +45,10 @@ public class ManualChangesIntegrationTest extends BuildDocumentIntegrationTest {
                 .positions(toAnnotationRectangles(biggerEntity.getPositionsOnPagePerPage().get(0)))
                 .status(AnnotationStatus.APPROVED)
                 .requestDate(OffsetDateTime.now())
+                .updateDictionary(false)
                 .build();
 
-        
-        kieSession.insert(document);
-        document.streamAllSubNodes().forEach(kieSession::insert);
-        kieSession.insert(manualResizeRedaction);
-        kieSession.fireAllRules();
-        kieSession.dispose();
+        doAnalysis(document, List.of(manualResizeRedaction));
 
         assertEquals(biggerEntity.getTextRange(), entity.getTextRange());
         assertEquals(biggerEntity.getDeepestFullyContainingNode(), entity.getDeepestFullyContainingNode());
@@ -153,12 +77,7 @@ public class ManualChangesIntegrationTest extends BuildDocumentIntegrationTest {
                 .requestDate(OffsetDateTime.now())
                 .build();
 
-        
-        kieSession.insert(manualForceRedaction);
-        kieSession.insert(document);
-        document.streamAllSubNodes().forEach(kieSession::insert);
-        kieSession.fireAllRules();
-        kieSession.dispose();
+        doAnalysis(document, List.of(manualForceRedaction));
 
         assertEquals(Paragraph.class, entity.getDeepestFullyContainingNode().getClass());
         assertFalse(entity.getIntersectingNodes().isEmpty());
@@ -183,12 +102,7 @@ public class ManualChangesIntegrationTest extends BuildDocumentIntegrationTest {
         String initialId = entity.getPositionsOnPagePerPage().get(0).getId();
         IdRemoval idRemoval = IdRemoval.builder().annotationId(initialId).status(AnnotationStatus.APPROVED).requestDate(OffsetDateTime.now()).build();
 
-        
-        kieSession.insert(document);
-        document.streamAllSubNodes().forEach(kieSession::insert);
-        kieSession.insert(idRemoval);
-        kieSession.fireAllRules();
-        kieSession.dispose();
+        doAnalysis(document, List.of(idRemoval));
 
         assertEquals("David Ksenia", entity.getValue());
         assertEquals(initialId, entity.getPositionsOnPagePerPage().get(0).getId());
@@ -213,13 +127,7 @@ public class ManualChangesIntegrationTest extends BuildDocumentIntegrationTest {
                 .requestDate(OffsetDateTime.now())
                 .build();
 
-        
-        kieSession.insert(document);
-        document.streamAllSubNodes().forEach(kieSession::insert);
-        kieSession.insert(idRemoval);
-        kieSession.insert(manualForceRedaction);
-        kieSession.fireAllRules();
-        kieSession.dispose();
+        doAnalysis(document, List.of(manualForceRedaction));
 
         assertEquals(Paragraph.class, entity.getDeepestFullyContainingNode().getClass());
         assertFalse(entity.getIntersectingNodes().isEmpty());
@@ -242,12 +150,7 @@ public class ManualChangesIntegrationTest extends BuildDocumentIntegrationTest {
         String initialId = entity.getPositionsOnPagePerPage().get(0).getId();
         IdRemoval idRemoval = IdRemoval.builder().annotationId(initialId).status(AnnotationStatus.REQUESTED).build();
 
-        
-        kieSession.insert(idRemoval);
-        kieSession.insert(document);
-        document.streamAllSubNodes().forEach(kieSession::insert);
-        kieSession.fireAllRules();
-        kieSession.dispose();
+        doAnalysis(document, List.of(idRemoval));
 
         assertEquals(Paragraph.class, entity.getDeepestFullyContainingNode().getClass());
         assertFalse(entity.getIntersectingNodes().isEmpty());

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualEntityTest.java

@@ -7,6 +7,7 @@ import static org.wildfly.common.Assert.assertFalse;
 import static org.wildfly.common.Assert.assertTrue;
 
 import java.awt.geom.Rectangle2D;
+import java.time.OffsetDateTime;
 import java.util.List;
 import java.util.Set;
 
@@ -19,12 +20,14 @@ import org.springframework.boot.test.mock.mockito.MockBean;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Point;
 import com.iqser.red.service.redaction.v1.server.document.graph.BuildDocumentIntegrationTest;
 import com.iqser.red.service.redaction.v1.server.model.ManualEntity;
 import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryVersion;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
+import com.iqser.red.service.redaction.v1.server.model.document.entity.IEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
 import com.iqser.red.service.redaction.v1.server.service.DictionaryService;
@@ -35,7 +38,7 @@ import com.iqser.red.service.redaction.v1.server.service.document.ManualEntityCr
 
 import lombok.SneakyThrows;
 
-public class ManualEntityCreationServiceTest extends BuildDocumentIntegrationTest {
+public class ManualEntityTest extends BuildDocumentIntegrationTest {
 
     @Autowired
     private EntityEnrichmentService entityEnrichmentService;
@@ -63,39 +66,7 @@ public class ManualEntityCreationServiceTest extends BuildDocumentIntegrationTes
     @SneakyThrows
     public void manualAddRedactionTest() {
 
-        Document document = buildGraph("files/new/VV-919901.pdf");
-        EntityCreationService entityCreationService = new EntityCreationService(entityEnrichmentService);
-
-        List<TextEntity> tempEntities = entityCreationService.byString("To: Syngenta Ltd.", "temp", EntityType.ENTITY, document).toList();
-        assertFalse(tempEntities.isEmpty());
-        var tempEntity = tempEntities.get(0);
-        List<Rectangle> positions = tempEntity.getPositionsOnPagePerPage()
-                .stream()
-                .flatMap(redactionPosition -> redactionPosition.getRectanglePerLine()
-                        .stream()
-                        .map(rectangle2D -> toAnnotationRectangle(rectangle2D, redactionPosition.getPage().getNumber())))
-                .toList();
-
-        ManualRedactionEntry manualRedactionEntry = ManualRedactionEntry.builder()
-                .type("manual")
-                .value(tempEntity.getValue())
-                .reason("reason")
-                .legalBasis("n-a")
-                .section(tempEntity.getDeepestFullyContainingNode().toString())
-                .rectangle(true)
-                .positions(positions)
-                .textAfter("")
-                .textBefore("")
-                .build();
-
-        tempEntity.removeFromGraph();
-        assertTrue(document.getEntities().isEmpty());
-
-        List<ManualEntity> notFoundManualEntities = manualEntityCreationService.createRedactionEntitiesIfFoundAndReturnNotFoundEntries(Set.of(manualRedactionEntry),
-                document,
-                TEST_DOSSIER_TEMPLATE_ID);
-        assertTrue(notFoundManualEntities.isEmpty());
-        assertEquals(1, document.getEntities().size());
+        createFoundManualRedaction();
     }
 
 
@@ -103,6 +74,34 @@ public class ManualEntityCreationServiceTest extends BuildDocumentIntegrationTes
     @SneakyThrows
     public void manualAddRedactionFailingTest() {
 
+        createNotFoundManualRedaction();
+    }
+
+
+    @Test
+    @SneakyThrows
+    public void testFoundManualAddRedactionAndRemovedHasStateRemoved() {
+
+        DocumentAndEntity context = createFoundManualRedaction();
+        IdRemoval removal = IdRemoval.builder().requestDate(OffsetDateTime.now()).build();
+        context.entity().getManualOverwrite().addChange(removal);
+        assertTrue(context.entity().removed());
+    }
+
+
+    @Test
+    @SneakyThrows
+    public void testNotFoundManualAddRedactionAndRemovedHasStateRemoved() {
+
+        DocumentAndEntity context = createNotFoundManualRedaction();
+        IdRemoval removal = IdRemoval.builder().requestDate(OffsetDateTime.now()).build();
+        context.entity().getManualOverwrite().addChange(removal);
+        assertTrue(context.entity().removed());
+    }
+
+
+    private DocumentAndEntity createNotFoundManualRedaction() {
+
         Document document = buildGraph("files/new/VV-919901.pdf");
         // This is important due to PDFTron Web Viewer reordering the content, such that this string is selectable.
         String value = "To: Syngenta Ltd. Jealott’s Hill";
@@ -115,6 +114,7 @@ public class ManualEntityCreationServiceTest extends BuildDocumentIntegrationTes
                 .section("n-a")
                 .rectangle(true)
                 .positions(List.of(new Rectangle(new Point(90, 262), 11, 88, 1), new Rectangle(new Point(90, 247), 11, 131, 1)))
+                .requestDate(OffsetDateTime.now())
                 .textAfter("")
                 .textBefore("")
                 .build();
@@ -136,6 +136,47 @@ public class ManualEntityCreationServiceTest extends BuildDocumentIntegrationTes
         assertEquals(1, redactionLogEntries.size());
         assertEquals(value, redactionLogEntries.get(0).getValue());
         assertEquals(type, redactionLogEntries.get(0).getType());
+        return new DocumentAndEntity(document, notFoundManualEntities.get(0));
+    }
+
+
+    private DocumentAndEntity createFoundManualRedaction() {
+
+        Document document = buildGraph("files/new/VV-919901.pdf");
+        EntityCreationService entityCreationService = new EntityCreationService(entityEnrichmentService);
+
+        List<TextEntity> tempEntities = entityCreationService.byString("To: Syngenta Ltd.", "temp", EntityType.ENTITY, document).toList();
+        assertFalse(tempEntities.isEmpty());
+        var tempEntity = tempEntities.get(0);
+        List<Rectangle> positions = tempEntity.getPositionsOnPagePerPage()
+                .stream()
+                .flatMap(redactionPosition -> redactionPosition.getRectanglePerLine()
+                        .stream()
+                        .map(rectangle2D -> toAnnotationRectangle(rectangle2D, redactionPosition.getPage().getNumber())))
+                .toList();
+
+        ManualRedactionEntry manualRedactionEntry = ManualRedactionEntry.builder()
+                .type("manual")
+                .value(tempEntity.getValue())
+                .reason("reason")
+                .legalBasis("n-a")
+                .section(tempEntity.getDeepestFullyContainingNode().toString())
+                .rectangle(true)
+                .positions(positions)
+                .requestDate(OffsetDateTime.now())
+                .textAfter("")
+                .textBefore("")
+                .build();
+
+        tempEntity.removeFromGraph();
+        assertTrue(document.getEntities().isEmpty());
+
+        List<ManualEntity> notFoundManualEntities = manualEntityCreationService.createRedactionEntitiesIfFoundAndReturnNotFoundEntries(Set.of(manualRedactionEntry),
+                document,
+                TEST_DOSSIER_TEMPLATE_ID);
+        assertTrue(notFoundManualEntities.isEmpty());
+        assertEquals(1, document.getEntities().size());
+        return new DocumentAndEntity(document, document.getEntities().stream().findFirst().get());
     }
 
 
@@ -147,4 +188,9 @@ public class ManualEntityCreationServiceTest extends BuildDocumentIntegrationTes
                 pageNumber);
     }
 
+
+    private record DocumentAndEntity(Document document, IEntity entity) {
+
+    }
+
 }

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/rules/Cbi11Test.java

@@ -0,0 +1,36 @@
+package com.iqser.red.service.redaction.v1.server.rules;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.wildfly.common.Assert.assertFalse;
+
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+
+import org.junit.jupiter.api.Test;
+
+import com.iqser.red.service.redaction.v1.server.model.dictionary.Dictionary;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
+import com.iqser.red.service.redaction.v1.server.utils.Patterns;
+
+public class Cbi11Test extends RulesIntegrationTest {
+
+    @Test
+    public void multiAuthorNameSplittingTest() {
+
+        Document document = buildGraph("files/Minimal Examples/multipleAuthorsInTable.pdf");
+
+        doAnalysis(document, Collections.emptyList());
+
+        List<String> authorNames = document.getEntities().stream().map(Dictionary::splitIntoAuthorNames).flatMap(Collection::stream).toList();
+
+        assertThat(authorNames).containsExactlyInAnyOrder("Cargile", "N.L.", "Ross", "J.A.", "Egli", "Ramsteiner");
+    }
+
+    @Test
+    public void testAndIsNotAnAuthorName() {
+
+        assertFalse(Patterns.AUTHOR_NAME_PATTERN.matcher("and").matches());
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/rules/RulesIntegrationTest.java

@@ -0,0 +1,80 @@
+package com.iqser.red.service.redaction.v1.server.rules;
+
+import java.util.Collection;
+
+import org.junit.jupiter.api.BeforeEach;
+import org.kie.api.KieServices;
+import org.kie.api.builder.KieBuilder;
+import org.kie.api.builder.KieFileSystem;
+import org.kie.api.builder.KieModule;
+import org.kie.api.runtime.KieContainer;
+import org.kie.api.runtime.KieSession;
+import org.kie.internal.io.ResourceFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.context.annotation.Import;
+
+import com.iqser.red.service.redaction.v1.server.document.graph.BuildDocumentIntegrationTest;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
+import com.iqser.red.service.redaction.v1.server.service.ManualChangesApplicationService;
+import com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService;
+import com.iqser.red.service.redaction.v1.server.service.document.EntityEnrichmentService;
+
+public class RulesIntegrationTest extends BuildDocumentIntegrationTest {
+
+    protected static final String RULES = "drools/rules.drl";
+
+    @Autowired
+    protected EntityEnrichmentService entityEnrichmentService;
+    protected EntityCreationService entityCreationService;
+    protected KieSession kieSession;
+
+    @Qualifier("kieContainer")
+    @Autowired
+    private KieContainer kieContainer;
+
+
+    protected void doAnalysis(Document document, Collection<Object> objectToInsert) {
+
+        kieSession.insert(document);
+        document.streamAllSubNodes().forEach(kieSession::insert);
+        objectToInsert.forEach(kieSession::insert);
+        kieSession.fireAllRules();
+        kieSession.dispose();
+    }
+
+
+    @Configuration
+    @Import(BuildDocumentIntegrationTest.TestConfiguration.class)
+    public static class TestConfiguration {
+
+        @Bean
+        public KieContainer kieContainer() {
+
+            KieServices kieServices = KieServices.Factory.get();
+
+            KieFileSystem kieFileSystem = kieServices.newKieFileSystem();
+            kieFileSystem.write(ResourceFactory.newClassPathResource(RULES));
+            KieBuilder kieBuilder = kieServices.newKieBuilder(kieFileSystem);
+            kieBuilder.buildAll();
+            KieModule kieModule = kieBuilder.getKieModule();
+
+            return kieServices.newKieContainer(kieModule.getReleaseId());
+        }
+
+    }
+
+
+    @BeforeEach
+    public void createServices() {
+
+        kieSession = kieContainer.newKieSession();
+        entityCreationService = new EntityCreationService(entityEnrichmentService, kieSession);
+        ManualChangesApplicationService manualChangesApplicationService = new ManualChangesApplicationService(entityCreationService);
+        kieSession.setGlobal("manualChangesApplicationService", manualChangesApplicationService);
+        kieSession.setGlobal("entityCreationService", entityCreationService);
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/service/document/UnprocessedChangesServiceTest.java

@@ -148,6 +148,10 @@ public class UnprocessedChangesServiceTest extends AbstractRedactionIntegrationT
         assertEquals(unprocessedManualEntity.getColor()[0], 0f);
         assertEquals(unprocessedManualEntity.getColor()[1], 0f);
         assertEquals(unprocessedManualEntity.getColor()[2], 0f);
+        assertEquals(unprocessedManualEntity.getPositions().get(0).x(), 355.53775f);
+        assertEquals(unprocessedManualEntity.getPositions().get(0).y(), 266.49002f);
+        assertEquals(unprocessedManualEntity.getPositions().get(0).w(), 29.322266f);
+        assertEquals(unprocessedManualEntity.getPositions().get(0).h(), 11.017679f);
 
         Optional<UnprocessedManualEntity> optionalNotFoundUnprocessedManualEntity = unprocessedManualEntities.stream().filter(manualEntity -> manualEntity.getAnnotationId().equals(notFoundId)).findFirst();
         assertTrue(optionalNotFoundUnprocessedManualEntity.isPresent());
@@ -163,5 +167,10 @@ public class UnprocessedChangesServiceTest extends AbstractRedactionIntegrationT
         assertEquals(unprocessedNotFoundManualEntity.getColor()[0], 0f);
         assertEquals(unprocessedNotFoundManualEntity.getColor()[1], 0f);
         assertEquals(unprocessedNotFoundManualEntity.getColor()[2], 0f);
+        assertEquals(unprocessedNotFoundManualEntity.getPositions().get(0).getPageNumber(), 1);
+        assertEquals(unprocessedNotFoundManualEntity.getPositions().get(0).getRectangle()[0], 1f);
+        assertEquals(unprocessedNotFoundManualEntity.getPositions().get(0).getRectangle()[1], 1f);
+        assertEquals(unprocessedNotFoundManualEntity.getPositions().get(0).getRectangle()[2], 1f);
+        assertEquals(unprocessedNotFoundManualEntity.getPositions().get(0).getRectangle()[3], 1f);
     }
 }

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/acceptance_rules.drl

@@ -82,25 +82,25 @@ rule "SYN.1.0: Recommend CTL/BL laboratory that start with BL or CTL"
 //------------------------------------ CBI rules ------------------------------------
 
 // Rule unit: CBI.0
-rule "CBI.0.0: Redact CBI Authors (Non Vertebrate Study)"
+rule "CBI.0.0: Redact CBI Authors (non vertebrate Study)"
     when
         not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
         $entity: TextEntity(type == "CBI_author", dictionaryEntry)
     then
-        $entity.apply("CBI.0.0", "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        $entity.redact("CBI.0.0", "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
     end
 
-rule "CBI.0.1: Redact CBI Authors (Vertebrate Study)"
+rule "CBI.0.1: Redact CBI Authors (vertebrate Study)"
     when
         FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
         $entity: TextEntity(type == "CBI_author", dictionaryEntry)
     then
-        $entity.apply("CBI.0.1", "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        $entity.redact("CBI.0.1", "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
     end
 
 
 // Rule unit: CBI.1
-rule "CBI.1.0: Don't redact CBI Address (Non Vertebrate Study)"
+rule "CBI.1.0: Do not redact CBI Address (non vertebrate Study)"
     when
         not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
         $entity: TextEntity(type == "CBI_address", dictionaryEntry)
@@ -108,19 +108,19 @@ rule "CBI.1.0: Don't redact CBI Address (Non Vertebrate Study)"
         $entity.skip("CBI.1.0", "Address found for Non Vertebrate Study");
     end
 
-rule "CBI.1.1: Redact CBI Address (Vertebrate Study)"
+rule "CBI.1.1: Redact CBI Address (vertebrate Study)"
     when
         FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
         $entity: TextEntity(type == "CBI_address", dictionaryEntry)
     then
-        $entity.apply("CBI.1.1", "Address found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        $entity.redact("CBI.1.1", "Address found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
     end
 
 
 // Rule unit: CBI.2
-rule "CBI.2.0: Don't redact genitive CBI_author"
+rule "CBI.2.0: Do not redact genitive CBI Author"
     when
-        $entity: TextEntity(type == "CBI_author", anyMatch(textAfter, "['’’'ʼˈ´`‘′ʻ’']s"), applied())
+        $entity: TextEntity(type == "CBI_author", anyMatch(textAfter, "['’’'ʼˈ´`‘′ʻ’']s"))
     then
         entityCreationService.byTextRange($entity.getTextRange(), "CBI_author", EntityType.FALSE_POSITIVE, document)
         .ifPresent(falsePositive -> falsePositive.skip("CBI.2.0", "Genitive Author found"));
@@ -128,7 +128,7 @@ rule "CBI.2.0: Don't redact genitive CBI_author"
 
 
 // Rule unit: CBI.7
-rule "CBI.7.0: Do not redact Names and Addresses if published information found in section without tables"
+rule "CBI.7.0: Do not redact Names and Addresses if published information found in Section without tables"
     when
         $section: Section(!hasTables(),
                           hasEntitiesOfType("published_information"),
@@ -146,22 +146,17 @@ rule "CBI.7.0: Do not redact Names and Addresses if published information found
 
 rule "CBI.7.1: Do not redact Names and Addresses if published information found in same table row"
     when
-        $table: Table(hasEntitiesOfType("published_information"),
-                     (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
+        $table: Table(hasEntitiesOfType("published_information"), hasEntitiesOfType("CBI_author"))
+        $cellsWithPublishedInformation: TableCell() from $table.streamTableCellsWhichContainType("published_information").toList()
+        $tableCell: TableCell(row == $cellsWithPublishedInformation.row) from $table.streamTableCells().toList()
+        $author: TextEntity(type == "CBI_author", active()) from $tableCell.getEntities()
     then
-        $table.streamEntitiesWhereRowContainsEntitiesOfType(List.of("CBI_author", "CBI_address"))
-            .forEach(redactionEntity -> {
-                redactionEntity.skipWithReferences(
-                        "CBI.7.1",
-                        "Published Information found in row",
-                        $table.getEntitiesOfTypeInSameRow("published_information", redactionEntity)
-                );
-            });
+        $author.skipWithReferences("CBI.7.1", "Published Information found in row", $table.getEntitiesOfTypeInSameRow("published_information", $author));
     end
 
 
 // Rule unit: CBI.9
-rule "CBI.9.0: Redact all Cell's with Header Author(s) as CBI_author (non vertebrate study)"
+rule "CBI.9.0: Redact all cells with Header Author(s) as CBI_author (non vertebrate study)"
     agenda-group "LOCAL_DICTIONARY_ADDS"
     when
         not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
@@ -171,10 +166,10 @@ rule "CBI.9.0: Redact all Cell's with Header Author(s) as CBI_author (non verteb
             .map(tableCell -> entityCreationService.bySemanticNode(tableCell, "CBI_author", EntityType.ENTITY))
             .filter(Optional::isPresent)
             .map(Optional::get)
-            .forEach(redactionEntity -> redactionEntity.apply("CBI.9.0", "Author(s) found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
+            .forEach(redactionEntity -> redactionEntity.redact("CBI.9.0", "Author(s) found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
     end
 
-rule "CBI.9.1: Redact all Cell's with Header Author as CBI_author (non vertebrate study)"
+rule "CBI.9.1: Redact all cells with Header Author as CBI_author (non vertebrate study)"
     agenda-group "LOCAL_DICTIONARY_ADDS"
     when
         not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
@@ -184,12 +179,12 @@ rule "CBI.9.1: Redact all Cell's with Header Author as CBI_author (non vertebrat
             .map(tableCell -> entityCreationService.bySemanticNode(tableCell, "CBI_author", EntityType.ENTITY))
             .filter(Optional::isPresent)
             .map(Optional::get)
-            .forEach(redactionEntity -> redactionEntity.apply("CBI.9.1", "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
+            .forEach(redactionEntity -> redactionEntity.redact("CBI.9.1", "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
     end
 
 
 // Rule unit: CBI.10
-rule "CBI.10.0: Redact all Cell's with Header Author(s) as CBI_author (vertebrate study)"
+rule "CBI.10.0: Redact all cells with Header Author(s) as CBI_author (vertebrate study)"
     agenda-group "LOCAL_DICTIONARY_ADDS"
     when
         FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
@@ -199,10 +194,10 @@ rule "CBI.10.0: Redact all Cell's with Header Author(s) as CBI_author (vertebrat
             .map(tableCell -> entityCreationService.bySemanticNode(tableCell, "CBI_author", EntityType.ENTITY))
             .filter(Optional::isPresent)
             .map(Optional::get)
-            .forEach(redactionEntity -> redactionEntity.apply("CBI.10.0", "Author(s) found", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
+            .forEach(redactionEntity -> redactionEntity.redact("CBI.10.0", "Author(s) found", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
     end
 
-rule "CBI.10.1: Redact all Cell's with Header Author as CBI_author (vertebrate study)"
+rule "CBI.10.1: Redact all cells with Header Author as CBI_author (vertebrate study)"
     agenda-group "LOCAL_DICTIONARY_ADDS"
     when
         FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
@@ -212,7 +207,7 @@ rule "CBI.10.1: Redact all Cell's with Header Author as CBI_author (vertebrate s
             .map(tableCell -> entityCreationService.bySemanticNode(tableCell, "CBI_author", EntityType.ENTITY))
             .filter(Optional::isPresent)
             .map(Optional::get)
-            .forEach(redactionEntity -> redactionEntity.apply("CBI.10.1", "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
+            .forEach(redactionEntity -> redactionEntity.redact("CBI.10.1", "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
     end
 
 
@@ -223,12 +218,12 @@ rule "CBI.11.0: Recommend all CBI_author entities in Table with Vertebrate Study
     when
         $table: Table(hasHeader("Author(s)") && hasHeader("Vertebrate Study Y/N"))
     then
-        $table.getEntitiesOfType("CBI_author").forEach(entity -> dictionary.addMultipleAuthorsAsRecommendation(entity));
+        $table.getEntitiesOfType("CBI_author").stream().filter(IEntity::applied).forEach(entity -> dictionary.addMultipleAuthorsAsRecommendation(entity));
     end
 
 
 // Rule unit: CBI.16
-rule "CBI.16.0: Add CBI_author with \"et al.\" Regex (non vertebrate study)"
+rule "CBI.16.0: Add CBI_author with \"et al.\" RegEx (non vertebrate study)"
     agenda-group "LOCAL_DICTIONARY_ADDS"
     when
         not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
@@ -236,12 +231,12 @@ rule "CBI.16.0: Add CBI_author with \"et al.\" Regex (non vertebrate study)"
     then
         entityCreationService.byRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, 1, $section)
             .forEach(entity -> {
-                entity.apply("CBI.16.0", "Author found by \"et al\" regex", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+                entity.redact("CBI.16.0", "Author found by \"et al\" regex", "Article 39(e)(3) of Regulation (EC) No 178/2002");
                 dictionary.recommendEverywhere(entity);
             });
     end
 
-rule "CBI.16.1: Add CBI_author with \"et al.\" Regex (vertebrate study)"
+rule "CBI.16.1: Add CBI_author with \"et al.\" RegEx (vertebrate study)"
     agenda-group "LOCAL_DICTIONARY_ADDS"
     when
         FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
@@ -249,7 +244,7 @@ rule "CBI.16.1: Add CBI_author with \"et al.\" Regex (vertebrate study)"
     then
         entityCreationService.byRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, 1, $section)
             .forEach(entity -> {
-                entity.apply("CBI.16.1", "Author found by \"et al\" regex", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+                entity.redact("CBI.16.1", "Author found by \"et al\" regex", "Article 39(e)(2) of Regulation (EC) No 178/2002");
                 dictionary.recommendEverywhere(entity);
             });
     end
@@ -295,7 +290,7 @@ rule "CBI.20.1: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJEC
     then
          entityCreationService.betweenStrings("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", EntityType.ENTITY, $section)
          .forEach(laboratoryEntity -> {
-             laboratoryEntity.apply("CBI.20.1", "PERFORMING LABORATORY was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+             laboratoryEntity.redact("CBI.20.1", "PERFORMING LABORATORY was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
              dictionary.recommendEverywhere(laboratoryEntity);
          });
     end
@@ -309,7 +304,7 @@ rule "PII.0.0: Redact all PII (non vertebrate study)"
         not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
         $pii: TextEntity(type == "PII", dictionaryEntry)
     then
-        $pii.apply("PII.0.0", "Personal Information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        $pii.redact("PII.0.0", "Personal Information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
     end
 
 rule "PII.0.1: Redact all PII (vertebrate study)"
@@ -317,7 +312,7 @@ rule "PII.0.1: Redact all PII (vertebrate study)"
         FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
         $pii: TextEntity(type == "PII", dictionaryEntry)
     then
-        $pii.apply("PII.0.1", "Personal Information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        $pii.redact("PII.0.1", "Personal Information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
     end
 
 
@@ -328,7 +323,7 @@ rule "PII.1.0: Redact Emails by RegEx (Non vertebrate study)"
         $section: Section(containsString("@"))
     then
         entityCreationService.byRegex("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", "PII", EntityType.ENTITY, 1, $section)
-            .forEach(emailEntity -> emailEntity.apply("PII.1.0", "Found by Email Regex", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
+            .forEach(emailEntity -> emailEntity.redact("PII.1.0", "Found by Email Regex", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
     end
 
 rule "PII.1.1: Redact Emails by RegEx (vertebrate study)"
@@ -337,7 +332,7 @@ rule "PII.1.1: Redact Emails by RegEx (vertebrate study)"
         $section: Section(containsString("@"))
     then
         entityCreationService.byRegex("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", "PII", EntityType.ENTITY, 1, $section)
-            .forEach(emailEntity -> emailEntity.apply("PII.1.1", "Found by Email Regex", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
+            .forEach(emailEntity -> emailEntity.redact("PII.1.1", "Found by Email Regex", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
     end
 
 
@@ -357,7 +352,7 @@ rule "PII.2.0: Redact Phone and Fax by RegEx (non vertebrate study)"
                               containsString("Fer"))
     then
         entityCreationService.byRegexIgnoreCase("\\b(contact|telephone|phone|ph\\.|fax|tel|ter|mobile|fel|fer)[a-zA-Z\\s]{0,10}[:.\\s]{0,3}([\\+\\d\\(][\\s\\d\\(\\)\\-\\/\\.]{4,100}\\d)\\b", "PII", EntityType.ENTITY, 2, $section)
-            .forEach(contactEntity -> contactEntity.apply("PII.2.0", "Found by Phone and Fax Regex", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
+            .forEach(contactEntity -> contactEntity.redact("PII.2.0", "Found by Phone and Fax Regex", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
     end
 
 rule "PII.2.1: Redact Phone and Fax by RegEx (vertebrate study)"
@@ -375,45 +370,27 @@ rule "PII.2.1: Redact Phone and Fax by RegEx (vertebrate study)"
                               containsString("Fer"))
     then
         entityCreationService.byRegexIgnoreCase("\\b(contact|telephone|phone|ph\\.|fax|tel|ter|mobile|fel|fer)[a-zA-Z\\s]{0,10}[:.\\s]{0,3}([\\+\\d\\(][\\s\\d\\(\\)\\-\\/\\.]{4,100}\\d)\\b", "PII", EntityType.ENTITY, 2, $section)
-            .forEach(contactEntity -> contactEntity.apply("PII.2.1", "Found by Phone and Fax Regex", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
+            .forEach(contactEntity -> contactEntity.redact("PII.2.1", "Found by Phone and Fax Regex", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
     end
 
 
 // Rule unit: PII.9
-rule "PII.9.0: Redact between \"AUTHOR(S)\" and \"COMPLETION DATE\" (non vertebrate study)"
+rule "PII.9.0: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\" (non vertebrate study)"
     when
         not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
-        $section: Section(!hasTables(), containsString("AUTHOR(S):"), containsString("COMPLETION DATE:"), !containsString("STUDY COMPLETION DATE:"))
+        $document: Document(containsStringIgnoreCase("AUTHOR(S)"), containsAnyStringIgnoreCase("COMPLETION DATE", "STUDY COMPLETION DATE"))
     then
-        entityCreationService.betweenStrings("AUTHOR(S):", "COMPLETION DATE:", "PII", EntityType.ENTITY, $section)
-         .forEach(authorEntity -> authorEntity.apply("PII.9.0", "AUTHOR(S) was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
+        entityCreationService.shortestBetweenAnyStringIgnoreCase(List.of("AUTHOR(S)", "AUTHOR(S):"), List.of("COMPLETION DATE", "COMPLETION DATE:", "STUDY COMPLETION DATE", "STUDY COMPLETION DATE:"), "PII", EntityType.ENTITY, $document)
+         .forEach(authorEntity -> authorEntity.redact("PII.9.0", "AUTHOR(S) was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
     end
 
-rule "PII.9.1: Redact between \"AUTHOR(S)\" and \"STUDY COMPLETION DATE\" (non vertebrate study)"
+rule "PII.9.1: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\" (vertebrate study)"
     when
         FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
-        $section: Section(!hasTables(), containsString("AUTHOR(S):"), containsString("COMPLETION DATE:"), !containsString("STUDY COMPLETION DATE:"))
+        $document: Document(containsStringIgnoreCase("AUTHOR(S)"), containsAnyStringIgnoreCase("COMPLETION DATE", "STUDY COMPLETION DATE"))
     then
-        entityCreationService.betweenStrings("AUTHOR(S):", "COMPLETION DATE:", "PII", EntityType.ENTITY, $section)
-         .forEach(authorEntity -> authorEntity.apply("PII.9.1", "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
-    end
-
-rule "PII.9.2: Redact between \"AUTHOR(S)\" and \"COMPLETION DATE\" (non vertebrate study)"
-    when
-        not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
-        $section: Section(!hasTables(), containsString("AUTHOR(S):"), containsString("STUDY COMPLETION DATE:"))
-    then
-        entityCreationService.betweenStrings("AUTHOR(S):", "STUDY COMPLETION DATE:", "PII", EntityType.ENTITY, $section)
-         .forEach(authorEntity -> authorEntity.apply("PII.9.2", "AUTHOR(S) was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
-    end
-
-rule "PII.9.3: Redact between \"AUTHOR(S)\" and \"STUDY COMPLETION DATE\" (vertebrate study)"
-    when
-        FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
-        $section: Section(!hasTables(), containsString("AUTHOR(S):"), containsString("STUDY COMPLETION DATE:"))
-    then
-        entityCreationService.betweenStrings("AUTHOR(S):", "STUDY COMPLETION DATE:", "PII", EntityType.ENTITY, $section)
-         .forEach(authorEntity -> authorEntity.apply("PII.9.3", "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
+        entityCreationService.shortestBetweenAnyStringIgnoreCase(List.of("AUTHOR(S)", "AUTHOR(S):"), List.of("COMPLETION DATE", "COMPLETION DATE:", "STUDY COMPLETION DATE", "STUDY COMPLETION DATE:"), "PII", EntityType.ENTITY, $document)
+        .forEach(authorEntity -> authorEntity.redact("PII.9.1", "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
     end
 
 
@@ -435,25 +412,33 @@ rule "ETC.2.0: Redact signatures (non vertebrate study)"
         not FileAttribute(label == "Vertebrate Study", value == "Yes")
         $signature: Image(imageType == ImageType.SIGNATURE)
     then
-        $signature.apply("ETC.2.0", "Signature Found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        $signature.redact("ETC.2.0", "Signature Found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+    end
+
+rule "ETC.2.1: Redact signatures (vertebrate study)"
+    when
+        FileAttribute(label == "Vertebrate Study", value == "Yes")
+        $signature: Image(imageType == ImageType.SIGNATURE)
+    then
+        $signature.redact("ETC.2.1", "Signature Found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
     end
 
 
 // Rule unit: ETC.3
-rule "ETC.3.0: Redact logos (vertebrate study)"
+rule "ETC.3.0: Skip logos (non vertebrate study)"
     when
         not FileAttribute(label == "Vertebrate Study", value == "Yes")
         $logo: Image(imageType == ImageType.LOGO)
     then
-        $logo.apply("ETC.3.0", "Logo Found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        $logo.skip("ETC.3.0", "Logo Found");
     end
 
-rule "ETC.3.1: Redact logos (non vertebrate study)"
+rule "ETC.3.1: Redact logos (vertebrate study)"
     when
         FileAttribute(label == "Vertebrate Study", value == "Yes")
         $logo: Image(imageType == ImageType.LOGO)
     then
-        $logo.apply("ETC.3.1", "Logo Found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        $logo.redact("ETC.3.1", "Logo Found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
     end
 
 
@@ -471,7 +456,7 @@ rule "ETC.5.0: Ignore dossier_redaction entries if confidentiality is not 'confi
 //------------------------------------ AI rules ------------------------------------
 
 // Rule unit: AI.0
-rule "AI.0.0: add all NER Entities of type CBI_author"
+rule "AI.0.0: Add all NER Entities of type CBI_author"
     salience 999
     when
         nerEntities: NerEntities(hasEntitiesOfType("CBI_author"))
@@ -482,7 +467,7 @@ rule "AI.0.0: add all NER Entities of type CBI_author"
 
 
 // Rule unit: AI.1
-rule "AI.1.0: combine and add NER Entities as CBI_address"
+rule "AI.1.0: Combine and add NER Entities as CBI_address"
     salience 999
     when
         nerEntities: NerEntities(hasEntitiesOfType("ORG") || hasEntitiesOfType("STREET") || hasEntitiesOfType("CITY"))
@@ -491,7 +476,7 @@ rule "AI.1.0: combine and add NER Entities as CBI_address"
     end
 
 
-//------------------------------------ Manual redaction rules ------------------------------------
+//------------------------------------ Manual changes rules ------------------------------------
 
 // Rule unit: MAN.0
 rule "MAN.0.0: Apply manual resize redaction"
@@ -636,7 +621,7 @@ rule "MAN.4.1: Apply legal basis change"
 //------------------------------------ Entity merging rules ------------------------------------
 
 // Rule unit: X.0
-rule "X.0.0: remove Entity contained by Entity of same type"
+rule "X.0.0: Remove Entity contained by Entity of same type"
     salience 65
     when
         $larger: TextEntity($type: type, $entityType: entityType, active())
@@ -648,7 +633,7 @@ rule "X.0.0: remove Entity contained by Entity of same type"
 
 
 // Rule unit: X.1
-rule "X.1.0: merge intersecting Entities of same type"
+rule "X.1.0: Merge intersecting Entities of same type"
     salience 64
     when
         $first: TextEntity($type: type, $entityType: entityType, !resized(), active())
@@ -664,7 +649,7 @@ rule "X.1.0: merge intersecting Entities of same type"
 
 
 // Rule unit: X.2
-rule "X.2.0: remove Entity of type ENTITY when contained by FALSE_POSITIVE"
+rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
     salience 64
     when
         $falsePositive: TextEntity($type: type, entityType == EntityType.FALSE_POSITIVE, active())
@@ -677,7 +662,7 @@ rule "X.2.0: remove Entity of type ENTITY when contained by FALSE_POSITIVE"
 
 
 // Rule unit: X.3
-rule "X.3.0: remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"
+rule "X.3.0: Remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"
     salience 64
     when
         $falseRecommendation: TextEntity($type: type, entityType == EntityType.FALSE_RECOMMENDATION, active())
@@ -689,7 +674,7 @@ rule "X.3.0: remove Entity of type RECOMMENDATION when contained by FALSE_RECOMM
 
 
 // Rule unit: X.4
-rule "X.4.0: remove Entity of type RECOMMENDATION when intersected by ENTITY with same type"
+rule "X.4.0: Remove Entity of type RECOMMENDATION when intersected by ENTITY with same type"
     salience 256
     when
         $entity: TextEntity($type: type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
@@ -702,7 +687,7 @@ rule "X.4.0: remove Entity of type RECOMMENDATION when intersected by ENTITY wit
 
 
 // Rule unit: X.5
-rule "X.5.0: remove Entity of type RECOMMENDATION when contained by ENTITY"
+rule "X.5.0: Remove Entity of type RECOMMENDATION when contained by ENTITY"
     salience 256
     when
         $entity: TextEntity((entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
@@ -714,7 +699,7 @@ rule "X.5.0: remove Entity of type RECOMMENDATION when contained by ENTITY"
 
 
 // Rule unit: X.6
-rule "X.6.0: remove Entity of lower rank, when contained by by entity of type ENTITY"
+rule "X.6.0: Remove Entity of lower rank, when contained by by entity of type ENTITY"
     salience 32
     when
         $higherRank: TextEntity($type: type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
@@ -729,7 +714,7 @@ rule "X.6.1: remove Entity of higher rank, when intersected by entity of type EN
     salience 32
     when
         $higherRank: TextEntity($type: type, $value: value, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
-        $lowerRank: TextEntity(intersects($higherRank), type != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active(), $lowerRank.getValue().length() > $value.length())
+        $lowerRank: TextEntity(intersects($higherRank), type != $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active(), $lowerRank.getValue().length() > $value.length())
     then
         $higherRank.getIntersectingNodes().forEach(node -> update(node));
         $higherRank.remove("X.6.1", "remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity");
@@ -740,7 +725,7 @@ rule "X.6.1: remove Entity of higher rank, when intersected by entity of type EN
 //------------------------------------ File attributes rules ------------------------------------
 
 // Rule unit: FA.1
-rule "FA.1.0: remove duplicate FileAttributes"
+rule "FA.1.0: Remove duplicate FileAttributes"
     salience 64
     when
         $fileAttribute: FileAttribute($label: label, $value: value)
@@ -761,7 +746,7 @@ rule "LDS.0.0: Run local dictionary search"
     then
         entityCreationService.bySearchImplementation($dictionaryModel.getLocalSearch(), $dictionaryModel.getType(), EntityType.RECOMMENDATION, document)
             .forEach(entity ->  {
-                Collection<MatchedRule> matchedRules = $dictionaryModel.getLocalEntriesWithMatchedRules().get(entity.getValue());
-                entity.addMatchedRules(matchedRules);
+                Collection<MatchedRule> matchedRules = $dictionaryModel.getMatchedRulesForLocalDictionaryEntry(entity.getValue());
+                matchedRules.forEach(matchedRule -> entity.addMatchedRule(matchedRule.asSkippedIfApplied()));
             });
     end

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/adama-pilot.drl

@@ -1,916 +0,0 @@
-package drools
-
-import static java.lang.String.format;
-import static com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility.anyMatch;
-import static com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility.exactMatch;
-
-import java.util.List;
-import java.util.LinkedList;
-import java.util.Set;
-import java.util.stream.Collectors;
-import java.util.Collection;
-import java.util.stream.Stream;
-import java.util.Optional;
-
-import com.iqser.red.service.redaction.v1.server.model.document.*;
-import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
-import com.iqser.red.service.redaction.v1.server.model.document.entity.*;
-import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
-import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule;
-import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity
-import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule
-import com.iqser.red.service.redaction.v1.server.model.document.nodes.*;
-import com.iqser.red.service.redaction.v1.server.model.document.nodes.Section;
-import com.iqser.red.service.redaction.v1.server.model.document.nodes.Table;
-import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
-import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
-import com.iqser.red.service.redaction.v1.server.model.document.nodes.Paragraph;
-import com.iqser.red.service.redaction.v1.server.model.document.nodes.Image;
-import com.iqser.red.service.redaction.v1.server.model.document.nodes.ImageType;
-import com.iqser.red.service.redaction.v1.server.model.document.nodes.Page;
-import com.iqser.red.service.redaction.v1.server.model.document.nodes.Headline;
-import com.iqser.red.service.redaction.v1.server.model.document.nodes.SectionIdentifier;
-import com.iqser.red.service.redaction.v1.server.model.document.nodes.Footer;
-import com.iqser.red.service.redaction.v1.server.model.document.nodes.Header;
-import com.iqser.red.service.redaction.v1.server.model.document.nodes.NodeType;
-import com.iqser.red.service.redaction.v1.server.model.document.textblock.*;
-import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
-import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlockCollector;
-import com.iqser.red.service.redaction.v1.server.model.document.textblock.AtomicTextBlock;
-import com.iqser.red.service.redaction.v1.server.model.document.textblock.ConcatenatedTextBlock;
-import com.iqser.red.service.redaction.v1.server.model.NerEntities;
-import com.iqser.red.service.redaction.v1.server.model.dictionary.Dictionary;
-import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryModel;
-import com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService;
-import com.iqser.red.service.redaction.v1.server.service.ManualChangesApplicationService;
-import com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility;
-
-import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus;
-
-global Document document
-global EntityCreationService entityCreationService
-global ManualChangesApplicationService manualChangesApplicationService
-global Dictionary dictionary
-
-//------------------------------------ queries ------------------------------------
-
-query "getFileAttributes"
-    $fileAttribute: FileAttribute()
-    end
-
-//---------------------------------------------------------------------------
-
-rule "H.0.0 retract table of contents page"
-    when
-        $page: Page(getMainBodyTextBlock().getSearchText().contains("........") || (getMainBodyTextBlock().getSearchText().contains("APPENDICES") && getMainBodyTextBlock().getSearchText().contains("TABLES")))
-        $node: SemanticNode(onPage($page.getNumber()), !onPage($page.getNumber() -1), getType() != NodeType.IMAGE)
-    then
-        retract($node);
-    end
-
-
-rule "H.0.0: Ignore Table of Contents"
-    salience 10
-    when
-        $tocHeadline: Headline(containsString("CONTENTS"))
-        $page: Page() from $tocHeadline.getParent().getPages()
-        $node: SemanticNode(this != $tocHeadline, getType() != NodeType.IMAGE, onPage($page.getNumber()), !onPage($page.getNumber() -1))
-    then
-        retract($node);
-    end
-
-//---------------------- CUSTOMER RULES -----------------------------------------------------
-
-rule "DOC.1.0: Adama number"
-    when
-       $paragraph: Paragraph(
-            containsString("R-")
-            && onPage(1)
-        )
-    then
-        entityCreationService
-            .byRegex(
-                "0?0?R\\-[l\\d]{3,5}\\p{Lu}?",
-                "adama_number",
-                EntityType.ENTITY,
-                $paragraph)
-            .findFirst()
-            .ifPresent(entity -> {
-                entity.apply("DOC.1.0", "Adama number found");
-                insert(FileAttribute.builder().label("Adama").value(entity.getValue()).build());
-            });
-    end
-
-rule "DOC.2.0: Study number by keyword"
-    when
-        $studyNumberKeyword: String() from List.of(
-            "BioChem project number:",
-            "DTI Report.",
-            "Final Report N°",
-            "LPT Report No.",
-            "Project identity",
-            "Project No.:",
-            "Protocol No.",
-            "PTRL Report No.",
-            "SLI Report #",
-            "SLI Study #",
-            "Specht Analytical Study Plan",
-            "SPL PROJECT NUMBER:",
-            "Study code:",
-            "Study N°",
-            "Study-No.",
-            "Study No:",
-            "Study number:",
-            "Study Number"
-            )
-        $excludeKeyWords: String() from List.of(
-            "Sponsors study number"
-        )
-        $paragraph: Paragraph(
-            containsStringIgnoreCase($studyNumberKeyword)
-            && !containsStringIgnoreCase($excludeKeyWords)
-            && onPage(1)
-        )
-    then
-        entityCreationService
-            .lineAfterStringIgnoreCase($studyNumberKeyword,"study_number",EntityType.ENTITY,$paragraph)
-            .findFirst()
-            .ifPresent(entity ->
-                {
-                    entity.apply("DOC.2.0", "Study number found");
-                    insert(FileAttribute.builder().label("Study").value(entity.getValue()).build());
-                }
-            );
-    end
-
-rule "DOC.2.1: Study number in header"
-    when
-        $studyNumberKeyword: String() from List.of(
-            "Final Report"
-            )
-        $header: Header(
-            containsStringIgnoreCase($studyNumberKeyword)
-            && onPage(2)
-        )
-    then
-        entityCreationService
-            .lineAfterStringIgnoreCase($studyNumberKeyword,"study_number",EntityType.ENTITY,$header)
-            .findFirst()
-            .ifPresent(entity ->
-                {
-                    entity.apply("DOC.2.1", "Study number found");
-                    insert(FileAttribute.builder().label("Study").value(entity.getValue()).build());
-                }
-            );
-    end
-
-rule "DOC.2.2: Project Number with Section Title"
-    when
-        $title: String() from List.of(
-            "LABORATORY PROJECT ID."
-        )
-        $paragraph: Paragraph(
-            containsStringIgnoreCase($title)
-            && onPage(1)
-        )
-    then
-        entityCreationService.semanticNodeAfterString(
-            $paragraph,
-            "LABORATORY PROJECT ID.",
-            "study_number",
-            EntityType.ENTITY
-            )
-            .ifPresent(
-                entity -> entity.apply("DOC.2.2","Study number by title found.")
-            );
-    end
-
-
-rule "DOC.3.0: Batch Material Number"
-    when
-        $hlKeyword: String() from List.of(
-            "formulation",
-            "formulation:",
-            "test item",
-            "test substance",
-            "test material"
-        )
-        $hlNoKeyword: String() from List.of(
-            "Preparation"
-        )
-        $headline: Headline(
-            containsStringIgnoreCase($hlKeyword)
-            && !containsStringIgnoreCase($hlNoKeyword)
-        )
-        $batchKeyword: String() from List.of(
-            "Batch number",
-            "Batch number:",
-            "Batch number* :",
-            "batch no",
-            "Batch no.",
-            "Lot.n.:"
-            )
-        $section: Section(
-            containsStringIgnoreCase($batchKeyword)
-            && (
-                getHeadline() == $headline
-                || (
-                    containsStringIgnoreCase($hlKeyword)
-                    && !containsStringIgnoreCase($hlNoKeyword)
-                )
-            )
-        )
-    then
-        entityCreationService
-            .lineAfterStringIgnoreCase($batchKeyword,"batch_material_number",EntityType.ENTITY,$section)
-            .forEach(
-                entity -> {
-                    entity.apply("DOC.3.0","Batch number found.");
-                    insert(FileAttribute.builder().label("Batch No").value(entity.getValue()).build());
-                }
-            );
-    end
-
-rule "DOC.4.0: study title as headline"
-    when
-        $hlKeyword: String() from List.of(
-            "Analyitical Method",
-            "Residues",
-            "Acute",
-            "Process",
-            "Application",
-            "Review",
-            "Examination",
-            "Toxicity",
-            "Method Validation",
-            "Explosion",
-            "Sensitisation",
-            "Determination",
-            "Test"
-        )
-        $hlNotKeyword: String() from List.of(
-            "Laboratoire",
-            "Laboratorien"
-        )
-        $species: TextEntity(type=="species")
-        $headline: Headline(
-            onPage(1)
-            && (
-                containsStringIgnoreCase($hlKeyword)
-                || entities contains $species
-            )
-            && !containsStringIgnoreCase($hlNotKeyword)
-        )
-    then
-        entityCreationService.bySemanticNode($headline, "title", EntityType.ENTITY).ifPresent(entity -> {
-            entity.apply("DOC.4.0", "Title found", "n-a");
-            insert(FileAttribute.builder().label("Title").value(entity.getValue()).build());
-        });
-     end
-
-rule "DOC.4.1: study title on cover page between sections"
-    when
-        not TextEntity(type=="title", applied())
-        $page: Page(
-            getNumber() == 1,
-            getMainBodyTextBlock().getSearchText().toLowerCase() (contains "Study Title".toLowerCase())
-            || getMainBodyTextBlock().getSearchText().toLowerCase() (contains "Title".toLowerCase())
-            || getMainBodyTextBlock().getSearchText().toLowerCase() (contains "Report 92 50 12 136".toLowerCase())
-            || getMainBodyTextBlock().getSearchText().toLowerCase() (contains "Final Report".toLowerCase())
-            || getMainBodyTextBlock().getSearchText().toLowerCase() (contains "Final Study Report".toLowerCase()),
-            getMainBodyTextBlock().getSearchText().toLowerCase() (contains "Guideline".toLowerCase())
-            || getMainBodyTextBlock().getSearchText().toLowerCase() (contains "Study Identification".toLowerCase())
-            // too many false positives due to term in header and cover page || getMainBodyTextBlock().getSearchText().toLowerCase() (contains "Final Report".toLowerCase())
-            || getMainBodyTextBlock().getSearchText().toLowerCase() (contains "Data Requirement".toLowerCase())
-            || getMainBodyTextBlock().getSearchText().toLowerCase() (contains "Submitted".toLowerCase())
-            || getMainBodyTextBlock().getSearchText().toLowerCase() (contains "Test Guideline".toLowerCase())
-            || getMainBodyTextBlock().getSearchText().toLowerCase() (contains "Study Director".toLowerCase())
-            || getMainBodyTextBlock().getSearchText().toLowerCase() (contains "Author".toLowerCase())
-            || getMainBodyTextBlock().getSearchText().toLowerCase() (contains "Including:".toLowerCase())
-            || getMainBodyTextBlock().getSearchText().toLowerCase() (contains "Laboratory Investigations".toLowerCase())
-            || getMainBodyTextBlock().getSearchText().toLowerCase() (contains "Test Article".toLowerCase())
-            || getMainBodyTextBlock().getSearchText().toLowerCase() (contains "HLS".toLowerCase())
-            || getMainBodyTextBlock().getSearchText().toLowerCase() (contains "Official Journal".toLowerCase())
-            )
-        then
-            List<String> startStrings = List.of("Study Title", "Study Title:", "Title", "Final Report", "Final Study Report", "Report 92 50 12 136");
-            List<String> stopStrings = List.of("Guideline", "Guidelines", "Study Identification", "Data Requirement", "Submitted", "Test Guideline",
-                                               "Study Director", "Author", "Including:", "Laboratory Investigations", "Test Article", "HLS", "Official Journal");
-            // too many false positives due to term in header and cover page  stopBoundaries.addAll(RedactionSearchUtility.findTextRangesByStringIgnoreCase("Final Report", $page.getMainBodyTextBlock()));
-
-            entityCreationService.shortestBetweenAnyString(startStrings, stopStrings, "title", EntityType.ENTITY, document).forEach(entity -> {
-                entity.apply("DOC.4.1", "Study title found", "n-a");
-                insert(FileAttribute.builder().label("Title").value(entity.getValue()).build());
-            });
-    end
-
-rule "DOC.4.2: Study title by paragraph including species on cover page"
-    when
-        not TextEntity(type=="title", applied())
-        $species: TextEntity(type=="species")
-        $paragraph: Paragraph(
-            onPage(1)
-            && entities contains $species
-        )
-    then
-        entityCreationService.bySemanticNode(
-            $paragraph,
-            "title",
-            EntityType.ENTITY
-        )
-        .ifPresent(entity -> {
-            entity.apply("DOC.4.2","Title with species found on cover page");
-            insert(FileAttribute.builder().label("Title").value(entity.getValue()).build());
-        });
-    end
-
-rule "DOC.5.0: Study Director on cover page in a paragraph on line"
-    when
-        not TextEntity(type=="author", applied())
-        $sectionTitle: String() from List.of(
-            "Study Director/Analyst:",
-            "Study Director:",
-            "Study Director :",
-            "Author:",
-            "Author "
-        )
-        $paragraph: Paragraph(
-            onPage(1)
-            && containsStringIgnoreCase($sectionTitle)
-        )
-    then
-        entityCreationService
-            .lineAfterStringIgnoreCase(
-                $sectionTitle,
-                "author",
-                EntityType.ENTITY,
-                $paragraph
-            )
-            .forEach(entity -> {
-                entity.apply("DOC.5.0","Study Director on cover page found");
-                insert(FileAttribute.builder().label("Author").value(entity.getValue()).build());
-            });
-    end
-
-rule "DOC.5.1: Author/Study Director listing on cover page"
-    when
-        not TextEntity(type=="author", applied())
-        $page: Page(
-            getNumber() == 1
-            && getMainBodyTextBlock().getSearchText().toLowerCase() (
-                contains "Author(s)".toLowerCase()
-                || contains "Authors".toLowerCase()
-                || contains "Author".toLowerCase()
-                || contains "Study Director".toLowerCase()
-                )
-            && getMainBodyTextBlock().getSearchText().toLowerCase() (
-                contains "Test Facility".toLowerCase()
-                || contains "Testing Facility".toLowerCase()
-                || contains "Study Initiation".toLowerCase()
-                || contains "Study Initiated".toLowerCase()
-                || contains "Study completed on".toLowerCase()
-                || contains "Study completion".toLowerCase()
-                || contains "Study amended".toLowerCase()
-                || contains "Report Issue Date".toLowerCase()
-                || contains "Document Date".toLowerCase()
-                || contains "Date".toLowerCase()
-                || contains "Defitrace".toLowerCase() // this should be solved with Organisation Entity as a TextRange
-                )
-            )
-    then
-        List<TextRange> startBoundaries = new LinkedList<>();
-        startBoundaries.addAll(RedactionSearchUtility.findTextRangesByStringIgnoreCase("Author(s)", $page.getMainBodyTextBlock()));
-        startBoundaries.addAll(RedactionSearchUtility.findTextRangesByStringIgnoreCase("Authors", $page.getMainBodyTextBlock()));
-        startBoundaries.addAll(RedactionSearchUtility.findTextRangesByStringIgnoreCase("Author", $page.getMainBodyTextBlock()));
-        startBoundaries.addAll(RedactionSearchUtility.findTextRangesByStringIgnoreCase("Study Director", $page.getMainBodyTextBlock()));
-
-        List<TextRange> stopBoundaries = new LinkedList<>();
-        stopBoundaries.addAll(RedactionSearchUtility.findTextRangesByStringIgnoreCase("Test Facility", $page.getMainBodyTextBlock()));
-        stopBoundaries.addAll(RedactionSearchUtility.findTextRangesByStringIgnoreCase("Testing Facility", $page.getMainBodyTextBlock()));
-        stopBoundaries.addAll(RedactionSearchUtility.findTextRangesByStringIgnoreCase("Study Initiation", $page.getMainBodyTextBlock()));
-        stopBoundaries.addAll(RedactionSearchUtility.findTextRangesByStringIgnoreCase("Study Initiated", $page.getMainBodyTextBlock()));
-        stopBoundaries.addAll(RedactionSearchUtility.findTextRangesByStringIgnoreCase("Study completed on", $page.getMainBodyTextBlock()));
-        stopBoundaries.addAll(RedactionSearchUtility.findTextRangesByStringIgnoreCase("Study completion", $page.getMainBodyTextBlock()));
-        stopBoundaries.addAll(RedactionSearchUtility.findTextRangesByStringIgnoreCase("Study amended", $page.getMainBodyTextBlock()));
-        stopBoundaries.addAll(RedactionSearchUtility.findTextRangesByStringIgnoreCase("Report Issue Date", $page.getMainBodyTextBlock()));
-        stopBoundaries.addAll(RedactionSearchUtility.findTextRangesByStringIgnoreCase("Document Date", $page.getMainBodyTextBlock()));
-        stopBoundaries.addAll(RedactionSearchUtility.findTextRangesByStringIgnoreCase("Date", $page.getMainBodyTextBlock()));
-        stopBoundaries.addAll(RedactionSearchUtility.findTextRangesByStringIgnoreCase("Defitrace", $page.getMainBodyTextBlock()));
-
-        entityCreationService.betweenTextRanges(startBoundaries, stopBoundaries, "author", EntityType.ENTITY, document).forEach(entity -> {
-            entity.apply("DOC.5.1", "Author list found", "n-a");
-            insert(FileAttribute.builder().label("Author").value(entity.getValue()).build());
-        });
-     end
-
-rule "DOC.5.2: Study Director on cover page in a paragraph"
-    when
-        not TextEntity(type=="author", applied())
-        $sectionTitle: String() from List.of(
-            "Study Director"
-        )
-        $node: Paragraph(
-            onPage(1)
-            && containsStringIgnoreCase($sectionTitle)
-        )
-    then
-        entityCreationService
-            .byRegexWithLineBreaksIgnoreCase(
-                "\\n([\\w\\(\\) .]{5,30})\\n",
-                "author",
-                EntityType.ENTITY,
-                1,
-                $node
-            )
-            .filter(
-                entity -> !entity.getValue().toLowerCase().contains($sectionTitle.toLowerCase())
-            )
-            .forEach(entity -> {
-                entity.apply("DOC.5.2","Study Director on cover page found");
-                insert(FileAttribute.builder().label("Author").value(entity.getValue()).build());
-            });
-    end
-
-rule "DOC.5.3: Study Director on cover page in a section"
-    when
-        not TextEntity(type=="author", applied())
-        $page: Page(getNumber() == 1)
-        $sectionTitle: String() from List.of(
-            "Study Director:",
-            "Study Director",
-            "Study Director/Author"
-        )
-        $node: Section(
-            onPage(1)
-            && containsStringIgnoreCase($sectionTitle)
-        )
-    then
-        entityCreationService
-            .byRegexWithLineBreaksIgnoreCase(
-                $sectionTitle+"\\n([\\w\\(\\) .]{5,30})\\n",
-                "author",
-                EntityType.ENTITY,
-                1,
-                $node
-            )
-            .filter(
-                entity -> entity.getPages().contains($page)
-            )
-            .forEach(entity -> {
-                entity.apply("DOC.5.3","Study Director on cover page found");
-                insert(FileAttribute.builder().label("Author").value(entity.getValue()).build());
-            });
-
-        entityCreationService
-            .lineAfterStringIgnoreCase($sectionTitle,"author",EntityType.ENTITY,$node)
-            .forEach(entity -> {
-                entity.apply("DOC.5.2","Study Director on cover page found");
-                insert(FileAttribute.builder().label("Author").value(entity.getValue()).build());
-            });
-    end
-
-rule "DOC.5.4: Author in document section with relevant headline"
-        when
-            not TextEntity(type == "author", applied())
-            $hlKeyword: String() from List.of(
-                "sponsor",
-                "personnel",
-                "staff involved",
-                "study details",
-                "management of study"
-            )
-            $keyword: String() from List.of(
-                "Study Director",
-                "Studv Director",
-                "Study Director:",
-                "Study Director :",
-                "stray Birector:",
-                "Author:",
-                "Author ",
-                "Author(s)"
-            )
-            $section: Section(
-                containsStringIgnoreCase($keyword)
-                && getHeadline().containsStringIgnoreCase($hlKeyword)
-            )
-        then
-            entityCreationService
-                .lineAfterStringIgnoreCase(
-                    $keyword,
-                    "author",
-                    EntityType.ENTITY,
-                    $section
-                )
-                .findFirst()
-                .ifPresent(
-                    entity -> {
-                        entity.apply("DOC.5.4","Author found.");
-                        insert(FileAttribute.builder().label("Author").value(entity.getValue()).build());
-                    }
-                );
-        end
-
-rule "DOC.5.5: Study Director from signature on compliance statement"
-    when
-        not TextEntity(type=="author", applied())
-        $sectionTitle: String() from List.of(
-            "Compliance Statement",
-            "Study Compliance",
-            "Statement of Compliance",
-            "Statement of Study Compliance",
-            "Contributing Scientist",
-            "Compliance with Good Laboratory Practice",
-            "Signatures and Approval"
-        )
-        $role: String() from List.of("Study Director")
-        $node: Section(
-            containsStringIgnoreCase($sectionTitle)
-            && containsStringIgnoreCase($role)
-            && getFirstPage().getNumber() > 4
-        )
-    then
-        entityCreationService
-            .byRegexWithLineBreaksIgnoreCase(
-                "([\\w\\(\\) .,]{5,35})[\\w\\/ ]{0,30}\\n"+$role,
-                "author",
-                EntityType.ENTITY,
-                1,
-                $node
-            )
-            .forEach(entity -> {
-                entity.apply("DOC.5.5","Study Director on compliance page found");
-                insert(FileAttribute.builder().label("Author").value(entity.getValue()).build());
-            });
-    end
-
-rule "DOC.5.6: Study Director from section with relevant headlining responsibility"
-    when
-        not TextEntity(type=="author", applied())
-        $sectionTitle: String() from List.of("Responsible personnel")
-        $role: String() from List.of("Study Director")
-        $node: Section(
-            getHeadline().containsStringIgnoreCase($sectionTitle)
-            && containsStringIgnoreCase($role)
-        )
-        $paragraph: Paragraph(
-            containsStringIgnoreCase($role)
-        )
-    then
-        entityCreationService
-            .lineAfterStringIgnoreCase(
-                $role,
-                "author",
-                EntityType.ENTITY,
-                $paragraph
-            )
-            .forEach(entity -> {
-                entity.apply("DOC.5.6","Study Director on compliance page found");
-                insert(FileAttribute.builder().label("Author").value(entity.getValue()).build());
-            });
-    end
-
-rule "DOC.5.7: Study Director from section with study director in headline"
-    when
-        not TextEntity(type=="author", applied())
-        $sectionTitle: String() from List.of("Study Director")
-        $section: Section(
-            getHeadline().containsStringIgnoreCase($sectionTitle)
-        )
-    then
-        entityCreationService
-            .bySemanticNodeParagraphsOnly(
-                $section,
-                "author",
-                EntityType.ENTITY
-            )
-            .forEach(entity -> {
-                entity.apply("DOC.5.7","Study Director found by headline");
-                insert(FileAttribute.builder().label("Author").value(entity.getValue()).build());
-            });
-    end
-
-rule "DOC.5.9: Remove skipped authors"
-    salience 9999
-    when
-        $author: TextEntity(type=="author", !applied())
-    then
-        $author.removeFromGraph();
-    end
-
-rule "DOC.14.0: GLP Study"
-     when
-        $hlKeywords: String() from List.of(
-            "Good Laboratory Practice",
-            "GLP"
-        )
-        $headline: Headline(
-            containsString($hlKeywords)
-        )
-        $compliance: String() from List.of(
-            "conducted in compliance with",
-            "conducted in accordance with",
-            "conducted and reported in compliance with",
-            "carried out in accordance with",
-            "meets the requirements",
-            "performed in compliance with"
-        )
-        $section: Section(
-            containsStringIgnoreCase($compliance)
-            && getHeadline() == $headline
-        )
-     then
-         entityCreationService.bySemanticNode($headline, "glp_study", EntityType.ENTITY).ifPresent(entity -> {
-             entity.apply("DOC.14.0", "GLP Study found", "n-a");
-             insert(FileAttribute.builder().label("GLP").value("TRUE").build());
-         });
-     end
-
-rule "DOC.14.1: GLP Study"
-     when
-        $compliance: String() from List.of(
-            "conducted in compliance with",
-            "conducted in accordance with",
-            "conducted and reported in compliance with",
-            "carried out in accordance with",
-            "meets the requirements",
-            "performed in compliance with",
-            "conduct"
-        )
-        $principle: String() from List.of(
-            "40 CFR Part 160",
-            "40CFR160",
-            "Good Laboratory Practice",
-            "FIFRA Good Laboratory Practice Standards"
-        )
-        $paragraph: Paragraph(
-            containsStringIgnoreCase($compliance)
-            && containsStringIgnoreCase($principle)
-        )
-     then
-        entityCreationService.betweenStringsIncludeStartAndEndIgnoreCase(
-            $compliance,
-            $principle,
-            "glp_study",
-            EntityType.ENTITY,
-            $paragraph
-            )
-            .forEach(entity -> {
-                entity.apply("DOC.14.1", "GLP Study found", "n-a");
-                insert(FileAttribute.builder().label("GLP").value("TRUE").build());
-            }
-        );
-     end
-
-//------------------------------------ Manual redaction rules ------------------------------------
-
-// Rule unit: MAN.0
-rule "MAN.0.0: Apply manual resize redaction"
-    salience 128
-    when
-        $resizeRedaction: ManualResizeRedaction($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate)
-        not ManualResizeRedaction(annotationId == $id, requestDate.isBefore($requestDate))
-        $entityToBeResized: TextEntity(matchesAnnotationId($id))
-    then
-        manualChangesApplicationService.resizeEntityAndReinsert($entityToBeResized, $resizeRedaction);
-        retract($resizeRedaction);
-        update($entityToBeResized);
-        $entityToBeResized.getIntersectingNodes().forEach(node -> update(node));
-    end
-
-rule "MAN.0.1: Apply manual resize redaction"
-    salience 128
-    when
-        $resizeRedaction: ManualResizeRedaction($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate)
-        not ManualResizeRedaction(annotationId == $id, requestDate.isBefore($requestDate))
-        $imageToBeResized: Image(id == $id)
-    then
-        manualChangesApplicationService.resizeImage($imageToBeResized, $resizeRedaction);
-        retract($resizeRedaction);
-        update($imageToBeResized);
-        update($imageToBeResized.getParent());
-    end
-
-
-// Rule unit: MAN.1
-rule "MAN.1.0: Apply id removals that are valid and not in forced redactions to Entity"
-    salience 128
-    when
-        $idRemoval: IdRemoval($id: annotationId, status == AnnotationStatus.APPROVED)
-        $entityToBeRemoved: TextEntity(matchesAnnotationId($id))
-    then
-        $entityToBeRemoved.getManualOverwrite().addChange($idRemoval);
-        update($entityToBeRemoved);
-        retract($idRemoval);
-        $entityToBeRemoved.getIntersectingNodes().forEach(node -> update(node));
-    end
-
-rule "MAN.1.1: Apply id removals that are valid and not in forced redactions to Image"
-    salience 128
-    when
-        $idRemoval: IdRemoval($id: annotationId, status == AnnotationStatus.APPROVED)
-        $imageEntityToBeRemoved: Image($id == id)
-    then
-        $imageEntityToBeRemoved.getManualOverwrite().addChange($idRemoval);
-        update($imageEntityToBeRemoved);
-        retract($idRemoval);
-        update($imageEntityToBeRemoved.getParent());
-    end
-
-
-// Rule unit: MAN.2
-rule "MAN.2.0: Apply force redaction"
-    salience 128
-    when
-        $force: ManualForceRedaction($id: annotationId, status == AnnotationStatus.APPROVED)
-        $entityToForce: TextEntity(matchesAnnotationId($id))
-    then
-        $entityToForce.getManualOverwrite().addChange($force);
-        update($entityToForce);
-        $entityToForce.getIntersectingNodes().forEach(node -> update(node));
-        retract($force);
-    end
-
-rule "MAN.2.1: Apply force redaction to images"
-    salience 128
-    when
-        $force: ManualForceRedaction($id: annotationId, status == AnnotationStatus.APPROVED)
-        $imageToForce: Image(id == $id)
-    then
-        $imageToForce.getManualOverwrite().addChange($force);
-        update($imageToForce);
-        update($imageToForce.getParent());
-        retract($force);
-    end
-
-
-// Rule unit: MAN.3
-rule "MAN.3.0: Apply entity recategorization"
-    salience 128
-    when
-        $recategorization: ManualRecategorization($id: annotationId, $type: type, status == AnnotationStatus.APPROVED, $requestDate: requestDate)
-        not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))
-        $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type != $type)
-    then
-        $entityToBeRecategorized.getIntersectingNodes().forEach(node -> update(node));
-        manualChangesApplicationService.recategorize($entityToBeRecategorized, $recategorization);
-        retract($recategorization);
-        // Entity is copied and inserted, so the old entity needs to be retracted to avoid duplication.
-        retract($entityToBeRecategorized);
-    end
-
-
-rule "MAN.3.1: Apply entity recategorization of same type"
-    salience 128
-    when
-        $recategorization: ManualRecategorization($id: annotationId, $type: type, status == AnnotationStatus.APPROVED, $requestDate: requestDate)
-        not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))
-        $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type == $type)
-    then
-        $entityToBeRecategorized.getManualOverwrite().addChange($recategorization);
-        retract($recategorization);
-    end
-
-
-rule "MAN.3.2: Apply image recategorization"
-    salience 128
-    when
-        $recategorization: ManualRecategorization($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate)
-        not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))
-        $imageToBeRecategorized: Image($id == id)
-    then
-        manualChangesApplicationService.recategorize($imageToBeRecategorized, $recategorization);
-        update($imageToBeRecategorized);
-        update($imageToBeRecategorized.getParent());
-        retract($recategorization);
-    end
-
-
-// Rule unit: MAN.4
-rule "MAN.4.0: Apply legal basis change"
-    salience 128
-    when
-        $legalbasisChange: ManualLegalBasisChange($id: annotationId, status == AnnotationStatus.APPROVED)
-        $imageToBeRecategorized: Image($id == id)
-    then
-        $imageToBeRecategorized.getManualOverwrite().addChange($legalbasisChange);
-    end
-
-rule "MAN.4.1: Apply legal basis change"
-    salience 128
-    when
-        $legalBasisChange: ManualLegalBasisChange($id: annotationId, status == AnnotationStatus.APPROVED)
-        $entityToBeChanged: TextEntity(matchesAnnotationId($id))
-    then
-        $entityToBeChanged.getManualOverwrite().addChange($legalBasisChange);
-    end
-
-
-//------------------------------------ Entity merging rules ------------------------------------
-
-// Rule unit: X.0
-rule "X.0.0: remove Entity contained by Entity of same type"
-    salience 65
-    when
-        $larger: TextEntity($type: type, $entityType: entityType, active())
-        $contained: TextEntity(containedBy($larger), type == $type, entityType == $entityType, this != $larger, !hasManualChanges(), active())
-    then
-        $contained.remove("X.0.0", "remove Entity contained by Entity of same type");
-        retract($contained);
-    end
-
-
-// Rule unit: X.2
-rule "X.2.0: remove Entity of type ENTITY when contained by FALSE_POSITIVE"
-    salience 64
-    when
-        $falsePositive: TextEntity($type: type, entityType == EntityType.FALSE_POSITIVE, active())
-        $entity: TextEntity(containedBy($falsePositive), type == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges(), active())
-    then
-        $entity.getIntersectingNodes().forEach(node -> update(node));
-        $entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE");
-        retract($entity)
-    end
-
-
-// Rule unit: X.3
-rule "X.3.0: remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"
-    salience 64
-    when
-        $falseRecommendation: TextEntity($type: type, entityType == EntityType.FALSE_RECOMMENDATION, active())
-        $recommendation: TextEntity(containedBy($falseRecommendation), type == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
-    then
-        $recommendation.remove("X.3.0", "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION");
-        retract($recommendation);
-    end
-
-
-// Rule unit: X.4
-rule "X.4.0: remove Entity of type RECOMMENDATION when intersected by ENTITY with same type"
-    salience 256
-    when
-        $entity: TextEntity($type: type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
-        $recommendation: TextEntity(intersects($entity), type == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
-    then
-        $entity.addEngines($recommendation.getEngines());
-        $recommendation.remove("X.4.0", "remove Entity of type RECOMMENDATION when intersected by ENTITY with same type");
-        retract($recommendation);
-    end
-
-
-// Rule unit: X.5
-rule "X.5.0: remove Entity of type RECOMMENDATION when contained by ENTITY"
-    salience 256
-    when
-        $entity: TextEntity((entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
-        $recommendation: TextEntity(containedBy($entity), entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
-    then
-        $recommendation.remove("X.5.0", "remove Entity of type RECOMMENDATION when contained by ENTITY");
-        retract($recommendation);
-    end
-
-
-// Rule unit: X.7
-rule "X.7.0: remove all images"
-    salience 512
-    when
-        $image: Image(imageType != ImageType.OCR, !hasManualChanges())
-    then
-        $image.remove("X.7.0", "remove all images");
-        retract($image);
-    end
-
-
-//------------------------------------ File attributes rules ------------------------------------
-
-// Rule unit: FA.1
-rule "FA.1.0: remove duplicate FileAttributes"
-
-    salience 64
-    when
-        $fileAttribute: FileAttribute($label: label, $value: value)
-        $duplicate: FileAttribute(this != $fileAttribute, label == $label, value == $value)
-    then
-        retract($duplicate);
-    end
-
-
-// Rule unit: LDS.0
-rule "LDS.0.0: Run local dictionary search"
-    agenda-group "LOCAL_DICTIONARY_ADDS"
-    salience -999
-    when
-        $dictionaryModel: DictionaryModel(!localEntriesWithMatchedRules.isEmpty()) from dictionary.getDictionaryModels()
-    then
-        entityCreationService.bySearchImplementation($dictionaryModel.getLocalSearch(), $dictionaryModel.getType(), EntityType.RECOMMENDATION, document)
-            .forEach(entity ->  {
-                Collection<MatchedRule> matchedRules = $dictionaryModel.getLocalEntriesWithMatchedRules().get(entity.getValue());
-                entity.addMatchedRules(matchedRules);
-            });
-    end

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/all_redact_manager_rules.drl

@@ -133,7 +133,7 @@ rule "CBI.1.1: Redact CBI Address (vertebrate Study)"
 // Rule unit: CBI.2
 rule "CBI.2.0: Do not redact genitive CBI Author"
     when
-        $entity: TextEntity(type == "CBI_author", anyMatch(textAfter, "['’’'ʼˈ´`‘′ʻ’']s"), applied())
+        $entity: TextEntity(type == "CBI_author", anyMatch(textAfter, "['’’'ʼˈ´`‘′ʻ’']s"))
     then
         entityCreationService.byTextRange($entity.getTextRange(), "CBI_author", EntityType.FALSE_POSITIVE, document)
         .ifPresent(falsePositive -> falsePositive.skip("CBI.2.0", "Genitive Author found"));
@@ -329,17 +329,12 @@ rule "CBI.7.0: Do not redact Names and Addresses if published information found
 
 rule "CBI.7.1: Do not redact Names and Addresses if published information found in same table row"
     when
-        $table: Table(hasEntitiesOfType("published_information"),
-                     (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
+        $table: Table(hasEntitiesOfType("published_information"), hasEntitiesOfType("CBI_author"))
+        $cellsWithPublishedInformation: TableCell() from $table.streamTableCellsWhichContainType("published_information").toList()
+        $tableCell: TableCell(row == $cellsWithPublishedInformation.row) from $table.streamTableCells().toList()
+        $author: TextEntity(type == "CBI_author", active()) from $tableCell.getEntities()
     then
-        $table.streamEntitiesWhereRowContainsEntitiesOfType(List.of("CBI_author", "CBI_address"))
-            .forEach(redactionEntity -> {
-                redactionEntity.skipWithReferences(
-                        "CBI.7.1",
-                        "Published Information found in row",
-                        $table.getEntitiesOfTypeInSameRow("published_information", redactionEntity)
-                );
-            });
+        $author.skipWithReferences("CBI.7.1", "Published Information found in row", $table.getEntitiesOfTypeInSameRow("published_information", $author));
     end
 
 
@@ -439,42 +434,59 @@ rule "CBI.11.0: Recommend all CBI_author entities in Table with Vertebrate Study
     when
         $table: Table(hasHeader("Author(s)") && hasHeader("Vertebrate Study Y/N"))
     then
-        $table.getEntitiesOfType("CBI_author").forEach(entity -> dictionary.addMultipleAuthorsAsRecommendation(entity));
+        $table.getEntitiesOfType("CBI_author").stream().filter(IEntity::applied).forEach(entity -> dictionary.addMultipleAuthorsAsRecommendation(entity));
     end
 
 
 // Rule unit: CBI.12
-rule "CBI.12.0: Add all cells with Header Author(s) as CBI_author"
-    salience 1
+rule "CBI.12.0: Redact and recommend TableCell with header 'Author' or 'Author(s)' and header 'Vertebrate study Y/N' with value 'Yes' (non vertebrate study)"
+    agenda-group "LOCAL_DICTIONARY_ADDS"
     when
-        $table: Table(hasHeader("Author(s)") || hasHeader("Author"))
+        not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
+        $table: Table(hasHeader("Author(s)") || hasHeader("Author"), hasHeaderIgnoreCase("Vertebrate Study Y/N"))
+        TableCell(header, containsAnyStringIgnoreCase("Author", "Author(s)"), $authorCol: col) from $table.streamHeaders().toList()
+        TableCell(header, containsStringIgnoreCase("Vertebrate study Y/N"), $vertebrateCol: col) from $table.streamHeaders().toList()
+        TableCell(!header, containsAnyString("Yes", "Y"), $rowWithYes: row) from $table.streamCol($vertebrateCol).toList()
+        $authorCell: TableCell(row == $rowWithYes) from $table.streamCol($authorCol).toList()
     then
-        Stream.concat(
-            $table.streamTableCellsWithHeader("Author(s)"),
-            $table.streamTableCellsWithHeader("Author")
-        )
-            .map(tableCell -> entityCreationService.bySemanticNode(tableCell, "CBI_author", EntityType.ENTITY))
-            .filter(Optional::isPresent)
-            .map(Optional::get)
-            .forEach(redactionEntity -> redactionEntity.skip("CBI.12.0", "Author(s) header found"));
+
+        entityCreationService.bySemanticNode($authorCell, "CBI_author", EntityType.ENTITY)
+          .ifPresent(authorEntity -> {
+              authorEntity.redact("CBI.12.0", "Redacted because it's row belongs to a vertebrate study", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+              dictionary.addMultipleAuthorsAsRecommendation(authorEntity);
+          });
+
     end
 
-rule "CBI.12.1: Do not redact CBI_author, if its row contains a cell with header \"Vertebrate study Y/N\" and value No"
+rule "CBI.12.1: Redact and recommend TableCell with header 'Author' or 'Author(s)' and header 'Vertebrate study Y/N' with value 'Yes' (vertebrate study)"
+    agenda-group "LOCAL_DICTIONARY_ADDS"
     when
-        $table: Table(hasRowWithHeaderAndValue("Vertebrate study Y/N", "N") || hasRowWithHeaderAndValue("Vertebrate study Y/N", "No"))
+        FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
+        $table: Table(hasHeader("Author(s)") || hasHeader("Author"), hasHeaderIgnoreCase("Vertebrate Study Y/N"))
+        TableCell(header, containsAnyStringIgnoreCase("Author", "Author(s)"), $authorCol: col) from $table.streamHeaders().toList()
+        TableCell(header, containsStringIgnoreCase("Vertebrate study Y/N"), $vertebrateCol: col) from $table.streamHeaders().toList()
+        TableCell(!header, containsAnyString("Yes", "Y"), $rowWithYes: row) from $table.streamCol($vertebrateCol).toList()
+        $authorCell: TableCell(row == $rowWithYes) from $table.streamCol($authorCol).toList()
     then
-        $table.streamEntitiesWhereRowHasHeaderAndAnyValue("Vertebrate study Y/N", List.of("N", "No"))
-            .filter(redactionEntity -> redactionEntity.isAnyType(List.of("CBI_author", "CBI_address")))
-            .forEach(authorEntity -> authorEntity.skip("CBI.12.1", "Not redacted because it's row does not belong to a vertebrate study"));
+
+        entityCreationService.bySemanticNode($authorCell, "CBI_author", EntityType.ENTITY)
+          .ifPresent(authorEntity -> {
+              authorEntity.redact("CBI.12.1", "Redacted because it's row belongs to a vertebrate study", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+              dictionary.addMultipleAuthorsAsRecommendation(authorEntity);
+          });
+
     end
 
-rule "CBI.12.2: Redact CBI_author, if its row contains a cell with header \"Vertebrate study Y/N\" and value Yes"
+rule "CBI.12.2: Skip TableCell with header 'Author' or 'Author(s)' and header 'Vertebrate study Y/N' with value 'No'"
     when
-        $table: Table(hasRowWithHeaderAndValue("Vertebrate study Y/N", "Y") || hasRowWithHeaderAndValue("Vertebrate study Y/N", "Yes"))
+        $table: Table(hasHeader("Author(s)") || hasHeader("Author"), hasHeaderIgnoreCase("Vertebrate Study Y/N"))
+        TableCell(header, containsAnyStringIgnoreCase("Author", "Author(s)"), $authorCol: col) from $table.streamHeaders().toList()
+        TableCell(header, containsStringIgnoreCase("Vertebrate study Y/N"), $vertebrateCol: col) from $table.streamHeaders().toList()
+        TableCell(!header, containsAnyString("No", "N"), $rowWithNo: row) from $table.streamCol($vertebrateCol).toList()
+        $authorCell: TableCell(row == $rowWithNo) from $table.streamCol($authorCol).toList()
     then
-        $table.streamEntitiesWhereRowHasHeaderAndAnyValue("Vertebrate study Y/N", List.of("Y", "Yes"))
-            .filter(redactionEntity -> redactionEntity.isAnyType(List.of("CBI_author", "CBI_address")))
-            .forEach(authorEntity -> authorEntity.redact("CBI.12.2", "Redacted because it's row belongs to a vertebrate study", "Reg (EC) No 1107/2009 Art. 63 (2g)"));
+        entityCreationService.bySemanticNode($authorCell, "CBI_author", EntityType.ENTITY)
+          .ifPresent(authorEntity -> authorEntity.skip("CBI.12.2", "Not redacted because it's row does not belong to a vertebrate study"));
     end
 
 
@@ -651,6 +663,7 @@ rule "CBI.20.1: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJEC
          });
     end
 
+
 // Rule unit: CBI.21
 rule "CBI.21.0: Redact short Authors section (non vertebrate study)"
     when
@@ -802,6 +815,7 @@ rule "PII.3.1: Redact telephone numbers by RegEx (vertebrate study)"
         .forEach(entity -> entity.redact("PII.3.1", "Telephone number found by regex", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
     end
 
+
 // Rule unit: PII.4
 rule "PII.4.0: Redact line after contact information keywords (non vertebrate study)"
     when
@@ -1039,6 +1053,7 @@ rule "PII.12.0: Expand PII entities with salutation prefix"
         .ifPresent(expandedEntity -> expandedEntity.addMatchedRules($entityToExpand.getMatchedRuleList()));
     end
 
+
 // Rule unit: PII.13
 rule "PII.13.0: Add recommendation for PII after Contact Person"
     when
@@ -1099,15 +1114,15 @@ rule "ETC.2.1: Redact signatures (vertebrate study)"
 
 
 // Rule unit: ETC.3
-rule "ETC.3.0: Redact logos (vertebrate study)"
+rule "ETC.3.0: Skip logos (non vertebrate study)"
     when
         not FileAttribute(label == "Vertebrate Study", value == "Yes")
         $logo: Image(imageType == ImageType.LOGO)
     then
-        $logo.redact("ETC.3.0", "Logo Found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        $logo.skip("ETC.3.0", "Logo Found");
     end
 
-rule "ETC.3.1: Redact logos (non vertebrate study)"
+rule "ETC.3.1: Redact logos (vertebrate study)"
     when
         FileAttribute(label == "Vertebrate Study", value == "Yes")
         $logo: Image(imageType == ImageType.LOGO)
@@ -1192,6 +1207,7 @@ rule "ETC.8.1: Redact formulas (non vertebrate study)"
         $logo.redact("ETC.8.1", "Logo Found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
     end
 
+
 // Rule unit: ETC.9
 rule "ETC.9.0: Redact skipped impurities"
     when
@@ -1209,6 +1225,7 @@ rule "ETC.9.1: Redact impurities"
         $skippedImpurities.redact("ETC.9.1", "Impurity found", "Article 63(2)(b) of Regulation (EC) No 1107/2009");
     end
 
+
 // Rule unit: ETC.10
 rule "ETC.10.0: Redact Product Composition Information"
     when
@@ -1217,6 +1234,7 @@ rule "ETC.10.0: Redact Product Composition Information"
         $compositionInformation.redact("ETC.10.0", "Product Composition Information found", "Article 63(2)(d) of Regulation (EC) No 1107/2009");
     end
 
+
 // Rule unit: ETC.11
 rule "ETC.11.0: Recommend first line in table cell with name and address of owner"
     when
@@ -1228,6 +1246,7 @@ rule "ETC.11.0: Recommend first line in table cell with name and address of owne
             .ifPresent(redactionEntity -> redactionEntity.redact("ETC.11.0", "Trial Site owner and address found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
     end
 
+
 //------------------------------------ AI rules ------------------------------------
 
 // Rule unit: AI.0
@@ -1273,7 +1292,8 @@ rule "AI.3.0: Recommend authors from AI as PII"
             .forEach(nerEntity -> entityCreationService.byNerEntity(nerEntity, "PII", EntityType.RECOMMENDATION, document));
     end
 
-//------------------------------------ Manual redaction rules ------------------------------------
+
+//------------------------------------ Manual changes rules ------------------------------------
 
 // Rule unit: MAN.0
 rule "MAN.0.0: Apply manual resize redaction"
@@ -1507,12 +1527,11 @@ rule "X.6.0: Remove Entity of lower rank, when contained by by entity of type EN
         retract($lowerRank);
     end
 
-
 rule "X.6.1: remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity"
     salience 32
     when
         $higherRank: TextEntity($type: type, $value: value, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
-        $lowerRank: TextEntity(intersects($higherRank), type != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active(), $lowerRank.getValue().length() > $value.length())
+        $lowerRank: TextEntity(intersects($higherRank), type != $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active(), $lowerRank.getValue().length() > $value.length())
     then
         $higherRank.getIntersectingNodes().forEach(node -> update(node));
         $higherRank.remove("X.6.1", "remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity");
@@ -1544,7 +1563,7 @@ rule "LDS.0.0: Run local dictionary search"
     then
         entityCreationService.bySearchImplementation($dictionaryModel.getLocalSearch(), $dictionaryModel.getType(), EntityType.RECOMMENDATION, document)
             .forEach(entity ->  {
-                Collection<MatchedRule> matchedRules = $dictionaryModel.getLocalEntriesWithMatchedRules().get(entity.getValue());
-                entity.addMatchedRules(matchedRules);
+                Collection<MatchedRule> matchedRules = $dictionaryModel.getMatchedRulesForLocalDictionaryEntry(entity.getValue());
+                matchedRules.forEach(matchedRule -> entity.addMatchedRule(matchedRule.asSkippedIfApplied()));
             });
     end

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora.drl

@@ -66,7 +66,7 @@ query "getFileAttributes"
     $fileAttribute: FileAttribute()
     end
 
-//------------------------------------ H rules ------------------------------------
+//------------------------------------ Headlines rules ------------------------------------
 
 // Rule unit: H.0
 rule "H.0.0: retract table of contents page"
@@ -116,6 +116,23 @@ rule "H.3.0: Study Type File Attribute"
              .ifPresent(fileAttribute -> insert(fileAttribute));
     end
 
+rule "H.3.1: Study Type File Attribute in Headlines"
+    when
+        not FileAttribute(label == "OECD Number", valueEqualsAnyOf("402","403","404","405","425","429","436","438","439","471","487"))
+        $page: Page($pageNumber:number,
+            getMainBodyTextBlock().getSearchText().contains("DATA REQUIREMENT")
+            || getMainBodyTextBlock().getSearchText().contains("TEST GUIDELINE")
+            || getMainBodyTextBlock().getSearchText().contains("MÉTODO(S) DE REFERÊNCIA(S):"))
+        $headline: Headline(onPage($pageNumber), containsString("OECD") || containsString("EPA")|| containsString("OPPTS"))
+    then
+      Stream.of(RedactionSearchUtility.findTextRangesByRegexIgnoreCase("(?<=OECD)(?:[\\w\\s,\\[\\]\\(\\)\\.]{1,10}|(?:.{5,40}(?:Number |Procedure |Guideline )))(4[\\d]{2})", 1, $headline.getTextBlock()),
+                RedactionSearchUtility.findTextRangesByRegexIgnoreCase("(?<=OECD).{5,40}Method (4[\\d]{2}).{1,65}(\\d{4})\\)", 1, $headline.getTextBlock()),
+                RedactionSearchUtility.findTextRangesByRegexIgnoreCase("(?<=OECD) Guideline (4\\d{2})", 1, $headline.getTextBlock())).flatMap(Collection::stream).findFirst()
+             .map(textRange -> $headline.getTextBlock().subSequence(textRange).toString())
+             .map(value -> FileAttribute.builder().label("OECD Number").value(value).build())
+             .ifPresent(fileAttribute -> insert(fileAttribute));
+    end
+
 
 //------------------------------------ General documine rules ------------------------------------
 
@@ -227,11 +244,63 @@ rule "DOC.1.3: Guidelines"
          )
      then
          $section.getEntitiesOfType(List.of("oecd_guideline", "ec_guideline", "epa_guideline")).forEach(entity -> {
-             entity.removeFromGraph();
+             entity.remove("DOC.1.3", "removed");
              retract(entity);
          });
      end
 
+rule "DOC.1.4: Guideline in Headlines"
+    when
+        $page: Page($pageNumber:number,
+            getMainBodyTextBlock().getSearchText().contains("DATA REQUIREMENT")
+            || getMainBodyTextBlock().getSearchText().contains("TEST GUIDELINE")
+            || getMainBodyTextBlock().getSearchText().contains("MÉTODO(S) DE REFERÊNCIA(S):"))
+        $headline: Headline(onPage($pageNumber), containsString("OECD") || containsString("EPA")|| containsString("OPPTS"))
+    then
+        entityCreationService.byRegex("OECD (No\\.? )?\\d{3}( \\(\\d{4}\\))?", "oecd_guideline", EntityType.ENTITY, $headline).forEach(entity ->
+            entity.apply("DOC.1.4", "OECD Guideline found", "n-a")
+        );
+        entityCreationService.byRegex("OECD[\\s,]{1}(?:.{1,40}.(?>Procedure|Method).{1,20}\\d{3,4}(?>.{1,100}\\d{4}\\))?|\\[.{1,20}.Skin.{1,20}\\]|[\\d\\s,\\(\\)]{7,10}|[\\w\\.\\s]{1,15}[\\d]{3}\\s\\(\\d{4}\\)|.{0,20}[N|n]umber\\s\\d{3}.{0,1}|Test Guideline \\d{3})", "oecd_guideline", EntityType.ENTITY, $headline).forEach(entity ->
+            entity.apply("DOC.1.4", "OECD Guideline found", "n-a")
+        );
+        entityCreationService.byRegex("EPA (OPPTS )?\\d{3}[. ]\\d{4}( \\(\\d{4}\\))?", "epa_guideline", EntityType.ENTITY, $headline).forEach(entity ->
+            entity.apply("DOC.1.4", "EPA Guideline found", "n-a")
+        );
+        entityCreationService.byRegex("EC (Directive )?(No\\.? )?\\d{3,4}\\/\\d{3,4}((,? B(\\.| )\\d{1,2}\\.?)? \\(\\d{4}\\))?", "ec_guideline", EntityType.ENTITY, $headline).forEach(entity ->
+            entity.apply("DOC.1.4", "EC Guideline found", "n-a")
+        );
+        entityCreationService.byRegex("Commission Regulation \\(EC\\) No \\d{3}\\/\\d{4}", "ec_guideline", EntityType.ENTITY, $headline).forEach(entity ->
+            entity.apply("DOC.1.4", "EC Guideline found", "n-a")
+        );
+        entityCreationService.byRegex("OECD Method 4\\d{2}.{5,40}\\(.{5,40}\\d{4}\\)", "oecd_guideline", EntityType.ENTITY, $headline).forEach(entity ->
+            entity.apply("DOC.1.4", "OECD Guideline year found", "n-a")
+        );
+        entityCreationService.byRegex("OPPTS (Guideline Number )?\\d{3}\\.\\d{4}( \\(\\d{4}\\))?", "epa_guideline", EntityType.ENTITY, $headline).forEach(entity ->
+            entity.apply("DOC.1.4", "EPA Guideline found", "n-a")
+        );
+        entityCreationService.byRegex("(?<=OECD)(?:[\\w\\s,\\[\\]\\(\\)\\.]{1,10}|.{5,40}(?:Number |Procedure |Guideline ))(4[\\d]{2})", "oecd_guideline_number", EntityType.ENTITY,1, $headline).forEach(entity ->
+            entity.apply("DOC.1.4", "OECD Guideline no. found", "n-a")
+        );
+        entityCreationService.byRegex("(?<=OECD)(?:[\\w\\s,\\[\\]\\(\\)\\.]{1,10}|.{5,40}(?:Number |Procedure |Guideline ))(4[\\d]{2}),?\\s\\(?(\\d{4})\\)?", "oecd_guideline_year", EntityType.ENTITY,2, $headline).forEach(entity ->
+            entity.apply("DOC.1.4", "OECD Guideline year found", "n-a")
+        );
+        entityCreationService.byRegex("(?<=OECD)[\\w\\s,\\[\\]]{1,10}\\((\\d{4})\\)\\s(4[\\d]{2})", "oecd_guideline_year", EntityType.ENTITY,1, $headline).forEach(entity ->
+            entity.apply("DOC.1.4", "OECD Guideline year found", "n-a")
+        );
+        entityCreationService.byRegex("(?<=OECD).{5,40}Method (4[\\d]{2}).{1,65}(\\d{4})\\)", "oecd_guideline_number", EntityType.ENTITY,1, $headline).forEach(entity ->
+             entity.apply("DOC.1.4", "OECD Guideline number found", "n-a")
+        );
+        entityCreationService.byRegex("(?<=OECD).{5,40}Method (4[\\d]{2}).{1,65}(\\d{4})\\)", "oecd_guideline_year", EntityType.ENTITY,2, $headline).forEach(entity ->
+            entity.apply("DOC.1.4", "OECD Guideline year found", "n-a")
+        );
+        entityCreationService.byRegex("(?<=OECD) Guideline (4\\d{2})", "oecd_guideline_number", EntityType.ENTITY,1, $headline).forEach(entity ->
+            entity.apply("DOC.1.4", "OECD Guideline number found", "n-a")
+        );
+        entityCreationService.byRegex("OECD Guideline 4\\d{2}", "oecd_guideline", EntityType.ENTITY, $headline).forEach(entity ->
+            entity.apply("DOC.1.4", "OECD Guideline found", "n-a")
+        );
+    end
+
 
 // Rule unit: DOC.2
 rule "DOC.2.0: Report number"
@@ -292,7 +361,7 @@ rule "DOC.5.0: Ignore species and strain in irrelevant study types"
          $section: Section(hasEntitiesOfType("species") || hasEntitiesOfType("strain"))
      then
          $section.getEntitiesOfType(List.of("species", "strain")).forEach(entity -> {
-             entity.removeFromGraph();
+             entity.remove("DOC.5.0", "removed");
              retract(entity);
          });
      end
@@ -311,7 +380,7 @@ rule "DOC.5.1: Hide all skipped species and strains except in the relevant secti
         )
      then
          $section.getEntitiesOfType(List.of("species", "strain")).forEach(entity -> {
-             entity.removeFromGraph();
+             entity.remove("DOC.5.1", "removed");
              retract(entity);
          });
      end
@@ -401,13 +470,13 @@ rule "DOC.7.1: Performing Laboratory (Country)"
                      .map(nerEntity -> entityCreationService.byNerEntity(nerEntity, "laboratory_country", EntityType.ENTITY, $section))
                      .forEach(entity -> {
                           entity.apply("DOC.7.1", "Performing Laboratory found", "n-a");
-                          });
-    end
+                     });
+     end
 
 rule "DOC.7.2: Performing Laboratory (Country & Name) from dict"
     when
         $section: Section(containsString("PERFORMING LABORATORY:") || (containsString("PERFORMING") && containsString("LABORATORY:")))
-        $countryOrNameFromDictionary: TextEntity(type == "laboratory_country" || type == "laboratory_name", $type: type, isDictionaryEntry())
+        $countryOrNameFromDictionary: TextEntity(type == "laboratory_country" || type == "laboratory_name", $type: type, isDictionaryEntry()) from $section.getEntities()
     then
         $countryOrNameFromDictionary.apply("DOC.7.2", "Performing " + $type + " dictionary entry found.");
     end
@@ -419,7 +488,10 @@ rule "DOC.7.3: Performing Laboratory (Country) from dict"
             && !(containsString("PERFORMING LABORATORY:") || (containsString("PERFORMING") && containsString("LABORATORY:")))
         )
     then
-        $section.getEntitiesOfType("laboratory_country", "laboratory_name").forEach(entity -> entity.remove("DOC.7.3", "Removed due to wrong section"));
+        $section.getEntitiesOfType(List.of("laboratory_country", "laboratory_name")).forEach(entity -> {
+            entity.remove("DOC.7.3", "removed");
+            retract(entity);
+        });
     end
 
 
@@ -953,7 +1025,6 @@ rule "DOC.28.1: No. Of animals - Fallback to appendix tables listing all individ
                 .map(Optional::get)
                 .forEach(entity -> {
                     entity.apply("DOC.28.1", "Animal number found.", "n-a");
-                    insert(entity);
                 });
      end
 
@@ -1034,7 +1105,6 @@ rule "DOC.33.0: Dose Mortality"
                             .map(Optional::get)
                             .forEach(entity -> {
                                 entity.apply("DOC.33.0", "Dose Mortality Data found.", "n-a");
-                                insert(entity);
                             });
 
         Stream.of($table.streamTableCellsWithHeader("Dose [mg/kg bodyweight]"),
@@ -1050,7 +1120,6 @@ rule "DOC.33.0: Dose Mortality"
                             .map(Optional::get)
                             .forEach(entity -> {
                                 entity.apply("DOC.33.0", "Dose Mortality Data found.", "n-a");
-                                insert(entity);
                             });
     end
 
@@ -1083,7 +1152,7 @@ rule "DOC.35.0: Doses (mg/kg bodyweight)"
     end
 
 
-//------------------------------------ Manual redaction rules ------------------------------------
+//------------------------------------ Manual changes rules ------------------------------------
 
 // Rule unit: MAN.0
 rule "MAN.0.0: Apply manual resize redaction"
@@ -1204,6 +1273,13 @@ rule "MAN.3.2: Apply image recategorization"
         retract($recategorization);
     end
 
+rule "MAN.3.3: Apply recategorization entities by default"
+    salience 128
+    when
+        $entity: IEntity(getManualOverwrite().getRecategorized().orElse(false))
+    then
+        $entity.apply("MAN.3.3", "Recategorized entities are applied by default.");
+    end
 
 // Rule unit: MAN.4
 rule "MAN.4.0: Apply legal basis change"
@@ -1290,7 +1366,7 @@ rule "X.5.0: Remove Entity of type RECOMMENDATION when contained by ENTITY"
 
 
 // Rule unit: X.7
-rule "X.7.0: remove all images"
+rule "X.7.0: Remove all images"
     salience 512
     when
         $image: Image(imageType != ImageType.OCR, !hasManualChanges())
@@ -1324,7 +1400,7 @@ rule "LDS.0.0: Run local dictionary search"
     then
         entityCreationService.bySearchImplementation($dictionaryModel.getLocalSearch(), $dictionaryModel.getType(), EntityType.RECOMMENDATION, document)
             .forEach(entity ->  {
-                Collection<MatchedRule> matchedRules = $dictionaryModel.getLocalEntriesWithMatchedRules().get(entity.getValue());
-                entity.addMatchedRules(matchedRules);
+                Collection<MatchedRule> matchedRules = $dictionaryModel.getMatchedRulesForLocalDictionaryEntry(entity.getValue());
+                matchedRules.forEach(matchedRule -> entity.addMatchedRule(matchedRule.asSkippedIfApplied()));
             });
     end

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora_components.drl

@@ -217,7 +217,7 @@ rule "GuidelineDeviation.0.0: Guideline deviation as sentences"
 
 rule "Species.0.0: First found species"
     when
-        $oecdNumber: String() from List.of("402", "403", "404", "405", "425", "429", "436", "471")
+        $oecdNumber: String() from List.of("402", "403", "404", "405", "425", "429", "436")
         FileAttribute(label == "OECD Number", value == $oecdNumber)
         $species: List() from collect (Entity(type == "species"))
     then
@@ -313,7 +313,7 @@ rule "Necropsy.2.0: Conducted with 4 hours of exposure as one block"
 
 rule "StudyDesign.0.0: Study design as one block"
     when
-        $oecdNumber: String() from List.of("404", "405", "429", "406", "428", "438", "439", "474", "487")
+        $oecdNumber: String() from List.of("404", "405", "406", "428", "429", "438", "439", "474", "487")
         FileAttribute(label == "OECD Number", value == $oecdNumber)
         $studyDesigns: List() from collect (Entity(type == "study_design"))
     then
@@ -497,15 +497,16 @@ rule "UsedApproach.1.0: Used approach not found and thus 'Individual'"
         componentCreationService.create("UsedApproach.1.0", "What_was_the_approach_used", "Individual", "'Group' when approach used is present, else 'Individual'");
     end
 
-/*
+
 rule "DefaultComponents.999.0: Create components for all unmapped entities."
     salience -999
     when
+        not FileAttribute(label == "OECD Number")
         $allEntities: List(!isEmpty()) from collect (Entity())
     then
         componentCreationService.createComponentsForUnMappedEntities("DefaultComponents.999.0", $allEntities);
     end
-*/
+
 
 //------------------------------------ Component merging rules ------------------------------------
 /*

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/manual_redaction_rules.drl

@@ -66,7 +66,7 @@ query "getFileAttributes"
     $fileAttribute: FileAttribute()
     end
 
-//------------------------------------ Manual redaction rules ------------------------------------
+//------------------------------------ Manual changes rules ------------------------------------
 
 // Rule unit: MAN.0
 rule "MAN.0.0: Apply manual resize redaction"
@@ -163,7 +163,6 @@ rule "MAN.3.0: Apply entity recategorization"
         retract($entityToBeRecategorized);
     end
 
-
 rule "MAN.3.1: Apply entity recategorization of same type"
     salience 128
     when
@@ -175,7 +174,6 @@ rule "MAN.3.1: Apply entity recategorization of same type"
         retract($recategorization);
     end
 
-
 rule "MAN.3.2: Apply image recategorization"
     salience 128
     when
@@ -208,6 +206,8 @@ rule "MAN.4.1: Apply legal basis change"
     then
         $entityToBeChanged.getManualOverwrite().addChange($legalBasisChange);
     end
+
+
 //------------------------------------ Local dictionary search rules ------------------------------------
 
 // Rule unit: LDS.0
@@ -219,7 +219,7 @@ rule "LDS.0.0: Run local dictionary search"
     then
         entityCreationService.bySearchImplementation($dictionaryModel.getLocalSearch(), $dictionaryModel.getType(), EntityType.RECOMMENDATION, document)
             .forEach(entity ->  {
-                Collection<MatchedRule> matchedRules = $dictionaryModel.getLocalEntriesWithMatchedRules().get(entity.getValue());
-                entity.addMatchedRules(matchedRules);
+                Collection<MatchedRule> matchedRules = $dictionaryModel.getMatchedRulesForLocalDictionaryEntry(entity.getValue());
+                matchedRules.forEach(matchedRule -> entity.addMatchedRule(matchedRule.asSkippedIfApplied()));
             });
     end

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl

@@ -84,7 +84,7 @@ rule "SYN.0.0: Redact if CTL/* or BL/* was found (Non Vertebrate Study)"
 //------------------------------------ CBI rules ------------------------------------
 
 // Rule unit: CBI.3
-rule "CBI.3.0: Redacted because Section contains Vertebrate"
+rule "CBI.3.0: Redacted because Section contains a vertebrate"
     when
         $section: Section(!hasTables(), hasEntitiesOfType("vertebrate"), (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
     then
@@ -99,7 +99,7 @@ rule "CBI.3.0: Redacted because Section contains Vertebrate"
             });
     end
 
-rule "CBI.3.1: Redacted because Table Row contains Vertebrate"
+rule "CBI.3.1: Redacted because table row contains a vertebrate"
     when
         $table: Table(hasEntitiesOfType("vertebrate"), (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
     then
@@ -115,7 +115,7 @@ rule "CBI.3.1: Redacted because Table Row contains Vertebrate"
             });
     end
 
-rule "CBI.3.2: Don't redact because Section doesn't contain Vertebrate"
+rule "CBI.3.2: Do not redact because Section does not contain a vertebrate"
     when
         $section: Section(!hasTables(), !hasEntitiesOfType("vertebrate"), (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
     then
@@ -123,7 +123,7 @@ rule "CBI.3.2: Don't redact because Section doesn't contain Vertebrate"
             .forEach(entity -> entity.skip("CBI.3.2", "No vertebrate found"));
     end
 
-rule "CBI.3.3: Dont redact because Table Row doesn't contain Vertebrate"
+rule "CBI.3.3: Do not redact because table row does not contain a vertebrate"
     when
         $table: Table(hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address"))
     then
@@ -134,7 +134,7 @@ rule "CBI.3.3: Dont redact because Table Row doesn't contain Vertebrate"
 
 
 // Rule unit: CBI.4
-rule "CBI.4.0: Dont redact Names and Addresses if no_redaction_indicator is found in Section"
+rule "CBI.4.0: Do not redact Names and Addresses if no_redaction_indicator is found in Section"
     when
         $section: Section(!hasTables(),
                           hasEntitiesOfType("vertebrate"),
@@ -151,7 +151,7 @@ rule "CBI.4.0: Dont redact Names and Addresses if no_redaction_indicator is foun
             });
     end
 
-rule "CBI.4.1: Dont redact Names and Addresses if no_redaction_indicator is found in Table Row"
+rule "CBI.4.1: Do not redact Names and Addresses if no_redaction_indicator is found in table row"
     when
         $table: Table(hasEntitiesOfType("no_redaction_indicator"),
                       hasEntitiesOfType("vertebrate"),
@@ -172,7 +172,7 @@ rule "CBI.4.1: Dont redact Names and Addresses if no_redaction_indicator is foun
 
 
 // Rule unit: CBI.5
-rule "CBI.5.0: Redact Names and Addresses if no_redaction_indicator but also redaction_indicator is found in section"
+rule "CBI.5.0: Redact Names and Addresses if no_redaction_indicator but also redaction_indicator is found in Section"
     when
         $section: Section(!hasTables(),
                           hasEntitiesOfType("redaction_indicator"),
@@ -192,7 +192,7 @@ rule "CBI.5.0: Redact Names and Addresses if no_redaction_indicator but also red
             });
     end
 
-rule "CBI.5.1: Redact Names and Addresses if no_redaction_indicator but also redaction_indicator is found in Table Row"
+rule "CBI.5.1: Redact Names and Addresses if no_redaction_indicator but also redaction_indicator is found in table row"
     when
         $table: Table(hasEntitiesOfType("no_redaction_indicator"),
                       hasEntitiesOfType("redaction_indicator"),
@@ -229,7 +229,7 @@ rule "CBI.8.0: Redacted because Section contains must_redact entity"
             });
     end
 
-rule "CBI.8.1: Redacted because Table Row contains must_redact entity"
+rule "CBI.8.1: Redacted because table row contains must_redact entity"
     when
         $table: Table(hasEntitiesOfType("must_redact"), (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
     then
@@ -247,7 +247,7 @@ rule "CBI.8.1: Redacted because Table Row contains must_redact entity"
 
 
 // Rule unit: CBI.9
-rule "CBI.9.0: Redact all Cell's with Header Author(s) as CBI_author (non vertebrate study)"
+rule "CBI.9.0: Redact all cells with Header Author(s) as CBI_author (non vertebrate study)"
     agenda-group "LOCAL_DICTIONARY_ADDS"
     when
         not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
@@ -257,10 +257,10 @@ rule "CBI.9.0: Redact all Cell's with Header Author(s) as CBI_author (non verteb
             .map(tableCell -> entityCreationService.bySemanticNode(tableCell, "CBI_author", EntityType.ENTITY))
             .filter(Optional::isPresent)
             .map(Optional::get)
-            .forEach(redactionEntity -> redactionEntity.apply("CBI.9.0", "Author(s) found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
+            .forEach(redactionEntity -> redactionEntity.redact("CBI.9.0", "Author(s) found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
     end
 
-rule "CBI.9.1: Redact all Cell's with Header Author as CBI_author (non vertebrate study)"
+rule "CBI.9.1: Redact all cells with Header Author as CBI_author (non vertebrate study)"
     agenda-group "LOCAL_DICTIONARY_ADDS"
     when
         not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
@@ -270,7 +270,7 @@ rule "CBI.9.1: Redact all Cell's with Header Author as CBI_author (non vertebrat
             .map(tableCell -> entityCreationService.bySemanticNode(tableCell, "CBI_author", EntityType.ENTITY))
             .filter(Optional::isPresent)
             .map(Optional::get)
-            .forEach(redactionEntity -> redactionEntity.apply("CBI.9.1", "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
+            .forEach(redactionEntity -> redactionEntity.redact("CBI.9.1", "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
     end
 
 
@@ -281,42 +281,59 @@ rule "CBI.11.0: Recommend all CBI_author entities in Table with Vertebrate Study
     when
         $table: Table(hasHeader("Author(s)") && hasHeader("Vertebrate Study Y/N"))
     then
-        $table.getEntitiesOfType("CBI_author").forEach(entity -> dictionary.addMultipleAuthorsAsRecommendation(entity));
+        $table.getEntitiesOfType("CBI_author").stream().filter(IEntity::applied).forEach(entity -> dictionary.addMultipleAuthorsAsRecommendation(entity));
     end
 
 
 // Rule unit: CBI.12
-rule "CBI.12.0: Add all Cell's with Header Author(s) as CBI_author"
-    salience 1
+rule "CBI.12.0: Redact and recommend TableCell with header 'Author' or 'Author(s)' and header 'Vertebrate study Y/N' with value 'Yes' (non vertebrate study)"
+    agenda-group "LOCAL_DICTIONARY_ADDS"
     when
-        $table: Table(hasHeader("Author(s)") || hasHeader("Author"))
+        not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
+        $table: Table(hasHeader("Author(s)") || hasHeader("Author"), hasHeaderIgnoreCase("Vertebrate Study Y/N"))
+        TableCell(header, containsAnyStringIgnoreCase("Author", "Author(s)"), $authorCol: col) from $table.streamHeaders().toList()
+        TableCell(header, containsStringIgnoreCase("Vertebrate study Y/N"), $vertebrateCol: col) from $table.streamHeaders().toList()
+        TableCell(!header, containsAnyString("Yes", "Y"), $rowWithYes: row) from $table.streamCol($vertebrateCol).toList()
+        $authorCell: TableCell(row == $rowWithYes) from $table.streamCol($authorCol).toList()
     then
-        Stream.concat(
-            $table.streamTableCellsWithHeader("Author(s)"),
-            $table.streamTableCellsWithHeader("Author")
-        )
-            .map(tableCell -> entityCreationService.bySemanticNode(tableCell, "CBI_author", EntityType.ENTITY))
-            .filter(Optional::isPresent)
-            .map(Optional::get)
-            .forEach(redactionEntity -> redactionEntity.skip("CBI.12.0", "Author(s) header found"));
+
+        entityCreationService.bySemanticNode($authorCell, "CBI_author", EntityType.ENTITY)
+          .ifPresent(authorEntity -> {
+              authorEntity.redact("CBI.12.0", "Redacted because it's row belongs to a vertebrate study", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+              dictionary.addMultipleAuthorsAsRecommendation(authorEntity);
+          });
+
     end
 
-rule "CBI.12.1: Dont redact CBI_author, if its row contains a cell with header \"Vertebrate study Y/N\" and value No"
+rule "CBI.12.1: Redact and recommend TableCell with header 'Author' or 'Author(s)' and header 'Vertebrate study Y/N' with value 'Yes' (vertebrate study)"
+    agenda-group "LOCAL_DICTIONARY_ADDS"
     when
-        $table: Table(hasRowWithHeaderAndValue("Vertebrate study Y/N", "N") || hasRowWithHeaderAndValue("Vertebrate study Y/N", "No"))
+        FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
+        $table: Table(hasHeader("Author(s)") || hasHeader("Author"), hasHeaderIgnoreCase("Vertebrate Study Y/N"))
+        TableCell(header, containsAnyStringIgnoreCase("Author", "Author(s)"), $authorCol: col) from $table.streamHeaders().toList()
+        TableCell(header, containsStringIgnoreCase("Vertebrate study Y/N"), $vertebrateCol: col) from $table.streamHeaders().toList()
+        TableCell(!header, containsAnyString("Yes", "Y"), $rowWithYes: row) from $table.streamCol($vertebrateCol).toList()
+        $authorCell: TableCell(row == $rowWithYes) from $table.streamCol($authorCol).toList()
     then
-        $table.streamEntitiesWhereRowHasHeaderAndAnyValue("Vertebrate study Y/N", List.of("N", "No"))
-            .filter(redactionEntity -> redactionEntity.isAnyType(List.of("CBI_author", "CBI_address")))
-            .forEach(authorEntity -> authorEntity.skip("CBI.12.1", "Not redacted because it's row does not belong to a vertebrate study"));
+
+        entityCreationService.bySemanticNode($authorCell, "CBI_author", EntityType.ENTITY)
+          .ifPresent(authorEntity -> {
+              authorEntity.redact("CBI.12.1", "Redacted because it's row belongs to a vertebrate study", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+              dictionary.addMultipleAuthorsAsRecommendation(authorEntity);
+          });
+
     end
 
-rule "CBI.12.2: Redact CBI_author, if its row contains a cell with header \"Vertebrate study Y/N\" and value Yes"
+rule "CBI.12.2: Skip TableCell with header 'Author' or 'Author(s)' and header 'Vertebrate study Y/N' with value 'No'"
     when
-        $table: Table(hasRowWithHeaderAndValue("Vertebrate study Y/N", "Y") || hasRowWithHeaderAndValue("Vertebrate study Y/N", "Yes"))
+        $table: Table(hasHeader("Author(s)") || hasHeader("Author"), hasHeaderIgnoreCase("Vertebrate Study Y/N"))
+        TableCell(header, containsAnyStringIgnoreCase("Author", "Author(s)"), $authorCol: col) from $table.streamHeaders().toList()
+        TableCell(header, containsStringIgnoreCase("Vertebrate study Y/N"), $vertebrateCol: col) from $table.streamHeaders().toList()
+        TableCell(!header, containsAnyString("No", "N"), $rowWithNo: row) from $table.streamCol($vertebrateCol).toList()
+        $authorCell: TableCell(row == $rowWithNo) from $table.streamCol($authorCol).toList()
     then
-        $table.streamEntitiesWhereRowHasHeaderAndAnyValue("Vertebrate study Y/N", List.of("Y", "Yes"))
-            .filter(redactionEntity -> redactionEntity.isAnyType(List.of("CBI_author", "CBI_address")))
-            .forEach(authorEntity -> authorEntity.apply("CBI.12.2", "Redacted because it's row belongs to a vertebrate study", "Reg (EC) No 1107/2009 Art. 63 (2g)"));
+        entityCreationService.bySemanticNode($authorCell, "CBI_author", EntityType.ENTITY)
+          .ifPresent(authorEntity -> authorEntity.skip("CBI.12.2", "Not redacted because it's row does not belong to a vertebrate study"));
     end
 
 
@@ -325,7 +342,7 @@ rule "CBI.14.0: Redact CBI_sponsor entities if preceded by \"batches produced at
     when
         $sponsorEntity: TextEntity(type == "CBI_sponsor", textBefore.contains("batches produced at"))
     then
-        $sponsorEntity.apply("CBI.14.0", "Redacted because it represents a sponsor company", "Reg (EC) No 1107/2009 Art. 63 (2g)");
+        $sponsorEntity.redact("CBI.14.0", "Redacted because it represents a sponsor company", "Reg (EC) No 1107/2009 Art. 63 (2g)");
     end
 
 
@@ -350,7 +367,7 @@ rule "CBI.15.0: Redact row if row contains \"determination of residues\" and liv
             .toList();
 
         $section.getEntitiesOfType(List.of($keyword, $residueKeyword))
-            .forEach(redactionEntity -> redactionEntity.apply("CBI.15.0", "Determination of residues and keyword \"" + $keyword + "\" was found.", "Reg (EC) No 1107/2009 Art. 63 (2g)"));
+            .forEach(redactionEntity -> redactionEntity.redact("CBI.15.0", "Determination of residues and keyword \"" + $keyword + "\" was found.", "Reg (EC) No 1107/2009 Art. 63 (2g)"));
     end
 
 rule "CBI.15.1: Redact CBI_author and CBI_address if row contains \"determination of residues\" and livestock keyword"
@@ -372,12 +389,12 @@ rule "CBI.15.1: Redact CBI_author and CBI_address if row contains \"determinatio
 
         $table.streamEntitiesWhereRowContainsStringsIgnoreCase(List.of($keyword, $residueKeyword))
             .filter(redactionEntity -> redactionEntity.isAnyType(List.of("CBI_author", "CBI_address")))
-            .forEach(redactionEntity -> redactionEntity.apply("CBI.15.1", "Determination of residues and keyword \"" + $keyword + "\" was found.", "Reg (EC) No 1107/2009 Art. 63 (2g)"));
+            .forEach(redactionEntity -> redactionEntity.redact("CBI.15.1", "Determination of residues and keyword \"" + $keyword + "\" was found.", "Reg (EC) No 1107/2009 Art. 63 (2g)"));
     end
 
 
 // Rule unit: CBI.16
-rule "CBI.16.0: Add CBI_author with \"et al.\" Regex (non vertebrate study)"
+rule "CBI.16.0: Add CBI_author with \"et al.\" RegEx (non vertebrate study)"
     agenda-group "LOCAL_DICTIONARY_ADDS"
     when
         not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
@@ -385,12 +402,12 @@ rule "CBI.16.0: Add CBI_author with \"et al.\" Regex (non vertebrate study)"
     then
         entityCreationService.byRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, 1, $section)
             .forEach(entity -> {
-                entity.apply("CBI.16.0", "Author found by \"et al\" regex", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+                entity.redact("CBI.16.0", "Author found by \"et al\" regex", "Article 39(e)(3) of Regulation (EC) No 178/2002");
                 dictionary.recommendEverywhere(entity);
             });
     end
 
-rule "CBI.16.1: Add CBI_author with \"et al.\" Regex (vertebrate study)"
+rule "CBI.16.1: Add CBI_author with \"et al.\" RegEx (vertebrate study)"
     agenda-group "LOCAL_DICTIONARY_ADDS"
     when
         FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
@@ -398,7 +415,7 @@ rule "CBI.16.1: Add CBI_author with \"et al.\" Regex (vertebrate study)"
     then
         entityCreationService.byRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, 1, $section)
             .forEach(entity -> {
-                entity.apply("CBI.16.1", "Author found by \"et al\" regex", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+                entity.redact("CBI.16.1", "Author found by \"et al\" regex", "Article 39(e)(2) of Regulation (EC) No 178/2002");
                 dictionary.recommendEverywhere(entity);
             });
     end
@@ -477,7 +494,7 @@ rule "CBI.20.1: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJEC
     then
          entityCreationService.betweenStrings("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", EntityType.ENTITY, $section)
          .forEach(laboratoryEntity -> {
-             laboratoryEntity.apply("CBI.20.1", "PERFORMING LABORATORY was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+             laboratoryEntity.redact("CBI.20.1", "PERFORMING LABORATORY was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
              dictionary.recommendEverywhere(laboratoryEntity);
          });
     end
@@ -491,7 +508,7 @@ rule "PII.0.0: Redact all PII (non vertebrate study)"
         not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
         $pii: TextEntity(type == "PII", dictionaryEntry)
     then
-        $pii.apply("PII.0.0", "Personal Information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        $pii.redact("PII.0.0", "Personal Information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
     end
 
 rule "PII.0.1: Redact all PII (vertebrate study)"
@@ -499,7 +516,7 @@ rule "PII.0.1: Redact all PII (vertebrate study)"
         FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
         $pii: TextEntity(type == "PII", dictionaryEntry)
     then
-        $pii.apply("PII.0.1", "Personal Information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        $pii.redact("PII.0.1", "Personal Information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
     end
 
 
@@ -510,7 +527,7 @@ rule "PII.1.0: Redact Emails by RegEx (Non vertebrate study)"
         $section: Section(containsString("@"))
     then
         entityCreationService.byRegex("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", "PII", EntityType.ENTITY, 1, $section)
-            .forEach(emailEntity -> emailEntity.apply("PII.1.0", "Found by Email Regex", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
+            .forEach(emailEntity -> emailEntity.redact("PII.1.0", "Found by Email Regex", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
     end
 
 rule "PII.1.1: Redact Emails by RegEx (vertebrate study)"
@@ -519,7 +536,7 @@ rule "PII.1.1: Redact Emails by RegEx (vertebrate study)"
         $section: Section(containsString("@"))
     then
         entityCreationService.byRegex("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", "PII", EntityType.ENTITY, 1, $section)
-            .forEach(emailEntity -> emailEntity.apply("PII.1.1", "Found by Email Regex", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
+            .forEach(emailEntity -> emailEntity.redact("PII.1.1", "Found by Email Regex", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
     end
 
 
@@ -549,12 +566,12 @@ rule "PII.4.0: Redact line after contact information keywords (non vertebrate st
         $section: Section(containsString($contactKeyword))
     then
         entityCreationService.lineAfterString($contactKeyword, "PII", EntityType.ENTITY, $section)
-            .forEach(contactEntity -> contactEntity.apply("PII.4.0", "Found after \"" + $contactKeyword + "\" contact keyword", "Reg (EC) No 1107/2009 Art. 63 (2e)"));
+            .forEach(contactEntity -> contactEntity.redact("PII.4.0", "Found after \"" + $contactKeyword + "\" contact keyword", "Reg (EC) No 1107/2009 Art. 63 (2e)"));
     end
 
-rule "PII.4.1: Redact line after contact information keywords (non vertebrate study)"
+rule "PII.4.1: Redact line after contact information keywords (vertebrate study)"
     when
-        not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
+        FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
         $contactKeyword: String() from List.of("Contact point:",
                                          "Contact:",
                                          "Alternative contact:",
@@ -577,12 +594,12 @@ rule "PII.4.1: Redact line after contact information keywords (non vertebrate st
         $section: Section(containsString($contactKeyword))
     then
         entityCreationService.lineAfterString($contactKeyword, "PII", EntityType.ENTITY, $section)
-            .forEach(contactEntity -> contactEntity.apply("PII.4.1", "Found after \"" + $contactKeyword + "\" contact keyword", "Reg (EC) No 1107/2009 Art. 63 (2e)"));
+            .forEach(contactEntity -> contactEntity.redact("PII.4.1", "Found after \"" + $contactKeyword + "\" contact keyword", "Reg (EC) No 1107/2009 Art. 63 (2e)"));
     end
 
 
 // Rule unit: PII.6
-rule "PII.6.0: redact line between contact keywords (non vertebrate study)"
+rule "PII.6.0: Redact line between contact keywords (non vertebrate study)"
     when
         not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
         $section: Section((containsString("No:") && containsString("Fax")) || (containsString("Contact:") && containsString("Tel")))
@@ -591,10 +608,10 @@ rule "PII.6.0: redact line between contact keywords (non vertebrate study)"
             entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section),
             entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section)
         )
-            .forEach(contactEntity -> contactEntity.apply("PII.6.0", "Found between contact keywords", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
+            .forEach(contactEntity -> contactEntity.redact("PII.6.0", "Found between contact keywords", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
     end
 
-rule "PII.6.1: redact line between contact keywords"
+rule "PII.6.1: Redact line between contact keywords (vertebrate study)"
     when
         FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
         $section: Section((containsString("No:") && containsString("Fax")) || (containsString("Contact:") && containsString("Tel")))
@@ -603,7 +620,7 @@ rule "PII.6.1: redact line between contact keywords"
             entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section),
             entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section)
         )
-            .forEach(contactEntity -> contactEntity.apply("PII.6.1", "Found between contact keywords", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
+            .forEach(contactEntity -> contactEntity.redact("PII.6.1", "Found between contact keywords", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
     end
 
 
@@ -623,10 +640,10 @@ rule "PII.7.0: Redact contact information if applicant is found (non vertebrate
                  entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section),
                  entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section)
              ))
-            .forEach(entity -> entity.apply("PII.7.0", "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
+            .forEach(entity -> entity.redact("PII.7.0", "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
     end
 
-rule "PII.7.1: Redact contact information if applicant is found (non vertebrate study)"
+rule "PII.7.1: Redact contact information if applicant is found (vertebrate study)"
     when
         FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
         $section: Section(getHeadline().containsString("applicant") ||
@@ -641,12 +658,12 @@ rule "PII.7.1: Redact contact information if applicant is found (non vertebrate
                  entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section),
                  entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section)
              ))
-            .forEach(entity -> entity.apply("PII.7.1", "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
+            .forEach(entity -> entity.redact("PII.7.1", "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
     end
 
 
 // Rule unit: PII.8
-rule "PII.8.0: Redact contact information if producer is found"
+rule "PII.8.0: Redact contact information if producer is found (non vertebrate study)"
     when
         not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
         $section: Section(containsStringIgnoreCase("producer of the plant protection") ||
@@ -661,10 +678,10 @@ rule "PII.8.0: Redact contact information if producer is found"
                  entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section),
                  entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section)
              ))
-            .forEach(entity -> entity.apply("PII.8.0", "Producer was found", "Reg (EC) No 1107/2009 Art. 63 (2e)"));
+            .forEach(entity -> entity.redact("PII.8.0", "Producer was found", "Reg (EC) No 1107/2009 Art. 63 (2e)"));
     end
 
-rule "PII.8.1: Redact contact information if producer is found"
+rule "PII.8.1: Redact contact information if producer is found (vertebrate study)"
     when
         FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
         $section: Section(containsStringIgnoreCase("producer of the plant protection") ||
@@ -679,45 +696,27 @@ rule "PII.8.1: Redact contact information if producer is found"
                  entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section),
                  entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section)
              ))
-            .forEach(entity -> entity.apply("PII.8.1", "Producer was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
+            .forEach(entity -> entity.redact("PII.8.1", "Producer was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
     end
 
 
 // Rule unit: PII.9
-rule "PII.9.0: Redact between \"AUTHOR(S)\" and \"COMPLETION DATE\" (non vertebrate study)"
+rule "PII.9.0: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\" (non vertebrate study)"
     when
         not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
-        $section: Section(!hasTables(), containsString("AUTHOR(S):"), containsString("COMPLETION DATE:"), !containsString("STUDY COMPLETION DATE:"))
+        $document: Document(containsStringIgnoreCase("AUTHOR(S)"), containsAnyStringIgnoreCase("COMPLETION DATE", "STUDY COMPLETION DATE"))
     then
-        entityCreationService.betweenStrings("AUTHOR(S):", "COMPLETION DATE:", "PII", EntityType.ENTITY, $section)
-         .forEach(authorEntity -> authorEntity.apply("PII.9.0", "AUTHOR(S) was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
+        entityCreationService.shortestBetweenAnyStringIgnoreCase(List.of("AUTHOR(S)", "AUTHOR(S):"), List.of("COMPLETION DATE", "COMPLETION DATE:", "STUDY COMPLETION DATE", "STUDY COMPLETION DATE:"), "PII", EntityType.ENTITY, $document)
+         .forEach(authorEntity -> authorEntity.redact("PII.9.0", "AUTHOR(S) was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
     end
 
-rule "PII.9.1: Redact between \"AUTHOR(S)\" and \"STUDY COMPLETION DATE\" (non vertebrate study)"
+rule "PII.9.1: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\" (vertebrate study)"
     when
         FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
-        $section: Section(!hasTables(), containsString("AUTHOR(S):"), containsString("COMPLETION DATE:"), !containsString("STUDY COMPLETION DATE:"))
+        $document: Document(containsStringIgnoreCase("AUTHOR(S)"), containsAnyStringIgnoreCase("COMPLETION DATE", "STUDY COMPLETION DATE"))
     then
-        entityCreationService.betweenStrings("AUTHOR(S):", "COMPLETION DATE:", "PII", EntityType.ENTITY, $section)
-         .forEach(authorEntity -> authorEntity.apply("PII.9.1", "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
-    end
-
-rule "PII.9.2: Redact between \"AUTHOR(S)\" and \"COMPLETION DATE\" (non vertebrate study)"
-    when
-        not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
-        $section: Section(!hasTables(), containsString("AUTHOR(S):"), containsString("STUDY COMPLETION DATE:"))
-    then
-        entityCreationService.betweenStrings("AUTHOR(S):", "STUDY COMPLETION DATE:", "PII", EntityType.ENTITY, $section)
-         .forEach(authorEntity -> authorEntity.apply("PII.9.2", "AUTHOR(S) was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
-    end
-
-rule "PII.9.3: Redact between \"AUTHOR(S)\" and \"STUDY COMPLETION DATE\" (vertebrate study)"
-    when
-        FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
-        $section: Section(!hasTables(), containsString("AUTHOR(S):"), containsString("STUDY COMPLETION DATE:"))
-    then
-        entityCreationService.betweenStrings("AUTHOR(S):", "STUDY COMPLETION DATE:", "PII", EntityType.ENTITY, $section)
-         .forEach(authorEntity -> authorEntity.apply("PII.9.3", "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
+        entityCreationService.shortestBetweenAnyStringIgnoreCase(List.of("AUTHOR(S)", "AUTHOR(S):"), List.of("COMPLETION DATE", "COMPLETION DATE:", "STUDY COMPLETION DATE", "STUDY COMPLETION DATE:"), "PII", EntityType.ENTITY, $document)
+        .forEach(authorEntity -> authorEntity.redact("PII.9.1", "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
     end
 
 
@@ -727,7 +726,7 @@ rule "PII.11.0: Redact On behalf of Sequani Ltd.:"
         $section: Section(!hasTables(), containsString("On behalf of Sequani Ltd.: Name Title"))
     then
         entityCreationService.betweenStrings("On behalf of Sequani Ltd.: Name Title", "On behalf of", "PII", EntityType.ENTITY, $section)
-         .forEach(authorEntity -> authorEntity.apply("PII.11.0", "On behalf of Sequani Ltd.: Name Title was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
+         .forEach(authorEntity -> authorEntity.redact("PII.11.0", "On behalf of Sequani Ltd.: Name Title was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
     end
 
 
@@ -749,7 +748,7 @@ rule "ETC.1.0: Redact Purity"
         $section: Section(containsStringIgnoreCase("purity"))
     then
         entityCreationService.byRegex("\\bPurity:\\s*(<?>?\\s*\\d{1,2}(?:\\.\\d{1,2})?\\s*%)", "purity", EntityType.ENTITY, 1, $section)
-            .forEach(entity -> entity.apply("ETC.1.0", "Purity found", "Reg (EC) No 1107/2009 Art. 63 (2a)"));
+            .forEach(entity -> entity.redact("ETC.1.0", "Purity found", "Reg (EC) No 1107/2009 Art. 63 (2a)"));
      end
 
 
@@ -759,7 +758,7 @@ rule "ETC.2.0: Redact signatures (non vertebrate study)"
         not FileAttribute(label == "Vertebrate Study", value == "Yes")
         $signature: Image(imageType == ImageType.SIGNATURE)
     then
-        $signature.apply("ETC.2.0", "Signature Found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        $signature.redact("ETC.2.0", "Signature Found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
     end
 
 rule "ETC.2.1: Redact signatures (vertebrate study)"
@@ -767,25 +766,25 @@ rule "ETC.2.1: Redact signatures (vertebrate study)"
         FileAttribute(label == "Vertebrate Study", value == "Yes")
         $signature: Image(imageType == ImageType.SIGNATURE)
     then
-        $signature.apply("ETC.2.1", "Signature Found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        $signature.redact("ETC.2.1", "Signature Found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
     end
 
 
 // Rule unit: ETC.3
-rule "ETC.3.0: Redact logos (vertebrate study)"
+rule "ETC.3.0: Skip logos (non vertebrate study)"
     when
         not FileAttribute(label == "Vertebrate Study", value == "Yes")
         $logo: Image(imageType == ImageType.LOGO)
     then
-        $logo.apply("ETC.3.0", "Logo Found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        $logo.skip("ETC.3.0", "Logo Found");
     end
 
-rule "ETC.3.1: Redact logos (non vertebrate study)"
+rule "ETC.3.1: Redact logos (vertebrate study)"
     when
         FileAttribute(label == "Vertebrate Study", value == "Yes")
         $logo: Image(imageType == ImageType.LOGO)
     then
-        $logo.apply("ETC.3.1", "Logo Found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        $logo.redact("ETC.3.1", "Logo Found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
     end
 
 
@@ -794,7 +793,7 @@ rule "ETC.4.0: Redact dossier dictionary entries"
     when
         $dossierRedaction: TextEntity(type == "dossier_redaction")
     then
-        $dossierRedaction.apply("ETC.4.0", "Specification of impurity found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        $dossierRedaction.redact("ETC.4.0", "Specification of impurity found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
     end
 
 
@@ -818,14 +817,14 @@ rule "ETC.6.0: Redact CAS Number"
             .map(tableCell -> entityCreationService.bySemanticNode(tableCell, "PII", EntityType.ENTITY))
             .filter(Optional::isPresent)
             .map(Optional::get)
-            .forEach(redactionEntity -> redactionEntity.apply("ETC.6.0", "Sample # found in Header", "Reg (EC) No 1107/2009 Art. 63 (2g)"));
+            .forEach(redactionEntity -> redactionEntity.redact("ETC.6.0", "Sample # found in Header", "Reg (EC) No 1107/2009 Art. 63 (2g)"));
     end
 
 
 // Rule unit: ETC.7
 rule "ETC.7.0: Guidelines FileAttributes"
     when
-        $section: Section(!hasTables(), (containsString("DATA REQUIREMENT(S):") || containsString("TEST GUIDELINE(S):")) && (containsString("OECD") || containsString("EPA") || containsString("OPPTS")))
+        $section: Section(!hasTables(), containsAnyString("DATA REQUIREMENT(S):", "TEST GUIDELINE(S):") && containsAnyString("OECD", "EPA", "OPPTS"))
     then
         RedactionSearchUtility.findTextRangesByRegex("OECD (No\\.? )?\\d{3}( \\(\\d{4}\\))?", $section.getTextBlock()).stream()
             .map(boundary -> $section.getTextBlock().subSequence(boundary).toString())
@@ -840,7 +839,7 @@ rule "ETC.8.0: Redact formulas (vertebrate study)"
         not FileAttribute(label == "Vertebrate Study", value == "Yes")
         $logo: Image(imageType == ImageType.FORMULA)
     then
-        $logo.apply("ETC.8.0", "Logo Found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        $logo.redact("ETC.8.0", "Logo Found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
     end
 
 rule "ETC.8.1: Redact formulas (non vertebrate study)"
@@ -848,14 +847,14 @@ rule "ETC.8.1: Redact formulas (non vertebrate study)"
         FileAttribute(label == "Vertebrate Study", value == "Yes")
         $logo: Image(imageType == ImageType.FORMULA)
     then
-        $logo.apply("ETC.8.1", "Logo Found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        $logo.redact("ETC.8.1", "Logo Found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
     end
 
 
 //------------------------------------ AI rules ------------------------------------
 
 // Rule unit: AI.0
-rule "AI.0.0: add all NER Entities of type CBI_author"
+rule "AI.0.0: Add all NER Entities of type CBI_author"
     salience 999
     when
         nerEntities: NerEntities(hasEntitiesOfType("CBI_author"))
@@ -866,7 +865,7 @@ rule "AI.0.0: add all NER Entities of type CBI_author"
 
 
 // Rule unit: AI.1
-rule "AI.1.0: combine and add NER Entities as CBI_address"
+rule "AI.1.0: Combine and add NER Entities as CBI_address"
     salience 999
     when
         nerEntities: NerEntities(hasEntitiesOfType("ORG") || hasEntitiesOfType("STREET") || hasEntitiesOfType("CITY"))
@@ -875,7 +874,7 @@ rule "AI.1.0: combine and add NER Entities as CBI_address"
     end
 
 
-//------------------------------------ Manual redaction rules ------------------------------------
+//------------------------------------ Manual changes rules ------------------------------------
 
 // Rule unit: MAN.0
 rule "MAN.0.0: Apply manual resize redaction"
@@ -1020,7 +1019,7 @@ rule "MAN.4.1: Apply legal basis change"
 //------------------------------------ Entity merging rules ------------------------------------
 
 // Rule unit: X.0
-rule "X.0.0: remove Entity contained by Entity of same type"
+rule "X.0.0: Remove Entity contained by Entity of same type"
     salience 65
     when
         $larger: TextEntity($type: type, $entityType: entityType, active())
@@ -1032,7 +1031,7 @@ rule "X.0.0: remove Entity contained by Entity of same type"
 
 
 // Rule unit: X.1
-rule "X.1.0: merge intersecting Entities of same type"
+rule "X.1.0: Merge intersecting Entities of same type"
     salience 64
     when
         $first: TextEntity($type: type, $entityType: entityType, !resized(), active())
@@ -1048,7 +1047,7 @@ rule "X.1.0: merge intersecting Entities of same type"
 
 
 // Rule unit: X.2
-rule "X.2.0: remove Entity of type ENTITY when contained by FALSE_POSITIVE"
+rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
     salience 64
     when
         $falsePositive: TextEntity($type: type, entityType == EntityType.FALSE_POSITIVE, active())
@@ -1061,7 +1060,7 @@ rule "X.2.0: remove Entity of type ENTITY when contained by FALSE_POSITIVE"
 
 
 // Rule unit: X.3
-rule "X.3.0: remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"
+rule "X.3.0: Remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"
     salience 64
     when
         $falseRecommendation: TextEntity($type: type, entityType == EntityType.FALSE_RECOMMENDATION, active())
@@ -1073,7 +1072,7 @@ rule "X.3.0: remove Entity of type RECOMMENDATION when contained by FALSE_RECOMM
 
 
 // Rule unit: X.4
-rule "X.4.0: remove Entity of type RECOMMENDATION when intersected by ENTITY with same type"
+rule "X.4.0: Remove Entity of type RECOMMENDATION when intersected by ENTITY with same type"
     salience 256
     when
         $entity: TextEntity($type: type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
@@ -1086,7 +1085,7 @@ rule "X.4.0: remove Entity of type RECOMMENDATION when intersected by ENTITY wit
 
 
 // Rule unit: X.5
-rule "X.5.0: remove Entity of type RECOMMENDATION when contained by ENTITY"
+rule "X.5.0: Remove Entity of type RECOMMENDATION when contained by ENTITY"
     salience 256
     when
         $entity: TextEntity((entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
@@ -1098,7 +1097,7 @@ rule "X.5.0: remove Entity of type RECOMMENDATION when contained by ENTITY"
 
 
 // Rule unit: X.6
-rule "X.6.0: remove Entity of lower rank, when contained by entity of type ENTITY"
+rule "X.6.0: Remove Entity of lower rank, when contained by by entity of type ENTITY"
     salience 32
     when
         $higherRank: TextEntity($type: type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
@@ -1113,7 +1112,7 @@ rule "X.6.1: remove Entity of higher rank, when intersected by entity of type EN
     salience 32
     when
         $higherRank: TextEntity($type: type, $value: value, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
-        $lowerRank: TextEntity(intersects($higherRank), type != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active(), $lowerRank.getValue().length() > $value.length())
+        $lowerRank: TextEntity(intersects($higherRank), type != $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active(), $lowerRank.getValue().length() > $value.length())
     then
         $higherRank.getIntersectingNodes().forEach(node -> update(node));
         $higherRank.remove("X.6.1", "remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity");
@@ -1124,7 +1123,7 @@ rule "X.6.1: remove Entity of higher rank, when intersected by entity of type EN
 //------------------------------------ File attributes rules ------------------------------------
 
 // Rule unit: FA.1
-rule "FA.1.0: remove duplicate FileAttributes"
+rule "FA.1.0: Remove duplicate FileAttributes"
     salience 64
     when
         $fileAttribute: FileAttribute($label: label, $value: value)
@@ -1145,7 +1144,7 @@ rule "LDS.0.0: Run local dictionary search"
     then
         entityCreationService.bySearchImplementation($dictionaryModel.getLocalSearch(), $dictionaryModel.getType(), EntityType.RECOMMENDATION, document)
             .forEach(entity ->  {
-                Collection<MatchedRule> matchedRules = $dictionaryModel.getLocalEntriesWithMatchedRules().get(entity.getValue());
-                entity.addMatchedRules(matchedRules);
+                Collection<MatchedRule> matchedRules = $dictionaryModel.getMatchedRulesForLocalDictionaryEntry(entity.getValue());
+                matchedRules.forEach(matchedRule -> entity.addMatchedRule(matchedRule.asSkippedIfApplied()));
             });
     end

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_v2.drl

@@ -60,16 +60,40 @@ global EntityCreationService entityCreationService
 global ManualChangesApplicationService manualChangesApplicationService
 global Dictionary dictionary
 
-// --------------------------------------- queries -------------------------------------------------------------------
+//------------------------------------ queries ------------------------------------
 
 query "getFileAttributes"
     $fileAttribute: FileAttribute()
     end
 
-// --------------------------------------- NER Entities rules -------------------------------------------------------------------
+//------------------------------------ CBI rules ------------------------------------
+
+// Rule unit: CBI.0
+rule "CBI.0.0: Redact CBI Authors (non vertebrate Study)"
+    when
+        not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
+        $entity: TextEntity(type == "CBI_author", dictionaryEntry)
+    then
+        $entity.redact("CBI.0.0", "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+    end
+
+
+//------------------------------------ PII rules ------------------------------------
+
+// Rule unit: PII.0
+rule "PII.0.0: Redact all PII (non vertebrate study)"
+    when
+        not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
+        $pii: TextEntity(type == "PII", dictionaryEntry)
+    then
+        $pii.redact("PII.0.0", "Personal Information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+    end
+
+
+//------------------------------------ AI rules ------------------------------------
 
 // Rule unit: AI.0
-rule "AI.0.0: add all NER Entities of type CBI_author"
+rule "AI.0.0: Add all NER Entities of type CBI_author"
     salience 999
     when
         nerEntities: NerEntities(hasEntitiesOfType("CBI_author"))
@@ -78,28 +102,8 @@ rule "AI.0.0: add all NER Entities of type CBI_author"
             .forEach(nerEntity -> entityCreationService.byNerEntity(nerEntity, EntityType.RECOMMENDATION, document));
     end
 
-// --------------------------------------- CBI rules -------------------------------------------------------------------
 
-rule "CBI.0.0: Always redact CBI_author"
-
-    when
-        $cbiAuthor: TextEntity(type == "CBI_author", entityType == EntityType.ENTITY)
-    then
-        $cbiAuthor.apply("CBI.0.0", "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
-    end
-
-// --------------------------------------- PII rules -------------------------------------------------------------------
-
-rule "PII.0.0: Always redact PII"
-
-    when
-        $cbiAuthor: TextEntity(type == "PII", entityType == EntityType.ENTITY)
-    then
-        $cbiAuthor.apply("PII.0.0", "PII found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
-    end
-
-
-//------------------------------------ Manual redaction rules ------------------------------------
+//------------------------------------ Manual changes rules ------------------------------------
 
 // Rule unit: MAN.0
 rule "MAN.0.0: Apply manual resize redaction"
@@ -157,7 +161,6 @@ rule "MAN.1.1: Apply id removals that are valid and not in forced redactions to
 
 // Rule unit: MAN.2
 rule "MAN.2.0: Apply force redaction"
-    no-loop true
     salience 128
     when
         $force: ManualForceRedaction($id: annotationId, status == AnnotationStatus.APPROVED)
@@ -166,10 +169,10 @@ rule "MAN.2.0: Apply force redaction"
         $entityToForce.getManualOverwrite().addChange($force);
         update($entityToForce);
         $entityToForce.getIntersectingNodes().forEach(node -> update(node));
+        retract($force);
     end
 
 rule "MAN.2.1: Apply force redaction to images"
-    no-loop true
     salience 128
     when
         $force: ManualForceRedaction($id: annotationId, status == AnnotationStatus.APPROVED)
@@ -178,6 +181,7 @@ rule "MAN.2.1: Apply force redaction to images"
         $imageToForce.getManualOverwrite().addChange($force);
         update($imageToForce);
         update($imageToForce.getParent());
+        retract($force);
     end
 
 
@@ -185,9 +189,9 @@ rule "MAN.2.1: Apply force redaction to images"
 rule "MAN.3.0: Apply entity recategorization"
     salience 128
     when
-        $recategorization: ManualRecategorization($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate)
+        $recategorization: ManualRecategorization($id: annotationId, $type: type, status == AnnotationStatus.APPROVED, $requestDate: requestDate)
         not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))
-        $entityToBeRecategorized: TextEntity(matchesAnnotationId($id))
+        $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type != $type)
     then
         $entityToBeRecategorized.getIntersectingNodes().forEach(node -> update(node));
         manualChangesApplicationService.recategorize($entityToBeRecategorized, $recategorization);
@@ -196,16 +200,14 @@ rule "MAN.3.0: Apply entity recategorization"
         retract($entityToBeRecategorized);
     end
 
-rule "MAN.3.1: Apply image recategorization"
+rule "MAN.3.1: Apply entity recategorization of same type"
     salience 128
     when
-        $recategorization: ManualRecategorization($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate)
+        $recategorization: ManualRecategorization($id: annotationId, $type: type, status == AnnotationStatus.APPROVED, $requestDate: requestDate)
         not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))
-        $imageToBeRecategorized: Image($id == id)
+        $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type == $type)
     then
-        manualChangesApplicationService.recategorize($imageToBeRecategorized, $recategorization);
-        update($imageToBeRecategorized);
-        update($imageToBeRecategorized.getParent());
+        $entityToBeRecategorized.getManualOverwrite().addChange($recategorization);
         retract($recategorization);
     end
 
@@ -233,7 +235,7 @@ rule "MAN.4.1: Apply legal basis change"
 //------------------------------------ Entity merging rules ------------------------------------
 
 // Rule unit: X.0
-rule "X.0.0: remove Entity contained by Entity of same type"
+rule "X.0.0: Remove Entity contained by Entity of same type"
     salience 65
     when
         $larger: TextEntity($type: type, $entityType: entityType, active())
@@ -245,7 +247,7 @@ rule "X.0.0: remove Entity contained by Entity of same type"
 
 
 // Rule unit: X.1
-rule "X.1.0: merge intersecting Entities of same type"
+rule "X.1.0: Merge intersecting Entities of same type"
     salience 64
     when
         $first: TextEntity($type: type, $entityType: entityType, !resized(), active())
@@ -261,7 +263,7 @@ rule "X.1.0: merge intersecting Entities of same type"
 
 
 // Rule unit: X.2
-rule "X.2.0: remove Entity of type ENTITY when contained by FALSE_POSITIVE"
+rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
     salience 64
     when
         $falsePositive: TextEntity($type: type, entityType == EntityType.FALSE_POSITIVE, active())
@@ -274,7 +276,7 @@ rule "X.2.0: remove Entity of type ENTITY when contained by FALSE_POSITIVE"
 
 
 // Rule unit: X.3
-rule "X.3.0: remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"
+rule "X.3.0: Remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"
     salience 64
     when
         $falseRecommendation: TextEntity($type: type, entityType == EntityType.FALSE_RECOMMENDATION, active())
@@ -286,7 +288,7 @@ rule "X.3.0: remove Entity of type RECOMMENDATION when contained by FALSE_RECOMM
 
 
 // Rule unit: X.4
-rule "X.4.0: remove Entity of type RECOMMENDATION when intersected by ENTITY with same type"
+rule "X.4.0: Remove Entity of type RECOMMENDATION when intersected by ENTITY with same type"
     salience 256
     when
         $entity: TextEntity($type: type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
@@ -299,7 +301,7 @@ rule "X.4.0: remove Entity of type RECOMMENDATION when intersected by ENTITY wit
 
 
 // Rule unit: X.5
-rule "X.5.0: remove Entity of type RECOMMENDATION when contained by ENTITY"
+rule "X.5.0: Remove Entity of type RECOMMENDATION when contained by ENTITY"
     salience 256
     when
         $entity: TextEntity((entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
@@ -311,7 +313,7 @@ rule "X.5.0: remove Entity of type RECOMMENDATION when contained by ENTITY"
 
 
 // Rule unit: X.6
-rule "X.6.0: remove Entity of lower rank, when contained by by entity of type ENTITY"
+rule "X.6.0: Remove Entity of lower rank, when contained by by entity of type ENTITY"
     salience 32
     when
         $higherRank: TextEntity($type: type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
@@ -326,7 +328,7 @@ rule "X.6.1: remove Entity of higher rank, when intersected by entity of type EN
     salience 32
     when
         $higherRank: TextEntity($type: type, $value: value, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
-        $lowerRank: TextEntity(intersects($higherRank), type != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active(), $lowerRank.getValue().length() > $value.length())
+        $lowerRank: TextEntity(intersects($higherRank), type != $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active(), $lowerRank.getValue().length() > $value.length())
     then
         $higherRank.getIntersectingNodes().forEach(node -> update(node));
         $higherRank.remove("X.6.1", "remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity");
@@ -337,7 +339,7 @@ rule "X.6.1: remove Entity of higher rank, when intersected by entity of type EN
 //------------------------------------ File attributes rules ------------------------------------
 
 // Rule unit: FA.1
-rule "FA.1.0: remove duplicate FileAttributes"
+rule "FA.1.0: Remove duplicate FileAttributes"
     salience 64
     when
         $fileAttribute: FileAttribute($label: label, $value: value)
@@ -358,7 +360,7 @@ rule "LDS.0.0: Run local dictionary search"
     then
         entityCreationService.bySearchImplementation($dictionaryModel.getLocalSearch(), $dictionaryModel.getType(), EntityType.RECOMMENDATION, document)
             .forEach(entity ->  {
-                Collection<MatchedRule> matchedRules = $dictionaryModel.getLocalEntriesWithMatchedRules().get(entity.getValue());
-                entity.addMatchedRules(matchedRules);
+                Collection<MatchedRule> matchedRules = $dictionaryModel.getMatchedRulesForLocalDictionaryEntry(entity.getValue());
+                matchedRules.forEach(matchedRule -> entity.addMatchedRule(matchedRule.asSkippedIfApplied()));
             });
     end

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/table_demo.drl

@@ -0,0 +1,487 @@
+package drools
+
+import static java.lang.String.format;
+import static com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility.anyMatch;
+import static com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility.exactMatch;
+
+import java.util.List;
+import java.util.LinkedList;
+import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.Collection;
+import java.util.stream.Stream;
+import java.util.Optional;
+
+import com.iqser.red.service.redaction.v1.server.model.document.*;
+import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
+import com.iqser.red.service.redaction.v1.server.model.document.entity.*;
+import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
+import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule;
+import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity
+import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.*;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.Section;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.Table;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.Paragraph;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.Image;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.ImageType;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.Page;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.Headline;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.SectionIdentifier;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.Footer;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.Header;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.NodeType;
+import com.iqser.red.service.redaction.v1.server.model.document.textblock.*;
+import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
+import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlockCollector;
+import com.iqser.red.service.redaction.v1.server.model.document.textblock.AtomicTextBlock;
+import com.iqser.red.service.redaction.v1.server.model.document.textblock.ConcatenatedTextBlock;
+import com.iqser.red.service.redaction.v1.server.model.NerEntities;
+import com.iqser.red.service.redaction.v1.server.model.dictionary.Dictionary;
+import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryModel;
+import com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService;
+import com.iqser.red.service.redaction.v1.server.service.ManualChangesApplicationService;
+import com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility;
+
+import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus;
+
+global Document document
+global EntityCreationService entityCreationService
+global ManualChangesApplicationService manualChangesApplicationService
+global Dictionary dictionary
+
+//------------------------------------ queries ------------------------------------
+
+query "getFileAttributes"
+    $fileAttribute: FileAttribute()
+    end
+
+//------------------------------------ Table extraction rules ------------------------------------
+
+// Rule unit: TAB.0
+rule "TAB.0.0: Study Type File Attribute"
+    when
+        not FileAttribute(label == "OECD Number", valueEqualsAnyOf("402","403","404","405","425","429","436","438","439","471","487"))
+        $section: Section(containsAnyString("DATA REQUIREMENT", "TEST GUIDELINE", "MÉTODO(S) DE REFERÊNCIA(S):")
+            && containsAnyString("OECD", "EPA", "OPPTS"))
+    then
+        RedactionSearchUtility.findTextRangesByRegexIgnoreCase("(?<=OECD)(?:[\\w\\s,\\[\\]\\(\\)\\.]{1,10}|(?:.{5,40}(?:Number |Procedure |Guideline )))(4[\\d]{2})", 1 ,$section.getTextBlock()).stream()
+            .map(boundary -> $section.getTextBlock().subSequence(boundary).toString())
+            .map(value -> FileAttribute.builder().label("OECD Number").value(value).build())
+            .forEach(fileAttribute -> insert(fileAttribute));
+        RedactionSearchUtility.findTextRangesByRegexIgnoreCase("(?<=OECD).{5,40}Method (4[\\d]{2}).{1,65}(\\d{4})\\)", 1, $section.getTextBlock()).stream()
+            .map(boundary -> $section.getTextBlock().subSequence(boundary).toString())
+            .map(value -> FileAttribute.builder().label("OECD Number").value(value).build())
+            .forEach(fileAttribute -> insert(fileAttribute));
+    end
+
+rule "TAB.0.1: Guidelines"
+    when
+        $section: Section(containsAnyString("DATA REQUIREMENT", "TEST GUIDELINE", "MÉTODO(S) DE REFERÊNCIA(S):") && containsAnyString("OECD", "EPA", "OPPTS"))
+    then
+        entityCreationService.byRegex("(?<=OECD)(?:[\\w\\s,\\[\\]\\(\\)\\.]{1,10}|.{5,40}(?:Number |Procedure |Guideline ))(4[\\d]{2})", "oecd_guideline_number", EntityType.ENTITY, 1, $section)
+            .forEach(guideline -> guideline.apply("TAB.0.1", "OECD Guideline no. found"));
+        entityCreationService.byRegex("(?<=OECD)(?:[\\w\\s,\\[\\]\\(\\)\\.]{1,10}|.{5,40}(?:Number |Procedure |Guideline ))(4[\\d]{2}),?\\s\\(?(\\d{4})\\)?", "oecd_guideline_year", EntityType.ENTITY, 2, $section)
+            .forEach(guideline -> guideline.apply("TAB.0.1", "OECD Guideline year found"));
+        entityCreationService.byRegex("(?<=OECD)[\\w\\s,\\[\\]]{1,10}\\((\\d{4})\\)\\s(4[\\d]{2})", "oecd_guideline_year", EntityType.ENTITY, 1, $section)
+            .forEach(guideline -> guideline.apply("TAB.0.1", "OECD Guideline year found"));
+        entityCreationService.byRegex("(?<=OECD).{5,40}Method (4[\\d]{2}).{1,65}(\\d{4})\\)", "oecd_guideline_number", EntityType.ENTITY, 1, $section)
+            .forEach(guideline -> guideline.apply("TAB.0.1", "OECD Guideline number found"));
+        entityCreationService.byRegex("(?<=OECD).{5,40}Method (4[\\d]{2}).{1,65}(\\d{4})\\)", "oecd_guideline_year", EntityType.ENTITY, 2, $section)
+            .forEach(guideline -> guideline.apply("TAB.0.1", "OECD Guideline year found"));
+    end
+
+
+// Rule unit: TAB.1
+rule "TAB.1.0: Full Table extraction (Guideline Deviation)"
+    when
+        FileAttribute(label == "OECD Number", valueEqualsAnyOf("425"))
+        $section: Section(getHeadline().containsString("Full Table"))
+        $table: Table() from $section.getParent().streamAllSubNodesOfType(NodeType.TABLE).toList()
+        $tableCell: TableCell(!header) from $table.streamTableCells().toList()
+    then
+        entityCreationService.bySemanticNode($tableCell, "full_table_row", EntityType.ENTITY)
+            .ifPresent(entity -> entity.apply("TAB.1.0", "full table extracted"));
+    end
+
+
+// Rule unit: TAB.2
+rule "TAB.2.0: Individual row extraction (Clinical Signs)"
+    when
+        FileAttribute(label == "OECD Number", valueEqualsAnyOf("425"))
+        $section: Section(getHeadline().containsString("Individual Rows Extraction"))
+        $table: Table(hasHeader("Animal No."), (hasRowWithHeaderAndAnyValue("Animal No.", List.of("120-2", "120-5")))) from $section.streamChildren().toList()
+        TableCell($row: row, containsAnyString("120-2", "120-5")) from $table.streamTableCellsWithHeader("Animal No.").toList()
+        $tableCell: TableCell($row == row) from $table.streamTableCells().toList()
+    then
+        entityCreationService.bySemanticNode($tableCell, "clinical_signs", EntityType.ENTITY)
+            .ifPresent(entity -> entity.apply("TAB.2.0", "Individual row based on animal number"));
+    end
+
+
+// Rule unit: TAB.3
+rule "TAB.3.0: Individual column extraction (Strain)"
+    when
+        FileAttribute(label == "OECD Number", valueEqualsAnyOf("425"))
+        $section: Section(getHeadline().containsString("Individual Column"))
+        $table: Table(hasHeader("Sex")) from $section.streamChildren().toList()
+    then
+        $table.streamTableCellsWithHeader("Sex")
+            .map(tableCell -> entityCreationService.bySemanticNode(tableCell, "dosages", EntityType.ENTITY))
+            .filter(Optional::isPresent)
+            .map(Optional::get)
+            .forEach(redactionEntity -> redactionEntity.apply("TAB.3.0", "Individual column based on column header"));
+    end
+
+
+// Rule unit: TAB.4
+rule "TAB.4.0: Combined Columns Extraction - Sex and Dosage"
+    when
+        FileAttribute(label == "OECD Number", valueEqualsAnyOf("425"))
+        $section: Section(getHeadline().containsString("Combined Columns"))
+        $table: Table(hasHeader("Dosage (mg/kg bw)")) from $section.getParent().streamAllSubNodesOfType(NodeType.TABLE).toList()
+    then
+        $table.streamTableCellsWithHeader("Dosage (mg/kg bw)")
+            .map(tableCell -> entityCreationService.bySemanticNode(tableCell, "dose_mortality_dose", EntityType.ENTITY))
+            .filter(Optional::isPresent)
+            .map(Optional::get)
+            .forEach(redactionEntity -> redactionEntity.apply("TAB.4.0", "Dose Mortality dose found."));
+    end
+
+rule "TAB.4.1: Combined Columns Extraction - Sex and Mortality"
+    when
+        FileAttribute(label == "OECD Number", valueEqualsAnyOf("425"))
+        $section: Section(getHeadline().containsString("Combined Columns"))
+        $table: Table(hasHeader("Mortality")) from $section.getParent().streamAllSubNodesOfType(NodeType.TABLE).toList()
+    then
+        $table.streamTableCellsWithHeader("Mortality")
+            .map(tableCell -> entityCreationService.bySemanticNode(tableCell, "dose_mortality", EntityType.ENTITY))
+            .filter(Optional::isPresent)
+            .map(Optional::get)
+            .forEach(redactionEntity -> redactionEntity.apply("TAB.4.1", "Dose Mortality found."));
+    end
+
+
+// Rule unit: TAB.5
+rule "TAB.5.0: Targeted cell extraction"
+    when
+        FileAttribute(label == "OECD Number", valueEqualsAnyOf("425"))
+        $section: Section(getHeadline().containsString("Value Extraction"))
+        $table: Table(hasHeader("Mortality"), hasRowWithHeaderAndAnyValue("Sex", List.of("male", "Male")), hasRowWithHeaderAndValue("Mortality", "Survived")) from $section.streamChildren().toList()
+        TableCell(containsWordIgnoreCase("Male"), $row: row) from $table.streamTableCellsWithHeader("Sex").toList()
+        TableCell($row == row, containsStringIgnoreCase("Survived")) from $table.streamTableCellsWithHeader("Mortality").toList()
+        $dosageCell: TableCell($row == row) from $table.streamTableCellsWithHeader("Dosage").toList()
+    then
+        entityCreationService.bySemanticNode($dosageCell,"doses_mg_kg_bw", EntityType.ENTITY)
+            .ifPresent(entity -> entity.apply("TAB.5.0", "Dosage found in row with survived male"));
+    end
+
+
+// Rule unit: TAB.6
+rule "TAB.6.0: Targeted cell extraction (Experimental Stop date)"
+    when
+        $section: Section(getHeadline().containsString("Advanced Table Extraction"), containsAllStrings("female", "Female", "Survived", "Group 2"))
+        $table: Table(hasHeader("Group 2")) from $section.streamChildren().toList()
+        TableCell(containsWordIgnoreCase("Female"), $row: row) from $table.streamTableCellsWithHeader("Group 2").toList()
+        TableCell($row == row, containsStringIgnoreCase("Survived")) from $table.streamTableCellsWithHeader("Group 2").toList()
+        $femaleSurvived: TableCell($row == row) from $table.streamTableCellsWithHeader("Group 2").toList()
+    then
+        entityCreationService.bySemanticNode($femaleSurvived, "experiment_female_survived", EntityType.ENTITY)
+            .ifPresent(entity -> entity.apply("TAB.6.0", "Female in group to experimental start date"));
+    end
+
+
+// Rule unit: TAB.7
+rule "TAB.7.0: Indicator (Species)"
+    when
+        FileAttribute(label == "OECD Number", valueEqualsAnyOf("425"))
+        $section: Section(getHeadline().containsString("Entity-Based"))
+        $table: Table() from $section.streamAllSubNodesOfType(NodeType.TABLE).toList()
+        TableCell(isHeader(), containsString("Title"), $col: col) from $table.streamTableCells().toList()
+        TableCell(hasEntitiesOfType("vertebrates"), $row: row) from $table.streamTableCells().toList()
+        $cell: TableCell($col == col, $row == row) from $table.streamTableCells().toList()
+    then
+        entityCreationService.bySemanticNode($cell, "study_design", EntityType.ENTITY)
+            .ifPresent(redactionEntity -> redactionEntity.apply("TAB.7.0", "Vertebrate study found"));
+    end
+
+
+//------------------------------------ Manual changes rules ------------------------------------
+
+// Rule unit: MAN.0
+rule "MAN.0.0: Apply manual resize redaction"
+    salience 128
+    when
+        $resizeRedaction: ManualResizeRedaction($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate)
+        not ManualResizeRedaction(annotationId == $id, requestDate.isBefore($requestDate))
+        $entityToBeResized: TextEntity(matchesAnnotationId($id))
+    then
+        manualChangesApplicationService.resizeEntityAndReinsert($entityToBeResized, $resizeRedaction);
+        retract($resizeRedaction);
+        update($entityToBeResized);
+        $entityToBeResized.getIntersectingNodes().forEach(node -> update(node));
+    end
+
+rule "MAN.0.1: Apply manual resize redaction"
+    salience 128
+    when
+        $resizeRedaction: ManualResizeRedaction($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate)
+        not ManualResizeRedaction(annotationId == $id, requestDate.isBefore($requestDate))
+        $imageToBeResized: Image(id == $id)
+    then
+        manualChangesApplicationService.resizeImage($imageToBeResized, $resizeRedaction);
+        retract($resizeRedaction);
+        update($imageToBeResized);
+        update($imageToBeResized.getParent());
+    end
+
+
+// Rule unit: MAN.1
+rule "MAN.1.0: Apply id removals that are valid and not in forced redactions to Entity"
+    salience 128
+    when
+        $idRemoval: IdRemoval($id: annotationId, status == AnnotationStatus.APPROVED)
+        $entityToBeRemoved: TextEntity(matchesAnnotationId($id))
+    then
+        $entityToBeRemoved.getManualOverwrite().addChange($idRemoval);
+        update($entityToBeRemoved);
+        retract($idRemoval);
+        $entityToBeRemoved.getIntersectingNodes().forEach(node -> update(node));
+    end
+
+rule "MAN.1.1: Apply id removals that are valid and not in forced redactions to Image"
+    salience 128
+    when
+        $idRemoval: IdRemoval($id: annotationId, status == AnnotationStatus.APPROVED)
+        $imageEntityToBeRemoved: Image($id == id)
+    then
+        $imageEntityToBeRemoved.getManualOverwrite().addChange($idRemoval);
+        update($imageEntityToBeRemoved);
+        retract($idRemoval);
+        update($imageEntityToBeRemoved.getParent());
+    end
+
+
+// Rule unit: MAN.2
+rule "MAN.2.0: Apply force redaction"
+    salience 128
+    when
+        $force: ManualForceRedaction($id: annotationId, status == AnnotationStatus.APPROVED)
+        $entityToForce: TextEntity(matchesAnnotationId($id))
+    then
+        $entityToForce.getManualOverwrite().addChange($force);
+        update($entityToForce);
+        $entityToForce.getIntersectingNodes().forEach(node -> update(node));
+        retract($force);
+    end
+
+rule "MAN.2.1: Apply force redaction to images"
+    salience 128
+    when
+        $force: ManualForceRedaction($id: annotationId, status == AnnotationStatus.APPROVED)
+        $imageToForce: Image(id == $id)
+    then
+        $imageToForce.getManualOverwrite().addChange($force);
+        update($imageToForce);
+        update($imageToForce.getParent());
+        retract($force);
+    end
+
+
+// Rule unit: MAN.3
+rule "MAN.3.0: Apply entity recategorization"
+    salience 128
+    when
+        $recategorization: ManualRecategorization($id: annotationId, $type: type, status == AnnotationStatus.APPROVED, $requestDate: requestDate)
+        not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))
+        $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type != $type)
+    then
+        $entityToBeRecategorized.getIntersectingNodes().forEach(node -> update(node));
+        manualChangesApplicationService.recategorize($entityToBeRecategorized, $recategorization);
+        retract($recategorization);
+        // Entity is copied and inserted, so the old entity needs to be retracted to avoid duplication.
+        retract($entityToBeRecategorized);
+    end
+
+rule "MAN.3.1: Apply entity recategorization of same type"
+    salience 128
+    when
+        $recategorization: ManualRecategorization($id: annotationId, $type: type, status == AnnotationStatus.APPROVED, $requestDate: requestDate)
+        not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))
+        $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type == $type)
+    then
+        $entityToBeRecategorized.getManualOverwrite().addChange($recategorization);
+        retract($recategorization);
+    end
+
+rule "MAN.3.2: Apply image recategorization"
+    salience 128
+    when
+        $recategorization: ManualRecategorization($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate)
+        not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))
+        $imageToBeRecategorized: Image($id == id)
+    then
+        manualChangesApplicationService.recategorize($imageToBeRecategorized, $recategorization);
+        update($imageToBeRecategorized);
+        update($imageToBeRecategorized.getParent());
+        retract($recategorization);
+    end
+
+
+// Rule unit: MAN.4
+rule "MAN.4.0: Apply legal basis change"
+    salience 128
+    when
+        $legalbasisChange: ManualLegalBasisChange($id: annotationId, status == AnnotationStatus.APPROVED)
+        $imageToBeRecategorized: Image($id == id)
+    then
+        $imageToBeRecategorized.getManualOverwrite().addChange($legalbasisChange);
+    end
+
+rule "MAN.4.1: Apply legal basis change"
+    salience 128
+    when
+        $legalBasisChange: ManualLegalBasisChange($id: annotationId, status == AnnotationStatus.APPROVED)
+        $entityToBeChanged: TextEntity(matchesAnnotationId($id))
+    then
+        $entityToBeChanged.getManualOverwrite().addChange($legalBasisChange);
+    end
+
+
+//------------------------------------ Entity merging rules ------------------------------------
+
+// Rule unit: X.0
+rule "X.0.0: Remove Entity contained by Entity of same type"
+    salience 65
+    when
+        $larger: TextEntity($type: type, $entityType: entityType, active())
+        $contained: TextEntity(containedBy($larger), type == $type, entityType == $entityType, this != $larger, !hasManualChanges(), active())
+    then
+        $contained.remove("X.0.0", "remove Entity contained by Entity of same type");
+        retract($contained);
+    end
+
+
+// Rule unit: X.2
+rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
+    salience 64
+    when
+        $falsePositive: TextEntity($type: type, entityType == EntityType.FALSE_POSITIVE, active())
+        $entity: TextEntity(containedBy($falsePositive), type == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges(), active())
+    then
+        $entity.getIntersectingNodes().forEach(node -> update(node));
+        $entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE");
+        retract($entity)
+    end
+
+
+// Rule unit: X.3
+rule "X.3.0: Remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"
+    salience 64
+    when
+        $falseRecommendation: TextEntity($type: type, entityType == EntityType.FALSE_RECOMMENDATION, active())
+        $recommendation: TextEntity(containedBy($falseRecommendation), type == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
+    then
+        $recommendation.remove("X.3.0", "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION");
+        retract($recommendation);
+    end
+
+
+// Rule unit: X.4
+rule "X.4.0: Remove Entity of type RECOMMENDATION when intersected by ENTITY with same type"
+    salience 256
+    when
+        $entity: TextEntity($type: type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
+        $recommendation: TextEntity(intersects($entity), type == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
+    then
+        $entity.addEngines($recommendation.getEngines());
+        $recommendation.remove("X.4.0", "remove Entity of type RECOMMENDATION when intersected by ENTITY with same type");
+        retract($recommendation);
+    end
+
+
+// Rule unit: X.5
+rule "X.5.0: Remove Entity of type RECOMMENDATION when contained by ENTITY"
+    salience 256
+    when
+        $entity: TextEntity((entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
+        $recommendation: TextEntity(containedBy($entity), entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
+    then
+        $recommendation.remove("X.5.0", "remove Entity of type RECOMMENDATION when contained by ENTITY");
+        retract($recommendation);
+    end
+
+
+// Rule unit: X.6
+rule "X.6.0: Remove Entity of lower rank, when contained by by entity of type ENTITY"
+    salience 32
+    when
+        $higherRank: TextEntity($type: type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
+        $lowerRank: TextEntity(containedBy($higherRank), type != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active())
+    then
+        $lowerRank.getIntersectingNodes().forEach(node -> update(node));
+        $lowerRank.remove("X.6.0", "remove Entity of lower rank, when contained by entity of type ENTITY");
+        retract($lowerRank);
+    end
+
+rule "X.6.1: remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity"
+    salience 32
+    when
+        $higherRank: TextEntity($type: type, $value: value, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
+        $lowerRank: TextEntity(intersects($higherRank), type != $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active(), $lowerRank.getValue().length() > $value.length())
+    then
+        $higherRank.getIntersectingNodes().forEach(node -> update(node));
+        $higherRank.remove("X.6.1", "remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity");
+        retract($higherRank);
+    end
+
+
+// Rule unit: X.7
+rule "X.7.0: Remove all images"
+    salience 512
+    when
+        $image: Image(imageType != ImageType.OCR, !hasManualChanges())
+    then
+        $image.remove("X.7.0", "remove all images");
+        retract($image);
+    end
+
+
+//------------------------------------ File attributes rules ------------------------------------
+
+// Rule unit: FA.1
+rule "FA.1.0: Remove duplicate FileAttributes"
+    salience 64
+    when
+        $fileAttribute: FileAttribute($label: label, $value: value)
+        $duplicate: FileAttribute(this != $fileAttribute, label == $label, value == $value)
+    then
+        retract($duplicate);
+    end
+
+
+//------------------------------------ Local dictionary search rules ------------------------------------
+
+// Rule unit: LDS.0
+rule "LDS.0.0: Run local dictionary search"
+    agenda-group "LOCAL_DICTIONARY_ADDS"
+    salience -999
+    when
+        $dictionaryModel: DictionaryModel(!localEntriesWithMatchedRules.isEmpty()) from dictionary.getDictionaryModels()
+    then
+        entityCreationService.bySearchImplementation($dictionaryModel.getLocalSearch(), $dictionaryModel.getType(), EntityType.RECOMMENDATION, document)
+            .forEach(entity ->  {
+                Collection<MatchedRule> matchedRules = $dictionaryModel.getMatchedRulesForLocalDictionaryEntry(entity.getValue());
+                matchedRules.forEach(matchedRule -> entity.addMatchedRule(matchedRule.asSkippedIfApplied()));
+            });
+    end

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/table_demo_components.drl

@@ -0,0 +1,126 @@
+package drools
+
+import static java.lang.String.format;
+import static com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility.anyMatch;
+import static com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility.exactMatch;
+
+import java.util.List;
+import java.util.LinkedList;
+import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.Collection;
+import java.util.stream.Stream;
+import java.util.Optional;
+
+import com.iqser.red.service.redaction.v1.server.model.component.Component;
+import com.iqser.red.service.redaction.v1.server.model.component.Entity;
+import com.iqser.red.service.redaction.v1.server.service.document.ComponentCreationService;
+
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Change;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryType;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualChange;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position;
+
+import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;
+
+global ComponentCreationService componentCreationService
+
+//------------------------------------ queries ------------------------------------
+
+query "getFileAttributes"
+    $fileAttribute: FileAttribute()
+    end
+
+query "getComponents"
+    $component: Component()
+    end
+
+//------------------------------------ Guideline mapping object ------------------------------------
+
+declare GuidelineMapping
+    number: String
+    year: String
+    guideline: String
+ end
+
+//------------------------------------ Demo Components rules ------------------------------------
+
+rule "FullTable.1.1: All rows of matching table"
+    when
+        $tableRowValues: List() from collect (Entity(type == "full_table_row"))
+    then
+        componentCreationService.joiningFromSameTableRow("FullTable.1.1", "1.1 Full Table", $tableRowValues);
+    end
+
+
+rule "IndivRowExtr.1.2: Individual rows of table"
+    when
+        $clinicalSigns: List() from collect (Entity(type == "clinical_signs"))
+    then
+        componentCreationService.joiningFromSameTableRow("IndivRowExtr.1.2", "1.2 Individual Rows", $clinicalSigns);
+    end
+
+
+rule "IndivColExtr.1.3: Individual column of table"
+    when
+        $tableColValues: List() from collect (Entity(type == "dosages"))
+    then
+        componentCreationService.joiningFromSameTableRow("IndivColExtr.1.3", "1.3 Individual Column", $tableColValues);
+    end
+
+
+rule "CombColExtr.2.1: Combined Columns Extraction"
+    when
+        $doseMortalities: List() from collect (Entity(type == "dose_mortality" || type == "dose_mortality_dose"))
+    then
+        componentCreationService.joiningFromSameTableRow("DoseMortality.0.0", "2.1 Combined Columns", $doseMortalities);
+    end
+
+
+rule "ValueExtrRef.2.2: Cells containing dose for survived males"
+    when
+        $tableValues: List() from collect (Entity(type == "doses_mg_kg_bw"))
+    then
+        componentCreationService.joining("ValueExtrRef.2.2", "2.2 Individual Table Values", $tableValues);
+    end
+
+
+rule "AdvTableExtr.2.3: Cells containing dose for survived males"
+    when
+        $tableValues: List() from collect (Entity(type == "experiment_female_survived"))
+    then
+        componentCreationService.rowValueCount("AdvTableExtr.2.3", "2.3 Advanced Table Values", $tableValues);
+    end
+
+
+rule "EntityBasedExtr.2.4: Cells containing dose for survived males"
+    when
+        $tableValues: List() from collect (Entity(type == "study_design"))
+    then
+        componentCreationService.joiningFromSameTableRow("EntityBasedExtr.2.4", "2.4 Entity-Based Values", $tableValues);
+    end
+
+
+rule "DefaultComponents.999.0: Create components for all unmapped entities."
+    salience -999
+    when
+        not FileAttribute(label == "OECD Number")
+        $allEntities: List(!isEmpty()) from collect (Entity())
+    then
+        componentCreationService.createComponentsForUnMappedEntities("DefaultComponents.999.0", $allEntities);
+    end
+
+
+//------------------------------------ Component merging rules ------------------------------------
+/*
+rule "X.0.0: merge duplicate component references"
+    when
+        $first: Component()
+        $duplicate: Component(this != $first, name == $first.name, value == $first.value)
+    then
+        $first.getReferences().addAll($duplicate.getReferences());
+        retract($duplicate);
+    end
+*/

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/test_components.drl

@@ -0,0 +1,55 @@
+package drools
+
+import static java.lang.String.format;
+import static com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility.anyMatch;
+import static com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility.exactMatch;
+
+import java.util.List;
+import java.util.LinkedList;
+import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.Collection;
+import java.util.stream.Stream;
+import java.util.Optional;
+
+import com.iqser.red.service.redaction.v1.server.model.component.Component;
+import com.iqser.red.service.redaction.v1.server.model.component.Entity;
+import com.iqser.red.service.redaction.v1.server.service.document.ComponentCreationService;
+
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Change;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryType;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualChange;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position;
+
+import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;
+
+global ComponentCreationService componentCreationService
+
+//------------------------------------ queries ------------------------------------
+
+query "getFileAttributes"
+    $fileAttribute: FileAttribute()
+    end
+
+query "getComponents"
+    $component: Component()
+    end
+
+//------------------------------------ Guideline mapping object ------------------------------------
+
+declare GuidelineMapping
+    number: String
+    year: String
+    guideline: String
+ end
+
+//------------------------------------ Test Components rules ------------------------------------
+
+rule "AdvTableExtr.2.3: Cells containing dose for survived males"
+    when
+        $tableValues: List() from collect (Entity(type == "test"))
+    then
+        componentCreationService.rowValueCount("AdvTableExtr.2.3", "2.3 Advanced Table Values", $tableValues);
+    end

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/test_rules.drl

@@ -0,0 +1,403 @@
+package drools
+
+import static java.lang.String.format;
+import static com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility.anyMatch;
+import static com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility.exactMatch;
+
+import java.util.List;
+import java.util.LinkedList;
+import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.Collection;
+import java.util.stream.Stream;
+import java.util.Optional;
+
+import com.iqser.red.service.redaction.v1.server.model.document.*;
+import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
+import com.iqser.red.service.redaction.v1.server.model.document.entity.*;
+import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
+import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule;
+import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity
+import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.*;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.Section;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.Table;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.Paragraph;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.Image;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.ImageType;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.Page;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.Headline;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.SectionIdentifier;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.Footer;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.Header;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.NodeType;
+import com.iqser.red.service.redaction.v1.server.model.document.textblock.*;
+import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
+import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlockCollector;
+import com.iqser.red.service.redaction.v1.server.model.document.textblock.AtomicTextBlock;
+import com.iqser.red.service.redaction.v1.server.model.document.textblock.ConcatenatedTextBlock;
+import com.iqser.red.service.redaction.v1.server.model.NerEntities;
+import com.iqser.red.service.redaction.v1.server.model.dictionary.Dictionary;
+import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryModel;
+import com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService;
+import com.iqser.red.service.redaction.v1.server.service.ManualChangesApplicationService;
+import com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility;
+
+import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus;
+
+global Document document
+global EntityCreationService entityCreationService
+global ManualChangesApplicationService manualChangesApplicationService
+global Dictionary dictionary
+
+//------------------------------------ queries ------------------------------------
+
+query "getFileAttributes"
+    $fileAttribute: FileAttribute()
+    end
+
+//------------------------------------ Table extraction rules ------------------------------------
+
+// Rule unit: TAB.0
+rule "TAB.0.0: Study Type File Attribute"
+    when
+        not FileAttribute(label == "OECD Number", valueEqualsAnyOf("402","403","404","405","425","429","436","438","439","471","487"))
+        $section: Section(containsAnyString("DATA REQUIREMENT", "TEST GUIDELINE", "MÉTODO(S) DE REFERÊNCIA(S):")
+            && containsAnyString("OECD", "EPA", "OPPTS"))
+    then
+        RedactionSearchUtility.findTextRangesByRegexIgnoreCase("(?<=OECD)(?:[\\w\\s,\\[\\]\\(\\)\\.]{1,10}|(?:.{5,40}(?:Number |Procedure |Guideline )))(4[\\d]{2})", 1 ,$section.getTextBlock()).stream()
+            .map(boundary -> $section.getTextBlock().subSequence(boundary).toString())
+            .map(value -> FileAttribute.builder().label("OECD Number").value(value).build())
+            .forEach(fileAttribute -> insert(fileAttribute));
+        RedactionSearchUtility.findTextRangesByRegexIgnoreCase("(?<=OECD).{5,40}Method (4[\\d]{2}).{1,65}(\\d{4})\\)", 1, $section.getTextBlock()).stream()
+            .map(boundary -> $section.getTextBlock().subSequence(boundary).toString())
+            .map(value -> FileAttribute.builder().label("OECD Number").value(value).build())
+            .forEach(fileAttribute -> insert(fileAttribute));
+    end
+
+rule "TAB.0.1: Guidelines"
+    when
+        $section: Section(containsAnyString("DATA REQUIREMENT", "TEST GUIDELINE", "MÉTODO(S) DE REFERÊNCIA(S):") && containsAnyString("OECD", "EPA", "OPPTS"))
+    then
+        entityCreationService.byRegex("(?<=OECD)(?:[\\w\\s,\\[\\]\\(\\)\\.]{1,10}|.{5,40}(?:Number |Procedure |Guideline ))(4[\\d]{2})", "oecd_guideline_number", EntityType.ENTITY, 1, $section)
+            .forEach(guideline -> guideline.apply("TAB.0.1", "OECD Guideline no. found"));
+        entityCreationService.byRegex("(?<=OECD)(?:[\\w\\s,\\[\\]\\(\\)\\.]{1,10}|.{5,40}(?:Number |Procedure |Guideline ))(4[\\d]{2}),?\\s\\(?(\\d{4})\\)?", "oecd_guideline_year", EntityType.ENTITY, 2, $section)
+            .forEach(guideline -> guideline.apply("TAB.0.1", "OECD Guideline year found"));
+        entityCreationService.byRegex("(?<=OECD)[\\w\\s,\\[\\]]{1,10}\\((\\d{4})\\)\\s(4[\\d]{2})", "oecd_guideline_year", EntityType.ENTITY, 1, $section)
+            .forEach(guideline -> guideline.apply("TAB.0.1", "OECD Guideline year found"));
+        entityCreationService.byRegex("(?<=OECD).{5,40}Method (4[\\d]{2}).{1,65}(\\d{4})\\)", "oecd_guideline_number", EntityType.ENTITY, 1, $section)
+            .forEach(guideline -> guideline.apply("TAB.0.1", "OECD Guideline number found"));
+        entityCreationService.byRegex("(?<=OECD).{5,40}Method (4[\\d]{2}).{1,65}(\\d{4})\\)", "oecd_guideline_year", EntityType.ENTITY, 2, $section)
+            .forEach(guideline -> guideline.apply("TAB.0.1", "OECD Guideline year found"));
+    end
+
+
+// Rule unit: TAB.6
+rule "TAB.6.0: Targeted cell extraction (Experimental Stop date)"
+    when
+        $section: Section(getHeadline().containsString("Advanced Table Extraction"), containsAllStrings("female", "Female", "Survived", "Group 2"))
+        $table: Table(hasHeader("Group 2")) from $section.streamChildren().toList()
+        TableCell(containsWordIgnoreCase("Female"), $row: row) from $table.streamTableCellsWithHeader("Group 2").toList()
+        TableCell($row == row, containsStringIgnoreCase("Survived")) from $table.streamTableCellsWithHeader("Group 2").toList()
+        $femaleSurvived: TableCell($row == row) from $table.streamTableCellsWithHeader("Group 2").toList()
+    then
+        entityCreationService.bySemanticNode($femaleSurvived, "experiment_female_survived", EntityType.ENTITY)
+            .ifPresent(entity -> entity.apply("TAB.6.0", "Female in group to experimental start date"));
+    end
+
+
+//------------------------------------ Manual changes rules ------------------------------------
+
+// Rule unit: MAN.0
+rule "MAN.0.0: Apply manual resize redaction"
+    salience 128
+    when
+        $resizeRedaction: ManualResizeRedaction($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate)
+        not ManualResizeRedaction(annotationId == $id, requestDate.isBefore($requestDate))
+        $entityToBeResized: TextEntity(matchesAnnotationId($id))
+    then
+        manualChangesApplicationService.resizeEntityAndReinsert($entityToBeResized, $resizeRedaction);
+        retract($resizeRedaction);
+        update($entityToBeResized);
+        $entityToBeResized.getIntersectingNodes().forEach(node -> update(node));
+    end
+
+rule "MAN.0.1: Apply manual resize redaction"
+    salience 128
+    when
+        $resizeRedaction: ManualResizeRedaction($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate)
+        not ManualResizeRedaction(annotationId == $id, requestDate.isBefore($requestDate))
+        $imageToBeResized: Image(id == $id)
+    then
+        manualChangesApplicationService.resizeImage($imageToBeResized, $resizeRedaction);
+        retract($resizeRedaction);
+        update($imageToBeResized);
+        update($imageToBeResized.getParent());
+    end
+
+
+// Rule unit: MAN.1
+rule "MAN.1.0: Apply id removals that are valid and not in forced redactions to Entity"
+    salience 128
+    when
+        $idRemoval: IdRemoval($id: annotationId, status == AnnotationStatus.APPROVED)
+        $entityToBeRemoved: TextEntity(matchesAnnotationId($id))
+    then
+        $entityToBeRemoved.getManualOverwrite().addChange($idRemoval);
+        update($entityToBeRemoved);
+        retract($idRemoval);
+        $entityToBeRemoved.getIntersectingNodes().forEach(node -> update(node));
+    end
+
+rule "MAN.1.1: Apply id removals that are valid and not in forced redactions to Image"
+    salience 128
+    when
+        $idRemoval: IdRemoval($id: annotationId, status == AnnotationStatus.APPROVED)
+        $imageEntityToBeRemoved: Image($id == id)
+    then
+        $imageEntityToBeRemoved.getManualOverwrite().addChange($idRemoval);
+        update($imageEntityToBeRemoved);
+        retract($idRemoval);
+        update($imageEntityToBeRemoved.getParent());
+    end
+
+
+// Rule unit: MAN.2
+rule "MAN.2.0: Apply force redaction"
+    salience 128
+    when
+        $force: ManualForceRedaction($id: annotationId, status == AnnotationStatus.APPROVED)
+        $entityToForce: TextEntity(matchesAnnotationId($id))
+    then
+        $entityToForce.getManualOverwrite().addChange($force);
+        update($entityToForce);
+        $entityToForce.getIntersectingNodes().forEach(node -> update(node));
+        retract($force);
+    end
+
+rule "MAN.2.1: Apply force redaction to images"
+    salience 128
+    when
+        $force: ManualForceRedaction($id: annotationId, status == AnnotationStatus.APPROVED)
+        $imageToForce: Image(id == $id)
+    then
+        $imageToForce.getManualOverwrite().addChange($force);
+        update($imageToForce);
+        update($imageToForce.getParent());
+        retract($force);
+    end
+
+
+// Rule unit: MAN.3
+rule "MAN.3.0: Apply entity recategorization"
+    salience 128
+    when
+        $recategorization: ManualRecategorization($id: annotationId, $type: type, status == AnnotationStatus.APPROVED, $requestDate: requestDate)
+        not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))
+        $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type != $type)
+    then
+        $entityToBeRecategorized.getIntersectingNodes().forEach(node -> update(node));
+        manualChangesApplicationService.recategorize($entityToBeRecategorized, $recategorization);
+        retract($recategorization);
+        // Entity is copied and inserted, so the old entity needs to be retracted to avoid duplication.
+        retract($entityToBeRecategorized);
+    end
+
+rule "MAN.3.1: Apply entity recategorization of same type"
+    salience 128
+    when
+        $recategorization: ManualRecategorization($id: annotationId, $type: type, status == AnnotationStatus.APPROVED, $requestDate: requestDate)
+        not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))
+        $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type == $type)
+    then
+        $entityToBeRecategorized.getManualOverwrite().addChange($recategorization);
+        retract($recategorization);
+    end
+
+rule "MAN.3.2: Apply image recategorization"
+    salience 128
+    when
+        $recategorization: ManualRecategorization($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate)
+        not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))
+        $imageToBeRecategorized: Image($id == id)
+    then
+        manualChangesApplicationService.recategorize($imageToBeRecategorized, $recategorization);
+        update($imageToBeRecategorized);
+        update($imageToBeRecategorized.getParent());
+        retract($recategorization);
+    end
+
+
+// Rule unit: MAN.4
+rule "MAN.4.0: Apply legal basis change"
+    salience 128
+    when
+        $legalbasisChange: ManualLegalBasisChange($id: annotationId, status == AnnotationStatus.APPROVED)
+        $imageToBeRecategorized: Image($id == id)
+    then
+        $imageToBeRecategorized.getManualOverwrite().addChange($legalbasisChange);
+    end
+
+rule "MAN.4.1: Apply legal basis change"
+    salience 128
+    when
+        $legalBasisChange: ManualLegalBasisChange($id: annotationId, status == AnnotationStatus.APPROVED)
+        $entityToBeChanged: TextEntity(matchesAnnotationId($id))
+    then
+        $entityToBeChanged.getManualOverwrite().addChange($legalBasisChange);
+    end
+
+
+//------------------------------------ Entity merging rules ------------------------------------
+
+// Rule unit: X.0
+rule "X.0.0: Remove Entity contained by Entity of same type"
+    salience 65
+    when
+        $larger: TextEntity($type: type, $entityType: entityType, active())
+        $contained: TextEntity(containedBy($larger), type == $type, entityType == $entityType, this != $larger, !hasManualChanges(), active())
+    then
+        $contained.remove("X.0.0", "remove Entity contained by Entity of same type");
+        retract($contained);
+    end
+
+
+// Rule unit: X.1
+rule "X.1.0: Merge intersecting Entities of same type"
+    salience 64
+    when
+        $first: TextEntity($type: type, $entityType: entityType, !resized(), active())
+        $second: TextEntity(intersects($first), type == $type, entityType == $entityType, this != $first, !hasManualChanges(), active())
+    then
+        TextEntity mergedEntity = entityCreationService.mergeEntitiesOfSameType(List.of($first, $second), $type, $entityType, document);
+        $first.remove("X.1.0", "merge intersecting Entities of same type");
+        $second.remove("X.1.0", "merge intersecting Entities of same type");
+        retract($first);
+        retract($second);
+        mergedEntity.getIntersectingNodes().forEach(node -> update(node));
+    end
+
+
+// Rule unit: X.2
+rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
+    salience 64
+    when
+        $falsePositive: TextEntity($type: type, entityType == EntityType.FALSE_POSITIVE, active())
+        $entity: TextEntity(containedBy($falsePositive), type == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges(), active())
+    then
+        $entity.getIntersectingNodes().forEach(node -> update(node));
+        $entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE");
+        retract($entity)
+    end
+
+
+// Rule unit: X.3
+rule "X.3.0: Remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"
+    salience 64
+    when
+        $falseRecommendation: TextEntity($type: type, entityType == EntityType.FALSE_RECOMMENDATION, active())
+        $recommendation: TextEntity(containedBy($falseRecommendation), type == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
+    then
+        $recommendation.remove("X.3.0", "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION");
+        retract($recommendation);
+    end
+
+
+// Rule unit: X.4
+rule "X.4.0: Remove Entity of type RECOMMENDATION when intersected by ENTITY with same type"
+    salience 256
+    when
+        $entity: TextEntity($type: type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
+        $recommendation: TextEntity(intersects($entity), type == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
+    then
+        $entity.addEngines($recommendation.getEngines());
+        $recommendation.remove("X.4.0", "remove Entity of type RECOMMENDATION when intersected by ENTITY with same type");
+        retract($recommendation);
+    end
+
+
+// Rule unit: X.5
+rule "X.5.0: Remove Entity of type RECOMMENDATION when contained by ENTITY"
+    salience 256
+    when
+        $entity: TextEntity((entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
+        $recommendation: TextEntity(containedBy($entity), entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
+    then
+        $recommendation.remove("X.5.0", "remove Entity of type RECOMMENDATION when contained by ENTITY");
+        retract($recommendation);
+    end
+
+
+// Rule unit: X.6
+rule "X.6.0: Remove Entity of lower rank, when contained by by entity of type ENTITY"
+    salience 32
+    when
+        $higherRank: TextEntity($type: type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
+        $lowerRank: TextEntity(containedBy($higherRank), type != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active())
+    then
+        $lowerRank.getIntersectingNodes().forEach(node -> update(node));
+        $lowerRank.remove("X.6.0", "remove Entity of lower rank, when contained by entity of type ENTITY");
+        retract($lowerRank);
+    end
+
+rule "X.6.1: remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity"
+    salience 32
+    when
+        $higherRank: TextEntity($type: type, $value: value, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
+        $lowerRank: TextEntity(intersects($higherRank), type != $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active(), $lowerRank.getValue().length() > $value.length())
+    then
+        $higherRank.getIntersectingNodes().forEach(node -> update(node));
+        $higherRank.remove("X.6.1", "remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity");
+        retract($higherRank);
+    end
+
+
+// Rule unit: X.7
+rule "X.7.0: Remove all images"
+    salience 512
+    when
+        $image: Image(imageType != ImageType.OCR, !hasManualChanges())
+    then
+        $image.remove("X.7.0", "remove all images");
+        retract($image);
+    end
+
+
+//------------------------------------ File attributes rules ------------------------------------
+
+// Rule unit: FA.1
+rule "FA.1.0: Remove duplicate FileAttributes"
+    salience 64
+    when
+        $fileAttribute: FileAttribute($label: label, $value: value)
+        $duplicate: FileAttribute(this != $fileAttribute, label == $label, value == $value)
+    then
+        retract($duplicate);
+    end
+
+
+//------------------------------------ Local dictionary search rules ------------------------------------
+
+// Rule unit: LDS.0
+rule "LDS.0.0: Run local dictionary search"
+    agenda-group "LOCAL_DICTIONARY_ADDS"
+    salience -999
+    when
+        $dictionaryModel: DictionaryModel(!localEntriesWithMatchedRules.isEmpty()) from dictionary.getDictionaryModels()
+    then
+        entityCreationService.bySearchImplementation($dictionaryModel.getLocalSearch(), $dictionaryModel.getType(), EntityType.RECOMMENDATION, document)
+            .forEach(entity ->  {
+                Collection<MatchedRule> matchedRules = $dictionaryModel.getMatchedRulesForLocalDictionaryEntry(entity.getValue());
+                matchedRules.forEach(matchedRule -> entity.addMatchedRule(matchedRule.asSkippedIfApplied()));
+            });
+    end

redaction-service-v1/redaction-service-server-v1/src/test/resources/files/migration/def8f960580f088b975ba806dfae1f87.IMAGE_INFO.json

@@ -0,0 +1,891 @@
+{
+  "dossierId": "12778ddd-0d87-4127-90f4-a15802e8f967",
+  "fileId": "4a15b5c89a4310eaef6243fb6f1f4f6f",
+  "targetFileExtension": "ORIGIN.pdf.gz",
+  "responseFileExtension": "IMAGE_INFO.json.gz",
+  "X-TENANT-ID": "documine",
+  "data": [
+    {
+      "classification": {
+        "label": "signature",
+        "probabilities": {
+          "signature": 0.7805,
+          "other": 0.1362,
+          "logo": 0.0629,
+          "formula": 0.0204
+        }
+      },
+      "representation": "F8F3FFCFCC30F0CFCFFCF37FF",
+      "position": {
+        "x1": 0,
+        "x2": 595,
+        "y1": 0,
+        "y2": 842,
+        "pageNumber": 1
+      },
+      "geometry": {
+        "width": 595,
+        "height": 842
+      },
+      "alpha": false,
+      "filters": {
+        "geometry": {
+          "imageSize": {
+            "quotient": 1.0,
+            "tooLarge": true,
+            "tooSmall": false
+          },
+          "imageFormat": {
+            "quotient": 0.7067,
+            "tooTall": false,
+            "tooWide": false
+          }
+        },
+        "probability": {
+          "unconfident": false
+        },
+        "allPassed": false
+      }
+    },
+    {
+      "classification": {
+        "label": "signature",
+        "probabilities": {
+          "signature": 0.9955,
+          "other": 0.0023,
+          "logo": 0.0022,
+          "formula": 0.0
+        }
+      },
+      "representation": "97FFFFFFFF10608F7E18107EF",
+      "position": {
+        "x1": 0,
+        "x2": 595,
+        "y1": 0,
+        "y2": 842,
+        "pageNumber": 2
+      },
+      "geometry": {
+        "width": 595,
+        "height": 842
+      },
+      "alpha": false,
+      "filters": {
+        "geometry": {
+          "imageSize": {
+            "quotient": 1.0,
+            "tooLarge": true,
+            "tooSmall": false
+          },
+          "imageFormat": {
+            "quotient": 0.7067,
+            "tooTall": false,
+            "tooWide": false
+          }
+        },
+        "probability": {
+          "unconfident": false
+        },
+        "allPassed": false
+      }
+    },
+    {
+      "classification": {
+        "label": "signature",
+        "probabilities": {
+          "signature": 0.9998,
+          "logo": 0.0001,
+          "other": 0.0001,
+          "formula": 0.0
+        }
+      },
+      "representation": "FFFFF10EFFF3E0810608307FF",
+      "position": {
+        "x1": 0,
+        "x2": 595,
+        "y1": 0,
+        "y2": 842,
+        "pageNumber": 3
+      },
+      "geometry": {
+        "width": 595,
+        "height": 842
+      },
+      "alpha": false,
+      "filters": {
+        "geometry": {
+          "imageSize": {
+            "quotient": 1.0,
+            "tooLarge": true,
+            "tooSmall": false
+          },
+          "imageFormat": {
+            "quotient": 0.7067,
+            "tooTall": false,
+            "tooWide": false
+          }
+        },
+        "probability": {
+          "unconfident": false
+        },
+        "allPassed": false
+      }
+    },
+    {
+      "classification": {
+        "label": "signature",
+        "probabilities": {
+          "signature": 0.9998,
+          "other": 0.0001,
+          "formula": 0.0,
+          "logo": 0.0
+        }
+      },
+      "representation": "93EFF76E08F0E88F1E187C7EF",
+      "position": {
+        "x1": 0,
+        "x2": 595,
+        "y1": 0,
+        "y2": 842,
+        "pageNumber": 4
+      },
+      "geometry": {
+        "width": 595,
+        "height": 842
+      },
+      "alpha": false,
+      "filters": {
+        "geometry": {
+          "imageSize": {
+            "quotient": 1.0,
+            "tooLarge": true,
+            "tooSmall": false
+          },
+          "imageFormat": {
+            "quotient": 0.7067,
+            "tooTall": false,
+            "tooWide": false
+          }
+        },
+        "probability": {
+          "unconfident": false
+        },
+        "allPassed": false
+      }
+    },
+    {
+      "classification": {
+        "label": "signature",
+        "probabilities": {
+          "signature": 0.9997,
+          "logo": 0.0001,
+          "other": 0.0001,
+          "formula": 0.0
+        }
+      },
+      "representation": "FFFFF33EFC3060818608787FF",
+      "position": {
+        "x1": 0,
+        "x2": 595,
+        "y1": 0,
+        "y2": 842,
+        "pageNumber": 5
+      },
+      "geometry": {
+        "width": 595,
+        "height": 842
+      },
+      "alpha": false,
+      "filters": {
+        "geometry": {
+          "imageSize": {
+            "quotient": 1.0,
+            "tooLarge": true,
+            "tooSmall": false
+          },
+          "imageFormat": {
+            "quotient": 0.7067,
+            "tooTall": false,
+            "tooWide": false
+          }
+        },
+        "probability": {
+          "unconfident": false
+        },
+        "allPassed": false
+      }
+    },
+    {
+      "classification": {
+        "label": "signature",
+        "probabilities": {
+          "signature": 0.9085,
+          "logo": 0.0453,
+          "other": 0.0413,
+          "formula": 0.005
+        }
+      },
+      "representation": "FFF3F506F81060CF1E18FC7FF",
+      "position": {
+        "x1": 0,
+        "x2": 595,
+        "y1": 0,
+        "y2": 842,
+        "pageNumber": 6
+      },
+      "geometry": {
+        "width": 595,
+        "height": 842
+      },
+      "alpha": false,
+      "filters": {
+        "geometry": {
+          "imageSize": {
+            "quotient": 1.0,
+            "tooLarge": true,
+            "tooSmall": false
+          },
+          "imageFormat": {
+            "quotient": 0.7067,
+            "tooTall": false,
+            "tooWide": false
+          }
+        },
+        "probability": {
+          "unconfident": false
+        },
+        "allPassed": false
+      }
+    },
+    {
+      "classification": {
+        "label": "signature",
+        "probabilities": {
+          "signature": 0.7596,
+          "other": 0.1218,
+          "logo": 0.1073,
+          "formula": 0.0113
+        }
+      },
+      "representation": "FFFFFFF70C1070C10F3C10FFF",
+      "position": {
+        "x1": 0,
+        "x2": 595,
+        "y1": 0,
+        "y2": 842,
+        "pageNumber": 7
+      },
+      "geometry": {
+        "width": 595,
+        "height": 842
+      },
+      "alpha": false,
+      "filters": {
+        "geometry": {
+          "imageSize": {
+            "quotient": 1.0,
+            "tooLarge": true,
+            "tooSmall": false
+          },
+          "imageFormat": {
+            "quotient": 0.7067,
+            "tooTall": false,
+            "tooWide": false
+          }
+        },
+        "probability": {
+          "unconfident": false
+        },
+        "allPassed": false
+      }
+    },
+    {
+      "classification": {
+        "label": "signature",
+        "probabilities": {
+          "signature": 0.4095,
+          "other": 0.2126,
+          "formula": 0.2008,
+          "logo": 0.1771
+        }
+      },
+      "representation": "FF70C1070C1070CFD70C10FFF",
+      "position": {
+        "x1": 0,
+        "x2": 595,
+        "y1": 0,
+        "y2": 842,
+        "pageNumber": 8
+      },
+      "geometry": {
+        "width": 595,
+        "height": 842
+      },
+      "alpha": false,
+      "filters": {
+        "geometry": {
+          "imageSize": {
+            "quotient": 1.0,
+            "tooLarge": true,
+            "tooSmall": false
+          },
+          "imageFormat": {
+            "quotient": 0.7067,
+            "tooTall": false,
+            "tooWide": false
+          }
+        },
+        "probability": {
+          "unconfident": true
+        },
+        "allPassed": false
+      }
+    },
+    {
+      "classification": {
+        "label": "signature",
+        "probabilities": {
+          "signature": 0.9282,
+          "logo": 0.0402,
+          "other": 0.0305,
+          "formula": 0.0011
+        }
+      },
+      "representation": "F4F1CF170CF170C1070CF7FFF",
+      "position": {
+        "x1": 0,
+        "x2": 595,
+        "y1": 0,
+        "y2": 842,
+        "pageNumber": 9
+      },
+      "geometry": {
+        "width": 595,
+        "height": 842
+      },
+      "alpha": false,
+      "filters": {
+        "geometry": {
+          "imageSize": {
+            "quotient": 1.0,
+            "tooLarge": true,
+            "tooSmall": false
+          },
+          "imageFormat": {
+            "quotient": 0.7067,
+            "tooTall": false,
+            "tooWide": false
+          }
+        },
+        "probability": {
+          "unconfident": false
+        },
+        "allPassed": false
+      }
+    },
+    {
+      "classification": {
+        "label": "signature",
+        "probabilities": {
+          "signature": 0.9699,
+          "logo": 0.02,
+          "other": 0.0097,
+          "formula": 0.0004
+        }
+      },
+      "representation": "FF70FF4E1D3070D1070CF0FFF",
+      "position": {
+        "x1": 0,
+        "x2": 595,
+        "y1": 0,
+        "y2": 842,
+        "pageNumber": 10
+      },
+      "geometry": {
+        "width": 595,
+        "height": 842
+      },
+      "alpha": false,
+      "filters": {
+        "geometry": {
+          "imageSize": {
+            "quotient": 1.0,
+            "tooLarge": true,
+            "tooSmall": false
+          },
+          "imageFormat": {
+            "quotient": 0.7067,
+            "tooTall": false,
+            "tooWide": false
+          }
+        },
+        "probability": {
+          "unconfident": false
+        },
+        "allPassed": false
+      }
+    },
+    {
+      "classification": {
+        "label": "other",
+        "probabilities": {
+          "other": 0.9181,
+          "signature": 0.0432,
+          "formula": 0.0355,
+          "logo": 0.0032
+        }
+      },
+      "representation": "F3F1F7DF1F10F0C1070CF3FFF",
+      "position": {
+        "x1": 0,
+        "x2": 595,
+        "y1": 0,
+        "y2": 842,
+        "pageNumber": 11
+      },
+      "geometry": {
+        "width": 595,
+        "height": 842
+      },
+      "alpha": false,
+      "filters": {
+        "geometry": {
+          "imageSize": {
+            "quotient": 1.0,
+            "tooLarge": true,
+            "tooSmall": false
+          },
+          "imageFormat": {
+            "quotient": 0.7067,
+            "tooTall": false,
+            "tooWide": false
+          }
+        },
+        "probability": {
+          "unconfident": false
+        },
+        "allPassed": false
+      }
+    },
+    {
+      "classification": {
+        "label": "signature",
+        "probabilities": {
+          "signature": 0.663,
+          "other": 0.137,
+          "logo": 0.1327,
+          "formula": 0.0674
+        }
+      },
+      "representation": "B770C10F7C1070C10F0C97FFF",
+      "position": {
+        "x1": 0,
+        "x2": 595,
+        "y1": 0,
+        "y2": 842,
+        "pageNumber": 12
+      },
+      "geometry": {
+        "width": 595,
+        "height": 842
+      },
+      "alpha": false,
+      "filters": {
+        "geometry": {
+          "imageSize": {
+            "quotient": 1.0,
+            "tooLarge": true,
+            "tooSmall": false
+          },
+          "imageFormat": {
+            "quotient": 0.7067,
+            "tooTall": false,
+            "tooWide": false
+          }
+        },
+        "probability": {
+          "unconfident": false
+        },
+        "allPassed": false
+      }
+    },
+    {
+      "classification": {
+        "label": "signature",
+        "probabilities": {
+          "signature": 0.5023,
+          "other": 0.2343,
+          "logo": 0.2255,
+          "formula": 0.0379
+        }
+      },
+      "representation": "B370C10FFC1070CF3F1CF0FFF",
+      "position": {
+        "x1": 0,
+        "x2": 595,
+        "y1": 0,
+        "y2": 842,
+        "pageNumber": 13
+      },
+      "geometry": {
+        "width": 595,
+        "height": 842
+      },
+      "alpha": false,
+      "filters": {
+        "geometry": {
+          "imageSize": {
+            "quotient": 1.0,
+            "tooLarge": true,
+            "tooSmall": false
+          },
+          "imageFormat": {
+            "quotient": 0.7067,
+            "tooTall": false,
+            "tooWide": false
+          }
+        },
+        "probability": {
+          "unconfident": false
+        },
+        "allPassed": false
+      }
+    },
+    {
+      "classification": {
+        "label": "signature",
+        "probabilities": {
+          "signature": 0.9999,
+          "other": 0.0001,
+          "formula": 0.0,
+          "logo": 0.0
+        }
+      },
+      "representation": "97FFFFFFFFFFFF810608907EF",
+      "position": {
+        "x1": 0,
+        "x2": 595,
+        "y1": 0,
+        "y2": 842,
+        "pageNumber": 14
+      },
+      "geometry": {
+        "width": 595,
+        "height": 842
+      },
+      "alpha": false,
+      "filters": {
+        "geometry": {
+          "imageSize": {
+            "quotient": 1.0,
+            "tooLarge": true,
+            "tooSmall": false
+          },
+          "imageFormat": {
+            "quotient": 0.7067,
+            "tooTall": false,
+            "tooWide": false
+          }
+        },
+        "probability": {
+          "unconfident": false
+        },
+        "allPassed": false
+      }
+    },
+    {
+      "classification": {
+        "label": "signature",
+        "probabilities": {
+          "signature": 0.9099,
+          "logo": 0.0548,
+          "other": 0.0339,
+          "formula": 0.0014
+        }
+      },
+      "representation": "FFFFF10F7DDD70C1070CF77FF",
+      "position": {
+        "x1": 0,
+        "x2": 595,
+        "y1": 0,
+        "y2": 842,
+        "pageNumber": 15
+      },
+      "geometry": {
+        "width": 595,
+        "height": 842
+      },
+      "alpha": false,
+      "filters": {
+        "geometry": {
+          "imageSize": {
+            "quotient": 1.0,
+            "tooLarge": true,
+            "tooSmall": false
+          },
+          "imageFormat": {
+            "quotient": 0.7067,
+            "tooTall": false,
+            "tooWide": false
+          }
+        },
+        "probability": {
+          "unconfident": false
+        },
+        "allPassed": false
+      }
+    },
+    {
+      "classification": {
+        "label": "signature",
+        "probabilities": {
+          "signature": 0.6538,
+          "other": 0.1607,
+          "logo": 0.1489,
+          "formula": 0.0366
+        }
+      },
+      "representation": "FF70CF370CFE70C1070CB3FFF",
+      "position": {
+        "x1": 0,
+        "x2": 595,
+        "y1": 0,
+        "y2": 842,
+        "pageNumber": 16
+      },
+      "geometry": {
+        "width": 595,
+        "height": 842
+      },
+      "alpha": false,
+      "filters": {
+        "geometry": {
+          "imageSize": {
+            "quotient": 1.0,
+            "tooLarge": true,
+            "tooSmall": false
+          },
+          "imageFormat": {
+            "quotient": 0.7067,
+            "tooTall": false,
+            "tooWide": false
+          }
+        },
+        "probability": {
+          "unconfident": false
+        },
+        "allPassed": false
+      }
+    },
+    {
+      "classification": {
+        "label": "signature",
+        "probabilities": {
+          "signature": 0.9637,
+          "logo": 0.0179,
+          "other": 0.0156,
+          "formula": 0.0028
+        }
+      },
+      "representation": "FF70CF170C10F0C1070CF7FFF",
+      "position": {
+        "x1": 0,
+        "x2": 595,
+        "y1": 0,
+        "y2": 842,
+        "pageNumber": 17
+      },
+      "geometry": {
+        "width": 595,
+        "height": 842
+      },
+      "alpha": false,
+      "filters": {
+        "geometry": {
+          "imageSize": {
+            "quotient": 1.0,
+            "tooLarge": true,
+            "tooSmall": false
+          },
+          "imageFormat": {
+            "quotient": 0.7067,
+            "tooTall": false,
+            "tooWide": false
+          }
+        },
+        "probability": {
+          "unconfident": false
+        },
+        "allPassed": false
+      }
+    },
+    {
+      "classification": {
+        "label": "other",
+        "probabilities": {
+          "other": 1.0,
+          "formula": 0.0,
+          "logo": 0.0,
+          "signature": 0.0
+        }
+      },
+      "representation": "FFFFF7070E1870E18608106FF",
+      "position": {
+        "x1": 0,
+        "x2": 595,
+        "y1": 0,
+        "y2": 842,
+        "pageNumber": 18
+      },
+      "geometry": {
+        "width": 595,
+        "height": 842
+      },
+      "alpha": false,
+      "filters": {
+        "geometry": {
+          "imageSize": {
+            "quotient": 1.0,
+            "tooLarge": true,
+            "tooSmall": false
+          },
+          "imageFormat": {
+            "quotient": 0.7067,
+            "tooTall": false,
+            "tooWide": false
+          }
+        },
+        "probability": {
+          "unconfident": false
+        },
+        "allPassed": false
+      }
+    },
+    {
+      "classification": {
+        "label": "other",
+        "probabilities": {
+          "other": 1.0,
+          "formula": 0.0,
+          "logo": 0.0,
+          "signature": 0.0
+        }
+      },
+      "representation": "9B70E1870E1870E1870810EFF",
+      "position": {
+        "x1": 0,
+        "x2": 595,
+        "y1": 0,
+        "y2": 842,
+        "pageNumber": 19
+      },
+      "geometry": {
+        "width": 595,
+        "height": 842
+      },
+      "alpha": false,
+      "filters": {
+        "geometry": {
+          "imageSize": {
+            "quotient": 1.0,
+            "tooLarge": true,
+            "tooSmall": false
+          },
+          "imageFormat": {
+            "quotient": 0.7067,
+            "tooTall": false,
+            "tooWide": false
+          }
+        },
+        "probability": {
+          "unconfident": false
+        },
+        "allPassed": false
+      }
+    },
+    {
+      "classification": {
+        "label": "signature",
+        "probabilities": {
+          "signature": 0.3619,
+          "other": 0.2659,
+          "logo": 0.2169,
+          "formula": 0.1552
+        }
+      },
+      "representation": "FFFFF30FE910ECF1E708106FF",
+      "position": {
+        "x1": 0,
+        "x2": 595,
+        "y1": 0,
+        "y2": 842,
+        "pageNumber": 20
+      },
+      "geometry": {
+        "width": 595,
+        "height": 842
+      },
+      "alpha": false,
+      "filters": {
+        "geometry": {
+          "imageSize": {
+            "quotient": 1.0,
+            "tooLarge": true,
+            "tooSmall": false
+          },
+          "imageFormat": {
+            "quotient": 0.7067,
+            "tooTall": false,
+            "tooWide": false
+          }
+        },
+        "probability": {
+          "unconfident": true
+        },
+        "allPassed": false
+      }
+    },
+    {
+      "classification": {
+        "label": "signature",
+        "probabilities": {
+          "signature": 0.9984,
+          "other": 0.001,
+          "logo": 0.0006,
+          "formula": 0.0
+        }
+      },
+      "representation": "B7E9F70F3C73F1C78F3F706EF",
+      "position": {
+        "x1": 0,
+        "x2": 595,
+        "y1": 0,
+        "y2": 842,
+        "pageNumber": 21
+      },
+      "geometry": {
+        "width": 595,
+        "height": 842
+      },
+      "alpha": false,
+      "filters": {
+        "geometry": {
+          "imageSize": {
+            "quotient": 1.0,
+            "tooLarge": true,
+            "tooSmall": false
+          },
+          "imageFormat": {
+            "quotient": 0.7067,
+            "tooTall": false,
+            "tooWide": false
+          }
+        },
+        "probability": {
+          "unconfident": false
+        },
+        "allPassed": false
+      }
+    }
+  ]
+}

redaction-service-v1/redaction-service-server-v1/src/test/resources/files/migration/def8f960580f088b975ba806dfae1f87.TABLES.json

@@ -0,0 +1,682 @@
+{
+  "dossierId": "12778ddd-0d87-4127-90f4-a15802e8f967",
+  "fileId": "4a15b5c89a4310eaef6243fb6f1f4f6f",
+  "targetFileExtension": "ORIGIN.pdf.gz",
+  "responseFileExtension": "TABLES.json.gz",
+  "X-TENANT-ID": "documine",
+  "operation": "table",
+  "data": [
+    {
+      "pageInfo": {
+        "number": 4,
+        "rotation": 0,
+        "width": 595.0,
+        "height": 842.0
+      },
+      "tableCells": [
+        {
+          "x0": 299.5199890136719,
+          "y0": 262.760009765625,
+          "x1": 359.6400146484375,
+          "y1": 284.0,
+          "width": 60.120025634765625,
+          "height": 21.239990234375
+        },
+        {
+          "x0": 360.7200012207031,
+          "y0": 262.760009765625,
+          "x1": 383.3999938964844,
+          "y1": 284.0,
+          "width": 22.67999267578125,
+          "height": 21.239990234375
+        },
+        {
+          "x0": 384.1199951171875,
+          "y0": 262.4000244140625,
+          "x1": 477.7200012207031,
+          "y1": 284.0,
+          "width": 93.60000610351562,
+          "height": 21.5999755859375
+        },
+        {
+          "x0": 114.12000274658203,
+          "y0": 264.20001220703125,
+          "x1": 140.0399932861328,
+          "y1": 274.6400146484375,
+          "width": 25.91999053955078,
+          "height": 10.44000244140625
+        },
+        {
+          "x0": 141.1199951171875,
+          "y0": 263.47998046875,
+          "x1": 208.8000030517578,
+          "y1": 274.6400146484375,
+          "width": 67.68000793457031,
+          "height": 11.1600341796875
+        },
+        {
+          "x0": 209.52000427246094,
+          "y0": 263.47998046875,
+          "x1": 256.67999267578125,
+          "y1": 274.6400146484375,
+          "width": 47.15998840332031,
+          "height": 11.1600341796875
+        },
+        {
+          "x0": 78.83999633789062,
+          "y0": 253.03997802734375,
+          "x1": 208.8000030517578,
+          "y1": 262.760009765625,
+          "width": 129.9600067138672,
+          "height": 9.72003173828125
+        },
+        {
+          "x0": 209.52000427246094,
+          "y0": 253.03997802734375,
+          "x1": 256.67999267578125,
+          "y1": 262.4000244140625,
+          "width": 47.15998840332031,
+          "height": 9.36004638671875
+        },
+        {
+          "x0": 299.5199890136719,
+          "y0": 250.52001953125,
+          "x1": 359.6400146484375,
+          "y1": 262.03997802734375,
+          "width": 60.120025634765625,
+          "height": 11.51995849609375
+        },
+        {
+          "x0": 399.6000061035156,
+          "y0": 250.52001953125,
+          "x1": 477.7200012207031,
+          "y1": 261.32000732421875,
+          "width": 78.1199951171875,
+          "height": 10.79998779296875
+        }
+      ]
+    },
+    {
+      "pageInfo": {
+        "number": 11,
+        "rotation": 0,
+        "width": 595.0,
+        "height": 842.0
+      },
+      "tableCells": [
+        {
+          "x0": 217.8000030517578,
+          "y0": 147.55999755859375,
+          "x1": 398.5199890136719,
+          "y1": 171.67999267578125,
+          "width": 180.71998596191406,
+          "height": 24.1199951171875
+        },
+        {
+          "x0": 229.67999267578125,
+          "y0": 121.280029296875,
+          "x1": 306.7200012207031,
+          "y1": 148.280029296875,
+          "width": 77.04000854492188,
+          "height": 27.0
+        },
+        {
+          "x0": 307.44000244140625,
+          "y0": 121.280029296875,
+          "x1": 399.239990234375,
+          "y1": 147.55999755859375,
+          "width": 91.79998779296875,
+          "height": 26.27996826171875
+        }
+      ]
+    },
+    {
+      "pageInfo": {
+        "number": 18,
+        "rotation": 0,
+        "width": 595.0,
+        "height": 842.0
+      },
+      "tableCells": [
+        {
+          "x0": 64.08000183105469,
+          "y0": 585.6799926757812,
+          "x1": 220.67999267578125,
+          "y1": 614.1199951171875,
+          "width": 156.59999084472656,
+          "height": 28.44000244140625
+        },
+        {
+          "x0": 221.75999450683594,
+          "y0": 584.9599609375,
+          "x1": 309.9599914550781,
+          "y1": 614.1199951171875,
+          "width": 88.19999694824219,
+          "height": 29.1600341796875
+        },
+        {
+          "x0": 311.3999938964844,
+          "y0": 584.239990234375,
+          "x1": 399.9599914550781,
+          "y1": 614.1199951171875,
+          "width": 88.55999755859375,
+          "height": 29.8800048828125
+        },
+        {
+          "x0": 401.0400085449219,
+          "y0": 583.52001953125,
+          "x1": 494.2799987792969,
+          "y1": 614.1199951171875,
+          "width": 93.239990234375,
+          "height": 30.5999755859375
+        },
+        {
+          "x0": 495.3599853515625,
+          "y0": 583.1600341796875,
+          "x1": 537.1199951171875,
+          "y1": 614.1199951171875,
+          "width": 41.760009765625,
+          "height": 30.9599609375
+        },
+        {
+          "x0": 64.08000183105469,
+          "y0": 262.760009765625,
+          "x1": 220.32000732421875,
+          "y1": 585.3200073242188,
+          "width": 156.24000549316406,
+          "height": 322.55999755859375
+        },
+        {
+          "x0": 221.39999389648438,
+          "y0": 543.5599975585938,
+          "x1": 309.9599914550781,
+          "y1": 584.239990234375,
+          "width": 88.55999755859375,
+          "height": 40.67999267578125
+        },
+        {
+          "x0": 311.0400085449219,
+          "y0": 542.47998046875,
+          "x1": 399.9599914550781,
+          "y1": 583.1600341796875,
+          "width": 88.91998291015625,
+          "height": 40.6800537109375
+        },
+        {
+          "x0": 400.67999267578125,
+          "y0": 541.760009765625,
+          "x1": 493.9200134277344,
+          "y1": 582.4400024414062,
+          "width": 93.24002075195312,
+          "height": 40.67999267578125
+        },
+        {
+          "x0": 495.0,
+          "y0": 541.760009765625,
+          "x1": 537.1199951171875,
+          "y1": 581.719970703125,
+          "width": 42.1199951171875,
+          "height": 39.9599609375
+        },
+        {
+          "x0": 220.67999267578125,
+          "y0": 485.6000061035156,
+          "x1": 309.239990234375,
+          "y1": 543.2000122070312,
+          "width": 88.55999755859375,
+          "height": 57.600006103515625
+        },
+        {
+          "x0": 310.32000732421875,
+          "y0": 484.8800048828125,
+          "x1": 399.239990234375,
+          "y1": 542.1199951171875,
+          "width": 88.91998291015625,
+          "height": 57.239990234375
+        },
+        {
+          "x0": 399.9599914550781,
+          "y0": 484.1600036621094,
+          "x1": 493.9200134277344,
+          "y1": 541.4000244140625,
+          "width": 93.96002197265625,
+          "height": 57.240020751953125
+        },
+        {
+          "x0": 494.6400146484375,
+          "y0": 483.79998779296875,
+          "x1": 537.1199951171875,
+          "y1": 540.6799926757812,
+          "width": 42.47998046875,
+          "height": 56.8800048828125
+        },
+        {
+          "x0": 219.9600067138672,
+          "y0": 438.0799865722656,
+          "x1": 308.8800048828125,
+          "y1": 485.239990234375,
+          "width": 88.91999816894531,
+          "height": 47.160003662109375
+        },
+        {
+          "x0": 309.6000061035156,
+          "y0": 437.3599853515625,
+          "x1": 398.8800048828125,
+          "y1": 484.1600036621094,
+          "width": 89.27999877929688,
+          "height": 46.800018310546875
+        },
+        {
+          "x0": 399.6000061035156,
+          "y0": 436.6400146484375,
+          "x1": 493.20001220703125,
+          "y1": 483.44000244140625,
+          "width": 93.60000610351562,
+          "height": 46.79998779296875
+        },
+        {
+          "x0": 493.9200134277344,
+          "y0": 436.2799987792969,
+          "x1": 537.1199951171875,
+          "y1": 482.7200012207031,
+          "width": 43.199981689453125,
+          "height": 46.44000244140625
+        },
+        {
+          "x0": 219.60000610351562,
+          "y0": 388.0400085449219,
+          "x1": 308.1600036621094,
+          "y1": 437.7200012207031,
+          "width": 88.55999755859375,
+          "height": 49.67999267578125
+        },
+        {
+          "x0": 309.239990234375,
+          "y0": 387.32000732421875,
+          "x1": 398.5199890136719,
+          "y1": 436.6400146484375,
+          "width": 89.27999877929688,
+          "height": 49.32000732421875
+        },
+        {
+          "x0": 398.8800048828125,
+          "y0": 386.6000061035156,
+          "x1": 492.8399963378906,
+          "y1": 435.9200134277344,
+          "width": 93.95999145507812,
+          "height": 49.32000732421875
+        },
+        {
+          "x0": 493.9200134277344,
+          "y0": 385.8800048828125,
+          "x1": 537.1199951171875,
+          "y1": 435.20001220703125,
+          "width": 43.199981689453125,
+          "height": 49.32000732421875
+        },
+        {
+          "x0": 218.8800048828125,
+          "y0": 339.0799865722656,
+          "x1": 307.79998779296875,
+          "y1": 387.67999267578125,
+          "width": 88.91998291015625,
+          "height": 48.600006103515625
+        },
+        {
+          "x0": 308.5199890136719,
+          "y0": 338.3599853515625,
+          "x1": 397.79998779296875,
+          "y1": 386.6000061035156,
+          "width": 89.27999877929688,
+          "height": 48.240020751953125
+        },
+        {
+          "x0": 398.5199890136719,
+          "y0": 337.6400146484375,
+          "x1": 492.4800109863281,
+          "y1": 385.8800048828125,
+          "width": 93.96002197265625,
+          "height": 48.239990234375
+        },
+        {
+          "x0": 493.20001220703125,
+          "y0": 337.2799987792969,
+          "x1": 537.1199951171875,
+          "y1": 385.1600036621094,
+          "width": 43.91998291015625,
+          "height": 47.8800048828125
+        },
+        {
+          "x0": 218.16000366210938,
+          "y0": 289.03997802734375,
+          "x1": 307.0799865722656,
+          "y1": 338.7200012207031,
+          "width": 88.91998291015625,
+          "height": 49.680023193359375
+        },
+        {
+          "x0": 308.1600036621094,
+          "y0": 288.32000732421875,
+          "x1": 397.44000244140625,
+          "y1": 337.6400146484375,
+          "width": 89.27999877929688,
+          "height": 49.32000732421875
+        },
+        {
+          "x0": 398.1600036621094,
+          "y0": 287.96002197265625,
+          "x1": 492.1199951171875,
+          "y1": 336.9200134277344,
+          "width": 93.95999145507812,
+          "height": 48.959991455078125
+        },
+        {
+          "x0": 492.8399963378906,
+          "y0": 287.239990234375,
+          "x1": 537.1199951171875,
+          "y1": 336.20001220703125,
+          "width": 44.279998779296875,
+          "height": 48.96002197265625
+        },
+        {
+          "x0": 217.8000030517578,
+          "y0": 262.760009765625,
+          "x1": 306.7200012207031,
+          "y1": 288.67999267578125,
+          "width": 88.91999816894531,
+          "height": 25.91998291015625
+        },
+        {
+          "x0": 307.79998779296875,
+          "y0": 262.760009765625,
+          "x1": 397.0799865722656,
+          "y1": 287.96002197265625,
+          "width": 89.27999877929688,
+          "height": 25.20001220703125
+        },
+        {
+          "x0": 397.79998779296875,
+          "y0": 262.760009765625,
+          "x1": 492.1199951171875,
+          "y1": 287.239990234375,
+          "width": 94.32000732421875,
+          "height": 24.47998046875
+        },
+        {
+          "x0": 492.8399963378906,
+          "y0": 262.760009765625,
+          "x1": 537.1199951171875,
+          "y1": 286.52001953125,
+          "width": 44.279998779296875,
+          "height": 23.760009765625
+        }
+      ]
+    },
+    {
+      "pageInfo": {
+        "number": 19,
+        "rotation": 0,
+        "width": 595.0,
+        "height": 842.0
+      },
+      "tableCells": [
+        {
+          "x0": 71.27999877929688,
+          "y0": 620.239990234375,
+          "x1": 231.47999572753906,
+          "y1": 643.280029296875,
+          "width": 160.1999969482422,
+          "height": 23.0400390625
+        },
+        {
+          "x0": 232.55999755859375,
+          "y0": 618.7999877929688,
+          "x1": 333.3599853515625,
+          "y1": 643.280029296875,
+          "width": 100.79998779296875,
+          "height": 24.48004150390625
+        },
+        {
+          "x0": 334.0799865722656,
+          "y0": 617.719970703125,
+          "x1": 434.1600036621094,
+          "y1": 643.280029296875,
+          "width": 100.08001708984375,
+          "height": 25.56005859375
+        },
+        {
+          "x0": 435.6000061035156,
+          "y0": 616.6400146484375,
+          "x1": 537.1199951171875,
+          "y1": 643.280029296875,
+          "width": 101.51998901367188,
+          "height": 26.6400146484375
+        },
+        {
+          "x0": 71.27999877929688,
+          "y0": 47.84002685546875,
+          "x1": 230.75999450683594,
+          "y1": 620.5999755859375,
+          "width": 159.47999572753906,
+          "height": 572.7599487304688
+        },
+        {
+          "x0": 231.83999633789062,
+          "y0": 559.4000244140625,
+          "x1": 333.0,
+          "y1": 618.7999877929688,
+          "width": 101.16000366210938,
+          "height": 59.39996337890625
+        },
+        {
+          "x0": 333.7200012207031,
+          "y0": 557.9599609375,
+          "x1": 433.79998779296875,
+          "y1": 617.3599853515625,
+          "width": 100.07998657226562,
+          "height": 59.4000244140625
+        },
+        {
+          "x0": 434.8800048828125,
+          "y0": 556.8800048828125,
+          "x1": 537.1199951171875,
+          "y1": 615.9199829101562,
+          "width": 102.239990234375,
+          "height": 59.03997802734375
+        },
+        {
+          "x0": 230.39999389648438,
+          "y0": 488.4800109863281,
+          "x1": 332.2799987792969,
+          "y1": 559.0400390625,
+          "width": 101.8800048828125,
+          "height": 70.56002807617188
+        },
+        {
+          "x0": 332.6400146484375,
+          "y0": 487.0400085449219,
+          "x1": 433.0799865722656,
+          "y1": 557.9599609375,
+          "width": 100.43997192382812,
+          "height": 70.91995239257812
+        },
+        {
+          "x0": 433.79998779296875,
+          "y0": 485.9599914550781,
+          "x1": 537.1199951171875,
+          "y1": 556.52001953125,
+          "width": 103.32000732421875,
+          "height": 70.56002807617188
+        },
+        {
+          "x0": 229.32000732421875,
+          "y0": 417.20001220703125,
+          "x1": 331.55999755859375,
+          "y1": 488.1199951171875,
+          "width": 102.239990234375,
+          "height": 70.91998291015625
+        },
+        {
+          "x0": 331.55999755859375,
+          "y0": 416.1199951171875,
+          "x1": 432.0,
+          "y1": 487.0400085449219,
+          "width": 100.44000244140625,
+          "height": 70.92001342773438
+        },
+        {
+          "x0": 432.7200012207031,
+          "y0": 415.0400085449219,
+          "x1": 537.1199951171875,
+          "y1": 485.6000061035156,
+          "width": 104.39999389648438,
+          "height": 70.55999755859375
+        },
+        {
+          "x0": 228.24000549316406,
+          "y0": 346.6400146484375,
+          "x1": 330.1199951171875,
+          "y1": 417.20001220703125,
+          "width": 101.87998962402344,
+          "height": 70.55999755859375
+        },
+        {
+          "x0": 330.4800109863281,
+          "y0": 345.20001220703125,
+          "x1": 431.2799987792969,
+          "y1": 416.1199951171875,
+          "width": 100.79998779296875,
+          "height": 70.91998291015625
+        },
+        {
+          "x0": 432.0,
+          "y0": 344.1199951171875,
+          "x1": 537.1199951171875,
+          "y1": 414.67999267578125,
+          "width": 105.1199951171875,
+          "height": 70.55999755859375
+        },
+        {
+          "x0": 227.16000366210938,
+          "y0": 276.08001708984375,
+          "x1": 329.3999938964844,
+          "y1": 346.2799987792969,
+          "width": 102.239990234375,
+          "height": 70.19998168945312
+        },
+        {
+          "x0": 329.760009765625,
+          "y0": 274.6400146484375,
+          "x1": 430.55999755859375,
+          "y1": 345.20001220703125,
+          "width": 100.79998779296875,
+          "height": 70.55999755859375
+        },
+        {
+          "x0": 431.2799987792969,
+          "y0": 273.55999755859375,
+          "x1": 537.1199951171875,
+          "y1": 343.760009765625,
+          "width": 105.83999633789062,
+          "height": 70.20001220703125
+        },
+        {
+          "x0": 226.0800018310547,
+          "y0": 205.15997314453125,
+          "x1": 328.32000732421875,
+          "y1": 275.719970703125,
+          "width": 102.24000549316406,
+          "height": 70.55999755859375
+        },
+        {
+          "x0": 328.67999267578125,
+          "y0": 203.719970703125,
+          "x1": 429.4800109863281,
+          "y1": 274.6400146484375,
+          "width": 100.80001831054688,
+          "height": 70.9200439453125
+        },
+        {
+          "x0": 430.55999755859375,
+          "y0": 203.0,
+          "x1": 537.1199951171875,
+          "y1": 273.20001220703125,
+          "width": 106.55999755859375,
+          "height": 70.20001220703125
+        },
+        {
+          "x0": 224.63999938964844,
+          "y0": 133.52001953125,
+          "x1": 327.239990234375,
+          "y1": 205.15997314453125,
+          "width": 102.59999084472656,
+          "height": 71.63995361328125
+        },
+        {
+          "x0": 327.6000061035156,
+          "y0": 132.08001708984375,
+          "x1": 428.760009765625,
+          "y1": 203.719970703125,
+          "width": 101.16000366210938,
+          "height": 71.63995361328125
+        },
+        {
+          "x0": 429.1199951171875,
+          "y0": 131.0,
+          "x1": 537.1199951171875,
+          "y1": 202.280029296875,
+          "width": 108.0,
+          "height": 71.280029296875
+        },
+        {
+          "x0": 223.9199981689453,
+          "y0": 61.8800048828125,
+          "x1": 326.5199890136719,
+          "y1": 133.52001953125,
+          "width": 102.59999084472656,
+          "height": 71.6400146484375
+        },
+        {
+          "x0": 326.5199890136719,
+          "y0": 60.44000244140625,
+          "x1": 427.67999267578125,
+          "y1": 132.44000244140625,
+          "width": 101.16000366210938,
+          "height": 72.0
+        },
+        {
+          "x0": 428.3999938964844,
+          "y0": 59.3599853515625,
+          "x1": 537.1199951171875,
+          "y1": 131.0,
+          "width": 108.72000122070312,
+          "height": 71.6400146484375
+        },
+        {
+          "x0": 223.55999755859375,
+          "y0": 47.84002685546875,
+          "x1": 325.79998779296875,
+          "y1": 61.8800048828125,
+          "width": 102.239990234375,
+          "height": 14.03997802734375
+        },
+        {
+          "x0": 326.1600036621094,
+          "y0": 47.84002685546875,
+          "x1": 426.9599914550781,
+          "y1": 60.44000244140625,
+          "width": 100.79998779296875,
+          "height": 12.5999755859375
+        },
+        {
+          "x0": 428.0400085449219,
+          "y0": 47.84002685546875,
+          "x1": 537.1199951171875,
+          "y1": 59.3599853515625,
+          "width": 109.07998657226562,
+          "height": 11.51995849609375
+        }
+      ]
+    }
+  ]
+}

redaction-service-v1/redaction-service-server-v1/src/test/resources/performance/dictionaries/EFSA_sanitisation_GFL_v1/rules.drl

@@ -142,17 +142,12 @@ rule "CBI.7.0: Do not redact Names and Addresses if published information found
 
 rule "CBI.7.1: Do not redact Names and Addresses if published information found in same table row"
     when
-        $table: Table(hasEntitiesOfType("published_information"),
-                     (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
+        $table: Table(hasEntitiesOfType("published_information"), hasEntitiesOfType("CBI_author"))
+        $cellsWithPublishedInformation: TableCell() from $table.streamTableCellsWhichContainType("published_information").toList()
+        $tableCell: TableCell(row == $cellsWithPublishedInformation.row) from $table.streamTableCells().toList()
+        $author: TextEntity(type == "CBI_author", active()) from $tableCell.getEntities()
     then
-        $table.streamEntitiesWhereRowContainsEntitiesOfType(List.of("CBI_author", "CBI_address"))
-            .forEach(redactionEntity -> {
-                redactionEntity.skipWithReferences(
-                        "CBI.7.1",
-                        "Published Information found in row",
-                        $table.getEntitiesOfTypeInSameRow("published_information", redactionEntity)
-                );
-            });
+        $author.skipWithReferences("CBI.7.1", "Published Information found in row", $table.getEntitiesOfTypeInSameRow("published_information", $author));
     end
 
 

redaction-service-v1/rules-management/README.md

@@ -0,0 +1,87 @@
+Rules Management CLI
+---
+The rules-management program is a command-line interface (CLI) tool written in Java. It is designed to generate Drools files based on rule classes and units. This README file provides an overview of the program and instructions for usage.
+
+Table of Contents  
+--
+* Introduction   
+* Installation  
+* Usage  
+* Options  
+* Examples  
+
+Introduction
+--
+The rules-management CLI program allows you to generate Drools files containing various rule classes and units. Each rule class represents a specific type of rule, such as SYN (Syngenta specific), CBI, PII, ETC (Other), AI, MAN (Manual redactions), X (Entity interactions), FA (FileAttribute interactions), and LDS (Local Dictionary Search).
+
+A rule unit is identified by a combination of the rule class and an integer. For example, CBI.1 refers to rule unit 1 in the CBI rule class. Each rule unit may contain multiple rules, which are also identified by integers. For instance, CBI.1.2 represents the second rule in rule unit 1 of the CBI rule class.
+
+The program provides options to specify the input type, input source, output destination, and additional flags for customization.
+
+Installation
+--
+To use the rules-management CLI, you need to have Java 17+ and gradle installed on your system. Follow these steps to get started:
+
+Clone the latest main branch.   
+compile with `gradle install`
+
+Testing
+--
+You can validate the rules you are changing if they are correctly written from a syntax point of view using the
+`DroolsCompilationTest` class. Make sure you also place the changes in the `test/resources/<rules_file>.drl` and then run the test. 
+
+Usage
+--
+To run the rules-management program, open a terminal or command prompt and navigate to the directory where the program files are located. Use the following command:
+
+
+`java -jar rules-management.jar [options]`  
+
+or directly with gradle  
+
+`gradle run --args='[options]'`
+
+Options
+--
+The rules-management program supports the following command-line options:
+
+-t, --type: Specifies the input type. Default is "list" (if the recursive flag is set: "file").  Available options:
+* "list" or "l": Expects a list of rule identifiers. For example: "MAN, X, LDS, CBI.0, PII.0.1", you can use wildcards such as * or leaving out digits with the identifiers to match multiple rule identifiers. This allows for convenient selection of multiple rule units or rules.   
+* "file" or "f": Expects the absolute path to a rule file. The file must contain the current Rule Identifiers. This option is useful for migrating rule files to the current state.
+* "old" or "o": Expects the absolute path to an old rule file. The program attempts to translate the old rules to new rules based on '"main/resources/old_rules_with_translations.csv"'.  
+
+
+-i, --input: Specifies the input source based on the chosen type. It can be:
+
+* A string of rule identifiers like `"MAN, X, LDS, CBI.0, SYN.*.0, PII.0.1"`  
+* path to a rules file  `/path/to/old_rules.txt`  
+* path to a folder containing rules files named "rules.txt" or "rules.drl" (if the "recursive" flag is set) `/path/to/rules_directory`   
+
+
+-o, --output: Specifies the output file path. If a directory is supplied as input, the folder structure is preserved.
+
+-n, --noop: If this flag is set, the input file will not be attempted to be translated or migrated, use this to only escape files. Currently, this only works with input type "file"
+
+-e, --escape: If this flag is set, the output files will be escaped and saved as ".txt", similar to dossier templates.
+
+-r, --recursive: Can only be set in combination with the "file" or "old" type and a directory as input. Searches recursively for files named "rules.txt" or "rules.drl" and attempts to migrate/translate them.
+The folder structure is preserved in the output folder.
+
+-a, --application: Indicate which application you are targeting. For redact manager use RM. For Documine use DM. If it's not specified it will default to RM.
+
+Examples
+--
+Generate Drools files from a list of rule identifiers and save them to the specified output file:  
+`gradle run --args='-t list -i "MAN, X, LDS, CBI.0, PII.0.1" -o /path/to/output/rules.drl'`  
+
+Migrate an old rule file to the current format and save the translated rules to the specified output file:  
+`gradle run --args='-t old -i /path/to/old_rules.drl -o /path/to/output/translated_rules.drl'`  
+
+Migrate multiple rule files in a directory (including subdirectories) and save the translated rules:  
+`gradle run --args='-t old -i /path/to/rules_directory -o /path/to/output/translated_rules -r'`  
+
+Generate Drools files from a rules file and escape the output:  
+`gradle run --args='-t file -i /path/to/rules.txt -o /path/to/output/escaped_rules -e'`
+
+Escape Drools file rules.drl as is:  
+`gradle run --args='-t f -n -e -i /path/to/rules.drl -o /path/to/output/rules.txt'`

redaction-service-v1/rules-management/build.gradle.kts

@@ -0,0 +1,52 @@
+plugins {
+    application
+}
+
+group = "com.knecon.fforesight.utility"
+version = "1.0-SNAPSHOT"
+
+repositories {
+    mavenCentral()
+    maven {
+        url = uri("https://nexus.knecon.com/repository/red-platform-releases/")
+        credentials {
+            username = providers.gradleProperty("mavenUser").getOrNull();
+            password = providers.gradleProperty("mavenPassword").getOrNull();
+        }
+    }
+}
+
+sourceSets {
+    test {
+        resources {
+            srcDirs("src/main/resources", "src/test/ressources") // add both so test can access the all_rules_file
+        }
+    }
+}
+
+dependencies {
+    implementation(project(":redaction-service-server-v1"))
+    implementation("org.projectlombok:lombok:1.18.28")
+    testImplementation(platform("org.junit:junit-bom:5.10.0"))
+    testImplementation("org.junit.jupiter:junit-jupiter")
+
+    implementation("com.github.javaparser:javaparser-core:3.25.3")
+    implementation("org.drools:drools-drl-parser:8.41.0.Final")
+    implementation("org.apache.commons:commons-csv:1.10.0")
+    implementation("commons-cli:commons-cli:1.5.0")
+
+
+    compileOnly("org.projectlombok:lombok:1.18.28")
+    annotationProcessor("org.projectlombok:lombok:1.18.28")
+
+    testCompileOnly("org.projectlombok:lombok:1.18.28")
+    testAnnotationProcessor("org.projectlombok:lombok:1.18.28")
+
+    implementation("org.kie:kie-api:9.44.0.Final")
+    implementation("org.drools:drools-compiler:9.44.0.Final")
+    testImplementation("org.kie:kie-ci:9.44.0.Final")
+}
+
+tasks.test {
+    useJUnitPlatform()
+}

redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/Main.java

@@ -0,0 +1,214 @@
+package com.knecon.fforesight.utility.rules.management;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Path;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.CommandLineParser;
+import org.apache.commons.cli.DefaultParser;
+import org.apache.commons.cli.HelpFormatter;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.Options;
+import org.apache.commons.cli.ParseException;
+
+import com.knecon.fforesight.utility.rules.management.factory.RuleFileFactory;
+import com.knecon.fforesight.utility.rules.management.factory.RuleFileParser;
+import com.knecon.fforesight.utility.rules.management.models.ApplicationType;
+import com.knecon.fforesight.utility.rules.management.models.RuleIdentifier;
+import com.knecon.fforesight.utility.rules.management.translation.OldRulesParser;
+import com.knecon.fforesight.utility.rules.management.utils.RuleFileIO;
+
+import lombok.SneakyThrows;
+
+public class Main {
+
+    public static void main(String[] args) {
+
+        Options options = buildOptions();
+
+        CommandLineParser parser = new DefaultParser();
+        HelpFormatter formatter = new HelpFormatter();
+        CommandLine cmd = null; //not a good practice, it serves its purpose
+
+        try {
+            cmd = parser.parse(options, args);
+        } catch (ParseException e) {
+            System.out.println(e.getMessage());
+            formatter.printHelp("utility-name", options);
+
+            System.exit(1);
+        }
+        final CommandLine finalCmd = cmd;
+
+        if (finalCmd.hasOption("r")) {
+            handleRecursiveFileCreation(finalCmd);
+        } else {
+            handleSingleFileCreation(finalCmd);
+        }
+    }
+
+
+    private static void handleSingleFileCreation(final CommandLine cmd) {
+
+        String ruleFileString;
+        String type = cmd.hasOption("t") ? cmd.getOptionValue("t") : "l";
+        String input = cmd.getOptionValue("input");
+        ApplicationType applicationType = getApplicationType(cmd);
+
+        if (cmd.hasOption("n")) {
+            if (!type.equals("file") && !type.equals("f")) {
+                System.out.println("No op can only be used with type \"file\"");
+                System.exit(1);
+            }
+            ruleFileString = RuleFileIO.getRulesString(input);
+        } else {
+            Set<RuleIdentifier> identifiers = switch (type) {
+                case "list", "l" -> RuleIdentifier.fromListOfIdentifiersString(input);
+                case "file", "f" -> RuleFileParser.parseRuleIdentifiersFromFile(input);
+                case "old", "o" -> OldRulesParser.translateOldRulesStringToNewIdentifiers(RuleFileIO.getRulesString(input), applicationType);
+                default -> throw new IllegalArgumentException(String.format("type \"%s\" is not valid", cmd.getOptionValue("t")));
+            };
+            ruleFileString = RuleFileFactory.createFileFromIdentifiers(identifiers, applicationType);
+        }
+
+        ruleFileString = escapeRuleStringIfOptionIsSet(cmd, ruleFileString);
+
+        try (var out = new FileOutputStream(cmd.getOptionValue("output"))) {
+            out.write(ruleFileString.getBytes(StandardCharsets.UTF_8));
+        } catch (IOException e) {
+            System.out.println(e.getMessage());
+            System.exit(1);
+        }
+    }
+
+
+    private static void handleRecursiveFileCreation(final CommandLine cmd) {
+
+        Path outputDir = Path.of(cmd.getOptionValue("output"));
+        if (!outputDir.toFile().isDirectory()) {
+            System.out.printf("%s is not a directory!%n", cmd.getOptionValue("output"));
+            System.exit(1);
+        }
+        outputDir.toFile().mkdirs();
+        ApplicationType applicationType = getApplicationType(cmd);
+
+        String type = cmd.hasOption("t") ? cmd.getOptionValue("t") : "f";
+        Map<Path, Set<RuleIdentifier>> identifiersPerFile = switch (type) {
+            case "file", "f" -> parseIdentifiersFromFiles(Path.of(cmd.getOptionValue("input")));
+            case "old", "o" -> translateIdentifiersFromOldFiles(Path.of(cmd.getOptionValue("input")), applicationType);
+            default -> throw new IllegalArgumentException(String.format("type \"%s\" is not valid in combination with the \"recursive\" flag", cmd.getOptionValue("t")));
+        };
+        identifiersPerFile.keySet().forEach(path -> {
+            String ruleFileString = RuleFileFactory.createFileFromIdentifiers(identifiersPerFile.get(path), applicationType);
+
+            ruleFileString = escapeRuleStringIfOptionIsSet(cmd, ruleFileString);
+            Path output = outputDir.resolve(path);
+            output.getParent().toFile().mkdirs();
+            try (var out = new FileOutputStream(output.toFile())) {
+                out.write(ruleFileString.getBytes(StandardCharsets.UTF_8));
+            } catch (IOException e) {
+                System.out.println(e.getMessage());
+                System.exit(1);
+            }
+        });
+    }
+
+
+    private static String escapeRuleStringIfOptionIsSet(CommandLine finalCmd, String ruleFileString) {
+
+        if (finalCmd.hasOption("e")) {
+            ruleFileString = RuleFileIO.escapeAndWrap(ruleFileString);
+        }
+        return ruleFileString;
+    }
+
+
+    private static Path getRelativizedPath(Path inputDirectory, File file) {
+
+        return inputDirectory.relativize(Path.of(file.getAbsolutePath()));
+    }
+
+
+    @SneakyThrows
+    private static Map<Path, Set<RuleIdentifier>> translateIdentifiersFromOldFiles(Path inputDirectory, ApplicationType applicationType) {
+
+        return RuleFileIO.streamAllRuleFilesInDirectory(inputDirectory)
+                .collect(Collectors.toMap(file -> getRelativizedPath(inputDirectory, file), e -> OldRulesParser.translateOldRulesFileToNewIdentifiers(e, applicationType)));
+    }
+
+
+    @SneakyThrows
+    private static Map<Path, Set<RuleIdentifier>> parseIdentifiersFromFiles(Path inputDirectory) {
+
+        return RuleFileIO.streamAllRuleFilesInDirectory(inputDirectory)
+                .collect(Collectors.toMap(file -> getRelativizedPath(inputDirectory, file), RuleFileParser::parseRuleIdentifiersFromFile));
+    }
+
+    private static ApplicationType getApplicationType(final CommandLine cmd) {
+
+        String application = cmd.hasOption("a") ? cmd.getOptionValue("a") : "RM";
+        ApplicationType applicationType;
+
+        if (application.equals("DM") || application.equals("RM")) {
+            applicationType = ApplicationType.valueOf(application);
+        } else {
+            throw new IllegalArgumentException(String.format("Application type \"%s\" is not valid, choose between \"RM\" or \"DM\"", cmd.getOptionValue("a")));
+        }
+
+        return applicationType;
+    }
+
+
+    private static Options buildOptions() {
+
+        Options options = new Options();
+
+        Option type = new Option("t",//
+                "type",//
+                true,//
+                "Please specify the input type, defaults to \"list\", \n" +//
+                        "current options:\n" +//
+                        "\"list\" or \"l\" (List of Rule identifiers e.g. \"MAN, X, LDS, CBI.0, PII.0.1\")\n" +//
+                        "\"file\" or \"f\" (Absolute Path to rule file, escaped or not, but the rule file must contain the current Rule Identifiers, the main use case is migrating rule files to the current state)\n" +//
+                        "\"old\" or \"o\" (Absolute Path to old rule file, escaped or not, the script will attempt to translate the old rules to new rules based on \"main/resources/old_rules_with_translations.csv\")\n");
+        options.addOption(type);
+        Option input = new Option("i",
+                "input",
+                true,
+                "Based on type expects: String of Rule identifiers, Absolute Path to a rules file or a folder containing rules files named rules.txt or rules.drl if \"recursive\" flag is set");
+        input.setRequired(true);
+        options.addOption(input);
+
+        Option output = new Option("o", "output", true, "output file path, if a directory is supplied, the folder structure is preserved");
+        output.setRequired(true);
+        options.addOption(output);
+
+        Option noTranslate = new Option("n",
+                "noop",
+                false,
+                "If this flag is set, the input file will not be attempted to be translated or migrated, use this to only escape files. Currently this only works with input type \"file\"");
+        options.addOption(noTranslate);
+        Option escapeOutput = new Option("e", "escape", false, "If this flag is set, the output files will be escaped and saved as \".txt\", like in the dossier templates");
+        options.addOption(escapeOutput);
+
+        Option recursive = new Option("r",
+                "recursive",
+                false,
+                "Can only be set in combination with type \"file\" or \"old\" Searches recursively for files named rules.txt or rules.drl and attempts to migrate/translate them.");
+        options.addOption(recursive);
+
+        Option application = new Option("a",
+                "application",
+                true,
+                "What application to update the rules for. For Redact Manager specify: RM. For Documine specify: DM. Defaults to RM.");
+        options.addOption(application);
+        return options;
+    }
+
+}

redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/RuleManagementResources.java

@@ -0,0 +1,55 @@
+package com.knecon.fforesight.utility.rules.management;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+
+import com.knecon.fforesight.utility.rules.management.models.ApplicationType;
+
+import lombok.SneakyThrows;
+
+public class RuleManagementResources {
+
+    public static InputStream getAllRulesInputStream(ApplicationType applicationType) {
+
+        if (applicationType == ApplicationType.RM) {
+            return RuleManagementResources.class.getClassLoader().getResourceAsStream("all_redact_manager_rules.drl");
+        }
+        return  RuleManagementResources.class.getClassLoader().getResourceAsStream("all_rules_documine.drl");
+    }
+
+
+    public static InputStream getDefaultRuleIdentifiesInputStream(ApplicationType applicationType) {
+
+        if (applicationType == ApplicationType.RM) {
+            return RuleManagementResources.class.getClassLoader().getResourceAsStream("default_rule_identifiers.txt");
+        } else {
+            return RuleManagementResources.class.getClassLoader().getResourceAsStream("default_rule_identifiers_dm.txt");
+        }
+    }
+
+
+    public static InputStream getTemplateInputStream() {
+
+        return RuleManagementResources.class.getClassLoader().getResourceAsStream("order_template.txt");
+    }
+
+
+    public static InputStream getOldRulesCsvInputStream() {
+
+        return RuleManagementResources.class.getClassLoader().getResourceAsStream("old_rules_with_translations.csv");
+    }
+
+
+    @SneakyThrows
+    public static String createTempOldRulesCsv(String formattedAsCsv) {
+
+        File csvFile = File.createTempFile("old_rules_with_translations-", ".csv");
+        try (var out = new FileOutputStream(csvFile)) {
+            out.write(formattedAsCsv.getBytes(StandardCharsets.UTF_8));
+        }
+        return csvFile.toString();
+    }
+
+}

redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/factory/DefaultRuleIdentifiersFactory.java

@@ -0,0 +1,26 @@
+package com.knecon.fforesight.utility.rules.management.factory;
+
+import java.util.Arrays;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import com.knecon.fforesight.utility.rules.management.RuleManagementResources;
+import com.knecon.fforesight.utility.rules.management.models.ApplicationType;
+import com.knecon.fforesight.utility.rules.management.models.RuleIdentifier;
+
+import lombok.SneakyThrows;
+import lombok.experimental.UtilityClass;
+
+@UtilityClass
+public class DefaultRuleIdentifiersFactory {
+
+    @SneakyThrows
+    public Set<RuleIdentifier> getDefaultRuleIdentifiers(ApplicationType applicationType) {
+
+        try (var defaultRuleIdentifiers = RuleManagementResources.getDefaultRuleIdentifiesInputStream(applicationType)) {
+            String ruleIdentifierString = new String(defaultRuleIdentifiers.readAllBytes());
+            return Arrays.stream(ruleIdentifierString.split("\n")).map(RuleIdentifier::fromString).collect(Collectors.toSet());
+        }
+    }
+
+}

redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/factory/RuleFileFactory.java

@@ -0,0 +1,120 @@
+package com.knecon.fforesight.utility.rules.management.factory;
+
+import static java.util.Collections.emptySet;
+
+import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import com.knecon.fforesight.utility.rules.management.RuleManagementResources;
+import com.knecon.fforesight.utility.rules.management.models.ApplicationType;
+import com.knecon.fforesight.utility.rules.management.models.RuleClass;
+import com.knecon.fforesight.utility.rules.management.models.RuleFileBluePrint;
+import com.knecon.fforesight.utility.rules.management.models.RuleIdentifier;
+import com.knecon.fforesight.utility.rules.management.models.RuleType;
+import com.knecon.fforesight.utility.rules.management.models.RuleUnit;
+
+import lombok.SneakyThrows;
+import lombok.experimental.UtilityClass;
+
+@UtilityClass
+public class RuleFileFactory {
+
+    public String createFileFromIdentifiers(String identifiers, ApplicationType applicationType) {
+
+        if (identifiers.isBlank() || identifiers.isEmpty()) {
+            return createFileFromIdentifiers(emptySet(), applicationType);
+        }
+        return createFileFromIdentifiers(Arrays.stream(identifiers.split(",")).map(String::trim).map(RuleIdentifier::fromString).collect(Collectors.toSet()), applicationType);
+    }
+
+
+    public String createFileFromIdentifiers(Set<RuleIdentifier> identifiers, ApplicationType applicationType) {
+
+        return createFileFromIdentifiers(identifiers, true, applicationType);
+    }
+
+
+    public String createFileFromIdentifiers(Set<RuleIdentifier> identifiers, boolean includeDefaults, ApplicationType applicationType) {
+
+        if (includeDefaults) {
+            identifiers.addAll(DefaultRuleIdentifiersFactory.getDefaultRuleIdentifiers(applicationType));
+        }
+        RuleFileBluePrint bluePrint = RuleFileParser.buildBluePrintFromAllRulesFile(applicationType);
+        RuleFileBluePrint filteredBluePrint = bluePrint.buildFilteredBluePrintByRuleIdentifiers(identifiers);
+        return buildRuleString(filteredBluePrint);
+    }
+
+
+    @SneakyThrows
+    public String buildRuleString(RuleFileBluePrint bluePrint) {
+
+        try (var templateInputStream = RuleManagementResources.getTemplateInputStream()) {
+            String template = new String(templateInputStream.readAllBytes(), StandardCharsets.UTF_8);
+            List<RuleType> templateRuleOrder = parseRuleOrder(template);
+            return buildBluePrintWithTemplateRuleOrder(bluePrint, templateRuleOrder);
+        }
+    }
+
+
+    private List<RuleType> parseRuleOrder(String template) {
+
+        String[] values = template.split("\n");
+        return Arrays.stream(values).map(RuleType::fromString).toList();
+    }
+
+
+    private String buildBluePrintWithTemplateRuleOrder(RuleFileBluePrint bluePrint, List<RuleType> ruleOrder) {
+
+        Set<RuleType> additionalRuleTypes = bluePrint.ruleClasses().stream().map(RuleClass::ruleType).filter(ruleType -> !ruleOrder.contains(ruleType)).collect(Collectors.toSet());
+
+        StringBuilder sb = new StringBuilder();
+        sb.append(bluePrint.imports());
+        sb.append("\n\n");
+        sb.append(bluePrint.globals());
+        sb.append("\n\n");
+        sb.append("//------------------------------------ queries ------------------------------------");
+        sb.append("\n\n");
+        sb.append(bluePrint.queries());
+        sb.append("\n\n");
+        for (RuleType ruleBlockType : ruleOrder) {
+            if (ruleBlockType.isWildCard()) {
+                additionalRuleTypes.stream().sorted(Comparator.comparing(RuleType::name)).forEach(ruleType -> writeRuleClass(bluePrint, ruleType, sb));
+                continue;
+            }
+            if (bluePrint.findRuleClassByType(ruleBlockType).isEmpty()) {
+                continue;
+            }
+            writeRuleClass(bluePrint, ruleBlockType, sb);
+        }
+        return sb.toString().trim() + "\n";
+    }
+
+
+    private static void writeRuleClass(RuleFileBluePrint bluePrint, RuleType ruleType, StringBuilder sb) {
+
+        sb.append("//------------------------------------ ");
+        sb.append(ruleType);
+        sb.append(" rules ------------------------------------");
+        sb.append("\n\n");
+        RuleClass ruleClass = bluePrint.findRuleClassByType(ruleType).orElseThrow();
+        List<RuleUnit> sortedRuleUnits = ruleClass.ruleUnits().stream().sorted(Comparator.comparingInt(RuleUnit::unit)).toList();
+        for (RuleUnit unit : sortedRuleUnits) {
+            if (unit.rules().isEmpty()) {
+                continue;
+            }
+            sb.append("// ");
+            sb.append(unit.rules().get(0).identifier().toRuleUnitString());
+            sb.append("\n");
+            unit.rules().stream().sorted(Comparator.comparingInt(rule -> rule.identifier().id())).forEach(rule -> {
+                sb.append(rule.code());
+                sb.append("\n\n");
+            });
+            sb.append("\n");
+        }
+    }
+
+}

redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/factory/RuleFileParser.java

@@ -0,0 +1,110 @@
+package com.knecon.fforesight.utility.rules.management.factory;
+
+import static java.util.stream.Collectors.groupingBy;
+
+import java.io.File;
+import java.nio.charset.StandardCharsets;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import org.drools.drl.ast.descr.ImportDescr;
+import org.drools.drl.ast.descr.PackageDescr;
+import org.drools.drl.ast.descr.RuleDescr;
+import org.drools.drl.parser.DrlParser;
+import org.kie.internal.builder.conf.LanguageLevelOption;
+
+import com.knecon.fforesight.utility.rules.management.RuleManagementResources;
+import com.knecon.fforesight.utility.rules.management.models.ApplicationType;
+import com.knecon.fforesight.utility.rules.management.models.BasicRule;
+import com.knecon.fforesight.utility.rules.management.models.RuleClass;
+import com.knecon.fforesight.utility.rules.management.models.RuleFileBluePrint;
+import com.knecon.fforesight.utility.rules.management.models.RuleIdentifier;
+import com.knecon.fforesight.utility.rules.management.models.RuleType;
+import com.knecon.fforesight.utility.rules.management.models.RuleUnit;
+import com.knecon.fforesight.utility.rules.management.utils.RuleFileIO;
+
+import lombok.SneakyThrows;
+import lombok.experimental.UtilityClass;
+
+@UtilityClass
+public class RuleFileParser {
+
+    @SneakyThrows
+    public RuleFileBluePrint buildBluePrintFromAllRulesFile(ApplicationType applicationType) {
+
+        try (var rulesInputStream = RuleManagementResources.getAllRulesInputStream(applicationType)) {
+            String rulesString = new String(rulesInputStream.readAllBytes(), StandardCharsets.UTF_8);
+            return buildBluePrintFromRulesString(rulesString);
+        }
+    }
+
+
+    @SneakyThrows
+    public RuleFileBluePrint buildBluePrintFromRulesString(String rulesString) {
+
+        DrlParser parser = new DrlParser(LanguageLevelOption.DRL6);
+        PackageDescr packageDescr = parser.parse(false, rulesString);
+        StringBuilder queryBuilder = new StringBuilder();
+        List<BasicRule> allRules = new LinkedList<>();
+        for (RuleDescr rule : packageDescr.getRules()) {
+            if (rule.isQuery()) {
+                queryBuilder.append(rulesString, rule.getStartCharacter(), rule.getEndCharacter());
+                queryBuilder.append("\n\n");
+                continue;
+            }
+            allRules.add(BasicRule.fromRuleDescr(rule, rulesString));
+        }
+        String imports = rulesString.substring(0, packageDescr.getImports().stream().mapToInt(ImportDescr::getEndCharacter).max().orElseThrow() + 1);
+        String globals = packageDescr.getGlobals()
+                .stream()
+                .map(globalDescr -> rulesString.substring(globalDescr.getStartCharacter(), globalDescr.getEndCharacter()))
+                .collect(Collectors.joining("\n"));
+
+        List<RuleClass> ruleClasses = buildRuleClasses(allRules);
+
+        return new RuleFileBluePrint(imports.trim(), globals.trim(), queryBuilder.toString().trim(), ruleClasses);
+    }
+
+
+    private List<RuleClass> buildRuleClasses(List<BasicRule> allRules) {
+
+        Map<RuleType, List<BasicRule>> rulesPerType = allRules.stream().collect(groupingBy(rule -> rule.identifier().type()));
+        return rulesPerType.keySet().stream().map(type -> new RuleClass(type, groupingByGroup(rulesPerType.get(type)))).collect(Collectors.toList());
+    }
+
+
+    private List<RuleUnit> groupingByGroup(List<BasicRule> rules) {
+
+        Map<Integer, List<BasicRule>> rulesPerUnit = rules.stream().collect(groupingBy(rule -> rule.identifier().unit()));
+        return rulesPerUnit.keySet().stream().sorted().map(unit -> new RuleUnit(unit, rulesPerUnit.get(unit))).collect(Collectors.toList());
+    }
+
+
+    public Set<RuleIdentifier> parseRuleIdentifiersFromFile(String input) {
+
+        return buildBluePrintFromRulesString(RuleFileIO.getRulesString(input)).getAllRuleIdentifiers();
+    }
+
+
+    public Set<RuleIdentifier> parseRuleIdentifiersFromFile(File file) {
+
+        return parseRuleIdentifiersFromFile(file.toString());
+    }
+
+
+    /**
+     * Creates a BluePrint from all redact manager, documine and component rule files
+     *
+     * @return RuleFileBluePrint containing all rules from any all rule files
+     */
+    public static RuleFileBluePrint buildBluePrintFromAllRuleFiles() {
+
+        RuleFileBluePrint bluePrint = buildBluePrintFromAllRulesFile(ApplicationType.DM);
+        bluePrint.addAllRulesFromBluePrint(buildBluePrintFromAllRulesFile(ApplicationType.RM));
+        return bluePrint;
+    }
+
+}

redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/migration/RuleFileMigrator.java

@@ -0,0 +1,43 @@
+package com.knecon.fforesight.utility.rules.management.migration;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.nio.charset.StandardCharsets;
+import java.util.List;
+
+import com.knecon.fforesight.utility.rules.management.factory.RuleFileFactory;
+import com.knecon.fforesight.utility.rules.management.factory.RuleFileParser;
+import com.knecon.fforesight.utility.rules.management.models.BasicRule;
+import com.knecon.fforesight.utility.rules.management.models.RuleFileBluePrint;
+import com.knecon.fforesight.utility.rules.management.utils.RuleFileIO;
+
+import lombok.SneakyThrows;
+import lombok.experimental.UtilityClass;
+
+/**
+ * This class is the primary class for migrating Files with the 4.0.0 format to newer versions. This assumes RuleIdentifiers are already formatted.
+ * It will search for any rule with the same Identifier in the base and replace it with the new version. If none is found, the rule is written as is.
+ */
+@UtilityClass
+public class RuleFileMigrator {
+
+    @SneakyThrows
+    public void migrateFile(File ruleFile) {
+
+        RuleFileBluePrint ruleFileBluePrint = RuleFileParser.buildBluePrintFromRulesString(RuleFileIO.getRulesString(ruleFile.getAbsolutePath()));
+        RuleFileBluePrint combinedBluePrint = RuleFileParser.buildBluePrintFromAllRuleFiles();
+
+        for (BasicRule ruleToReplace : ruleFileBluePrint.getAllRules()) {
+            List<BasicRule> rulesToAdd = combinedBluePrint.findRuleByIdentifier(ruleToReplace.identifier());
+            ruleFileBluePrint.removeRule(ruleToReplace.identifier());
+            rulesToAdd.forEach(ruleFileBluePrint::addRule);
+        }
+
+        String migratedRulesString = RuleFileFactory.buildRuleString(ruleFileBluePrint);
+        String migratedFilePath = ruleFile.getAbsolutePath();
+        try (var out = new FileOutputStream(migratedFilePath)) {
+            out.write(migratedRulesString.getBytes(StandardCharsets.UTF_8));
+        }
+    }
+
+}

redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/migration/RuleIdentifierMigrator.java

@@ -0,0 +1,156 @@
+package com.knecon.fforesight.utility.rules.management.migration;
+
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.Collection;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import com.knecon.fforesight.utility.rules.management.RuleManagementResources;
+import com.knecon.fforesight.utility.rules.management.factory.RuleFileFactory;
+import com.knecon.fforesight.utility.rules.management.factory.RuleFileParser;
+import com.knecon.fforesight.utility.rules.management.models.ApplicationType;
+import com.knecon.fforesight.utility.rules.management.models.BasicRule;
+import com.knecon.fforesight.utility.rules.management.models.RuleClass;
+import com.knecon.fforesight.utility.rules.management.models.RuleFileBluePrint;
+import com.knecon.fforesight.utility.rules.management.models.RuleIdentifier;
+import com.knecon.fforesight.utility.rules.management.models.RuleUnit;
+import com.knecon.fforesight.utility.rules.management.translation.OldRulesParser;
+
+import lombok.SneakyThrows;
+import lombok.experimental.UtilityClass;
+
+@UtilityClass
+public class RuleIdentifierMigrator {
+
+    public static void main(String[] args) {
+
+        migrateAllRuleIdentifiers();
+    }
+
+
+    @SneakyThrows
+    public void migrateAllRuleIdentifiers() {
+
+        List<OldRulesParser.OldRulesCsvRecord> parsedRecords = OldRulesParser.getOldRulesCsvRecords(RuleManagementResources.getOldRulesCsvInputStream());
+        RuleFileBluePrint bluePrint = getBluePrint(ApplicationType.RM);
+
+//        migrateIdentifier(RuleIdentifier.fromString("PII.10.0"), RuleIdentifier.fromString("CBI.20.0"), bluePrint, parsedRecords);
+//        migrateIdentifier(RuleIdentifier.fromString("PII.10.1"), RuleIdentifier.fromString("CBI.20.1"), bluePrint, parsedRecords);
+//        migrateIdentifier(RuleIdentifier.fromString("PII.11.0"), RuleIdentifier.fromString("PII.10.0"), bluePrint, parsedRecords);
+//        migrateIdentifier(RuleIdentifier.fromString("PII.12.0"), RuleIdentifier.fromString("PII.11.0"), bluePrint, parsedRecords);
+//        migrateIdentifier(RuleIdentifier.fromString("PII.13.0"), RuleIdentifier.fromString("PII.12.0"), bluePrint, parsedRecords);
+
+        bluePrint = migrateMatchedRuleForAllRules(bluePrint);
+
+        String ruleString = RuleFileFactory.buildRuleString(bluePrint);
+        try (var out = new FileOutputStream("/tmp/all_redact_manager_rules.drl")) {
+            out.write(ruleString.getBytes(StandardCharsets.UTF_8));
+        }
+        String csvString = OldRulesParser.formatAsCsv(parsedRecords);
+        try (var out = new FileOutputStream("/tmp/old_rules_with_translations.csv")) {
+            out.write(csvString.getBytes(StandardCharsets.UTF_8));
+        }
+    }
+
+
+    private static RuleFileBluePrint migrateMatchedRuleForAllRules(RuleFileBluePrint bluePrint) {
+
+        List<BasicRule> migratedRules = bluePrint.ruleClasses()
+                .stream()
+                .map(RuleClass::ruleUnits)
+                .flatMap(Collection::stream)
+                .map(RuleUnit::rules)
+                .flatMap(Collection::stream)
+                .map(RuleIdentifierMigrator::migrateMatchedRule)
+                .toList();
+        RuleFileBluePrint migratedBluePrint = new RuleFileBluePrint(bluePrint.imports(), bluePrint.globals(), bluePrint.queries(), new LinkedList<>());
+        migratedRules.forEach(migratedBluePrint::addRule);
+        return migratedBluePrint;
+    }
+
+
+    private static BasicRule migrateMatchedRule(BasicRule basicRule) {
+
+        String redactionReason = findByRegex("\\.setRedactionReason\\(\"(.*)\"\\)", basicRule.code(), 1);
+        String legalBasis = findByRegex("\\.setLegalBasis\\(\"(.*)\"\\)", basicRule.code(), 1);
+
+        String migratedCode = basicRule.code()
+                .replaceAll("\\.addMatchedRule\\(.*\\)",
+                        Matcher.quoteReplacement(String.format(".addMatchedRule(\"%s\", \"%s\", \"%s\")", basicRule.identifier().toString(), redactionReason, legalBasis)));
+        migratedCode = migratedCode.replaceAll("\\.setMatchedRule\\(.*\\)",
+                Matcher.quoteReplacement(String.format(".addMatchedRule(\"%s\", \"%s\", \"%s\")", basicRule.identifier().toString(), redactionReason, legalBasis)));
+        migratedCode = migratedCode.replaceAll("\\.setRedactionReason\\(\".*\"\\)", "");
+        migratedCode = migratedCode.replaceAll("\\.setLegalBasis\\(\".*\"\\)", "");
+        migratedCode = migratedCode.replaceAll("\\$entity;\n", "");
+        migratedCode = migratedCode.replaceAll("\\$redactionEntity;", "");
+        migratedCode = migratedCode.replaceAll("\\$laboratoryEntity;", "");
+        migratedCode = migratedCode.replaceAll("\\$contactEntity;", "");
+        return new BasicRule(basicRule.identifier(), basicRule.name(), migratedCode);
+    }
+
+
+    private static String findByRegex(String regex, String searchText, int group) {
+
+        Pattern pattern = Pattern.compile(regex);
+        Matcher matcher = pattern.matcher(searchText);
+        if (matcher.find()) {
+            return searchText.substring(matcher.start(group), matcher.end(group));
+        }
+        return "";
+    }
+
+
+    private static RuleFileBluePrint getBluePrint(ApplicationType applicationType) throws IOException {
+
+        RuleFileBluePrint bluePrint;
+        try (var inputStream = RuleManagementResources.getAllRulesInputStream(applicationType)) {
+            String rulesString = new String(inputStream.readAllBytes());
+            bluePrint = RuleFileParser.buildBluePrintFromRulesString(rulesString);
+        }
+        return bluePrint;
+    }
+
+
+    public void migrateIdentifier(RuleIdentifier oldIdentifier, RuleIdentifier newIdentifier, RuleFileBluePrint bluePrint, List<OldRulesParser.OldRulesCsvRecord> records) {
+
+        BasicRule oldRule = bluePrint.findRuleClassByType(oldIdentifier.type()).orElseThrow()
+                .findRuleUnitByInteger(oldIdentifier.unit()).orElseThrow()
+                .rules()
+                .stream()
+                .filter(rule -> rule.identifier().equals(oldIdentifier))
+                .findFirst()
+                .orElseThrow();
+
+        BasicRule newRule = new BasicRule(newIdentifier, oldRule.name(), migrateCode(oldRule.name(), oldRule.code(), oldIdentifier, newIdentifier));
+        bluePrint.removeRule(oldIdentifier);
+        bluePrint.addRule(newRule);
+
+        records.stream()
+                .filter(record -> record.translatesTo().contains(oldIdentifier))
+                .forEach(record -> replaceOldIdentifier(oldIdentifier, newIdentifier, record.translatesTo()));
+    }
+
+
+    private String migrateCode(String name, String code, RuleIdentifier oldIdentifier, RuleIdentifier newIdentifier) {
+
+        return code.replace(String.format(".addMatchedRule(%d);", oldIdentifier.unit()), String.format(".addMatchedRule(%d);", newIdentifier.unit()))
+                .replace(oldIdentifier.toString(), newIdentifier.toString());
+    }
+
+
+    private static void replaceOldIdentifier(RuleIdentifier oldIdentifier, RuleIdentifier newIdentifier, List<RuleIdentifier> oldIdentifiers) {
+
+        oldIdentifiers.remove(oldIdentifier);
+        oldIdentifiers.add(newIdentifier);
+    }
+
+
+    public record Context(RuleFileBluePrint bluePrint, List<OldRulesParser.OldRulesCsvRecord> records) {
+
+    }
+
+}

redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/ApplicationType.java

@@ -0,0 +1,17 @@
+package com.knecon.fforesight.utility.rules.management.models;
+
+import lombok.Getter;
+
+public enum ApplicationType {
+
+    RM("RM"),
+    DM("DM");
+
+    @Getter
+    private final String value;
+
+    ApplicationType(String value) {
+
+        this.value = value;
+    }
+}

redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/BasicRule.java

@@ -0,0 +1,21 @@
+package com.knecon.fforesight.utility.rules.management.models;
+
+import org.drools.drl.ast.descr.RuleDescr;
+
+import com.knecon.fforesight.utility.rules.management.utils.RuleFileIO;
+
+public record BasicRule(RuleIdentifier identifier, String name, String code) {
+    public static BasicRule fromRuleDescr(RuleDescr rule, String rulesString) {
+
+        RuleIdentifier identifier = RuleIdentifier.fromName(rule.getName());
+        String nameWithoutIdentifier = rule.getName().replace(identifier + ":", "");
+        String code = rulesString.substring(rule.getStartCharacter(), rule.getEndCharacter());
+        return new BasicRule(identifier, nameWithoutIdentifier, code);
+    }
+
+    public String toEscapedCode() {
+
+        return RuleFileIO.escapeAndWrap(code());
+    }
+
+}

redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/OldRule.java

@@ -0,0 +1,21 @@
+package com.knecon.fforesight.utility.rules.management.models;
+
+import org.drools.drl.ast.descr.RuleDescr;
+
+public record OldRule(String name, String code) {
+
+    private static final int RULE_PREFIX_LENGTH = "rule \"\"".length();
+
+
+    public static OldRule fromRuleDescr(RuleDescr ruleDescr, String oldRuleString) {
+
+        return new OldRule(ruleDescr.getName(), getCodeSubstring(ruleDescr, oldRuleString));
+    }
+
+
+    private static String getCodeSubstring(RuleDescr rule, String oldRuleString) {
+
+        return oldRuleString.substring(rule.getStartCharacter() + rule.getName().length() + RULE_PREFIX_LENGTH, rule.getEndCharacter());
+    }
+
+}

redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/RuleClass.java

@@ -0,0 +1,23 @@
+package com.knecon.fforesight.utility.rules.management.models;
+
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Objects;
+import java.util.Optional;
+
+public record RuleClass(RuleType ruleType, List<RuleUnit> ruleUnits) {
+
+    public Optional<RuleUnit> findRuleUnitByInteger(Integer unit) {
+
+        return ruleUnits.stream().filter(ruleUnit -> Objects.equals(ruleUnit.unit(), unit)).findFirst();
+    }
+
+
+    public RuleUnit createNewRuleUnit(Integer unit) {
+
+        var ruleUnit = new RuleUnit(unit, new LinkedList<>());
+        ruleUnits.add(ruleUnit);
+        return ruleUnit;
+    }
+
+}

redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/RuleFileBluePrint.java

@@ -0,0 +1,121 @@
+package com.knecon.fforesight.utility.rules.management.models;
+
+import java.util.Collection;
+import java.util.Collections;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+public record RuleFileBluePrint(String imports, String globals, String queries, List<RuleClass> ruleClasses) {
+
+    public void removeRule(RuleIdentifier ruleIdentifier) {
+
+        findRuleClassByType(ruleIdentifier.type()).ifPresent(ruleClass -> ruleClass.findRuleUnitByInteger(ruleIdentifier.unit()).ifPresent(ruleUnit -> {
+            ruleUnit.rules().removeIf(rule -> rule.identifier().matches(ruleIdentifier));
+            if (ruleUnit.rules().isEmpty()) {
+                ruleClass.ruleUnits().remove(ruleUnit);
+            }
+            if (ruleClass.ruleUnits().isEmpty()) {
+                ruleClasses().remove(ruleClass);
+            }
+        }));
+
+    }
+
+
+    public Optional<RuleClass> findRuleClassByType(RuleType ruleType) {
+
+        return ruleClasses.stream().filter(ruleClass -> Objects.equals(ruleClass.ruleType(), ruleType)).findFirst();
+    }
+
+
+    public Set<BasicRule> getAllRules() {
+
+        return ruleClasses().stream().map(RuleClass::ruleUnits).flatMap(Collection::stream).map(RuleUnit::rules).flatMap(Collection::stream).collect(Collectors.toSet());
+    }
+
+
+    /**
+     * Adds all rules from a given RuleFileBluePrint to the current instance.
+     *
+     * @param ruleFileBluePrint The RuleFileBluePrint that contains the rules to be added.
+     * @throws IllegalArgumentException If the RuleFileBluePrints have non-matching rules with the same identifier and cannot be merged.
+     */
+    public void addAllRulesFromBluePrint(RuleFileBluePrint ruleFileBluePrint) {
+
+        Set<RuleIdentifier> newRuleIdentifiers = ruleFileBluePrint.getAllRuleIdentifiers();
+        Set<RuleIdentifier> existingRuleIdentifiers = this.getAllRuleIdentifiers();
+        Set<RuleIdentifier> duplicatedRuleIdentifiers = existingRuleIdentifiers.stream().filter(newRuleIdentifiers::contains).collect(Collectors.toSet());
+        for (RuleIdentifier duplicatedRuleIdentifier : duplicatedRuleIdentifiers) {
+            List<BasicRule> thisRules = findRuleByIdentifier(duplicatedRuleIdentifier);
+            List<BasicRule> otherRules = ruleFileBluePrint.findRuleByIdentifier(duplicatedRuleIdentifier);
+            assert thisRules.size() == 1;
+            assert otherRules.size() == 1;
+            if (!thisRules.get(0).equals(otherRules.get(0))) {
+                throw new IllegalArgumentException("RuleFileBluePrints have non matching rules with the same identifier, cannot be merged!");
+            }
+        }
+        newRuleIdentifiers.removeAll(existingRuleIdentifiers);
+        newRuleIdentifiers.forEach(identifierToAdd -> ruleFileBluePrint.findRuleByIdentifier(identifierToAdd).forEach(this::addRule));
+    }
+
+
+    public List<BasicRule> findRuleByIdentifier(RuleIdentifier ruleIdentifier) {
+
+        if (Objects.isNull(ruleIdentifier.unit())) {
+            return findRuleClassByType(ruleIdentifier.type()).map(ruleClass -> ruleClass.ruleUnits().stream().flatMap(ruleUnit -> ruleUnit.rules().stream()).toList())
+                    .orElse(Collections.emptyList());
+        }
+        return findRuleClassByType(ruleIdentifier.type())//
+                .map(ruleClass -> ruleClass.findRuleUnitByInteger(ruleIdentifier.unit())//
+                        .map(ruleUnit -> ruleUnit.rules().stream().filter(rule -> rule.identifier().matches(ruleIdentifier)).toList())//
+                        .orElse(Collections.emptyList()))//
+                .orElse(Collections.emptyList());
+    }
+
+
+    public Set<RuleIdentifier> getAllRuleIdentifiers() {
+
+        return ruleClasses().stream()
+                .map(RuleClass::ruleUnits)
+                .flatMap(Collection::stream)
+                .map(RuleUnit::rules)
+                .flatMap(Collection::stream)
+                .map(BasicRule::identifier)
+                .collect(Collectors.toSet());
+    }
+
+
+    public void addRule(BasicRule rule) {
+
+        RuleClass ruleClass = findRuleClassByType(rule.identifier().type()).orElseGet(() -> createNewRuleClass(rule));
+
+        RuleUnit ruleUnit = ruleClass.findRuleUnitByInteger(rule.identifier().unit()).orElseGet(() -> ruleClass.createNewRuleUnit(rule.identifier().unit()));
+
+        ruleUnit.rules().add(rule);
+    }
+
+
+    public RuleFileBluePrint buildFilteredBluePrintByRuleIdentifiers(Set<RuleIdentifier> identifiers) {
+
+        RuleFileBluePrint filteredBluePrint = new RuleFileBluePrint(imports(), globals(), queries(), new LinkedList<>());
+        ruleClasses().stream()
+                .flatMap(ruleClass -> ruleClass.ruleUnits().stream())
+                .flatMap(ruleUnit -> ruleUnit.rules().stream())
+                .filter(rule -> identifiers.stream().anyMatch(identifier -> rule.identifier().matches(identifier)))
+                .forEach(filteredBluePrint::addRule);
+        return filteredBluePrint;
+    }
+
+
+    private RuleClass createNewRuleClass(BasicRule rule) {
+
+        var newClass = new RuleClass(rule.identifier().type(), new LinkedList<>());
+        ruleClasses.add(newClass);
+        return newClass;
+    }
+
+}

redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/RuleIdentifier.java

@@ -0,0 +1,90 @@
+package com.knecon.fforesight.utility.rules.management.models;
+
+import java.util.Arrays;
+import java.util.Objects;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import lombok.NonNull;
+
+/*
+This is used to filter rules, if the field Integer unit or Integer id is null, the field will match any other RuleIdentifier.
+Therefore, to compare RuleIdentifiers one should always use the function matches.
+ */
+public record RuleIdentifier(@NonNull RuleType type, Integer unit, Integer id) {
+
+    public static RuleIdentifier fromName(String name) {
+
+        String identifier = name.split(":")[0];
+        return fromString(identifier);
+    }
+
+    public static RuleIdentifier fromRuleType(RuleType ruleType) {
+
+        return new RuleIdentifier(ruleType, null, null);
+    }
+
+
+    public static RuleIdentifier fromString(String identifier) {
+        String[] values = identifier.split("\\.");
+        RuleType type = RuleType.fromString(values[0]);
+        Integer group = values.length > 1 ? parseIntOrStar(values[1]) : null;
+        Integer id = values.length > 2 ? parseIntOrStar(values[2]) : null;
+        return new RuleIdentifier(type, group, id);
+    }
+
+    public static Set<RuleIdentifier> fromListOfIdentifiersString(String input) {
+
+        return Arrays.stream(input.split(",")).map(String::trim).map(RuleIdentifier::fromString).collect(Collectors.toSet());
+    }
+
+
+    private static Integer parseIntOrStar(String value) {
+        value = value.replaceAll("\r","");
+        return !value.equals("*") ? Integer.parseInt(value) : null;
+    }
+
+
+    /**
+     * This is used to filter rules, if the field Integer unit or Integer id is null, the field will match any other RuleIdentifier.
+     * Therefore, to compare RuleIdentifiers one should always use the function matches.
+     *
+     * @param ruleIdentifier RuleIdentifier to check if this one matches it.
+     * @return true, if all fields match. If a field is null, it matches any field.
+     */
+    public boolean matches(RuleIdentifier ruleIdentifier) {
+
+        return ruleIdentifier.type().equals(this.type()) && //
+                (Objects.isNull(ruleIdentifier.unit()) || Objects.isNull(this.unit()) || Objects.equals(this.unit(), ruleIdentifier.unit())) && //
+                (Objects.isNull(ruleIdentifier.id()) || Objects.isNull(this.id()) || Objects.equals(this.id(), ruleIdentifier.id()));
+
+    }
+
+
+    @Override
+    public String toString() {
+
+        StringBuilder sb = new StringBuilder();
+        sb.append(type().name());
+        if (Objects.nonNull(unit())) {
+            sb.append(".");
+            sb.append(unit());
+        } else {
+            sb.append(".*");
+        }
+        if (Objects.nonNull(id())) {
+            sb.append(".");
+            sb.append(id());
+        } else {
+            sb.append(".*");
+        }
+        return sb.toString();
+    }
+
+
+    public String toRuleUnitString() {
+
+        return String.format("Rule unit: %s.%d", type.name(), unit);
+    }
+
+}

redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/RuleType.java

@@ -0,0 +1,54 @@
+package com.knecon.fforesight.utility.rules.management.models;
+
+import java.util.Map;
+
+import lombok.AccessLevel;
+import lombok.AllArgsConstructor;
+import lombok.EqualsAndHashCode;
+import lombok.experimental.FieldDefaults;
+
+@AllArgsConstructor
+@EqualsAndHashCode(onlyExplicitlyIncluded = true)
+@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
+public class RuleType {
+
+    static Map<String, String> fullNameMap = Map.of("SYN", "Syngenta specific",//
+            "ETC", "Other",//
+            "MAN", "Manual changes",//
+            "X", "Entity merging",//
+            "FA", "File attributes",//
+            "LDS", "Local dictionary search",//
+            "TAB", "Table extraction",//
+            "H", "Headlines",//
+            "DOC", "General documine");
+
+    @EqualsAndHashCode.Include
+    String name;
+
+
+    public static RuleType fromString(String value) {
+
+        value = value.replaceAll("\r", "");
+        return new RuleType(value);
+    }
+
+
+    public String name() {
+
+        return name;
+    }
+
+
+    public boolean isWildCard() {
+
+        return name.equals("*");
+    }
+
+
+    @Override
+    public String toString() {
+
+        return fullNameMap.getOrDefault(name, name);
+    }
+
+}