test publish stuff

RED-8590 - Missing reason for added CBI Address, recategorized LOGO and signature

Closes RED-8590

See merge request redactmanager/redaction-service!293

.gitlab-ci.yml

@@ -21,3 +21,16 @@ deploy:
     - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
     - if: $CI_COMMIT_BRANCH =~ /^release/
     - if: $CI_COMMIT_TAG
+
+pages:
+  stage: build
+  tags:
+    - dind
+  script:
+    - echo "Generating Javadoc..."
+    - gradle generateJavaDoc -PjavadocDestinationDir="javadoc"
+    - mkdir public
+    - mv redaction-service-v1/redaction-service-server-v1/javadoc/* public/
+  artifacts:
+    paths:
+      - public

redaction-service-v1/redaction-service-api-v1/build.gradle.kts

@@ -7,7 +7,7 @@ description = "redaction-service-api-v1"
 
 dependencies {
     implementation("org.springframework:spring-web:6.0.12")
-    implementation("com.iqser.red.service:persistence-service-internal-api-v1:2.338.0")
+    implementation("com.iqser.red.service:persistence-service-internal-api-v1:2.351.0")
 }
 
 publishing {

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/DroolsSyntaxDeprecatedWarnings.java

@@ -0,0 +1,20 @@
+package com.iqser.red.service.redaction.v1.model;
+
+import lombok.AccessLevel;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+import lombok.experimental.FieldDefaults;
+
+@Data
+@Builder
+@AllArgsConstructor
+@NoArgsConstructor
+@FieldDefaults(level = AccessLevel.PRIVATE)
+public class DroolsSyntaxDeprecatedWarnings {
+
+    Integer line;
+    Integer column;
+    String message;
+}

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/DroolsSyntaxValidation.java

@@ -19,6 +19,8 @@ public class DroolsSyntaxValidation {
 
     @Builder.Default
     List<DroolsSyntaxErrorMessage> droolsSyntaxErrorMessages = new LinkedList<>();
+    @Builder.Default
+    List<DroolsSyntaxDeprecatedWarnings> droolsSyntaxDeprecatedWarnings = new LinkedList<>();
 
 
     public void addErrorMessage(int line, int column, String message) {

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/MigrationRequest.java

@@ -1,7 +1,6 @@
 package com.iqser.red.service.redaction.v1.model;
 
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog;
 
 import lombok.AllArgsConstructor;
 import lombok.Builder;

redaction-service-v1/redaction-service-server-v1/build.gradle.kts

@@ -12,11 +12,11 @@ plugins {
 description = "redaction-service-server-v1"
 
 
-val layoutParserVersion = "0.86.0"
+val layoutParserVersion = "0.94.0"
 val jacksonVersion = "2.15.2"
 val droolsVersion = "9.44.0.Final"
 val pdfBoxVersion = "3.0.0"
-val persistenceServiceVersion = "2.338.0"
+val persistenceServiceVersion = "2.360.0"
 val springBootStarterVersion = "3.1.5"
 
 configurations {
@@ -60,11 +60,14 @@ dependencies {
     implementation("net.logstash.logback:logstash-logback-encoder:7.4")
     implementation("ch.qos.logback:logback-classic")
 
+    implementation("org.reflections:reflections:0.10.2")
+
     testImplementation(project(":rules-management"))
     testImplementation("org.apache.pdfbox:pdfbox:${pdfBoxVersion}")
     testImplementation("org.apache.pdfbox:pdfbox-tools:${pdfBoxVersion}")
 
     testImplementation("org.springframework.boot:spring-boot-starter-test:${springBootStarterVersion}")
+    testImplementation("com.knecon.fforesight:viewer-doc-processor:${layoutParserVersion}")
     testImplementation("com.knecon.fforesight:layoutparser-service-processor:${layoutParserVersion}") {
         exclude(
             group = "com.iqser.red.service",
@@ -81,9 +84,6 @@ tasks.test {
     maxHeapSize = "1024m"
 }
 
-
-
-
 tasks.named<BootBuildImage>("bootBuildImage") {
 
     environment.put("BPE_DELIM_JAVA_TOOL_OPTIONS", " ")
@@ -110,3 +110,40 @@ tasks.named<BootBuildImage>("bootBuildImage") {
         tags.set(listOf(dockerTag))
     }
 }
+
+fun parseDroolsImports(droolsFilePath: String): List<String> {
+    val imports = mutableListOf<String>()
+    val importPattern = Regex("^import\\s+(com\\.iqser\\.red\\.service\\.redaction\\.v1\\.[\\w.]+);")
+    val desiredPrefix = "com.iqser.red.service.redaction.v1"
+
+    File(droolsFilePath).forEachLine { line ->
+        importPattern.find(line)?.let { matchResult ->
+            val importPath = matchResult.groupValues[1].trim()
+            if (importPath.startsWith(desiredPrefix)) {
+                val formattedPath = importPath.replace('.', '/')
+                imports.add("$formattedPath.java")
+            }
+        }
+    }
+
+    return imports
+}
+
+val droolsImports = parseDroolsImports("redaction-service-v1/redaction-service-server-v1/src/main/resources/drools/all_rules_documine.drl")
+
+tasks.register("generateJavaDoc", Javadoc::class) {
+    dependsOn("compileJava")
+    dependsOn("delombok")
+    classpath = project.sourceSets["main"].runtimeClasspath
+    source = fileTree("${buildDir}/generated/sources/delombok/java/main") {
+        include(droolsImports)
+    }
+    destinationDir = file(project.findProperty("javadocDestinationDir")?.toString() ?: "")
+
+    options.memberLevel = JavadocMemberLevel.PUBLIC
+    (options as StandardJavadocDocletOptions).apply {
+        header = "Redaction Service ${project.version}"
+        footer = "Redaction Service ${project.version}"
+        title = "API Documentation for Redaction Service ${project.version}"
+    }
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/Application.java

@@ -46,10 +46,18 @@ public class Application {
         return new TimedAspect(registry);
     }
 
+
     @Bean
     public DictionaryMergeService dictionaryMergeService() {
 
         return new DictionaryMergeService();
     }
 
+
+    @Bean(initMethod = "findDeprecatedMethods")
+    public DeprecatedElementsFinder deprecatedElementsFinder() {
+
+        return new DeprecatedElementsFinder();
+    }
+
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/DeprecatedElementsFinder.java

@@ -0,0 +1,108 @@
+package com.iqser.red.service.redaction.v1.server;
+
+import static java.util.stream.Collectors.groupingBy;
+
+import java.lang.reflect.Method;
+import java.lang.reflect.Type;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import org.reflections.Reflections;
+import org.reflections.scanners.Scanners;
+import org.reflections.util.ConfigurationBuilder;
+
+import com.iqser.red.service.redaction.v1.server.model.dictionary.SearchImplementation;
+
+import lombok.Getter;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+public class DeprecatedElementsFinder {
+
+    public static final String PACKAGE_NAME = "com.iqser.red.service.redaction.v1.server";
+    private Set<Method> deprecatedMethods;
+    @Getter
+    private Map<String, String> deprecatedMethodsSignaturesMap;
+    private Set<Class<?>> deprecatedClasses;
+    @Getter
+    private SearchImplementation classesSearchImplementation;
+    @Getter
+    private SearchImplementation methodsSearchImplementation;
+
+
+    public void findDeprecatedMethods() {
+
+        deprecatedMethods = new HashSet<>();
+        deprecatedClasses = new HashSet<>();
+        deprecatedMethodsSignaturesMap = new HashMap<>();
+
+        Reflections reflections = new Reflections(new ConfigurationBuilder().forPackage(PACKAGE_NAME)
+                                                          .setExpandSuperTypes(true)
+                                                          .setScanners(Scanners.MethodsAnnotated, Scanners.TypesAnnotated, Scanners.SubTypes));
+
+        deprecatedMethods = reflections.get(Scanners.MethodsAnnotated.with(Deprecated.class).as(Method.class));
+
+        deprecatedClasses = reflections.get(Scanners.TypesAnnotated.with(Deprecated.class).asClass());
+
+        if (!deprecatedClasses.isEmpty()) {
+            deprecatedClasses.forEach(deprecatedType -> deprecatedMethods.addAll(Arrays.stream(deprecatedType.getDeclaredMethods())
+                                                                                         .collect(Collectors.toSet())));
+            classesSearchImplementation = new SearchImplementation(deprecatedClasses.stream()
+                                                                           .map(Class::getName)
+                                                                           .collect(Collectors.toSet()), false);
+        }
+
+        if (!deprecatedMethods.isEmpty()) {
+            methodsSearchImplementation = new SearchImplementation(deprecatedMethods.stream()
+                                                                           .map(Method::getName)
+                                                                           .collect(Collectors.toSet()), false);
+            deprecatedMethodsSignaturesMap = new HashMap<>();
+            Map<String, List<Method>> deprecatedMethodsNameMap = deprecatedMethods.stream()
+                    .collect(groupingBy(Method::getName));
+            for (String deprecatedMethod : deprecatedMethodsNameMap.keySet()) {
+                deprecatedMethodsSignaturesMap.computeIfAbsent(deprecatedMethod,
+                                                               key -> deprecatedMethodsNameMap.get(deprecatedMethod)
+                                                                       .stream()
+                                                                       .map(this::getMethodSignature)
+                                                                       .collect(Collectors.joining("\n")));
+            }
+
+        }
+
+        log.info("For package: {} found deprecated classes {} deprecated methods {}", PACKAGE_NAME, deprecatedClasses.size(), deprecatedMethods.size());
+    }
+
+
+    public Set<Method> getDeprecatedMethods() {
+
+        if (this.deprecatedMethods == null) {
+            findDeprecatedMethods();
+        }
+        return this.deprecatedMethods;
+    }
+
+
+    public Set<Class<?>> getDeprecatedClasses() {
+
+        if (this.deprecatedClasses == null) {
+            findDeprecatedMethods();
+        }
+        return this.deprecatedClasses;
+    }
+
+    private String getMethodSignature(Method method) {
+
+        String methodName = method.getName();
+        String parameterTypes = Arrays.stream(method.getGenericParameterTypes())
+                .map(Type::getTypeName)
+                .collect(Collectors.joining(", "));
+
+        return method.getGenericReturnType() + " " + methodName + "(" + parameterTypes + ")";
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/migration/LegacyRedactionLogMergeService.java

@@ -45,7 +45,7 @@ public class LegacyRedactionLogMergeService {
     public RedactionLog addManualAddEntriesAndRemoveSkippedImported(RedactionLog redactionLog, ManualRedactions manualRedactions, String dossierTemplateId) {
 
         Set<String> skippedImportedRedactions = new HashSet<>();
-        log.info("Merging Redaction log with manual redactions");
+        log.info("Adding manual add Entries and removing skipped or imported entries");
         if (manualRedactions != null) {
 
             var manualRedactionLogEntries = addManualAddEntries(manualRedactions.getEntriesToAdd(), redactionLog.getAnalysisNumber());
@@ -92,6 +92,10 @@ public class LegacyRedactionLogMergeService {
         return redactionLog;
     }
 
+    public long getNumberOfAffectedAnnotations(ManualRedactions manualRedactions) {
+
+        return createManualRedactionWrappers(manualRedactions).stream().map(ManualRedactionWrapper::getId).distinct().count();
+    }
 
     private List<ManualRedactionWrapper> createManualRedactionWrappers(ManualRedactions manualRedactions) {
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/migration/MigrationMapper.java

@@ -0,0 +1,93 @@
+package com.iqser.red.service.redaction.v1.server.migration;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ChangeType;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualChange;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Change;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ManualRedactionType;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
+
+public class MigrationMapper {
+
+    public static com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Change toEntityLogChanges(Change change) {
+
+        return new com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Change(change.getAnalysisNumber(),
+                                                                                                              toEntityLogType(change.getType()),
+                                                                                                              change.getDateTime());
+    }
+
+
+    public static com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualChange toEntityLogManualChanges(com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ManualChange manualChange) {
+
+        return new com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualChange(toManualRedactionType(manualChange.getManualRedactionType()),
+                                                                                                                    manualChange.getProcessedDate(),
+                                                                                                                    manualChange.getRequestedDate(),
+                                                                                                                    manualChange.getUserId(),
+                                                                                                                    manualChange.getPropertyChanges());
+    }
+
+
+    public static ChangeType toEntityLogType(com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ChangeType type) {
+
+        return switch (type) {
+            case ADDED -> ChangeType.ADDED;
+            case REMOVED -> ChangeType.REMOVED;
+            case CHANGED -> ChangeType.CHANGED;
+        };
+    }
+
+
+    public static com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType toManualRedactionType(ManualRedactionType manualRedactionType) {
+
+        return switch (manualRedactionType) {
+            case ADD_LOCALLY -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.ADD;
+            case ADD_TO_DICTIONARY -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.ADD_TO_DICTIONARY;
+            case REMOVE_LOCALLY -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.REMOVE;
+            case REMOVE_FROM_DICTIONARY -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.REMOVE_FROM_DICTIONARY;
+            case FORCE_REDACT -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.FORCE;
+            case FORCE_HINT -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.FORCE;
+            case RECATEGORIZE -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.RECATEGORIZE;
+            case LEGAL_BASIS_CHANGE -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.LEGAL_BASIS_CHANGE;
+            case RESIZE -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.RESIZE;
+        };
+    }
+
+
+    public static com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine toEntityLogEngine(Engine engine) {
+
+        return switch (engine) {
+            case DICTIONARY -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine.DICTIONARY;
+            case NER -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine.NER;
+            case RULE -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine.RULE;
+        };
+    }
+
+
+    public static Set<com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine> getMigratedEngines(RedactionLogEntry entry) {
+
+        if (entry.getEngines() == null) {
+            return Collections.emptySet();
+        }
+        return entry.getEngines()
+                .stream()
+                .map(MigrationMapper::toEntityLogEngine)
+                .collect(Collectors.toSet());
+    }
+
+
+    public List<ManualChange> migrateManualChanges(List<com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ManualChange> manualChanges) {
+
+        if (manualChanges == null) {
+            return Collections.emptyList();
+        }
+        return manualChanges.stream()
+                .map(MigrationMapper::toEntityLogManualChanges)
+                .toList();
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/migration/MigrationMessageReceiver.java

@@ -58,19 +58,28 @@ public class MigrationMessageReceiver {
         if (redactionLog.getAnalysisVersion() == 0) {
             redactionLog = legacyVersion0MigrationService.mergeDuplicateAnnotationIds(redactionLog);
         } else if (migrationRequest.getManualRedactions() != null) {
-            redactionLog = legacyRedactionLogMergeService.addManualAddEntriesAndRemoveSkippedImported(redactionLog, migrationRequest.getManualRedactions(), migrationRequest.getDossierTemplateId());
+            redactionLog = legacyRedactionLogMergeService.addManualAddEntriesAndRemoveSkippedImported(redactionLog,
+                                                                                                      migrationRequest.getManualRedactions(),
+                                                                                                      migrationRequest.getDossierTemplateId());
         }
 
-        MigratedEntityLog migratedEntityLog = redactionLogToEntityLogMigrationService.migrate(redactionLog, document, migrationRequest.getDossierTemplateId(), migrationRequest.getManualRedactions());
+        MigratedEntityLog migratedEntityLog = redactionLogToEntityLogMigrationService.migrate(redactionLog,
+                                                                                              document,
+                                                                                              migrationRequest.getDossierTemplateId(),
+                                                                                              migrationRequest.getManualRedactions(),
+                                                                                              migrationRequest.getFileId());
 
+        log.info("Storing migrated entityLog and ids to migrate in DB for file {}", migrationRequest.getFileId());
         redactionStorageService.storeObject(migrationRequest.getDossierId(), migrationRequest.getFileId(), FileType.ENTITY_LOG, migratedEntityLog.getEntityLog());
         redactionStorageService.storeObject(migrationRequest.getDossierId(), migrationRequest.getFileId(), FileType.MIGRATED_IDS, migratedEntityLog.getMigratedIds());
 
         sendFinished(MigrationResponse.builder().dossierId(migrationRequest.getDossierId()).fileId(migrationRequest.getFileId()).build());
-        log.info("Migrated {} redactionLog entries for dossierId {} and fileId {}",
-                migratedEntityLog.getEntityLog().getEntityLogEntry().size(),
-                migrationRequest.getDossierId(),
-                migrationRequest.getFileId());
+        log.info("Migrated {} redactionLog entries, found {} annotation ids for migration in the db, {} new manual entries, for dossierId {} and fileId {}",
+                 migratedEntityLog.getEntityLog().getEntityLogEntry().size(),
+                 migratedEntityLog.getMigratedIds().getMappings().size(),
+                 migratedEntityLog.getMigratedIds().getManualRedactionEntriesToAdd().size(),
+                 migrationRequest.getDossierId(),
+                 migrationRequest.getFileId());
         log.info("");
     }
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/migration/RedactionLogToEntityLogMigrationService.java

@@ -19,29 +19,24 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.migration.MigratedIds;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.BaseAnnotation;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ManualRedactionType;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogLegalBasis;
-import com.iqser.red.service.redaction.v1.model.MigrationRequest;
-import com.iqser.red.service.redaction.v1.server.model.PrecursorEntity;
 import com.iqser.red.service.redaction.v1.server.model.MigratedEntityLog;
 import com.iqser.red.service.redaction.v1.server.model.MigrationEntity;
+import com.iqser.red.service.redaction.v1.server.model.PrecursorEntity;
 import com.iqser.red.service.redaction.v1.server.model.RectangleWithPage;
-import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
-import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Image;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.ImageType;
-import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
 import com.iqser.red.service.redaction.v1.server.service.DictionaryService;
 import com.iqser.red.service.redaction.v1.server.service.ManualChangesApplicationService;
-import com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService;
 import com.iqser.red.service.redaction.v1.server.service.document.EntityEnrichmentService;
 import com.iqser.red.service.redaction.v1.server.service.document.EntityFindingUtility;
+import com.iqser.red.service.redaction.v1.server.service.document.EntityFromPrecursorCreationService;
 import com.iqser.red.service.redaction.v1.server.utils.IdBuilder;
 import com.iqser.red.service.redaction.v1.server.utils.MigratedIdsCollector;
 
@@ -64,13 +59,16 @@ public class RedactionLogToEntityLogMigrationService {
     ManualChangesApplicationService manualChangesApplicationService;
 
 
-    public MigratedEntityLog migrate(RedactionLog redactionLog, Document document, String dossierTemplateId, ManualRedactions manualRedactions) {
+    public MigratedEntityLog migrate(RedactionLog redactionLog, Document document, String dossierTemplateId, ManualRedactions manualRedactions, String fileId) {
+
+        log.info("Migrating entities for file {}", fileId);
+        List<MigrationEntity> entitiesToMigrate = calculateMigrationEntitiesFromRedactionLog(redactionLog, document, dossierTemplateId, fileId);
 
-        List<MigrationEntity> entitiesToMigrate = calculateMigrationEntitiesFromRedactionLog(redactionLog, document, dossierTemplateId);
         MigratedIds migratedIds = entitiesToMigrate.stream()
                 .collect(new MigratedIdsCollector());
 
         applyManualChanges(entitiesToMigrate, manualRedactions);
+        log.info("applying manual changes to migrated entities for file {}", fileId);
 
         EntityLog entityLog = new EntityLog();
         entityLog.setAnalysisNumber(redactionLog.getAnalysisNumber());
@@ -85,6 +83,8 @@ public class RedactionLogToEntityLogMigrationService {
                                         .toList());
 
         Map<String, String> oldToNewIDMapping = migratedIds.buildOldToNewMapping();
+
+        log.info("Writing migrated entities to entityLog for file {}", fileId);
         entityLog.setEntityLogEntry(entitiesToMigrate.stream()
                                             .map(migrationEntity -> migrationEntity.toEntityLogEntry(oldToNewIDMapping))
                                             .toList());
@@ -102,6 +102,13 @@ public class RedactionLogToEntityLogMigrationService {
                 .filter(m -> !m.getOldId().equals(m.getNewId()))
                 .collect(new MigratedIdsCollector());
 
+        List<ManualRedactionEntry> manualRedactionEntriesToAdd = entitiesToMigrate.stream()
+                .filter(MigrationEntity::needsManualEntry)
+                .map(MigrationEntity::buildManualRedactionEntry)
+                .toList();
+
+        idsToMigrateInDb.setManualRedactionEntriesToAdd(manualRedactionEntriesToAdd);
+
         return new MigratedEntityLog(idsToMigrateInDb, entityLog);
     }
 
@@ -117,27 +124,14 @@ public class RedactionLogToEntityLogMigrationService {
                                                                                    manualRedactions.getForceRedactions(),
                                                                                    manualRedactions.getResizeRedactions(),
                                                                                    manualRedactions.getLegalBasisChanges(),
-                                                                                   manualRedactions.getRecategorizations(),
-                                                                                   manualRedactions.getLegalBasisChanges())
+                                                                                   manualRedactions.getRecategorizations())
                 .flatMap(Collection::stream)
                 .collect(Collectors.groupingBy(BaseAnnotation::getAnnotationId));
 
-        entitiesToMigrate.forEach(migrationEntity -> manualChangesPerAnnotationId.getOrDefault(migrationEntity.getOldId(), Collections.emptyList())
-                .forEach(manualChange -> {
-                    if (manualChange instanceof ManualResizeRedaction manualResizeRedaction && migrationEntity.getMigratedEntity() instanceof TextEntity textEntity) {
-                        ManualResizeRedaction migratedManualResizeRedaction = ManualResizeRedaction.builder()
-                                .positions(manualResizeRedaction.getPositions())
-                                .annotationId(migrationEntity.getNewId())
-                                .updateDictionary(manualResizeRedaction.getUpdateDictionary())
-                                .addToAllDossiers(manualResizeRedaction.isAddToAllDossiers())
-                                .textAfter(manualResizeRedaction.getTextAfter())
-                                .textBefore(manualResizeRedaction.getTextBefore())
-                                .build();
-                        manualChangesApplicationService.resize(textEntity, migratedManualResizeRedaction);
-                    } else {
-                        migrationEntity.getMigratedEntity().getManualOverwrite().addChange(manualChange);
-                    }
-                }));
+        entitiesToMigrate.forEach(migrationEntity -> migrationEntity.applyManualChanges(manualChangesPerAnnotationId.getOrDefault(migrationEntity.getOldId(),
+                                                                                                                                  Collections.emptyList()),
+                                                                                        manualChangesApplicationService));
+
     }
 
 
@@ -147,10 +141,10 @@ public class RedactionLogToEntityLogMigrationService {
     }
 
 
-    private List<MigrationEntity> calculateMigrationEntitiesFromRedactionLog(RedactionLog redactionLog, Document document, String dossierTemplateId) {
+    private List<MigrationEntity> calculateMigrationEntitiesFromRedactionLog(RedactionLog redactionLog, Document document, String dossierTemplateId, String fileId) {
 
-        List<MigrationEntity> images = getImageBasedMigrationEntities(redactionLog, document, dossierTemplateId);
-        List<MigrationEntity> textMigrationEntities = getTextBasedMigrationEntities(redactionLog, document, dossierTemplateId);
+        List<MigrationEntity> images = getImageBasedMigrationEntities(redactionLog, document, fileId);
+        List<MigrationEntity> textMigrationEntities = getTextBasedMigrationEntities(redactionLog, document, dossierTemplateId, fileId);
         return Stream.of(textMigrationEntities.stream(), images.stream())
                 .flatMap(Function.identity())
                 .toList();
@@ -163,7 +157,7 @@ public class RedactionLogToEntityLogMigrationService {
     }
 
 
-    private List<MigrationEntity> getImageBasedMigrationEntities(RedactionLog redactionLog, Document document, String dossierTemplateId) {
+    private List<MigrationEntity> getImageBasedMigrationEntities(RedactionLog redactionLog, Document document, String fileId) {
 
         List<Image> images = document.streamAllImages()
                 .collect(Collectors.toList());
@@ -195,7 +189,8 @@ public class RedactionLogToEntityLogMigrationService {
             }
 
             String ruleIdentifier;
-            String reason = Optional.ofNullable(redactionLogImage.getReason()).orElse("");
+            String reason = Optional.ofNullable(redactionLogImage.getReason())
+                    .orElse("");
             if (redactionLogImage.getMatchedRule().isBlank() || redactionLogImage.getMatchedRule() == null) {
                 ruleIdentifier = "OLDIMG.0.0";
             } else {
@@ -209,7 +204,7 @@ public class RedactionLogToEntityLogMigrationService {
             } else {
                 closestImage.skip(ruleIdentifier, reason);
             }
-            migrationEntities.add(new MigrationEntity(null, redactionLogImage, closestImage, redactionLogImage.getId(), closestImage.getId()));
+            migrationEntities.add(MigrationEntity.fromRedactionLogImage(redactionLogImage, closestImage, fileId));
         }
         return migrationEntities;
     }
@@ -250,40 +245,20 @@ public class RedactionLogToEntityLogMigrationService {
     }
 
 
-    private List<MigrationEntity> getTextBasedMigrationEntities(RedactionLog redactionLog, Document document, String dossierTemplateId) {
+    private List<MigrationEntity> getTextBasedMigrationEntities(RedactionLog redactionLog, Document document, String dossierTemplateId, String fileId) {
 
         List<MigrationEntity> entitiesToMigrate = redactionLog.getRedactionLogEntry()
                 .stream()
                 .filter(redactionLogEntry -> !redactionLogEntry.isImage())
-                .map(entry -> MigrationEntity.fromRedactionLogEntry(entry, dictionaryService.isHint(entry.getType(), dossierTemplateId)))
-                .peek(migrationEntity -> {
-                    if (migrationEntity.getPrecursorEntity().getEntityType().equals(EntityType.HINT) &&//
-                        !migrationEntity.getRedactionLogEntry().isHint() &&//
-                        !migrationEntity.getRedactionLogEntry().isRedacted()) {
-                        migrationEntity.getPrecursorEntity().ignore(migrationEntity.getPrecursorEntity().getRuleIdentifier(), migrationEntity.getPrecursorEntity().getReason());
-                    } else if (migrationEntity.getRedactionLogEntry().lastChangeIsRemoved()) {
-                        migrationEntity.getPrecursorEntity().remove(migrationEntity.getPrecursorEntity().getRuleIdentifier(), migrationEntity.getPrecursorEntity().getReason());
-                    } else if (lastManualChangeIsRemove(migrationEntity)) {
-                        migrationEntity.getPrecursorEntity().ignore(migrationEntity.getPrecursorEntity().getRuleIdentifier(), migrationEntity.getPrecursorEntity().getReason());
-                    } else if (migrationEntity.getPrecursorEntity().isApplied() && migrationEntity.getRedactionLogEntry().isRecommendation()) {
-                        migrationEntity.getPrecursorEntity()
-                                .skip(migrationEntity.getPrecursorEntity().getRuleIdentifier(), migrationEntity.getPrecursorEntity().getReason());
-                    } else if (migrationEntity.getPrecursorEntity().isApplied()) {
-                        migrationEntity.getPrecursorEntity()
-                                .apply(migrationEntity.getPrecursorEntity().getRuleIdentifier(),
-                                       migrationEntity.getPrecursorEntity().getReason(),
-                                       migrationEntity.getPrecursorEntity().getLegalBasis());
-                    } else {
-                        migrationEntity.getPrecursorEntity()
-                                .skip(migrationEntity.getPrecursorEntity().getRuleIdentifier(), migrationEntity.getPrecursorEntity().getReason());
-                    }
-                })
+                .map(entry -> MigrationEntity.fromRedactionLogEntry(entry, dictionaryService.isHint(entry.getType(), dossierTemplateId), fileId))
                 .toList();
 
-        Map<String, List<TextEntity>> tempEntitiesByValue = entityFindingUtility.findAllPossibleEntitiesAndGroupByValue(document,
-                                                                                                                        entitiesToMigrate.stream()
-                                                                                                                                .map(MigrationEntity::getPrecursorEntity)
-                                                                                                                                .toList());
+        List<PrecursorEntity> precursorEntities = entitiesToMigrate.stream()
+                .map(MigrationEntity::getPrecursorEntity)
+                .toList();
+
+        log.info("Finding all possible entities");
+        Map<String, List<TextEntity>> tempEntitiesByValue = entityFindingUtility.findAllPossibleEntitiesAndGroupByValue(document, precursorEntities);
 
         for (MigrationEntity migrationEntity : entitiesToMigrate) {
             Optional<TextEntity> optionalTextEntity = entityFindingUtility.findClosestEntityAndReturnEmptyIfNotFound(migrationEntity.getPrecursorEntity(),
@@ -297,45 +272,19 @@ public class RedactionLogToEntityLogMigrationService {
                 continue;
             }
 
-            TextEntity entity = createCorrectEntity(migrationEntity.getPrecursorEntity(), document, optionalTextEntity.get().getTextRange());
-            migrationEntity.setMigratedEntity(entity);
-            migrationEntity.setOldId(migrationEntity.getPrecursorEntity().getId());
-            migrationEntity.setNewId(entity.getId()); // Can only be on one page, since redactionLogEntries can only be on one page
+            TextEntity migratedEntity = EntityFromPrecursorCreationService.createCorrectEntity(migrationEntity.getPrecursorEntity(), optionalTextEntity.get(), true);
 
+            migrationEntity.setMigratedEntity(migratedEntity);
+            migrationEntity.setOldId(migrationEntity.getPrecursorEntity().getId());
+            migrationEntity.setNewId(migratedEntity.getId());
         }
 
         tempEntitiesByValue.values()
                 .stream()
                 .flatMap(Collection::stream)
                 .forEach(TextEntity::removeFromGraph);
+
         return entitiesToMigrate;
     }
 
-
-    private static boolean lastManualChangeIsRemove(MigrationEntity migrationEntity) {
-
-        if (migrationEntity.getRedactionLogEntry().getManualChanges() == null) {
-            return false;
-        }
-
-        return migrationEntity.getRedactionLogEntry().getManualChanges()
-                .stream()
-                .reduce((a, b) -> b)
-                .map(m -> m.getManualRedactionType().equals(ManualRedactionType.REMOVE_LOCALLY))
-                .orElse(false);
-    }
-
-
-    private TextEntity createCorrectEntity(PrecursorEntity precursorEntity, SemanticNode node, TextRange closestTextRange) {
-
-        EntityCreationService entityCreationService = new EntityCreationService(entityEnrichmentService);
-        TextEntity correctEntity = entityCreationService.forceByTextRange(closestTextRange, precursorEntity.getType(), precursorEntity.getEntityType(), node);
-
-        correctEntity.addMatchedRules(precursorEntity.getMatchedRuleList());
-        correctEntity.setDictionaryEntry(precursorEntity.isDictionaryEntry());
-        correctEntity.setDossierDictionaryEntry(precursorEntity.isDossierDictionaryEntry());
-        correctEntity.getManualOverwrite().addChanges(precursorEntity.getManualOverwrite().getManualChangeLog());
-        return correctEntity;
-    }
-
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/MigratedEntityLog.java

@@ -1,7 +1,10 @@
 package com.iqser.red.service.redaction.v1.server.model;
 
+import java.util.List;
+
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.migration.MigratedIds;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
 
 import lombok.AllArgsConstructor;
 import lombok.Builder;
@@ -16,5 +19,4 @@ public class MigratedEntityLog {
 
     MigratedIds migratedIds;
     EntityLog entityLog;
-
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/MigrationEntity.java

@@ -1,34 +1,45 @@
 package com.iqser.red.service.redaction.v1.server.model;
 
+import static com.iqser.red.service.redaction.v1.server.service.EntityLogCreatorService.buildEntryState;
+import static com.iqser.red.service.redaction.v1.server.service.EntityLogCreatorService.buildEntryType;
+
+import java.awt.geom.Rectangle2D;
 import java.util.Collections;
+import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
 import java.util.Set;
 import java.util.stream.Collectors;
 
-import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ChangeType;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryType;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Change;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ManualChange;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.BaseAnnotation;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ManualRedactionType;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
+import com.iqser.red.service.redaction.v1.server.migration.MigrationMapper;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.IEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.ManualChangeOverwrite;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Image;
 import com.iqser.red.service.redaction.v1.server.service.ManualChangeFactory;
+import com.iqser.red.service.redaction.v1.server.service.ManualChangesApplicationService;
 
 import lombok.AllArgsConstructor;
+import lombok.Builder;
 import lombok.Data;
 import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
 
+@Slf4j
 @Data
+@Builder
 @AllArgsConstructor
 @RequiredArgsConstructor
 public final class MigrationEntity {
@@ -38,28 +49,73 @@ public final class MigrationEntity {
     private IEntity migratedEntity;
     private String oldId;
     private String newId;
+    private String fileId;
+
+    @Builder.Default
+    List<BaseAnnotation> manualChanges = new LinkedList<>();
 
 
-    public static MigrationEntity fromRedactionLogEntry(RedactionLogEntry redactionLogEntry, boolean hint) {
+    public static MigrationEntity fromRedactionLogEntry(RedactionLogEntry redactionLogEntry, boolean hint, String fileId) {
 
-        return new MigrationEntity(createPrecursorEntity(redactionLogEntry, hint), redactionLogEntry);
+        PrecursorEntity precursorEntity = createPrecursorEntity(redactionLogEntry, hint);
+
+        if (precursorEntity.getEntityType().equals(EntityType.HINT) && !redactionLogEntry.isHint() && !redactionLogEntry.isRedacted()) {
+            precursorEntity.ignore(precursorEntity.getRuleIdentifier(), precursorEntity.getReason());
+        } else if (redactionLogEntry.lastChangeIsRemoved()) {
+            precursorEntity.remove(precursorEntity.getRuleIdentifier(), precursorEntity.getReason());
+        } else if (lastManualChangeIsRemove(redactionLogEntry)) {
+            precursorEntity.ignore(precursorEntity.getRuleIdentifier(), precursorEntity.getReason());
+        } else if (precursorEntity.isApplied() && redactionLogEntry.isRecommendation()) {
+            precursorEntity.skip(precursorEntity.getRuleIdentifier(), precursorEntity.getReason());
+        } else if (precursorEntity.isApplied()) {
+            precursorEntity.apply(precursorEntity.getRuleIdentifier(), precursorEntity.getReason(), precursorEntity.getLegalBasis());
+        } else {
+            precursorEntity.skip(precursorEntity.getRuleIdentifier(), precursorEntity.getReason());
+        }
+
+        return MigrationEntity.builder().precursorEntity(precursorEntity).redactionLogEntry(redactionLogEntry).oldId(redactionLogEntry.getId()).fileId(fileId).build();
+    }
+
+
+    public static MigrationEntity fromRedactionLogImage(RedactionLogEntry redactionLogImage, Image image, String fileId) {
+
+        return MigrationEntity.builder().redactionLogEntry(redactionLogImage).migratedEntity(image).oldId(redactionLogImage.getId()).newId(image.getId()).fileId(fileId).build();
+    }
+
+
+    private static boolean lastManualChangeIsRemove(RedactionLogEntry redactionLogEntry) {
+
+        if (redactionLogEntry.getManualChanges() == null) {
+            return false;
+        }
+
+        return redactionLogEntry.getManualChanges()
+                .stream()
+                .reduce((a, b) -> b)
+                .map(m -> m.getManualRedactionType().equals(ManualRedactionType.REMOVE_LOCALLY))
+                .orElse(false);
     }
 
 
     public static PrecursorEntity createPrecursorEntity(RedactionLogEntry redactionLogEntry, boolean hint) {
 
         String ruleIdentifier = buildRuleIdentifier(redactionLogEntry);
-        List<RectangleWithPage> rectangleWithPages = redactionLogEntry.getPositions().stream().map(RectangleWithPage::fromRedactionLogRectangle).toList();
+        List<RectangleWithPage> rectangleWithPages = redactionLogEntry.getPositions()
+                .stream()
+                .map(RectangleWithPage::fromRedactionLogRectangle)
+                .toList();
         EntityType entityType = getEntityType(redactionLogEntry, hint);
         return PrecursorEntity.builder()
                 .id(redactionLogEntry.getId())
                 .value(redactionLogEntry.getValue())
                 .entityPosition(rectangleWithPages)
                 .ruleIdentifier(ruleIdentifier)
-                .reason(Optional.ofNullable(redactionLogEntry.getReason()).orElse(""))
+                .reason(Optional.ofNullable(redactionLogEntry.getReason())
+                                .orElse(""))
                 .legalBasis(redactionLogEntry.getLegalBasis())
                 .type(redactionLogEntry.getType())
                 .section(redactionLogEntry.getSection())
+                .engines(MigrationMapper.getMigratedEngines(redactionLogEntry))
                 .entityType(entityType)
                 .applied(redactionLogEntry.isRedacted())
                 .isDictionaryEntry(redactionLogEntry.isDictionaryEntry())
@@ -100,14 +156,6 @@ public final class MigrationEntity {
     }
 
 
-    private static com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Change toEntityLogChanges(Change change) {
-
-        return new com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Change(change.getAnalysisNumber(),
-                toEntityLogType(change.getType()),
-                change.getDateTime());
-    }
-
-
     private static EntryType getEntryType(EntityType entityType) {
 
         return switch (entityType) {
@@ -120,42 +168,6 @@ public final class MigrationEntity {
     }
 
 
-    private static com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualChange toEntityLogManualChanges(ManualChange manualChange) {
-
-        return new com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualChange(toManualRedactionType(manualChange.getManualRedactionType()),
-                manualChange.getProcessedDate(),
-                manualChange.getRequestedDate(),
-                manualChange.getUserId(),
-                manualChange.getPropertyChanges());
-    }
-
-
-    private static ChangeType toEntityLogType(com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ChangeType type) {
-
-        return switch (type) {
-            case ADDED -> ChangeType.ADDED;
-            case REMOVED -> ChangeType.REMOVED;
-            case CHANGED -> ChangeType.CHANGED;
-        };
-    }
-
-
-    private static com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType toManualRedactionType(ManualRedactionType manualRedactionType) {
-
-        return switch (manualRedactionType) {
-            case ADD_LOCALLY -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.ADD_LOCALLY;
-            case ADD_TO_DICTIONARY -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.ADD_TO_DICTIONARY;
-            case REMOVE_LOCALLY -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.REMOVE_LOCALLY;
-            case REMOVE_FROM_DICTIONARY -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.REMOVE_FROM_DICTIONARY;
-            case FORCE_REDACT -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.FORCE_REDACT;
-            case FORCE_HINT -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.FORCE_HINT;
-            case RECATEGORIZE -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.RECATEGORIZE;
-            case LEGAL_BASIS_CHANGE -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.LEGAL_BASIS_CHANGE;
-            case RESIZE -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.RESIZE;
-        };
-    }
-
-
     public EntityLogEntry toEntityLogEntry(Map<String, String> oldToNewIdMapping) {
 
         EntityLogEntry entityLogEntry;
@@ -171,10 +183,13 @@ public final class MigrationEntity {
 
         entityLogEntry.setManualChanges(ManualChangeFactory.toManualChangeList(migratedEntity.getManualOverwrite().getManualChangeLog(), redactionLogEntry.isHint()));
         entityLogEntry.setColor(redactionLogEntry.getColor());
-        entityLogEntry.setChanges(redactionLogEntry.getChanges().stream().map(MigrationEntity::toEntityLogChanges).toList());
+        entityLogEntry.setChanges(redactionLogEntry.getChanges()
+                                          .stream()
+                                          .map(MigrationMapper::toEntityLogChanges)
+                                          .toList());
         entityLogEntry.setReference(migrateSetOfIds(redactionLogEntry.getReference(), oldToNewIdMapping));
         entityLogEntry.setImportedRedactionIntersections(migrateSetOfIds(redactionLogEntry.getImportedRedactionIntersections(), oldToNewIdMapping));
-        entityLogEntry.setEngines(getMigratedEngines(redactionLogEntry));
+        entityLogEntry.setEngines(MigrationMapper.getMigratedEngines(redactionLogEntry));
         if (redactionLogEntry.getLegalBasis() != null) {
             entityLogEntry.setLegalBasis(redactionLogEntry.getLegalBasis());
         }
@@ -192,53 +207,26 @@ public final class MigrationEntity {
         return entityLogEntry.getManualChanges()
                 .stream()
                 .reduce((a, b) -> b)
-                .filter(mc -> mc.getManualRedactionType()
-                        .equals(com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.REMOVE_LOCALLY))
+                .filter(mc -> mc.getManualRedactionType().equals(com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.REMOVE))
                 .isPresent();
     }
 
 
-    private List<com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualChange> migrateManualChanges(List<ManualChange> manualChanges) {
-
-        if (manualChanges == null) {
-            return Collections.emptyList();
-        }
-        return manualChanges.stream().map(MigrationEntity::toEntityLogManualChanges).toList();
-    }
-
-
-    private static Set<com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine> getMigratedEngines(RedactionLogEntry entry) {
-
-        if (entry.getEngines() == null) {
-            return Collections.emptySet();
-        }
-        return entry.getEngines().stream().map(MigrationEntity::toEntityLogEngine).collect(Collectors.toSet());
-    }
-
-
     private Set<String> migrateSetOfIds(Set<String> ids, Map<String, String> oldToNewIdMapping) {
 
         if (ids == null) {
             return Collections.emptySet();
         }
-        return ids.stream().map(oldToNewIdMapping::get).collect(Collectors.toSet());
-    }
-
-
-    private static com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine toEntityLogEngine(Engine engine) {
-
-        return switch (engine) {
-            case DICTIONARY -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine.DICTIONARY;
-            case NER -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine.NER;
-            case RULE -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine.RULE;
-        };
+        return ids.stream()
+                .map(oldToNewIdMapping::get)
+                .collect(Collectors.toSet());
     }
 
 
     public EntityLogEntry createEntityLogEntry(Image image) {
 
         List<Position> positions = getPositionsFromOverride(image).orElse(List.of(new Position(image.getPosition(), image.getPage().getNumber())));
-                return EntityLogEntry.builder()
+        return EntityLogEntry.builder()
                 .id(image.getId())
                 .value(image.value())
                 .type(image.type())
@@ -249,7 +237,8 @@ public final class MigrationEntity {
                 .positions(positions)
                 .containingNodeId(image.getTreeId())
                 .closestHeadline(image.getHeadline().getTextBlock().getSearchText())
-                .section(redactionLogEntry.getSection())
+                .section(image.getManualOverwrite().getSection()
+                                 .orElse(redactionLogEntry.getSection()))
                 .textAfter(redactionLogEntry.getTextAfter())
                 .textBefore(redactionLogEntry.getTextBefore())
                 .imageHasTransparency(image.isTransparent())
@@ -270,7 +259,8 @@ public final class MigrationEntity {
                 .type(precursorEntity.type())
                 .state(buildEntryState(precursorEntity))
                 .entryType(buildEntryType(precursorEntity))
-                .section(redactionLogEntry.getSection())
+                .section(precursorEntity.getManualOverwrite().getSection()
+                                 .orElse(redactionLogEntry.getSection()))
                 .textAfter(redactionLogEntry.getTextAfter())
                 .textBefore(redactionLogEntry.getTextBefore())
                 .containingNodeId(Collections.emptyList())
@@ -280,12 +270,11 @@ public final class MigrationEntity {
                 .dossierDictionaryEntry(precursorEntity.isDossierDictionaryEntry())
                 .startOffset(-1)
                 .endOffset(-1)
-                .positions(precursorEntity.getManualOverwrite()
-                        .getPositions()
-                        .orElse(precursorEntity.getEntityPosition())
-                        .stream()
-                        .map(entityPosition -> new Position(entityPosition.rectangle2D(), entityPosition.pageNumber()))
-                        .toList())
+                .positions(precursorEntity.getManualOverwrite().getPositions()
+                                   .orElse(precursorEntity.getEntityPosition())
+                                   .stream()
+                                   .map(entityPosition -> new Position(entityPosition.rectangle2D(), entityPosition.pageNumber()))
+                                   .toList())
                 .engines(Collections.emptySet())
                 .build();
     }
@@ -300,11 +289,13 @@ public final class MigrationEntity {
                 .positions(rectanglesPerLine)
                 .reason(entity.buildReasonWithManualChangeDescriptions())
                 .legalBasis(entity.legalBasis())
-                .value(entity.getManualOverwrite().getValue().orElse(entity.getMatchedRule().isWriteValueWithLineBreaks() ? entity.getValueWithLineBreaks() : entity.getValue()))
+                .value(entity.getManualOverwrite().getValue()
+                               .orElse(entity.getMatchedRule().isWriteValueWithLineBreaks() ? entity.getValueWithLineBreaks() : entity.getValue()))
                 .type(entity.type())
-                .section(redactionLogEntry.getSection())
-                .textAfter(redactionLogEntry.getTextAfter())
-                .textBefore(redactionLogEntry.getTextBefore())
+                .section(entity.getManualOverwrite().getSection()
+                                 .orElse(redactionLogEntry.getSection()))
+                .textAfter(entity.getTextAfter())
+                .textBefore(entity.getTextBefore())
                 .containingNodeId(entity.getDeepestFullyContainingNode().getTreeId())
                 .closestHeadline(entity.getDeepestFullyContainingNode().getHeadline().getTextBlock().getSearchText())
                 .matchedRule(entity.getMatchedRule().getRuleIdentifier().toString())
@@ -322,54 +313,129 @@ public final class MigrationEntity {
     private static List<Position> getRectanglesPerLine(TextEntity entity) {
 
         return getPositionsFromOverride(entity).orElse(entity.getPositionsOnPagePerPage()
-                .get(0)
-                .getRectanglePerLine()
-                .stream()
-                .map(rectangle2D -> new Position(rectangle2D, entity.getPositionsOnPagePerPage().get(0).getPage().getNumber()))
-                .toList());
+                                                               .get(0).getRectanglePerLine()
+                                                               .stream()
+                                                               .map(rectangle2D -> new Position(rectangle2D,
+                                                                                                entity.getPositionsOnPagePerPage()
+                                                                                                        .get(0).getPage().getNumber()))
+                                                               .toList());
     }
 
 
     private static Optional<List<Position>> getPositionsFromOverride(IEntity entity) {
 
-        return entity.getManualOverwrite().getPositions().map(rects -> rects.stream().map(r -> new Position(r.rectangle2D(), r.pageNumber())).toList());
-    }
-
-
-    private EntryState buildEntryState(IEntity entity) {
-
-        if (entity.applied() && entity.active()) {
-            return EntryState.APPLIED;
-        } else if (entity.skipped() && entity.active()) {
-            return EntryState.SKIPPED;
-        } else if (entity.ignored()) {
-            return EntryState.IGNORED;
-        } else {
-            return EntryState.REMOVED;
-        }
-    }
-
-
-    private EntryType buildEntryType(IEntity entity) {
-
-        if (entity instanceof TextEntity textEntity) {
-            return getEntryType(textEntity.getEntityType());
-        } else if (entity instanceof PrecursorEntity precursorEntity) {
-            if (precursorEntity.isRectangle()) {
-                return EntryType.AREA;
-            }
-            return getEntryType(precursorEntity.getEntityType());
-        } else if (entity instanceof Image) {
-            return EntryType.IMAGE;
-        }
-        throw new UnsupportedOperationException(String.format("Entity subclass %s is not implemented!", entity.getClass()));
+        return entity.getManualOverwrite().getPositions()
+                .map(rects -> rects.stream()
+                        .map(r -> new Position(r.rectangle2D(), r.pageNumber()))
+                        .toList());
     }
 
 
     public boolean hasManualChangesOrComments() {
 
         return !(redactionLogEntry.getManualChanges() == null || redactionLogEntry.getManualChanges().isEmpty()) || //
-                !(redactionLogEntry.getComments() == null || redactionLogEntry.getComments().isEmpty());
+               !(redactionLogEntry.getComments() == null || redactionLogEntry.getComments().isEmpty()) //
+               || hasManualChanges();
+    }
+
+
+    public boolean hasManualChanges() {
+
+        return !manualChanges.isEmpty();
+    }
+
+
+    public void applyManualChanges(List<BaseAnnotation> manualChangesToApply, ManualChangesApplicationService manualChangesApplicationService) {
+
+        manualChanges.addAll(manualChangesToApply);
+        manualChangesToApply.forEach(manualChange -> {
+            if (manualChange instanceof ManualResizeRedaction manualResizeRedaction && migratedEntity instanceof TextEntity textEntity) {
+                // Due to the value in the old redaction log already being resized, there is no way to find the original entity ID and therefore to migrate the resize annotation correctly.
+                // Instead, we add an add_locally change to the db.
+                ManualResizeRedaction migratedManualResizeRedaction = ManualResizeRedaction.builder()
+                        .positions(manualResizeRedaction.getPositions())
+                        .annotationId(getNewId())
+                        .updateDictionary(manualResizeRedaction.getUpdateDictionary())
+                        .addToAllDossiers(manualResizeRedaction.isAddToAllDossiers())
+                        .textAfter(manualResizeRedaction.getTextAfter())
+                        .textBefore(manualResizeRedaction.getTextBefore())
+                        .build();
+                manualChangesApplicationService.resize(textEntity, migratedManualResizeRedaction);
+            } else {
+                migratedEntity.getManualOverwrite().addChange(manualChange);
+            }
+        });
+    }
+
+
+    public ManualRedactionEntry buildManualRedactionEntry() {
+
+        assert hasManualChanges();
+
+        // currently we need to insert a manual redaction entry, whenever an entity has been resized.
+        String user = manualChanges.stream()
+                .filter(mc -> mc instanceof ManualResizeRedaction)
+                .findFirst()
+                .orElse(manualChanges.get(0)).getUser();
+
+        return ManualRedactionEntry.builder()
+                .annotationId(newId)
+                .fileId(fileId)
+                .type(redactionLogEntry.getType())
+                .value(redactionLogEntry.getValue())
+                .reason(redactionLogEntry.getReason())
+                .legalBasis(redactionLogEntry.getLegalBasis())
+                .section(redactionLogEntry.getSection())
+                .addToDictionary(false)
+                .addToDossierDictionary(false)
+                .rectangle(false)
+                .positions(buildPositions(migratedEntity))
+                .user(user)
+                .build();
+    }
+
+
+    private List<Rectangle> buildPositions(IEntity entity) {
+
+        if (entity instanceof TextEntity textEntity) {
+
+            var positionsOnPage = textEntity.getPositionsOnPagePerPage()
+                    .get(0);
+            return positionsOnPage.getRectanglePerLine()
+                    .stream()
+                    .map(p -> new Rectangle((float) p.getX(), (float) p.getY(), (float) p.getWidth(), (float) p.getHeight(), positionsOnPage.getPage().getNumber()))
+                    .toList();
+        }
+        if (entity instanceof PrecursorEntity pEntity) {
+
+            return pEntity.getManualOverwrite().getPositions()
+                    .orElse(pEntity.getEntityPosition())
+                    .stream()
+                    .map(p -> new Rectangle((float) p.rectangle2D().getX(),
+                                            (float) p.rectangle2D().getY(),
+                                            (float) p.rectangle2D().getWidth(),
+                                            (float) p.rectangle2D().getHeight(),
+                                            p.pageNumber()))
+                    .toList();
+        }
+        if (entity instanceof Image image) {
+
+            Rectangle2D position = image.getManualOverwrite().getPositions()
+                    .map(p -> p.get(0).rectangle2D())
+                    .orElse(image.getPosition());
+
+            return List.of(new Rectangle((float) position.getX(), (float) position.getY(), (float) position.getWidth(), (float) position.getHeight(), image.getPage().getNumber()));
+
+        } else {
+            throw new UnsupportedOperationException();
+        }
+    }
+
+
+    public boolean needsManualEntry() {
+
+        return manualChanges.stream()
+                       .anyMatch(mc -> mc instanceof ManualResizeRedaction && !((ManualResizeRedaction) mc).getUpdateDictionary()) && !(migratedEntity instanceof Image);
     }
 
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/PrecursorEntity.java

@@ -43,7 +43,6 @@ public class PrecursorEntity implements IEntity {
     String type;
     String section;
     EntityType entityType;
-    EntryType entryType;
     boolean applied;
     boolean isDictionaryEntry;
     boolean isDossierDictionaryEntry;
@@ -61,8 +60,8 @@ public class PrecursorEntity implements IEntity {
                 .stream()
                 .map(RectangleWithPage::fromAnnotationRectangle)
                 .toList();
+
         var entityType = hint ? EntityType.HINT : EntityType.ENTITY;
-        var entryType = hint ? EntryType.HINT : (manualRedactionEntry.isRectangle() ? EntryType.AREA : EntryType.ENTITY);
         ManualChangeOverwrite manualChangeOverwrite = new ManualChangeOverwrite(entityType);
         manualChangeOverwrite.addChange(manualRedactionEntry);
         return PrecursorEntity.builder()
@@ -75,7 +74,6 @@ public class PrecursorEntity implements IEntity {
                 .type(manualRedactionEntry.getType())
                 .section(manualRedactionEntry.getSection())
                 .entityType(entityType)
-                .entryType(entryType)
                 .applied(true)
                 .isDictionaryEntry(false)
                 .isDossierDictionaryEntry(false)
@@ -103,7 +101,6 @@ public class PrecursorEntity implements IEntity {
                 .type(entityLogEntry.getType())
                 .section(entityLogEntry.getSection())
                 .entityType(entityType)
-                .entryType(entityLogEntry.getEntryType())
                 .isDictionaryEntry(entityLogEntry.isDictionaryEntry())
                 .isDossierDictionaryEntry(entityLogEntry.isDossierDictionaryEntry())
                 .manualOverwrite(new ManualChangeOverwrite(entityType))
@@ -134,7 +131,6 @@ public class PrecursorEntity implements IEntity {
                 .type(Optional.ofNullable(importedRedaction.getType())
                               .orElse(IMPORTED_REDACTION_TYPE))
                 .entityType(entityType)
-                .entryType(entryType)
                 .isDictionaryEntry(false)
                 .isDossierDictionaryEntry(false)
                 .rectangle(value.isBlank() || entryType.equals(EntryType.IMAGE) || entryType.equals(EntryType.IMAGE_HINT) || entryType.equals(EntryType.AREA))

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/ManualChangeOverwrite.java

@@ -27,12 +27,12 @@ import lombok.experimental.FieldDefaults;
 public class ManualChangeOverwrite {
 
     private static final Map<Class<? extends BaseAnnotation>, String> MANUAL_CHANGE_DESCRIPTIONS = Map.of(//
-            ManualRedactionEntry.class, "created by manual change", //
-            ManualLegalBasisChange.class, "legal basis was manually changed", //
-            ManualResizeRedaction.class, "resized by manual override", //
-            ManualForceRedaction.class, "forced by manual override", //
-            IdRemoval.class, "removed by manual override", //
-            ManualRecategorization.class, "recategorized by manual override");
+                                                                                                          ManualRedactionEntry.class, "created by manual change", //
+                                                                                                          ManualLegalBasisChange.class, "legal basis was manually changed", //
+                                                                                                          ManualResizeRedaction.class, "resized by manual override", //
+                                                                                                          ManualForceRedaction.class, "forced by manual override", //
+                                                                                                          IdRemoval.class, "removed by manual override", //
+                                                                                                          ManualRecategorization.class, "recategorized by manual override");
 
     List<BaseAnnotation> manualChanges = new LinkedList<>();
     boolean changed;
@@ -80,7 +80,8 @@ public class ManualChangeOverwrite {
         manualChanges.sort(Comparator.comparing(BaseAnnotation::getRequestDate));
         updateFields(manualChanges);
         // make list unmodifiable.
-        return manualChanges.stream().toList();
+        return manualChanges.stream()
+                .toList();
     }
 
 
@@ -121,14 +122,18 @@ public class ManualChangeOverwrite {
                 resized = true;
                 // This is only for not found Manual Entities.
                 value = manualResizeRedaction.getValue();
-                positions = manualResizeRedaction.getPositions().stream().map(RectangleWithPage::fromAnnotationRectangle).toList();
+                positions = manualResizeRedaction.getPositions()
+                        .stream()
+                        .map(RectangleWithPage::fromAnnotationRectangle)
+                        .toList();
             }
 
             if (manualChange instanceof ManualRecategorization recategorization) {
-                // recategorization logic happens in ManualChangesApplicationService.
                 recategorized = true;
-                // this is only relevant for ManualEntities. Image and TextEntity is recategorized in the ManualChangesApplicationService.
                 type = recategorization.getType();
+                if (recategorization.getLegalBasis() != null && !recategorization.getLegalBasis().isEmpty()) {
+                    legalBasis = recategorization.getLegalBasis();
+                }
             }
 
             descriptions.add(MANUAL_CHANGE_DESCRIPTIONS.get(manualChange.getClass()));

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Document.java

@@ -14,6 +14,7 @@ import com.iqser.red.service.redaction.v1.server.model.document.DocumentTree;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlockCollector;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
 
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
@@ -33,6 +34,8 @@ public class Document implements GenericSemanticNode {
 
     @EqualsAndHashCode.Include
     List<Integer> treeId = Collections.emptyList();
+    @Builder.Default
+    Set<LayoutEngine> engines = new HashSet<>(Set.of(LayoutEngine.ALGORITHM));
 
     Set<Page> pages;
     DocumentTree documentTree;

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Footer.java

@@ -9,6 +9,7 @@ import java.util.Set;
 import com.iqser.red.service.redaction.v1.server.model.document.DocumentTree;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
 
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
@@ -27,6 +28,8 @@ import lombok.experimental.FieldDefaults;
 public class Footer implements GenericSemanticNode {
 
     final static SectionIdentifier sectionIdentifier = SectionIdentifier.empty();
+    @Builder.Default
+    Set<LayoutEngine> engines = new HashSet<>(Set.of(LayoutEngine.ALGORITHM));
 
     @EqualsAndHashCode.Include
     List<Integer> treeId;

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Header.java

@@ -9,6 +9,7 @@ import java.util.Set;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.DocumentTree;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
 
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
@@ -27,6 +28,8 @@ import lombok.experimental.FieldDefaults;
 public class Header implements GenericSemanticNode {
 
     final static SectionIdentifier sectionIdentifier = SectionIdentifier.empty();
+    @Builder.Default
+    Set<LayoutEngine> engines = new HashSet<>(Set.of(LayoutEngine.ALGORITHM));
 
     @EqualsAndHashCode.Include
     List<Integer> treeId;

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Headline.java

@@ -10,6 +10,7 @@ import com.iqser.red.service.redaction.v1.server.model.document.DocumentTree;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.AtomicTextBlock;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
 
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
@@ -27,6 +28,8 @@ import lombok.experimental.FieldDefaults;
 @EqualsAndHashCode(onlyExplicitlyIncluded = true)
 public class Headline implements GenericSemanticNode {
 
+    @Builder.Default
+    Set<LayoutEngine> engines = new HashSet<>(Set.of(LayoutEngine.ALGORITHM));
     @EqualsAndHashCode.Include
     List<Integer> treeId;
     TextBlock leafTextBlock;

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Image.java

@@ -18,6 +18,7 @@ import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRu
 import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlockCollector;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
 
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
@@ -35,6 +36,9 @@ import lombok.experimental.FieldDefaults;
 @EqualsAndHashCode(onlyExplicitlyIncluded = true)
 public class Image implements GenericSemanticNode, IEntity {
 
+    @Builder.Default
+    Set<LayoutEngine> engines = new HashSet<>(Set.of(LayoutEngine.ALGORITHM));
+
     @EqualsAndHashCode.Include
     List<Integer> treeId;
     String id;
@@ -70,7 +74,9 @@ public class Image implements GenericSemanticNode, IEntity {
     @Override
     public TextBlock getTextBlock() {
 
-        return streamAllSubNodes().filter(SemanticNode::isLeaf).map(SemanticNode::getLeafTextBlock).collect(new TextBlockCollector());
+        return streamAllSubNodes().filter(SemanticNode::isLeaf)
+                .map(SemanticNode::getLeafTextBlock)
+                .collect(new TextBlockCollector());
     }
 
 
@@ -91,7 +97,8 @@ public class Image implements GenericSemanticNode, IEntity {
     @Override
     public String type() {
 
-        return getManualOverwrite().getType().orElse(imageType.toString());
+        return getManualOverwrite().getType()
+                .orElse(imageType.toString());
     }
 
 
@@ -124,6 +131,32 @@ public class Image implements GenericSemanticNode, IEntity {
     }
 
 
+    public boolean mostlyContainedBy(Image image, double containmentThreshold) {
+
+        Map<Page, Rectangle2D> bboxImage = image.getBBox();
+        Map<Page, Rectangle2D> bbox = this.getBBox();
+        //image needs to be on the same page
+        if(bboxImage.get(this.page) != null) {
+            Rectangle2D intersection = bboxImage.get(this.page).createIntersection(bbox.get(this.page));
+            double calculatedIntersection = intersection.getWidth() * intersection.getHeight();
+            double area = bbox.get(this.page).getWidth() * bbox.get(this.page).getHeight();
+            return (calculatedIntersection / area) > containmentThreshold;
+        }
+        return false;
+    }
+
+
+    public boolean mostlyContains(Image image, double containmentThreshold) {
+
+        Map<Page, Rectangle2D> bboxImage = image.getBBox();
+        Map<Page, Rectangle2D> bbox = this.getBBox();
+        Rectangle2D intersection = bbox.get(this.page).createIntersection(bboxImage.get(this.page));
+        double calculatedIntersection = intersection.getWidth() * intersection.getHeight();
+        double area = bbox.get(this.page).getWidth() * bbox.get(this.page).getHeight();
+        return (area / calculatedIntersection) > containmentThreshold;
+    }
+
+
     public int length() {
 
         return 0;

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/ImageType.java

@@ -6,10 +6,22 @@ public enum ImageType {
     LOGO,
     FORMULA,
     SIGNATURE,
-    OTHER,
+    OTHER {
+        @Override
+        public String toString() {
+
+            return "image";
+        }
+    },
     OCR;
 
 
+    public String toString() {
+
+        return name().toLowerCase(Locale.ENGLISH);
+    }
+
+
     public static ImageType fromString(String imageType) {
 
         return switch (imageType.toLowerCase(Locale.ROOT)) {

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Paragraph.java

@@ -9,6 +9,7 @@ import java.util.Set;
 import com.iqser.red.service.redaction.v1.server.model.document.DocumentTree;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
 
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
@@ -24,6 +25,9 @@ import lombok.experimental.FieldDefaults;
 @EqualsAndHashCode(onlyExplicitlyIncluded = true)
 public class Paragraph implements GenericSemanticNode {
 
+    @Builder.Default
+    Set<LayoutEngine> engines = new HashSet<>(Set.of(LayoutEngine.ALGORITHM));
+
     @EqualsAndHashCode.Include
     List<Integer> treeId;
     TextBlock leafTextBlock;

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Section.java

@@ -10,12 +10,14 @@ import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntit
 import com.iqser.red.service.redaction.v1.server.model.document.DocumentTree;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlockCollector;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
 
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;
 import lombok.EqualsAndHashCode;
+import lombok.RequiredArgsConstructor;
 import lombok.experimental.FieldDefaults;
 import lombok.extern.slf4j.Slf4j;
 
@@ -26,6 +28,7 @@ import lombok.extern.slf4j.Slf4j;
 @FieldDefaults(level = AccessLevel.PRIVATE)
 @EqualsAndHashCode(onlyExplicitlyIncluded = true)
 public class Section implements GenericSemanticNode {
+
     @EqualsAndHashCode.Include
     List<Integer> treeId;
 
@@ -37,6 +40,9 @@ public class Section implements GenericSemanticNode {
 
     Map<Page, Rectangle2D> bBoxCache;
 
+    @Builder.Default
+    Set<LayoutEngine> engines = new HashSet<>(Set.of(LayoutEngine.ALGORITHM));
+
 
     @Override
     public NodeType getType() {

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/SemanticNode.java

@@ -22,6 +22,7 @@ import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBl
 import com.iqser.red.service.redaction.v1.server.service.document.NodeVisitor;
 import com.iqser.red.service.redaction.v1.server.utils.RectangleTransformations;
 import com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
 
 public interface SemanticNode {
 
@@ -356,6 +357,13 @@ public interface SemanticNode {
         return getTextBlock().getSearchText().contains(string);
     }
 
+    Set<LayoutEngine> getEngines();
+
+    default void addEngine(LayoutEngine engine) {
+        getEngines().add(engine);
+    }
+
+
 
     /**
      * Checks whether this SemanticNode contains all the provided Strings.
@@ -554,7 +562,6 @@ public interface SemanticNode {
             if (textBlock.containsTextRange(textEntity.getTextRange())) {
                 textEntity.setDeepestFullyContainingNode(this);
             }
-
             textEntity.addIntersectingNode(this);
             streamChildren().filter(semanticNode -> semanticNode.getTextRange().intersects(textEntity.getTextRange()))
                     .forEach(node -> node.addThisToEntityIfIntersects(textEntity));

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Table.java

@@ -9,6 +9,7 @@ import java.util.List;
 import java.util.Locale;
 import java.util.Map;
 import java.util.Set;
+import java.util.stream.Collectors;
 import java.util.stream.IntStream;
 import java.util.stream.Stream;
 
@@ -16,6 +17,7 @@ import com.iqser.red.service.redaction.v1.server.model.document.DocumentTree;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlockCollector;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
 
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
@@ -31,6 +33,8 @@ import lombok.experimental.FieldDefaults;
 @EqualsAndHashCode(onlyExplicitlyIncluded = true)
 public class Table implements SemanticNode {
 
+    @Builder.Default
+    Set<LayoutEngine> engines = new HashSet<>(Set.of(LayoutEngine.ALGORITHM));
     @EqualsAndHashCode.Include
     List<Integer> treeId;
     DocumentTree documentTree;
@@ -64,8 +68,7 @@ public class Table implements SemanticNode {
      */
     public Stream<TextEntity> streamEntitiesWhereRowContainsStringsIgnoreCase(List<String> strings) {
 
-        return IntStream.range(0, numberOfRows)
-                .boxed()
+        return IntStream.range(0, numberOfRows).boxed()
                 .filter(row -> rowContainsStringsIgnoreCase(row, strings))
                 .flatMap(this::streamRow)
                 .map(TableCell::getEntities)
@@ -82,8 +85,11 @@ public class Table implements SemanticNode {
      */
     public boolean rowContainsStringsIgnoreCase(Integer row, List<String> strings) {
 
-        String rowText = streamRow(row).map(TableCell::getTextBlock).collect(new TextBlockCollector()).getSearchText().toLowerCase(Locale.ROOT);
-        return strings.stream().map(String::toLowerCase).allMatch(rowText::contains);
+        String rowText = streamRow(row).map(TableCell::getTextBlock)
+                .collect(new TextBlockCollector()).getSearchText().toLowerCase(Locale.ROOT);
+        return strings.stream()
+                .map(String::toLowerCase)
+                .allMatch(rowText::contains);
     }
 
 
@@ -96,9 +102,13 @@ public class Table implements SemanticNode {
      */
     public Stream<TextEntity> streamEntitiesWhereRowHasHeaderAndValue(String header, String value) {
 
-        List<Integer> vertebrateStudyCols = streamHeaders().filter(headerNode -> headerNode.containsString(header)).map(TableCell::getCol).toList();
+        List<Integer> vertebrateStudyCols = streamHeaders().filter(headerNode -> headerNode.containsString(header))
+                .map(TableCell::getCol)
+                .toList();
         return streamTableCells().filter(tableCellNode -> vertebrateStudyCols.stream()
-                .anyMatch(vertebrateStudyCol -> getCell(tableCellNode.getRow(), vertebrateStudyCol).containsString(value))).map(TableCell::getEntities).flatMap(Collection::stream);
+                        .anyMatch(vertebrateStudyCol -> getCell(tableCellNode.getRow(), vertebrateStudyCol).containsString(value)))
+                .map(TableCell::getEntities)
+                .flatMap(Collection::stream);
     }
 
 
@@ -111,9 +121,13 @@ public class Table implements SemanticNode {
      */
     public Stream<TextEntity> streamEntitiesWhereRowHasHeaderAndAnyValue(String header, List<String> values) {
 
-        List<Integer> colsWithHeader = streamHeaders().filter(headerNode -> headerNode.containsString(header)).map(TableCell::getCol).toList();
+        List<Integer> colsWithHeader = streamHeaders().filter(headerNode -> headerNode.containsString(header))
+                .map(TableCell::getCol)
+                .toList();
         return streamTableCells().filter(tableCellNode -> colsWithHeader.stream()
-                .anyMatch(colWithHeader -> getCell(tableCellNode.getRow(), colWithHeader).containsAnyString(values))).map(TableCell::getEntities).flatMap(Collection::stream);
+                        .anyMatch(colWithHeader -> getCell(tableCellNode.getRow(), colWithHeader).containsAnyString(values)))
+                .map(TableCell::getEntities)
+                .flatMap(Collection::stream);
     }
 
 
@@ -126,16 +140,33 @@ public class Table implements SemanticNode {
      */
     public Stream<TextEntity> streamEntitiesWhereRowContainsEntitiesOfType(List<String> types) {
 
-        List<Integer> rowsWithEntityOfType = getEntities().stream()
-                .filter(TextEntity::active)
-                .filter(redactionEntity -> types.stream().anyMatch(type -> type.equals(redactionEntity.type())))
-                .map(TextEntity::getIntersectingNodes)
-                .filter(node -> node instanceof TableCell)
-                .map(node -> (TableCell) node)
-                .map(TableCell::getRow)
-                .toList();
+        return IntStream.range(0, numberOfRows).boxed()
+                .filter(rowNumber -> streamTextEntitiesInRow(rowNumber).map(TextEntity::type)
+                        .anyMatch(types::contains))
+                .flatMap(this::streamRow)
+                .map(TableCell::getEntities)
+                .flatMap(Collection::stream);
+    }
 
-        return rowsWithEntityOfType.stream().flatMap(this::streamRow).map(TableCell::getEntities).flatMap(Collection::stream);
+
+    /**
+     * Streams all entities in this table, that appear in a row, which contains at least one entity of each of the provided types.
+     * Ignores Entity with ignored == true or removed == true.
+     *
+     * @param types type strings to check whether a row contains an entity like them
+     * @return Stream of all entities in this table, that appear in a row, which contains at least one entity of each of the provided types.
+     */
+    public Stream<TextEntity> streamEntitiesWhereRowContainsEntitiesOfEachType(List<String> types) {
+
+        return IntStream.range(0, numberOfRows).boxed()
+                .filter(rowNumber -> {
+                    Set<String> entityTypes = streamTextEntitiesInRow(rowNumber).map(TextEntity::type)
+                            .collect(Collectors.toSet());
+                    return entityTypes.containsAll(types);
+                })
+                .flatMap(this::streamRow)
+                .map(TableCell::getEntities)
+                .flatMap(Collection::stream);
     }
 
 
@@ -148,18 +179,43 @@ public class Table implements SemanticNode {
      */
     public Stream<TextEntity> streamEntitiesWhereRowContainsNoEntitiesOfType(List<String> types) {
 
-        return IntStream.range(0, numberOfRows)
-                .boxed()
-                .filter(rowNumber -> streamRow(rowNumber).map(TableCell::getEntities)
-                        .flatMap(Collection::stream)
-                        .filter(TextEntity::active)
-                        .noneMatch(entity -> types.contains(entity.type())))
+        return IntStream.range(0, numberOfRows).boxed()
+                .filter(rowNumber -> streamTextEntitiesInRow(rowNumber).map(TextEntity::type)
+                        .noneMatch(types::contains))
                 .flatMap(this::streamRow)
                 .map(TableCell::getEntities)
                 .flatMap(Collection::stream);
     }
 
 
+    /**
+     * Streams all Entities in the given row.
+     *
+     * @param rowNumber the row number to look for
+     * @return stream of TextEntities occurring in row
+     */
+    public Stream<TextEntity> streamTextEntitiesInRow(int rowNumber) {
+
+        return streamRow(rowNumber).map(TableCell::getEntities)
+                .flatMap(Collection::stream)
+                .filter(TextEntity::active);
+    }
+
+
+    /**
+     * Streams all Entities in the given col.
+     *
+     * @param colNumber the column number to look for
+     * @return stream of TextEntities occurring in row
+     */
+    public Stream<TextEntity> streamTextEntitiesInCol(int colNumber) {
+
+        return streamCol(colNumber).map(TableCell::getEntities)
+                .flatMap(Collection::stream)
+                .filter(TextEntity::active);
+    }
+
+
     /**
      * Returns a TableCell at the provided row and column location.
      *
@@ -173,7 +229,8 @@ public class Table implements SemanticNode {
             throw new IllegalArgumentException(format("row %d, col %d is out of bounds for number of rows of %d and number of cols %d", row, col, numberOfRows, numberOfCols));
         }
         int idx = row * numberOfCols + col;
-        return (TableCell) documentTree.getEntryById(treeId).getChildren().get(idx).getNode();
+        return (TableCell) documentTree.getEntryById(treeId).getChildren()
+                .get(idx).getNode();
     }
 
 
@@ -196,7 +253,7 @@ public class Table implements SemanticNode {
      */
     public Stream<TableCell> streamTableCellsWhichContainType(String type) {
 
-        return streamTableCells().filter(tableCell -> tableCell.getEntities().stream().filter(TextEntity::active).anyMatch(entity -> entity.type().equals(type)));
+        return streamTableCells().filter(tableCell -> tableCell.hasEntitiesOfType(type));
     }
 
 
@@ -222,7 +279,8 @@ public class Table implements SemanticNode {
      */
     public Stream<TableCell> streamCol(int col) {
 
-        return IntStream.range(0, numberOfRows).boxed().map(row -> getCell(row, col));
+        return IntStream.range(0, numberOfRows).boxed()
+                .map(row -> getCell(row, col));
     }
 
 
@@ -234,7 +292,8 @@ public class Table implements SemanticNode {
      */
     public Stream<TableCell> streamRow(int row) {
 
-        return IntStream.range(0, numberOfCols).boxed().map(col -> getCell(row, col));
+        return IntStream.range(0, numberOfCols).boxed()
+                .map(col -> getCell(row, col));
     }
 
 
@@ -258,7 +317,8 @@ public class Table implements SemanticNode {
      */
     public Stream<TableCell> streamHeadersForCell(int row, int col) {
 
-        return Stream.concat(streamRow(row), streamCol(col)).filter(TableCell::isHeader);
+        return Stream.concat(streamRow(row), streamCol(col))
+                .filter(TableCell::isHeader);
     }
 
 
@@ -348,7 +408,9 @@ public class Table implements SemanticNode {
     public TextBlock getTextBlock() {
 
         if (textBlock == null) {
-            textBlock = streamAllSubNodes().filter(SemanticNode::isLeaf).map(SemanticNode::getLeafTextBlock).collect(new TextBlockCollector());
+            textBlock = streamAllSubNodes().filter(SemanticNode::isLeaf)
+                    .map(SemanticNode::getLeafTextBlock)
+                    .collect(new TextBlockCollector());
         }
         return textBlock;
     }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/TableCell.java

@@ -11,6 +11,7 @@ import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntit
 import com.iqser.red.service.redaction.v1.server.model.document.DocumentTree;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlockCollector;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
 
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
@@ -26,6 +27,9 @@ import lombok.experimental.FieldDefaults;
 @EqualsAndHashCode(onlyExplicitlyIncluded = true)
 public class TableCell implements GenericSemanticNode {
 
+    @Builder.Default
+    Set<LayoutEngine> engines = new HashSet<>(Set.of(LayoutEngine.ALGORITHM));
+
     @EqualsAndHashCode.Include
     List<Integer> treeId;
     int row;

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityChangeLogService.java

@@ -26,10 +26,9 @@ import lombok.extern.slf4j.Slf4j;
 @Slf4j
 @Service
 @RequiredArgsConstructor
-@FieldDefaults(makeFinal=true, level= AccessLevel.PRIVATE)
+@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
 public class EntityChangeLogService {
 
-
     @Timed("redactmanager_computeChanges")
     public boolean computeChanges(List<EntityLogEntry> previousEntityLogEntries, List<EntityLogEntry> newEntityLogEntries, ManualRedactions manualRedactions, int analysisNumber) {
 
@@ -42,7 +41,9 @@ public class EntityChangeLogService {
         boolean hasChanges = false;
 
         for (EntityLogEntry entityLogEntry : newEntityLogEntries) {
-            Optional<EntityLogEntry> optionalPreviousEntity = previousEntityLogEntries.stream().filter(entry -> entry.getId().equals(entityLogEntry.getId())).findAny();
+            Optional<EntityLogEntry> optionalPreviousEntity = previousEntityLogEntries.stream()
+                    .filter(entry -> entry.getId().equals(entityLogEntry.getId()))
+                    .findAny();
             if (optionalPreviousEntity.isEmpty()) {
                 hasChanges = true;
                 entityLogEntry.getChanges().add(new Change(analysisNumber, ChangeType.ADDED, now));
@@ -56,45 +57,29 @@ public class EntityChangeLogService {
                 ChangeType changeType = calculateChangeType(entityLogEntry.getState(), previousEntity.getState());
                 entityLogEntry.getChanges().add(new Change(analysisNumber, changeType, now));
             }
-
-            addManualChanges(entityLogEntry, previousEntity);
         }
         addRemovedEntriesAsRemoved(previousEntityLogEntries, newEntityLogEntries, manualRedactions, analysisNumber, now);
         return hasChanges;
     }
 
 
-    // If a manual change is present in the previous entity but not in the new entity, add it to the new one and
-    // sort them, so they are displayed in the correct order.
-    private void addManualChanges(EntityLogEntry entityLogEntry, EntityLogEntry previousEntity) {
-
-        Comparator<ManualChange> manualChangeComparator =
-                Comparator.comparing(ManualChange::getManualRedactionType)
-                        .thenComparing(ManualChange::getRequestedDate);
-
-        previousEntity.getManualChanges().forEach(manualChange -> {
-            boolean contains = entityLogEntry.getManualChanges()
-                    .stream()
-                    .anyMatch(existingChange -> manualChangeComparator.compare(existingChange, manualChange) == 0);
-
-            if (!contains) {
-                entityLogEntry.getManualChanges().add(manualChange);
-                entityLogEntry.getManualChanges().sort(Comparator.comparing(ManualChange::getRequestedDate));
-            }
-        });
-    }
-
-
     private void addRemovedEntriesAsRemoved(List<EntityLogEntry> previousEntityLogEntries,
-                                                   List<EntityLogEntry> newEntityLogEntries,
-                                                   ManualRedactions manualRedactions,
-                                                   int analysisNumber,
-                                                   OffsetDateTime now) {
+                                            List<EntityLogEntry> newEntityLogEntries,
+                                            ManualRedactions manualRedactions,
+                                            int analysisNumber,
+                                            OffsetDateTime now) {
 
-        Set<String> existingIds = newEntityLogEntries.stream().map(EntityLogEntry::getId).collect(Collectors.toSet());
+        Set<String> existingIds = newEntityLogEntries.stream()
+                .map(EntityLogEntry::getId)
+                .collect(Collectors.toSet());
         List<EntityLogEntry> removedEntries = previousEntityLogEntries.stream()
                 .filter(entry -> !existingIds.contains(entry.getId()))
+                .collect(Collectors.toList());
+        List<EntityLogEntry> removedDossierRedaction = removedEntries.stream()
+                .filter(e -> e.getState() == EntryState.REMOVED && e.getType().equals("dossier_redaction"))
                 .toList();
+        previousEntityLogEntries.removeAll(removedDossierRedaction);
+        removedEntries.removeAll(removedDossierRedaction);
         removedEntries.forEach(entry -> entry.getChanges().add(new Change(analysisNumber, ChangeType.REMOVED, now)));
         removedEntries.forEach(entry -> entry.setState(EntryState.REMOVED));
         removedEntries.forEach(entry -> addManualChangeForDictionaryRemovals(entry, manualRedactions));
@@ -108,13 +93,15 @@ public class EntityChangeLogService {
             return;
         }
 
-        manualRedactions.getIdsToRemove().stream()
+        manualRedactions.getIdsToRemove()
+                .stream()
                 .filter(IdRemoval::isRemoveFromDictionary)//
                 .filter(removed -> removed.getAnnotationId().equals(entry.getId()))//
                 .findFirst()//
                 .ifPresent(idRemove -> entry.getManualChanges().add(ManualChangeFactory.toManualChange(idRemove, false)));
     }
 
+
     private ChangeType calculateChangeType(EntryState state, EntryState previousState) {
 
         if (state.equals(previousState)) {

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java

@@ -11,6 +11,7 @@ import java.util.stream.Collectors;
 import org.springframework.stereotype.Service;
 
 import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogChanges;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
@@ -18,7 +19,6 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryType;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.legalbasis.LegalBasis;
 import com.iqser.red.service.redaction.v1.server.RedactionServiceSettings;
 import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient;
@@ -26,6 +26,7 @@ import com.iqser.red.service.redaction.v1.server.model.PrecursorEntity;
 import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryVersion;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.IEntity;
+import com.iqser.red.service.redaction.v1.server.model.document.entity.ManualChangeOverwrite;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.PositionOnPage;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
@@ -66,20 +67,19 @@ public class EntityLogCreatorService {
         List<EntityLogEntry> entityLogEntries = createEntityLogEntries(document, analyzeRequest, notFoundEntities);
 
         List<LegalBasis> legalBasis = legalBasisClient.getLegalBasisMapping(analyzeRequest.getDossierTemplateId());
-        EntityLog entityLog = new EntityLog(redactionServiceSettings.getAnalysisVersion(),
-                analyzeRequest.getAnalysisNumber(),
-                entityLogEntries,
-                toEntityLogLegalBasis(legalBasis),
-                dictionaryVersion.getDossierTemplateVersion(),
-                dictionaryVersion.getDossierVersion(),
-                rulesVersion,
-                legalBasisClient.getVersion(analyzeRequest.getDossierTemplateId()));
 
         List<EntityLogEntry> previousExistingEntityLogEntries = getPreviousEntityLogEntries(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
 
         entityChangeLogService.computeChanges(previousExistingEntityLogEntries, entityLogEntries, analyzeRequest.getManualRedactions(), analyzeRequest.getAnalysisNumber());
 
-        return entityLog;
+        return new EntityLog(redactionServiceSettings.getAnalysisVersion(),
+                             analyzeRequest.getAnalysisNumber(),
+                             entityLogEntries,
+                             toEntityLogLegalBasis(legalBasis),
+                             dictionaryVersion.getDossierTemplateVersion(),
+                             dictionaryVersion.getDossierVersion(),
+                             rulesVersion,
+                             legalBasisClient.getVersion(analyzeRequest.getDossierTemplateId()));
     }
 
 
@@ -114,21 +114,24 @@ public class EntityLogCreatorService {
                                                     DictionaryVersion dictionaryVersion) {
 
         List<EntityLogEntry> newEntityLogEntries = createEntityLogEntries(document, analyzeRequest, notFoundEntries).stream()
-                .filter(entry -> entry.getContainingNodeId().isEmpty() || sectionsToReanalyseIds.contains(entry.getContainingNodeId().get(0)))
+                .filter(entry -> entry.getContainingNodeId().isEmpty() || sectionsToReanalyseIds.contains(entry.getContainingNodeId()
+                                                                                                                  .get(0)))
                 .collect(Collectors.toList());
-        Set<String> newEntityIds = newEntityLogEntries.stream().map(EntityLogEntry::getId).collect(Collectors.toSet());
+        Set<String> newEntityIds = newEntityLogEntries.stream()
+                .map(EntityLogEntry::getId)
+                .collect(Collectors.toSet());
 
         List<EntityLogEntry> previousEntriesFromReAnalyzedSections = previousEntityLog.getEntityLogEntry()
                 .stream()
                 .filter(entry -> (newEntityIds.contains(entry.getId()) || entry.getContainingNodeId().isEmpty() || sectionsToReanalyseIds.contains(entry.getContainingNodeId()
-                        .get(0))))
-                .toList();
+                                                                                                                                                           .get(0))))
+                .collect(Collectors.toList());
         previousEntityLog.getEntityLogEntry().removeAll(previousEntriesFromReAnalyzedSections);
 
         boolean hasChanges = entityChangeLogService.computeChanges(previousEntriesFromReAnalyzedSections,
-                newEntityLogEntries,
-                analyzeRequest.getManualRedactions(),
-                analyzeRequest.getAnalysisNumber());
+                                                                   newEntityLogEntries,
+                                                                   analyzeRequest.getManualRedactions(),
+                                                                   analyzeRequest.getAnalysisNumber());
         previousEntityLog.getEntityLogEntry().addAll(newEntityLogEntries);
 
         return updateVersionsAndReturnChanges(previousEntityLog, dictionaryVersion, analyzeRequest, hasChanges);
@@ -137,22 +140,6 @@ public class EntityLogCreatorService {
 
     private List<EntityLogEntry> createEntityLogEntries(Document document, AnalyzeRequest analyzeRequest, List<PrecursorEntity> notFoundPrecursorEntries) {
 
-        Set<ManualRedactionEntry> dictionaryEntries;
-        Set<String> dictionaryEntriesValues;
-
-        if (analyzeRequest.getManualRedactions() != null && !analyzeRequest.getManualRedactions().getEntriesToAdd().isEmpty()) {
-            dictionaryEntries = analyzeRequest.getManualRedactions().getEntriesToAdd()
-                    .stream()
-                    .filter(e -> e.isAddToDictionary() || e.isAddToDossierDictionary())
-                    .collect(Collectors.toSet());
-            dictionaryEntriesValues = dictionaryEntries.stream()
-                    .map(ManualRedactionEntry::getValue)
-                    .collect(Collectors.toSet());
-        } else {
-            dictionaryEntriesValues = new HashSet<>();
-            dictionaryEntries = new HashSet<>();
-        }
-
         String dossierTemplateId = analyzeRequest.getDossierTemplateId();
 
         List<EntityLogEntry> entries = new ArrayList<>();
@@ -162,22 +149,21 @@ public class EntityLogCreatorService {
                 .filter(entity -> !entity.getValue().isEmpty())
                 .filter(EntityLogCreatorService::notFalsePositiveOrFalseRecommendation)
                 .filter(entity -> !entity.removed())
-                .forEach(entityNode -> entries.addAll(toEntityLogEntries(entityNode, dictionaryEntries, dictionaryEntriesValues)));
-        document.streamAllImages().filter(entity -> !entity.removed()).forEach(imageNode -> entries.add(createEntityLogEntry(imageNode, dossierTemplateId)));
-        notFoundPrecursorEntries.stream().filter(entity -> !entity.removed()).forEach(precursorEntity -> entries.add(createEntityLogEntry(precursorEntity, dossierTemplateId)));
+                .forEach(entityNode -> entries.addAll(toEntityLogEntries(entityNode)));
+        document.streamAllImages()
+                .filter(entity -> !entity.removed())
+                .forEach(imageNode -> entries.add(createEntityLogEntry(imageNode, dossierTemplateId)));
+        notFoundPrecursorEntries.stream()
+                .filter(entity -> !entity.removed())
+                .forEach(precursorEntity -> entries.add(createEntityLogEntry(precursorEntity, dossierTemplateId)));
         return entries;
     }
 
 
-    private List<EntityLogEntry> toEntityLogEntries(TextEntity textEntity, Set<ManualRedactionEntry> dictionaryEntries, Set<String> dictionaryEntriesValues) {
+    private List<EntityLogEntry> toEntityLogEntries(TextEntity textEntity) {
 
         List<EntityLogEntry> entityLogEntries = new ArrayList<>();
 
-        // Adding ADD_TO_DICTIONARY manual change to the entity's manual overwrite
-        if (dictionaryEntriesValues.contains(textEntity.getValue())) {
-            textEntity.getManualOverwrite().addChange(dictionaryEntries.stream().filter(entry -> entry.getValue().equals(textEntity.getValue())).findFirst().get());
-        }
-
         // split entity into multiple entries if it occurs on multiple pages, since FE can't handle multi page entities
         for (PositionOnPage positionOnPage : textEntity.getPositionsOnPagePerPage()) {
 
@@ -200,12 +186,11 @@ public class EntityLogCreatorService {
 
     private EntityLogEntry createEntityLogEntry(Image image, String dossierTemplateId) {
 
-        String imageType = image.getImageType().equals(ImageType.OTHER) ? "image" : image.getImageType().toString().toLowerCase(Locale.ENGLISH);
-        boolean isHint = dictionaryService.isHint(imageType, dossierTemplateId);
+        boolean isHint = dictionaryService.isHint(image.type(), dossierTemplateId);
         return EntityLogEntry.builder()
                 .id(image.getId())
                 .value(image.value())
-                .type(imageType)
+                .type(image.type())
                 .reason(image.buildReasonWithManualChangeDescriptions())
                 .legalBasis(image.legalBasis())
                 .matchedRule(image.getMatchedRule().getRuleIdentifier().toString())
@@ -213,11 +198,13 @@ public class EntityLogCreatorService {
                 .positions(List.of(new Position(image.getPosition(), image.getPage().getNumber())))
                 .containingNodeId(image.getTreeId())
                 .closestHeadline(image.getHeadline().getTextBlock().getSearchText())
-                .section(image.getManualOverwrite().getSection().orElse(image.getParent().toString()))
+                .section(image.getManualOverwrite().getSection()
+                                 .orElse(image.getParent().toString()))
                 .imageHasTransparency(image.isTransparent())
                 .manualChanges(ManualChangeFactory.toManualChangeList(image.getManualOverwrite().getManualChangeLog(), isHint))
                 .state(buildEntryState(image))
                 .entryType(isHint ? EntryType.IMAGE_HINT : EntryType.IMAGE)
+                .engines(getEngines(null, image.getManualOverwrite()))
                 .build();
 
     }
@@ -225,7 +212,8 @@ public class EntityLogCreatorService {
 
     private EntityLogEntry createEntityLogEntry(PrecursorEntity precursorEntity, String dossierTemplateId) {
 
-        String type = precursorEntity.getManualOverwrite().getType().orElse(precursorEntity.getType());
+        String type = precursorEntity.getManualOverwrite().getType()
+                .orElse(precursorEntity.getType());
         boolean isHint = isHint(precursorEntity.getEntityType());
         return EntityLogEntry.builder()
                 .id(precursorEntity.getId())
@@ -235,7 +223,8 @@ public class EntityLogCreatorService {
                 .type(type)
                 .state(buildEntryState(precursorEntity))
                 .entryType(buildEntryType(precursorEntity))
-                .section(precursorEntity.getManualOverwrite().getSection().orElse(precursorEntity.getSection()))
+                .section(precursorEntity.getManualOverwrite().getSection()
+                                 .orElse(precursorEntity.getSection()))
                 .containingNodeId(Collections.emptyList())
                 .closestHeadline("")
                 .matchedRule(precursorEntity.getMatchedRule().getRuleIdentifier().toString())
@@ -245,13 +234,12 @@ public class EntityLogCreatorService {
                 .textBefore("")
                 .startOffset(-1)
                 .endOffset(-1)
-                .positions(precursorEntity.getManualOverwrite()
-                        .getPositions()
-                        .orElse(precursorEntity.getEntityPosition())
-                        .stream()
-                        .map(entityPosition -> new Position(entityPosition.rectangle2D(), entityPosition.pageNumber()))
-                        .toList())
-                .engines(precursorEntity.getEngines())
+                .positions(precursorEntity.getManualOverwrite().getPositions()
+                                   .orElse(precursorEntity.getEntityPosition())
+                                   .stream()
+                                   .map(entityPosition -> new Position(entityPosition.rectangle2D(), entityPosition.pageNumber()))
+                                   .toList())
+                .engines(getEngines(precursorEntity.getEngines(), precursorEntity.getManualOverwrite()))
                 //imported is no longer used, frontend should check engines
                 //(was .imported(precursorEntity.getEngines() != null && precursorEntity.getEngines().contains(Engine.IMPORTED)))
                 .imported(false)
@@ -264,14 +252,20 @@ public class EntityLogCreatorService {
     private EntityLogEntry createEntityLogEntry(TextEntity entity) {
 
         Set<String> referenceIds = new HashSet<>();
-        entity.references().stream().filter(TextEntity::active).forEach(ref -> ref.getPositionsOnPagePerPage().forEach(pos -> referenceIds.add(pos.getId())));
+        entity.references()
+                .stream()
+                .filter(TextEntity::active)
+                .forEach(ref -> ref.getPositionsOnPagePerPage()
+                        .forEach(pos -> referenceIds.add(pos.getId())));
         boolean isHint = isHint(entity.getEntityType());
         return EntityLogEntry.builder()
                 .reason(entity.buildReasonWithManualChangeDescriptions())
                 .legalBasis(entity.legalBasis())
-                .value(entity.getManualOverwrite().getValue().orElse(entity.getMatchedRule().isWriteValueWithLineBreaks() ? entity.getValueWithLineBreaks() : entity.getValue()))
+                .value(entity.getManualOverwrite().getValue()
+                               .orElse(entity.getMatchedRule().isWriteValueWithLineBreaks() ? entity.getValueWithLineBreaks() : entity.getValue()))
                 .type(entity.type())
-                .section(entity.getManualOverwrite().getSection().orElse(entity.getDeepestFullyContainingNode().toString()))
+                .section(entity.getManualOverwrite().getSection()
+                                 .orElse(entity.getDeepestFullyContainingNode().toString()))
                 .containingNodeId(entity.getDeepestFullyContainingNode().getTreeId())
                 .closestHeadline(entity.getDeepestFullyContainingNode().getHeadline().getTextBlock().getSearchText())
                 .matchedRule(entity.getMatchedRule().getRuleIdentifier().toString())
@@ -281,7 +275,7 @@ public class EntityLogCreatorService {
                 .startOffset(entity.getTextRange().start())
                 .endOffset(entity.getTextRange().end())
                 .dossierDictionaryEntry(entity.isDossierDictionaryEntry())
-                .engines(entity.getEngines() != null ? entity.getEngines() : Collections.emptySet())
+                .engines(getEngines(entity.getEngines(), entity.getManualOverwrite()))
                 //imported is no longer used, frontend should check engines
                 //(was .imported(entity.getEngines() != null && entity.getEngines().contains(Engine.IMPORTED)))
                 .imported(false)
@@ -293,13 +287,24 @@ public class EntityLogCreatorService {
     }
 
 
+    private Set<Engine> getEngines(Set<Engine> currentEngines, ManualChangeOverwrite manualChangeOverwrite) {
+
+        Set<Engine> engines = currentEngines != null ? new HashSet<>(currentEngines) : new HashSet<>();
+
+        if (manualChangeOverwrite != null && !manualChangeOverwrite.getManualChangeLog().isEmpty()) {
+            engines.add(Engine.MANUAL);
+        }
+        return engines;
+    }
+
+
     private boolean isHint(EntityType entityType) {
 
         return entityType.equals(EntityType.HINT);
     }
 
 
-    private EntryState buildEntryState(IEntity entity) {
+    public static EntryState buildEntryState(IEntity entity) {
 
         if (entity.applied() && entity.active()) {
             return EntryState.APPLIED;
@@ -313,12 +318,17 @@ public class EntityLogCreatorService {
     }
 
 
-    private EntryType buildEntryType(IEntity entity) {
+    public static EntryType buildEntryType(IEntity entity) {
 
         if (entity instanceof TextEntity textEntity) {
             return getEntryType(textEntity.getEntityType());
         } else if (entity instanceof PrecursorEntity precursorEntity) {
-            return precursorEntity.getEntryType();
+            if (precursorEntity.isRectangle()) {
+                return EntryType.AREA;
+            }
+            return getEntryType(precursorEntity.getEntityType());
+        } else if (entity instanceof Image) {
+            return EntryType.IMAGE;
         }
         throw new UnsupportedOperationException(String.format("Entity subclass %s is not implemented!", entity.getClass()));
     }
@@ -338,7 +348,9 @@ public class EntityLogCreatorService {
 
     private List<EntityLogLegalBasis> toEntityLogLegalBasis(List<LegalBasis> legalBasis) {
 
-        return legalBasis.stream().map(l -> new EntityLogLegalBasis(l.getName(), l.getDescription(), l.getReason())).collect(Collectors.toList());
+        return legalBasis.stream()
+                .map(l -> new EntityLogLegalBasis(l.getName(), l.getDescription(), l.getReason()))
+                .collect(Collectors.toList());
     }
 
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/ManualChangeFactory.java

@@ -4,8 +4,8 @@ import java.time.OffsetDateTime;
 import java.util.List;
 import java.util.stream.Collectors;
 
-import org.springframework.stereotype.Service;
-
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualChange;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.BaseAnnotation;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction;
@@ -13,8 +13,6 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualChange;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType;
 
 import lombok.experimental.UtilityClass;
 
@@ -23,7 +21,9 @@ public class ManualChangeFactory {
 
     public List<ManualChange> toManualChangeList(List<BaseAnnotation> manualChanges, boolean isHint) {
 
-        return manualChanges.stream().map(baseAnnotation -> toManualChange(baseAnnotation, isHint)).collect(Collectors.toList());
+        return manualChanges.stream()
+                .map(baseAnnotation -> toManualChange(baseAnnotation, isHint))
+                .collect(Collectors.toList());
     }
 
 
@@ -33,13 +33,14 @@ public class ManualChangeFactory {
         if (baseAnnotation instanceof ManualRecategorization imageRecategorization) {
             manualChange.withManualRedactionType(ManualRedactionType.RECATEGORIZE).withChange("type", imageRecategorization.getType());
         } else if (baseAnnotation instanceof IdRemoval manualRemoval) {
-            manualChange.withManualRedactionType(manualRemoval.isRemoveFromDictionary() ? ManualRedactionType.REMOVE_FROM_DICTIONARY : ManualRedactionType.REMOVE_LOCALLY);
-        } else if (baseAnnotation instanceof ManualForceRedaction) {
-            manualChange.withManualRedactionType(isHint ? ManualRedactionType.FORCE_HINT : ManualRedactionType.FORCE_REDACT);
+            manualChange.withManualRedactionType(manualRemoval.isRemoveFromDictionary() ? ManualRedactionType.REMOVE_FROM_DICTIONARY : ManualRedactionType.REMOVE);
+        } else if (baseAnnotation instanceof ManualForceRedaction manualForceRedaction) {
+            manualChange.withManualRedactionType(ManualRedactionType.FORCE).withChange("legalBasis", manualForceRedaction.getLegalBasis());
         } else if (baseAnnotation instanceof ManualResizeRedaction manualResizeRedact) {
-            manualChange.withManualRedactionType(manualResizeRedact.getUpdateDictionary() ? ManualRedactionType.RESIZE_IN_DICTIONARY : ManualRedactionType.RESIZE).withChange("value", manualResizeRedact.getValue());
+            manualChange.withManualRedactionType(manualResizeRedact.getUpdateDictionary() ? ManualRedactionType.RESIZE_IN_DICTIONARY : ManualRedactionType.RESIZE)
+                    .withChange("value", manualResizeRedact.getValue());
         } else if (baseAnnotation instanceof ManualRedactionEntry manualRedactionEntry) {
-            manualChange.withManualRedactionType(manualRedactionEntry.isAddToDictionary() ? ManualRedactionType.ADD_TO_DICTIONARY : ManualRedactionType.ADD_LOCALLY)
+            manualChange.withManualRedactionType(manualRedactionEntry.isAddToDictionary() ? ManualRedactionType.ADD_TO_DICTIONARY : ManualRedactionType.ADD)
                     .withChange("value", manualRedactionEntry.getValue());
         } else if (baseAnnotation instanceof ManualLegalBasisChange manualLegalBasisChange) {
             manualChange.withManualRedactionType(ManualRedactionType.LEGAL_BASIS_CHANGE)

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/UnprocessedChangesService.java

@@ -16,6 +16,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequ
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.BaseAnnotation;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
 import com.iqser.red.service.redaction.v1.model.AnalyzeResponse;
@@ -25,15 +26,12 @@ import com.iqser.red.service.redaction.v1.server.model.PrecursorEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
 import com.iqser.red.service.redaction.v1.server.service.document.DocumentGraphMapper;
-import com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService;
-import com.iqser.red.service.redaction.v1.server.service.document.EntityEnrichmentService;
 import com.iqser.red.service.redaction.v1.server.service.document.EntityFindingUtility;
 import com.iqser.red.service.redaction.v1.server.service.document.EntityFromPrecursorCreationService;
 import com.iqser.red.service.redaction.v1.server.storage.ObservedStorageService;
 import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
 
 import io.micrometer.observation.annotation.Observed;
-import jakarta.annotation.PostConstruct;
 import lombok.AccessLevel;
 import lombok.RequiredArgsConstructor;
 import lombok.experimental.FieldDefaults;
@@ -51,20 +49,10 @@ public class UnprocessedChangesService {
     final ObservedStorageService observedStorageService;
     final EntityFindingUtility entityFindingUtility;
     final RedactionStorageService redactionStorageService;
-    final EntityEnrichmentService entityEnrichmentService;
     final EntityFromPrecursorCreationService entityFromPrecursorCreationService;
     final DictionaryService dictionaryService;
     final ManualChangesApplicationService manualChangesApplicationService;
 
-    EntityCreationService entityCreationService;
-
-
-    @PostConstruct
-    public void initEntityCreationService() {
-
-        entityCreationService = new EntityCreationService(entityEnrichmentService);
-    }
-
 
     @Observed(name = "UnprocessedChangesService", contextualName = "analyse-surrounding-text")
     public void analyseSurroundingText(AnalyzeRequest analyzeRequest) {
@@ -76,11 +64,19 @@ public class UnprocessedChangesService {
         EntityLog previousEntityLog = redactionStorageService.getEntityLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
         Document document = DocumentGraphMapper.toDocumentGraph(observedStorageService.getDocumentData(analyzeRequest.getDossierId(), analyzeRequest.getFileId()));
 
-        Set<String> allAnnotationIds = analyzeRequest.getManualRedactions().getEntriesToAdd().stream().map(ManualRedactionEntry::getAnnotationId).collect(Collectors.toSet());
-        Set<String> resizeIds = analyzeRequest.getManualRedactions().getResizeRedactions().stream().map(ManualResizeRedaction::getAnnotationId).collect(Collectors.toSet());
+        Set<String> allAnnotationIds = analyzeRequest.getManualRedactions().getEntriesToAdd()
+                .stream()
+                .map(ManualRedactionEntry::getAnnotationId)
+                .collect(Collectors.toSet());
+        Set<String> resizeIds = analyzeRequest.getManualRedactions().getResizeRedactions()
+                .stream()
+                .map(ManualResizeRedaction::getAnnotationId)
+                .collect(Collectors.toSet());
         allAnnotationIds.addAll(resizeIds);
 
-        List<ManualResizeRedaction> manualResizeRedactions = analyzeRequest.getManualRedactions().getResizeRedactions().stream().toList();
+        List<ManualResizeRedaction> manualResizeRedactions = analyzeRequest.getManualRedactions().getResizeRedactions()
+                .stream()
+                .toList();
         List<PrecursorEntity> manualEntitiesToBeResized = previousEntityLog.getEntityLogEntry()
                 .stream()
                 .filter(entityLogEntry -> resizeIds.contains(entityLogEntry.getId()))
@@ -99,31 +95,36 @@ public class UnprocessedChangesService {
             notFoundManualEntities = entityFromPrecursorCreationService.toTextEntity(manualEntities, document);
         }
 
-        document.getEntities().forEach(textEntity -> {
-            Set<String> processedIds = new HashSet<>();
-            for (var positionsOnPerPage : textEntity.getPositionsOnPagePerPage()) {
-                if (processedIds.contains(positionsOnPerPage.getId())) {
-                    continue;
-                }
-                processedIds.add(positionsOnPerPage.getId());
-                List<Position> positions = positionsOnPerPage.getRectanglePerLine()
-                        .stream()
-                        .map(rectangle2D -> new Position(rectangle2D, positionsOnPerPage.getPage().getNumber()))
-                        .collect(Collectors.toList());
-                unprocessedManualEntities.add(UnprocessedManualEntity.builder()
-                        .annotationId(allAnnotationIds.stream().filter(textEntity::matchesAnnotationId).findFirst().orElse(""))
-                        .textBefore(textEntity.getTextBefore())
-                        .textAfter(textEntity.getTextAfter())
-                        .section(textEntity.getManualOverwrite().getSection().orElse(textEntity.getDeepestFullyContainingNode().toString()))
-                        .positions(positions)
-                        .build());
-            }
-        });
+        document.getEntities()
+                .forEach(textEntity -> {
+                    Set<String> processedIds = new HashSet<>();
+                    for (var positionsOnPerPage : textEntity.getPositionsOnPagePerPage()) {
+                        if (processedIds.contains(positionsOnPerPage.getId())) {
+                            continue;
+                        }
+                        processedIds.add(positionsOnPerPage.getId());
+                        List<Position> positions = positionsOnPerPage.getRectanglePerLine()
+                                .stream()
+                                .map(rectangle2D -> new Position(rectangle2D, positionsOnPerPage.getPage().getNumber()))
+                                .collect(Collectors.toList());
+                        unprocessedManualEntities.add(UnprocessedManualEntity.builder()
+                                                              .annotationId(allAnnotationIds.stream()
+                                                                                    .filter(textEntity::matchesAnnotationId)
+                                                                                    .findFirst()
+                                                                                    .orElse(""))
+                                                              .textBefore(textEntity.getTextBefore())
+                                                              .textAfter(textEntity.getTextAfter())
+                                                              .section(textEntity.getManualOverwrite().getSection()
+                                                                               .orElse(textEntity.getDeepestFullyContainingNode().toString()))
+                                                              .positions(positions)
+                                                              .build());
+                    }
+                });
 
         notFoundManualEntities.forEach(manualEntity -> unprocessedManualEntities.add(builDefaultUnprocessedManualEntity(manualEntity)));
 
         rabbitTemplate.convertAndSend(QueueNames.REDACTION_ANALYSIS_RESPONSE_QUEUE,
-                AnalyzeResponse.builder().fileId(analyzeRequest.getFileId()).unprocessedManualEntities(unprocessedManualEntities).build());
+                                      AnalyzeResponse.builder().fileId(analyzeRequest.getFileId()).unprocessedManualEntities(unprocessedManualEntities).build());
     }
 
 
@@ -143,13 +144,13 @@ public class UnprocessedChangesService {
                 continue;
             }
 
-            TextEntity correctEntity = createCorrectEntity(precursorEntity, optionalTextEntity.get());
+            TextEntity correctEntity = EntityFromPrecursorCreationService.createCorrectEntity(precursorEntity, optionalTextEntity.get());
             Optional<ManualResizeRedaction> optionalManualResizeRedaction = manualResizeRedactions.stream()
                     .filter(manualResizeRedaction -> manualResizeRedaction.getAnnotationId().equals(precursorEntity.getId()))
                     .findFirst();
             if (optionalManualResizeRedaction.isPresent()) {
                 ManualResizeRedaction manualResizeRedaction = optionalManualResizeRedaction.get();
-                manualChangesApplicationService.resizeEntityAndReinsert(correctEntity, manualResizeRedaction);
+                manualChangesApplicationService.resize(correctEntity, manualResizeRedaction);
 
                 // If the entity's value is not the same as the manual resize request's value it means we didn't find it anywhere and we want to remove it
                 // from the graph, so it does not get processed and sent back to persistence-service to update its value.
@@ -160,60 +161,38 @@ public class UnprocessedChangesService {
         }
 
         // remove all temp entities from the graph
-        tempEntities.values().stream().flatMap(Collection::stream).forEach(TextEntity::removeFromGraph);
+        tempEntities.values()
+                .stream()
+                .flatMap(Collection::stream)
+                .forEach(TextEntity::removeFromGraph);
     }
 
 
-    private TextEntity createCorrectEntity(PrecursorEntity precursorEntity, TextEntity closestEntity) {
+    private UnprocessedManualEntity builDefaultUnprocessedManualEntity(PrecursorEntity precursorEntity) {
 
-        TextEntity correctEntity = TextEntity.initialEntityNode(closestEntity.getTextRange(), precursorEntity.type(), precursorEntity.getEntityType(), precursorEntity.getId());
-
-        correctEntity.setDeepestFullyContainingNode(closestEntity.getDeepestFullyContainingNode());
-        correctEntity.setIntersectingNodes(new ArrayList<>(closestEntity.getIntersectingNodes()));
-        correctEntity.setDuplicateTextRanges(new ArrayList<>(closestEntity.getDuplicateTextRanges()));
-        correctEntity.setPages(new HashSet<>(closestEntity.getPages()));
-
-        correctEntity.setValue(closestEntity.getValue());
-        correctEntity.setTextAfter(closestEntity.getTextAfter());
-        correctEntity.setTextBefore(closestEntity.getTextBefore());
-
-        correctEntity.getIntersectingNodes().forEach(n -> n.getEntities().add(correctEntity));
-        correctEntity.getPages().forEach(page -> page.getEntities().add(correctEntity));
-
-    correctEntity.addMatchedRules(precursorEntity.getMatchedRuleList());
-    correctEntity.setDictionaryEntry(precursorEntity.isDictionaryEntry());
-    correctEntity.setDossierDictionaryEntry(precursorEntity.isDossierDictionaryEntry());
-    correctEntity.getManualOverwrite().addChanges(precursorEntity.getManualOverwrite().getManualChangeLog());
-
-    return correctEntity;
-}
+        return UnprocessedManualEntity.builder()
+                .annotationId(precursorEntity.getId())
+                .textAfter("")
+                .textBefore("")
+                .section("")
+                .positions(precursorEntity.getManualOverwrite().getPositions()
+                                   .orElse(precursorEntity.getEntityPosition())
+                                   .stream()
+                                   .map(entityPosition -> new Position(entityPosition.rectangle2D(), entityPosition.pageNumber()))
+                                   .toList())
+                .build();
+    }
 
 
-private UnprocessedManualEntity builDefaultUnprocessedManualEntity(PrecursorEntity precursorEntity) {
+    private List<PrecursorEntity> manualEntitiesConverter(ManualRedactions manualRedactions, String dossierTemplateId) {
 
-    return UnprocessedManualEntity.builder()
-            .annotationId(precursorEntity.getId())
-            .textAfter("")
-            .textBefore("")
-            .section("")
-            .positions(precursorEntity.getManualOverwrite()
-                    .getPositions()
-                    .orElse(precursorEntity.getEntityPosition())
-                    .stream()
-                    .map(entityPosition -> new Position(entityPosition.rectangle2D(), entityPosition.pageNumber()))
-                    .toList())
-            .build();
-}
-
-
-private List<PrecursorEntity> manualEntitiesConverter(ManualRedactions manualRedactions, String dossierTemplateId) {
-
-    return manualRedactions.getEntriesToAdd()
-            .stream()
-            .filter(manualRedactionEntry -> manualRedactionEntry.getPositions() != null && !manualRedactionEntry.getPositions().isEmpty())
-            .map(manualRedactionEntry -> PrecursorEntity.fromManualRedactionEntry(manualRedactionEntry,
-                    dictionaryService.isHint(manualRedactionEntry.getType(), dossierTemplateId)))
-            .toList();
-}
+        return manualRedactions.getEntriesToAdd()
+                .stream()
+                .filter(manualRedactionEntry -> manualRedactionEntry.getPositions() != null && !manualRedactionEntry.getPositions().isEmpty())
+                .filter(BaseAnnotation::isLocal)
+                .map(manualRedactionEntry -> PrecursorEntity.fromManualRedactionEntry(manualRedactionEntry,
+                                                                                      dictionaryService.isHint(manualRedactionEntry.getType(), dossierTemplateId)))
+                .toList();
+    }
 
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/DocumentGraphMapper.java

@@ -6,6 +6,7 @@ import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
 import java.util.NoSuchElementException;
+import java.util.Set;
 
 import com.iqser.red.service.redaction.v1.server.model.document.DocumentTree;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Footer;
@@ -76,6 +77,7 @@ public class DocumentGraphMapper {
                 node.setLeafTextBlock(textBlock);
             }
             List<Integer> treeId = Arrays.stream(entryData.getTreeId()).boxed().toList();
+            entryData.getEngines().forEach(engine -> node.addEngine(engine));
             node.setTreeId(treeId);
 
             switch (entryData.getType()) {

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityFromPrecursorCreationService.java

@@ -51,18 +51,14 @@ public class EntityFromPrecursorCreationService {
         Set<IdRemoval> idRemovals = manualRedactions.getIdsToRemove();
         List<PrecursorEntity> manualEntities = manualRedactions.getEntriesToAdd()
                 .stream()
-                .filter(manualRedactionEntry -> !(idRemovals.stream()
-                        .map(BaseAnnotation::getAnnotationId)
-                        .toList()
-                        .contains(manualRedactionEntry.getAnnotationId()) && manualRedactionEntry.getRequestDate()
-                        .isBefore(idRemovals.stream()
-                                .filter(idRemoval -> idRemoval.getAnnotationId().equals(manualRedactionEntry.getAnnotationId()))
-                                .findFirst()
-                                .get()
-                                .getRequestDate())))
-                .filter(manualRedactionEntry -> !(manualRedactionEntry.isAddToDictionary() || manualRedactionEntry.isAddToDossierDictionary()))
-                .map(manualRedactionEntry -> PrecursorEntity.fromManualRedactionEntry(manualRedactionEntry,
-                        dictionaryService.isHint(manualRedactionEntry.getType(), dossierTemplateId)))
+                .filter(BaseAnnotation::isLocal)
+                .filter(manualRedactionEntry -> idRemovals.stream()
+                        .filter(idRemoval -> idRemoval.getAnnotationId().equals(manualRedactionEntry.getAnnotationId()))
+                        .filter(idRemoval -> idRemoval.getRequestDate().isBefore(manualRedactionEntry.getRequestDate()))
+                        .findAny()//
+                        .isEmpty())
+                .map(manualRedactionEntry -> //
+                             PrecursorEntity.fromManualRedactionEntry(manualRedactionEntry, dictionaryService.isHint(manualRedactionEntry.getType(), dossierTemplateId)))
                 .peek(manualEntity -> {
                     if (manualEntity.getEntityType().equals(EntityType.HINT)) {
                         manualEntity.skip("MAN.5.1", "manual hint is skipped by default");
@@ -71,7 +67,6 @@ public class EntityFromPrecursorCreationService {
                     }
                 })
                 .toList();
-
         return toTextEntity(manualEntities, node);
     }
 
@@ -90,8 +85,14 @@ public class EntityFromPrecursorCreationService {
 
     public List<PrecursorEntity> toTextEntity(List<PrecursorEntity> precursorEntities, SemanticNode node) {
 
-        var notFoundEntities = precursorEntities.stream().filter(PrecursorEntity::isRectangle).collect(Collectors.toList());
-        var findableEntities = precursorEntities.stream().filter(precursorEntity -> !precursorEntity.isRectangle()).toList();
+        var notFoundEntities = precursorEntities.stream()
+                .filter(PrecursorEntity::isRectangle)
+                .collect(Collectors.toList());
+
+        var findableEntities = precursorEntities.stream()
+                .filter(precursorEntity -> !precursorEntity.isRectangle())
+                .toList();
+
         Map<String, List<TextEntity>> tempEntitiesByValue = entityFindingUtility.findAllPossibleEntitiesAndGroupByValue(node, findableEntities);
 
         for (PrecursorEntity precursorEntity : findableEntities) {
@@ -102,7 +103,12 @@ public class EntityFromPrecursorCreationService {
             }
             createCorrectEntity(precursorEntity, optionalClosestEntity.get());
         }
-        tempEntitiesByValue.values().stream().flatMap(Collection::stream).forEach(TextEntity::removeFromGraph);
+
+        tempEntitiesByValue.values()
+                .stream()
+                .flatMap(Collection::stream)
+                .forEach(TextEntity::removeFromGraph);
+
         return notFoundEntities;
     }
 
@@ -113,9 +119,23 @@ public class EntityFromPrecursorCreationService {
      * @param precursorEntity The entity identifier for the RedactionEntity.
      * @param closestEntity   The closest Boundary to the RedactionEntity.
      */
-    private void createCorrectEntity(PrecursorEntity precursorEntity, TextEntity closestEntity) {
+    public static TextEntity createCorrectEntity(PrecursorEntity precursorEntity, TextEntity closestEntity) {
 
-        TextEntity correctEntity = TextEntity.initialEntityNode(closestEntity.getTextRange(), precursorEntity.type(), precursorEntity.getEntityType(), precursorEntity.getId());
+        return createCorrectEntity(precursorEntity, closestEntity, false);
+    }
+
+
+    public static TextEntity createCorrectEntity(PrecursorEntity precursorEntity, TextEntity closestEntity, boolean generateId) {
+
+        TextEntity correctEntity;
+        if (generateId) {
+            correctEntity = TextEntity.initialEntityNode(closestEntity.getTextRange(),
+                                                         precursorEntity.type(),
+                                                         precursorEntity.getEntityType(),
+                                                         closestEntity.getDeepestFullyContainingNode());
+        } else {
+            correctEntity = TextEntity.initialEntityNode(closestEntity.getTextRange(), precursorEntity.type(), precursorEntity.getEntityType(), precursorEntity.getId());
+        }
         correctEntity.setDeepestFullyContainingNode(closestEntity.getDeepestFullyContainingNode());
         correctEntity.setIntersectingNodes(new ArrayList<>(closestEntity.getIntersectingNodes()));
         correctEntity.setDuplicateTextRanges(new ArrayList<>(closestEntity.getDuplicateTextRanges()));
@@ -125,14 +145,17 @@ public class EntityFromPrecursorCreationService {
         correctEntity.setTextAfter(closestEntity.getTextAfter());
         correctEntity.setTextBefore(closestEntity.getTextBefore());
 
-        correctEntity.getIntersectingNodes().forEach(n -> n.getEntities().add(correctEntity));
-        correctEntity.getPages().forEach(page -> page.getEntities().add(correctEntity));
+        correctEntity.getIntersectingNodes()
+                .forEach(n -> n.getEntities().add(correctEntity));
+        correctEntity.getPages()
+                .forEach(page -> page.getEntities().add(correctEntity));
 
         correctEntity.addMatchedRules(precursorEntity.getMatchedRuleList());
         correctEntity.setDictionaryEntry(precursorEntity.isDictionaryEntry());
         correctEntity.setDossierDictionaryEntry(precursorEntity.isDossierDictionaryEntry());
         correctEntity.getManualOverwrite().addChanges(precursorEntity.getManualOverwrite().getManualChangeLog());
         correctEntity.addEngines(precursorEntity.getEngines());
+        return correctEntity;
     }
 
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/SectionFinderService.java

@@ -57,18 +57,23 @@ public class SectionFinderService {
                 if (entry.getContainingNodeId().isEmpty()) {
                     continue; // Empty list means either Entity has not been found or it is between main sections. Thus, this might lead to wrong reanalysis.
                 }
-                sectionsToReanalyse.add(entry.getContainingNodeId().get(0));
+                sectionsToReanalyse.add(entry.getContainingNodeId()
+                                                .get(0));
             }
         }
 
-        var dictionaryIncrementsSearch = new SearchImplementation(dictionaryIncrement.getValues().stream().map(DictionaryIncrementValue::getValue).collect(Collectors.toList()),
-                true);
+        var dictionaryIncrementsSearch = new SearchImplementation(dictionaryIncrement.getValues()
+                                                                          .stream()
+                                                                          .map(DictionaryIncrementValue::getValue)
+                                                                          .toList(), true);
 
-        document.streamChildren().forEach(mainNode -> {
-            if (dictionaryIncrementsSearch.atLeastOneMatches(mainNode.getTextBlock().getSearchText())) {
-                sectionsToReanalyse.add(mainNode.getTreeId().get(0));
-            }
-        });
+        document.streamChildren()
+                .forEach(mainNode -> {
+                    if (dictionaryIncrementsSearch.atLeastOneMatches(mainNode.getTextBlock().getSearchText())) {
+                        sectionsToReanalyse.add(mainNode.getTreeId()
+                                                        .get(0));
+                    }
+                });
 
         Set<Integer> relevantPagesForReanalysis = getRelevantPageNumbersForAddRedactions(analyzeRequest);
         relevantPagesForReanalysis.addAll(getRelevantPageNumbersForImportedRedactions(importedRedactions, relevantManuallyModifiedAnnotationIds));
@@ -87,8 +92,11 @@ public class SectionFinderService {
         return document.getPages()
                 .stream()
                 .filter(page -> relevantPagesForReanalysis.contains(page.getNumber()))
-                .flatMap(page -> Stream.concat(page.getMainBody().stream().filter(node -> node.getType().equals(NodeType.SECTION)), Stream.of(page.getHeader(), page.getFooter())))
-                .map(node -> node.getTreeId().get(0))
+                .flatMap(page -> Stream.concat(page.getMainBody()
+                                                       .stream()
+                                                       .filter(node -> node.getType().equals(NodeType.SECTION)), Stream.of(page.getHeader(), page.getFooter())))
+                .map(node -> node.getTreeId()
+                        .get(0))
                 .toList();
     }
 
@@ -99,8 +107,7 @@ public class SectionFinderService {
             return Collections.emptySet();
         }
 
-        return analyzeRequest.getManualRedactions()
-                .getEntriesToAdd()
+        return analyzeRequest.getManualRedactions().getEntriesToAdd()
                 .stream()
                 .filter(addRedaction -> !(addRedaction.isAddToDictionary() || addRedaction.isAddToDossierDictionary()))
                 .map(ManualRedactionEntry::getPositions)
@@ -132,11 +139,22 @@ public class SectionFinderService {
             return new HashSet<>();
         }
 
-        return Stream.concat(manualRedactions.getResizeRedactions().stream().map(ManualResizeRedaction::getAnnotationId),
-                Stream.concat(manualRedactions.getLegalBasisChanges().stream().map(ManualLegalBasisChange::getAnnotationId),
-                        Stream.concat(manualRedactions.getRecategorizations().stream().map(ManualRecategorization::getAnnotationId),
-                                Stream.concat(manualRedactions.getIdsToRemove().stream().map(IdRemoval::getAnnotationId),
-                                        manualRedactions.getForceRedactions().stream().map(ManualForceRedaction::getAnnotationId))))).collect(Collectors.toSet());
+        return Stream.concat(manualRedactions.getResizeRedactions()
+                                     .stream()
+                                     .map(ManualResizeRedaction::getAnnotationId),
+                             Stream.concat(manualRedactions.getLegalBasisChanges()
+                                                   .stream()
+                                                   .map(ManualLegalBasisChange::getAnnotationId),
+                                           Stream.concat(manualRedactions.getRecategorizations()
+                                                                 .stream()
+                                                                 .map(ManualRecategorization::getAnnotationId),
+                                                         Stream.concat(manualRedactions.getIdsToRemove()
+                                                                               .stream()
+                                                                               .map(IdRemoval::getAnnotationId),
+                                                                       manualRedactions.getForceRedactions()
+                                                                               .stream()
+                                                                               .map(ManualForceRedaction::getAnnotationId)))))
+                .collect(Collectors.toSet());
     }
 
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/DroolsSyntaxValidationService.java

@@ -1,6 +1,9 @@
 package com.iqser.red.service.redaction.v1.server.service.drools;
 
+import java.util.ArrayList;
+import java.util.Comparator;
 import java.util.List;
+import java.util.Map;
 import java.util.Set;
 import java.util.regex.Pattern;
 import java.util.stream.Collectors;
@@ -12,24 +15,34 @@ import org.springframework.stereotype.Service;
 
 import com.google.common.collect.Sets;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType;
+import com.iqser.red.service.redaction.v1.model.DroolsSyntaxDeprecatedWarnings;
 import com.iqser.red.service.redaction.v1.model.DroolsSyntaxErrorMessage;
 import com.iqser.red.service.redaction.v1.model.DroolsSyntaxValidation;
 import com.iqser.red.service.redaction.v1.model.RuleValidationModel;
+import com.iqser.red.service.redaction.v1.server.DeprecatedElementsFinder;
+import com.iqser.red.service.redaction.v1.server.model.dictionary.SearchImplementation;
 import com.iqser.red.service.redaction.v1.server.model.drools.BasicQuery;
+import com.iqser.red.service.redaction.v1.server.model.drools.BasicRule;
+import com.iqser.red.service.redaction.v1.server.model.drools.RuleClass;
 import com.iqser.red.service.redaction.v1.server.model.drools.RuleFileBluePrint;
+import com.iqser.red.service.redaction.v1.server.model.drools.RuleUnit;
 import com.iqser.red.service.redaction.v1.server.storage.RuleManagementResources;
 
 import lombok.RequiredArgsConstructor;
 import lombok.SneakyThrows;
+import lombok.extern.slf4j.Slf4j;
 
 @Service
 @RequiredArgsConstructor
+@Slf4j
 public class DroolsSyntaxValidationService {
-    
+
     private static final Pattern allowedImportsPattern = Pattern.compile("^(?:import\\s+static\\s+)?(?:import\\s+)?(?:com\\.knecon\\.fforesight|com\\.iqser\\.red)\\..*;$");
 
     private final KieContainerCreationService kieContainerCreationService;
 
+    private final DeprecatedElementsFinder deprecatedElementsFinder;
+
 
     @SneakyThrows
     public DroolsSyntaxValidation testRules(RuleValidationModel rules) {
@@ -43,15 +56,74 @@ public class DroolsSyntaxValidationService {
         }
         DroolsSyntaxValidation droolsCompilerSyntaxValidation = buildDroolsCompilerSyntaxValidation(rules);
         droolsCompilerSyntaxValidation.getDroolsSyntaxErrorMessages().addAll(customDroolsSyntaxValidation.getDroolsSyntaxErrorMessages());
+        droolsCompilerSyntaxValidation.getDroolsSyntaxDeprecatedWarnings().addAll(customDroolsSyntaxValidation.getDroolsSyntaxDeprecatedWarnings());
         return droolsCompilerSyntaxValidation;
     }
 
+    private DroolsSyntaxDeprecatedWarnings getWarningsForDeprecatedImports(RuleFileBluePrint ruleFileBluePrint) {
+
+        if (!deprecatedElementsFinder.getDeprecatedClasses().isEmpty()) {
+            String imports = ruleFileBluePrint.getImports();
+            SearchImplementation classesSearchImplementation = deprecatedElementsFinder.getClassesSearchImplementation();
+            List<SearchImplementation.MatchPosition> matches = classesSearchImplementation.getMatches(imports);
+            if (!matches.isEmpty()) {
+                String sb = "Following imports are deprecated: \n" + matches.stream()
+                        .map(m -> imports.substring(m.startIndex(), m.endIndex()))
+                        .collect(Collectors.joining("\n"));
+                return DroolsSyntaxDeprecatedWarnings.builder().line(ruleFileBluePrint.getImportLine()).column(0).message(sb)
+                                            .build();
+            }
+        }
+        return null;
+    }
+
+    private List<DroolsSyntaxDeprecatedWarnings> getWarningsForDeprecatedRules(RuleFileBluePrint ruleFileBluePrint) {
+
+        List<DroolsSyntaxDeprecatedWarnings> warningMessages = new ArrayList<>();
+
+        // checks the rules for deprecateMethods
+        if (!deprecatedElementsFinder.getDeprecatedMethods().isEmpty()) {
+            SearchImplementation methodsSearchImplementation = deprecatedElementsFinder.getMethodsSearchImplementation();
+            Map<String, String> deprecatedMethodsSignatureMap = deprecatedElementsFinder.getDeprecatedMethodsSignaturesMap();
+
+            for (RuleClass ruleClass : ruleFileBluePrint.getRuleClasses()) {
+                for (RuleUnit ruleUnit : ruleClass.ruleUnits()) {
+                    for (BasicRule basicRule : ruleUnit.rules()) {
+                        List<SearchImplementation.MatchPosition> matches = methodsSearchImplementation.getMatches(basicRule.getCode());
+                        if (!matches.isEmpty()) {
+                            String warningMessage = matches.stream()
+                                    .map(m -> basicRule.getCode().substring(m.startIndex(), m.endIndex()))
+                                    .distinct()
+                                    .map(dm -> String.format("Method %s might be deprecated because of \n %s \n", dm, deprecatedMethodsSignatureMap.get(dm)))
+                                    .collect(Collectors.joining("\n"));
+                            warningMessages.add(DroolsSyntaxDeprecatedWarnings.builder().line(basicRule.getLine()).column(0).message(warningMessage)
+                                                        .build());
+                        }
+                    }
+                }
+            }
+
+        }
+
+        return warningMessages.stream()
+                .sorted(Comparator.comparingInt(DroolsSyntaxDeprecatedWarnings::getLine))
+                .collect(Collectors.toList());
+    }
+
 
     private DroolsSyntaxValidation buildCustomDroolsSyntaxValidation(String ruleString, RuleFileType ruleFileType) throws DroolsParserException {
 
         RuleFileBluePrint ruleFileBluePrint = RuleFileParser.buildBluePrintFromRulesString(ruleString);
+
         DroolsSyntaxValidation customSyntaxValidation = ruleFileBluePrint.getDroolsSyntaxValidation();
 
+        // find deprecated elements in the ruleFileBluePrint
+        DroolsSyntaxDeprecatedWarnings warningMessageForImports = getWarningsForDeprecatedImports(ruleFileBluePrint);
+        if (warningMessageForImports != null) {
+            customSyntaxValidation.getDroolsSyntaxDeprecatedWarnings().add(warningMessageForImports);
+        }
+        customSyntaxValidation.getDroolsSyntaxDeprecatedWarnings().addAll(getWarningsForDeprecatedRules(ruleFileBluePrint));
+
         RuleFileBluePrint baseRuleFileBluePrint = switch (ruleFileType) {
             case ENTITY -> RuleFileParser.buildBluePrintFromRulesString(RuleManagementResources.getBaseRuleFileString());
             case COMPONENT -> RuleFileParser.buildBluePrintFromRulesString(RuleManagementResources.getBaseComponentRuleFileString());
@@ -60,35 +132,36 @@ public class DroolsSyntaxValidationService {
         if (!importsAreValid(baseRuleFileBluePrint, ruleFileBluePrint)) {
             customSyntaxValidation.getDroolsSyntaxErrorMessages()
                     .add(DroolsSyntaxErrorMessage.builder()
-                            .line(ruleFileBluePrint.getImportLine())
-                            .column(0)
-                            .message(String.format("Changing the imports is not allowed! Must be: %n%s", baseRuleFileBluePrint.getImports()))
-                            .build());
+                                 .line(ruleFileBluePrint.getImportLine())
+                                 .column(0)
+                                 .message(String.format("Changing the imports is not allowed! Must be: %n%s", baseRuleFileBluePrint.getImports()))
+                                 .build());
         }
         if (!ruleFileBluePrint.getGlobals().equals(baseRuleFileBluePrint.getGlobals())) {
             customSyntaxValidation.getDroolsSyntaxErrorMessages()
                     .add(DroolsSyntaxErrorMessage.builder()
-                            .line(ruleFileBluePrint.getGlobalsLine())
-                            .column(0)
-                            .message(String.format("Changing the globals is not allowed! Must be: %n%s", baseRuleFileBluePrint.getGlobals()))
-                            .build());
+                                 .line(ruleFileBluePrint.getGlobalsLine())
+                                 .column(0)
+                                 .message(String.format("Changing the globals is not allowed! Must be: %n%s", baseRuleFileBluePrint.getGlobals()))
+                                 .build());
         }
-        baseRuleFileBluePrint.getQueries().forEach(basicQuery -> {
-            if (!validateQueryIsPresent(basicQuery, ruleFileBluePrint)) {
-                customSyntaxValidation.getDroolsSyntaxErrorMessages()
-                        .add(DroolsSyntaxErrorMessage.builder()
-                                .line(basicQuery.getLine())
-                                .column(0)
-                                .message(String.format("Changing or removing the query %s is not allowed! Must be: %n%s", basicQuery.getName(), basicQuery.getCode()))
-                                .build());
-            }
-        });
+        baseRuleFileBluePrint.getQueries()
+                .forEach(basicQuery -> {
+                    if (!validateQueryIsPresent(basicQuery, ruleFileBluePrint)) {
+                        customSyntaxValidation.getDroolsSyntaxErrorMessages()
+                                .add(DroolsSyntaxErrorMessage.builder()
+                                             .line(basicQuery.getLine())
+                                             .column(0)
+                                             .message(String.format("Changing or removing the query %s is not allowed! Must be: %n%s", basicQuery.getName(), basicQuery.getCode()))
+                                             .build());
+                    }
+                });
         if (ruleFileType.equals(RuleFileType.ENTITY)) {
             String requiredAgendaGroup = "LOCAL_DICTIONARY_ADDS";
             if (!validateAgendaGroupIsPresent(ruleFileBluePrint, requiredAgendaGroup)) {
-                customSyntaxValidation.getDroolsSyntaxErrorMessages().add(DroolsSyntaxErrorMessage.builder().line(0)
-                                .column(0).message(String.format("At least one rule with Agenda-Group '%s' required!", requiredAgendaGroup))
-                                .build());
+                customSyntaxValidation.getDroolsSyntaxErrorMessages()
+                        .add(DroolsSyntaxErrorMessage.builder().line(0).column(0).message(String.format("At least one rule with Agenda-Group '%s' required!", requiredAgendaGroup))
+                                     .build());
             }
         }
         return customSyntaxValidation;
@@ -97,7 +170,8 @@ public class DroolsSyntaxValidationService {
 
     private boolean validateAgendaGroupIsPresent(RuleFileBluePrint ruleFileBluePrint, String agendaGroupName) {
 
-        return ruleFileBluePrint.streamAllRules().anyMatch(basicRule -> basicRule.getAgendaGroup().equals(agendaGroupName));
+        return ruleFileBluePrint.streamAllRules()
+                .anyMatch(basicRule -> basicRule.getAgendaGroup().equals(agendaGroupName));
     }
 
 
@@ -109,13 +183,16 @@ public class DroolsSyntaxValidationService {
 
         Set<String> additionalImports = Sets.difference(imports, baseImports);
 
-        return additionalImports.stream().allMatch(additionalImport -> allowedImportsPattern.matcher(additionalImport).matches());
+        return additionalImports.stream()
+                .allMatch(additionalImport -> allowedImportsPattern.matcher(additionalImport).matches());
     }
 
 
     private static boolean validateQueryIsPresent(BasicQuery queryToCheckFor, RuleFileBluePrint ruleFileBluePrint) {
 
-        return ruleFileBluePrint.getQueries().stream().anyMatch(query -> query.getName().equals(queryToCheckFor.getName()) && query.getCode().equals(queryToCheckFor.getCode()));
+        return ruleFileBluePrint.getQueries()
+                .stream()
+                .anyMatch(query -> query.getName().equals(queryToCheckFor.getName()) && query.getCode().equals(queryToCheckFor.getCode()));
     }
 
 
@@ -124,9 +201,9 @@ public class DroolsSyntaxValidationService {
         var versionId = System.currentTimeMillis();
         var testRules = "test-rules";
         KieBuilder kieBuilder = kieContainerCreationService.registerNewKieContainerVersion(testRules,
-                versionId,
-                rules.getRulesString(),
-                RuleFileType.valueOf(rules.getRuleFileType()));
+                                                                                           versionId,
+                                                                                           rules.getRulesString(),
+                                                                                           RuleFileType.valueOf(rules.getRuleFileType()));
         return buildDroolsCompilerSyntaxValidation(kieBuilder);
     }
 
@@ -134,14 +211,18 @@ public class DroolsSyntaxValidationService {
     private DroolsSyntaxValidation buildDroolsCompilerSyntaxValidation(KieBuilder kieBuilder) {
 
         List<Message> errorMessages = kieBuilder.getResults().getMessages(Message.Level.ERROR);
-        List<DroolsSyntaxErrorMessage> droolsSyntaxErrorMessages = errorMessages.stream().map(this::buildDroolsSyntaxErrorMessage).collect(Collectors.toList());
-        return DroolsSyntaxValidation.builder().droolsSyntaxErrorMessages(droolsSyntaxErrorMessages).build();
+        List<DroolsSyntaxErrorMessage> droolsSyntaxErrorMessages = errorMessages.stream()
+                .map(this::buildDroolsSyntaxErrorMessage)
+                .collect(Collectors.toList());
+        return DroolsSyntaxValidation.builder().droolsSyntaxErrorMessages(droolsSyntaxErrorMessages)
+                .build();
     }
 
 
     private DroolsSyntaxErrorMessage buildDroolsSyntaxErrorMessage(Message message) {
 
-        return DroolsSyntaxErrorMessage.builder().line(message.getLine()).column(message.getColumn()).message(message.getText()).build();
+        return DroolsSyntaxErrorMessage.builder().line(message.getLine()).column(message.getColumn()).message(message.getText())
+                .build();
     }
 
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/EntityDroolsExecutionService.java

@@ -16,6 +16,7 @@ import org.springframework.stereotype.Service;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.BaseAnnotation;
 import com.iqser.red.service.redaction.v1.server.RedactionServiceSettings;
 import com.iqser.red.service.redaction.v1.server.model.NerEntities;
 import com.iqser.red.service.redaction.v1.server.model.dictionary.Dictionary;
@@ -55,7 +56,14 @@ public class EntityDroolsExecutionService {
                                             ManualRedactions manualRedactions,
                                             NerEntities nerEntities) {
 
-        return executeRules(kieContainer, document, document.streamChildren().toList(), dictionary, fileAttributes, manualRedactions, nerEntities);
+        return executeRules(kieContainer,
+                            document,
+                            document.streamChildren()
+                                    .toList(),
+                            dictionary,
+                            fileAttributes,
+                            manualRedactions,
+                            nerEntities);
     }
 
 
@@ -80,19 +88,28 @@ public class EntityDroolsExecutionService {
         kieSession.setGlobal("dictionary", dictionary);
 
         kieSession.insert(document);
-        document.getEntities().forEach(kieSession::insert);
+
+        document.getEntities()
+                .forEach(kieSession::insert);
+
         sectionsToAnalyze.forEach(kieSession::insert);
-        sectionsToAnalyze.stream().flatMap(SemanticNode::streamAllSubNodes).forEach(kieSession::insert);
-        document.getPages().forEach(kieSession::insert);
-        fileAttributes.stream().filter(f -> f.getValue() != null).forEach(kieSession::insert);
+
+        sectionsToAnalyze.stream()
+                .flatMap(SemanticNode::streamAllSubNodes)
+                .forEach(kieSession::insert);
+
+        document.getPages()
+                .forEach(kieSession::insert);
+
+        fileAttributes.stream()
+                .filter(f -> f.getValue() != null)
+                .forEach(kieSession::insert);
 
         if (manualRedactions != null) {
-            manualRedactions.getResizeRedactions().forEach(kieSession::insert);
-            manualRedactions.getRecategorizations().forEach(kieSession::insert);
-            manualRedactions.getEntriesToAdd().forEach(kieSession::insert);
-            manualRedactions.getForceRedactions().forEach(kieSession::insert);
-            manualRedactions.getIdsToRemove().forEach(kieSession::insert);
-            manualRedactions.getLegalBasisChanges().forEach(kieSession::insert);
+            manualRedactions.buildAll()
+                    .stream()
+                    .filter(BaseAnnotation::isLocal)
+                    .forEach(kieSession::insert);
         }
 
         kieSession.insert(nerEntities);
@@ -105,7 +122,8 @@ public class EntityDroolsExecutionService {
         });
 
         try {
-            completableFuture.orTimeout(settings.getDroolsExecutionTimeoutSecs(), TimeUnit.SECONDS).get();
+            completableFuture.orTimeout(settings.getDroolsExecutionTimeoutSecs(), TimeUnit.SECONDS)
+                    .get();
         } catch (ExecutionException e) {
             kieSession.dispose();
             if (e.getCause() instanceof TimeoutException) {

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/utils/MigratedIdsCollector.java

@@ -1,5 +1,6 @@
 package com.iqser.red.service.redaction.v1.server.utils;
 
+import java.util.Collections;
 import java.util.LinkedList;
 import java.util.Set;
 import java.util.function.BiConsumer;
@@ -17,7 +18,7 @@ public class MigratedIdsCollector implements Collector<MigrationEntity, Migrated
     @Override
     public Supplier<MigratedIds> supplier() {
 
-        return () -> new MigratedIds(new LinkedList<>());
+        return () -> new MigratedIds(new LinkedList<>(), Collections.emptyList());
     }
 
 

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/MigrationIntegrationTest.java

@@ -31,6 +31,7 @@ import org.springframework.test.context.junit.jupiter.SpringExtension;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.migration.MigratedIds;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
@@ -49,7 +50,6 @@ import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.OsUtils;
 import com.iqser.red.service.redaction.v1.server.service.DictionaryService;
 import com.iqser.red.service.redaction.v1.server.service.document.EntityFindingUtility;
-import com.iqser.red.service.redaction.v1.server.utils.RectangleTransformations;
 import com.knecon.fforesight.tenantcommons.TenantContext;
 
 import lombok.SneakyThrows;
@@ -107,7 +107,7 @@ public class MigrationIntegrationTest extends BuildDocumentIntegrationTest {
     @SneakyThrows
     public void testSave() {
 
-        MigratedIds ids = new MigratedIds(new LinkedList<>());
+        MigratedIds ids = new MigratedIds(new LinkedList<>(), null);
         ids.addMapping("123", "321");
         ids.addMapping("123", "321");
         ids.addMapping("123", "321");
@@ -173,7 +173,11 @@ public class MigrationIntegrationTest extends BuildDocumentIntegrationTest {
             mergedRedactionLog = redactionLog;
         }
 
-        MigratedEntityLog migratedEntityLog = redactionLogToEntityLogMigrationService.migrate(mergedRedactionLog, document, TEST_DOSSIER_TEMPLATE_ID, manualRedactions);
+        MigratedEntityLog migratedEntityLog = redactionLogToEntityLogMigrationService.migrate(mergedRedactionLog,
+                                                                                              document,
+                                                                                              TEST_DOSSIER_TEMPLATE_ID,
+                                                                                              manualRedactions,
+                                                                                              TEST_FILE_ID);
 
         redactionStorageService.storeObject(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.ENTITY_LOG, migratedEntityLog.getEntityLog());
         assertEquals(mergedRedactionLog.getRedactionLogEntry().size(), migratedEntityLog.getEntityLog().getEntityLogEntry().size());
@@ -187,10 +191,11 @@ public class MigrationIntegrationTest extends BuildDocumentIntegrationTest {
         assertEquals(mergedRedactionLog.getLegalBasis().size(), entityLog.getLegalBasis().size());
 
         Map<String, String> migratedIds = migratedEntityLog.getMigratedIds().buildOldToNewMapping();
+//        assertEquals(legacyRedactionLogMergeService.getNumberOfAffectedAnnotations(manualRedactions), migratedIds.size());
+
         migratedIds.forEach((oldId, newId) -> assertEntryIsEqual(oldId, newId, mergedRedactionLog, entityLog, migratedIds));
 
-        AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder().dossierId(TEST_DOSSIER_ID).fileId(TEST_FILE_ID)
-                                                                               .build());
+        AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder().dossierId(TEST_DOSSIER_ID).fileId(TEST_FILE_ID).build());
         File outputFile = Path.of(OsUtils.getTemporaryDirectory()).resolve(Path.of(fileName.replaceAll(".pdf", "_MIGRATED.pdf")).getFileName()).toFile();
 
         try (FileOutputStream fileOutputStream = new FileOutputStream(outputFile)) {
@@ -268,13 +273,24 @@ public class MigrationIntegrationTest extends BuildDocumentIntegrationTest {
         if (!redactionLogEntry.isImage()) {
             assertEquals(redactionLogEntry.getValue().toLowerCase(Locale.ENGLISH), entityLogEntry.getValue().toLowerCase(Locale.ENGLISH));
         }
+        if (entityLogEntry.getManualChanges()
+                .stream()
+                .noneMatch(mc -> mc.getManualRedactionType().equals(ManualRedactionType.RECATEGORIZE))) {
+            assertEquals(redactionLogEntry.getType(), entityLogEntry.getType());
+        }
         assertEquals(redactionLogEntry.getChanges().size(), entityLogEntry.getChanges().size());
         assertTrue(redactionLogEntry.getManualChanges().size() <= entityLogEntry.getManualChanges().size());
         assertEquals(redactionLogEntry.getPositions().size(), entityLogEntry.getPositions().size());
-        assertTrue(positionsAlmostEqual(redactionLogEntry.getPositions(), entityLogEntry.getPositions()));
-//        assertEquals(redactionLogEntry.getColor(), entityLogEntry.getColor());
-        assertEqualsNullSafe(redactionLogEntry.getLegalBasis(), entityLogEntry.getLegalBasis());
-//        assertEqualsNullSafe(redactionLogEntry.getReason(), entityLogEntry.getReason());
+        if (entityLogEntry.getManualChanges()
+                .stream()
+                .noneMatch(mc -> mc.getManualRedactionType().equals(ManualRedactionType.RESIZE) || mc.getManualRedactionType().equals(ManualRedactionType.RESIZE_IN_DICTIONARY))) {
+            assertTrue(positionsAlmostEqual(redactionLogEntry.getPositions(), entityLogEntry.getPositions()));
+        }
+        if (entityLogEntry.getManualChanges()
+                .stream()
+                .noneMatch(mc -> mc.getManualRedactionType().equals(ManualRedactionType.FORCE))) {
+            assertEqualsNullSafe(redactionLogEntry.getLegalBasis(), entityLogEntry.getLegalBasis());
+        }
         assertReferencesEqual(redactionLogEntry.getReference(), entityLogEntry.getReference(), oldToNewMapping);
         assertEquals(redactionLogEntry.isDictionaryEntry(), entityLogEntry.isDictionaryEntry());
         assertEquals(redactionLogEntry.isDossierDictionaryEntry(), entityLogEntry.isDossierDictionaryEntry());

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java

@@ -119,15 +119,15 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
 
         when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L);
         when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, true)).thenReturn(List.of(Type.builder()
-                .id(DOSSIER_REDACTIONS_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID)
-                .type(DOSSIER_REDACTIONS_INDICATOR)
-                .dossierTemplateId(TEST_DOSSIER_ID)
-                .hexColor("#ffe187")
-                .isHint(hintTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
-                .isCaseInsensitive(caseInSensitiveMap.get(DOSSIER_REDACTIONS_INDICATOR))
-                .isRecommendation(recommendationTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
-                .rank(rankTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
-                .build()));
+                                                                                                       .id(DOSSIER_REDACTIONS_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID)
+                                                                                                       .type(DOSSIER_REDACTIONS_INDICATOR)
+                                                                                                       .dossierTemplateId(TEST_DOSSIER_ID)
+                                                                                                       .hexColor("#ffe187")
+                                                                                                       .isHint(hintTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
+                                                                                                       .isCaseInsensitive(caseInSensitiveMap.get(DOSSIER_REDACTIONS_INDICATOR))
+                                                                                                       .isRecommendation(recommendationTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
+                                                                                                       .rank(rankTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
+                                                                                                       .build()));
 
         mockDictionaryCalls(null);
 
@@ -169,9 +169,10 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
 
         var entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
 
-        entityLog.getEntityLogEntry().forEach(entry -> {
-            duplicates.computeIfAbsent(entry.getId(), v -> new ArrayList<>()).add(entry);
-        });
+        entityLog.getEntityLogEntry()
+                .forEach(entry -> {
+                    duplicates.computeIfAbsent(entry.getId(), v -> new ArrayList<>()).add(entry);
+                });
 
         duplicates.forEach((key, value) -> assertThat(value.size()).isEqualTo(1));
 
@@ -216,12 +217,12 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
 
         ManualRedactions manualRedactions = ManualRedactions.builder()
                 .resizeRedactions(Set.of(ManualResizeRedaction.builder()
-                        .annotationId("c6be5277f5ee60dc3d83527798b7fe02")
-                        .value("Dr. Alan")
-                        .positions(List.of(new Rectangle(236.8f, 182.90005f, 40.584f, 12.642f, 7)))
-                        .requestDate(OffsetDateTime.now())
-                        .updateDictionary(false)
-                        .build()))
+                                                 .annotationId("c6be5277f5ee60dc3d83527798b7fe02")
+                                                 .value("Dr. Alan")
+                                                 .positions(List.of(new Rectangle(236.8f, 182.90005f, 40.584f, 12.642f, 7)))
+                                                 .requestDate(OffsetDateTime.now())
+                                                 .updateDictionary(false)
+                                                 .build()))
                 .build();
 
         request.setManualRedactions(manualRedactions);
@@ -256,7 +257,10 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
 
         var redactionLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
 
-        var values = redactionLog.getEntityLogEntry().stream().map(EntityLogEntry::getValue).collect(Collectors.toList());
+        var values = redactionLog.getEntityLogEntry()
+                .stream()
+                .map(EntityLogEntry::getValue)
+                .collect(Collectors.toList());
 
         assertThat(values).containsExactlyInAnyOrder("Lastname M.", "Doe", "Doe J.", "M. Mustermann", "Mustermann M.", "F. Lastname");
     }
@@ -268,8 +272,8 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
         ClassPathResource importedRedactionClasspathResource = new ClassPathResource(
                 "files/ImportedRedactions/18 Chlorothalonil RAR 08 Volume 3CA B 6a Oct 2017.IMPORTED_REDACTIONS.json");
         storageService.storeObject(TenantContext.getTenantId(),
-                RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.IMPORTED_REDACTIONS),
-                importedRedactionClasspathResource.getInputStream());
+                                   RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.IMPORTED_REDACTIONS),
+                                   importedRedactionClasspathResource.getInputStream());
 
         AnalyzeRequest request = uploadFileToStorage("files/ImportedRedactions/18 Chlorothalonil RAR 08 Volume 3CA B 6a Oct 2017.pdf");
         System.out.println("Start Full integration test");
@@ -353,10 +357,18 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
 
         var mergedEntityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
 
-        var cbiAddressBeforeHintRemoval = entityLog.getEntityLogEntry().stream().filter(re -> re.getType().equalsIgnoreCase("CBI_Address")).findAny().get();
+        var cbiAddressBeforeHintRemoval = entityLog.getEntityLogEntry()
+                .stream()
+                .filter(re -> re.getType().equalsIgnoreCase("CBI_Address"))
+                .findAny()
+                .get();
         assertThat(cbiAddressBeforeHintRemoval.getState().equals(EntryState.APPLIED)).isFalse();
 
-        var cbiAddressAfterHintRemoval = mergedEntityLog.getEntityLogEntry().stream().filter(re -> re.getType().equalsIgnoreCase("CBI_Address")).findAny().get();
+        var cbiAddressAfterHintRemoval = mergedEntityLog.getEntityLogEntry()
+                .stream()
+                .filter(re -> re.getType().equalsIgnoreCase("CBI_Address"))
+                .findAny()
+                .get();
         assertThat(cbiAddressAfterHintRemoval.getState().equals(EntryState.APPLIED)).isTrue();
     }
 
@@ -386,9 +398,10 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
 
                     var entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
 
-                    entityLog.getEntityLogEntry().forEach(entry -> {
-                        duplicates.computeIfAbsent(entry.getId(), v -> new ArrayList<>()).add(entry);
-                    });
+                    entityLog.getEntityLogEntry()
+                            .forEach(entry -> {
+                                duplicates.computeIfAbsent(entry.getId(), v -> new ArrayList<>()).add(entry);
+                            });
 
                     duplicates.forEach((id, redactionLogEntries) -> assertThat(redactionLogEntries.size()).isEqualTo(1));
 
@@ -421,11 +434,11 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
         AnalyzeRequest request = uploadFileToStorage(fileName);
 
         request.setFileAttributes(List.of(FileAttribute.builder()
-                .id("fileAttributeId")
-                .label("Vertebrate Study")
-                .placeholder("{fileattributes.vertebrateStudy}")
-                .value("true")
-                .build()));
+                                                  .id("fileAttributeId")
+                                                  .label("Vertebrate Study")
+                                                  .placeholder("{fileattributes.vertebrateStudy}")
+                                                  .value("true")
+                                                  .build()));
 
         analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request);
         AnalyzeResult result = analyzeService.analyze(request);
@@ -449,7 +462,10 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
                     correctFound++;
                     continue loop;
                 }
-                if (Objects.equals(entityLogEntry.getContainingNodeId().get(0), section.getTreeId().get(0))) {
+                if (Objects.equals(entityLogEntry.getContainingNodeId()
+                                           .get(0),
+                                   section.getTreeId()
+                                           .get(0))) {
                     String value = section.getTextBlock().subSequence(new TextRange(entityLogEntry.getStartOffset(), entityLogEntry.getEndOffset())).toString();
                     if (entityLogEntry.getValue().equalsIgnoreCase(value)) {
                         correctFound++;
@@ -481,12 +497,12 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
         ManualRedactions manualRedactions = new ManualRedactions();
 
         manualRedactions.setEntriesToAdd(Set.of(ManualRedactionEntry.builder()
-                .value("Redact")
-                .addToDictionary(true)
-                .addToDossierDictionary(true)
-                .positions(List.of(new Rectangle(new Point(95.96979999999999f, 515.7984f), 19.866899999999987f, 46.953f, 2)))
-                .type("dossier_redaction")
-                .build()));
+                                                        .value("Redact")
+                                                        .addToDictionary(true)
+                                                        .addToDossierDictionary(true)
+                                                        .positions(List.of(new Rectangle(new Point(95.96979999999999f, 515.7984f), 19.866899999999987f, 46.953f, 2)))
+                                                        .type("dossier_redaction")
+                                                        .build()));
 
         request.setManualRedactions(manualRedactions);
 
@@ -548,7 +564,11 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
 
         var entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
 
-        var changes = entityLog.getEntityLogEntry().stream().filter(entry -> entry.getValue() != null && entry.getValue().equals("report")).findFirst().get().getChanges();
+        var changes = entityLog.getEntityLogEntry()
+                .stream()
+                .filter(entry -> entry.getValue() != null && entry.getValue().equals("report"))
+                .findFirst()
+                .get().getChanges();
 
         assertThat(changes.size()).isEqualTo(2);
 
@@ -568,18 +588,18 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
 
         ClassPathResource responseJson = new ClassPathResource("files/crafted_document.NER_ENTITIES.json");
         storageService.storeObject(TenantContext.getTenantId(),
-                RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.NER_ENTITIES),
-                responseJson.getInputStream());
+                                   RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.NER_ENTITIES),
+                                   responseJson.getInputStream());
 
         long start = System.currentTimeMillis();
         AnalyzeRequest request = uploadFileToStorage(fileName);
 
         request.setFileAttributes(List.of(FileAttribute.builder()
-                .id("fileAttributeId")
-                .label("Vertebrate Study")
-                .placeholder("{fileattributes.vertebrateStudy}")
-                .value("true")
-                .build()));
+                                                  .id("fileAttributeId")
+                                                  .label("Vertebrate Study")
+                                                  .placeholder("{fileattributes.vertebrateStudy}")
+                                                  .value("true")
+                                                  .build()));
 
         analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request);
         AnalyzeResult result = analyzeService.analyze(request);
@@ -601,7 +621,11 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
                 .map(redactionLogEntry -> new TextRange(redactionLogEntry.getStartOffset(), redactionLogEntry.getEndOffset()))
                 .map(boundary -> documentGraph.getTextBlock().subSequence(boundary).toString())
                 .toList();
-        List<String> valuesInRedactionLog = entityLog.getEntityLogEntry().stream().filter(e -> !e.getEntryType().equals(EntryType.IMAGE)).map(EntityLogEntry::getValue).toList();
+        List<String> valuesInRedactionLog = entityLog.getEntityLogEntry()
+                .stream()
+                .filter(e -> !e.getEntryType().equals(EntryType.IMAGE))
+                .map(EntityLogEntry::getValue)
+                .toList();
 
         assertEquals(valuesInRedactionLog, valuesInDocument);
 
@@ -628,11 +652,11 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
         ManualRedactions manualRedactions = new ManualRedactions();
 
         manualRedactions.setRecategorizations(Set.of(ManualRecategorization.builder()
-                .annotationId("37eee3e9d589a5cc529bfec38c3ba479")
-                .fileId("fileId")
-                .type("signature")
-                .requestDate(OffsetDateTime.now())
-                .build()));
+                                                             .annotationId("37eee3e9d589a5cc529bfec38c3ba479")
+                                                             .fileId("fileId")
+                                                             .type("signature")
+                                                             .requestDate(OffsetDateTime.now())
+                                                             .build()));
 
         request.setManualRedactions(manualRedactions);
 
@@ -683,40 +707,40 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
         ManualRedactions manualRedactions = new ManualRedactions();
         manualRedactions.getIdsToRemove()
                 .add(IdRemoval.builder()
-                        .annotationId("308dab9015bfafd911568cffe0a7f7de")
-                        .fileId(TEST_FILE_ID)
-                        .requestDate(OffsetDateTime.of(2022, 05, 23, 8, 30, 07, 475479, ZoneOffset.UTC))
-                        .processedDate(OffsetDateTime.of(2022, 05, 23, 8, 30, 07, 483651, ZoneOffset.UTC))
-                        .build());
+                             .annotationId("308dab9015bfafd911568cffe0a7f7de")
+                             .fileId(TEST_FILE_ID)
+                             .requestDate(OffsetDateTime.of(2022, 05, 23, 8, 30, 07, 475479, ZoneOffset.UTC))
+                             .processedDate(OffsetDateTime.of(2022, 05, 23, 8, 30, 07, 483651, ZoneOffset.UTC))
+                             .build());
 
         manualRedactions.getForceRedactions()
                 .add(ManualForceRedaction.builder()
-                        .annotationId("0b56ea1a87c83f351df177315af94f0d")
-                        .fileId(TEST_FILE_ID)
-                        .legalBasis("Something")
-                        .requestDate(OffsetDateTime.of(2022, 05, 23, 9, 30, 15, 4653, ZoneOffset.UTC))
-                        .processedDate(OffsetDateTime.of(2022, 05, 23, 9, 30, 15, 794, ZoneOffset.UTC))
-                        .build());
+                             .annotationId("0b56ea1a87c83f351df177315af94f0d")
+                             .fileId(TEST_FILE_ID)
+                             .legalBasis("Something")
+                             .requestDate(OffsetDateTime.of(2022, 05, 23, 9, 30, 15, 4653, ZoneOffset.UTC))
+                             .processedDate(OffsetDateTime.of(2022, 05, 23, 9, 30, 15, 794, ZoneOffset.UTC))
+                             .build());
 
         manualRedactions.getIdsToRemove()
                 .add(IdRemoval.builder()
-                        .annotationId("0b56ea1a87c83f351df177315af94f0d")
-                        .fileId(TEST_FILE_ID)
-                        .requestDate(OffsetDateTime.of(2022, 05, 23, 8, 30, 23, 961721, ZoneOffset.UTC))
-                        .processedDate(OffsetDateTime.of(2022, 05, 23, 8, 30, 23, 96528, ZoneOffset.UTC))
-                        .build());
+                             .annotationId("0b56ea1a87c83f351df177315af94f0d")
+                             .fileId(TEST_FILE_ID)
+                             .requestDate(OffsetDateTime.of(2022, 05, 23, 8, 30, 23, 961721, ZoneOffset.UTC))
+                             .processedDate(OffsetDateTime.of(2022, 05, 23, 8, 30, 23, 96528, ZoneOffset.UTC))
+                             .build());
 
         request.setManualRedactions(manualRedactions);
 
         AnalyzeResult result = analyzeService.analyze(request);
 
         AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder()
-                .manualRedactions(manualRedactions)
-                .colors(colors)
-                .types(types)
-                .dossierId(TEST_DOSSIER_ID)
-                .fileId(TEST_FILE_ID)
-                .build());
+                                                                               .manualRedactions(manualRedactions)
+                                                                               .colors(colors)
+                                                                               .types(types)
+                                                                               .dossierId(TEST_DOSSIER_ID)
+                                                                               .fileId(TEST_FILE_ID)
+                                                                               .build());
 
         try (FileOutputStream fileOutputStream = new FileOutputStream(OsUtils.getTemporaryDirectory() + "/Annotated.pdf")) {
             fileOutputStream.write(annotateResponse.getDocument());
@@ -932,12 +956,12 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
         var entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
 
         AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder()
-                .manualRedactions(manualRedactions)
-                .colors(colors)
-                .types(types)
-                .dossierId(TEST_DOSSIER_ID)
-                .fileId(TEST_FILE_ID)
-                .build());
+                                                                               .manualRedactions(manualRedactions)
+                                                                               .colors(colors)
+                                                                               .types(types)
+                                                                               .dossierId(TEST_DOSSIER_ID)
+                                                                               .fileId(TEST_FILE_ID)
+                                                                               .build());
 
         try (FileOutputStream fileOutputStream = new FileOutputStream(OsUtils.getTemporaryDirectory() + "/Annotated.pdf")) {
             fileOutputStream.write(annotateResponse.getDocument());
@@ -960,15 +984,16 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
 
         var redactionLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
 
-        redactionLog.getEntityLogEntry().forEach(entry -> {
-            if (!entry.getEntryType().equals(EntryType.HINT)) {
-                if (entry.getType().equals("CBI_author")) {
-                    assertThat(entry.getReason()).isEqualTo("Not redacted because it's row does not belong to a vertebrate study");
-                } else if (entry.getType().equals("CBI_address")) {
-                    assertThat(entry.getReason()).isEqualTo("No vertebrate found");
-                }
-            }
-        });
+        redactionLog.getEntityLogEntry()
+                .forEach(entry -> {
+                    if (!entry.getEntryType().equals(EntryType.HINT)) {
+                        if (entry.getType().equals("CBI_author")) {
+                            assertThat(entry.getReason()).isEqualTo("Not redacted because it's row does not belong to a vertebrate study");
+                        } else if (entry.getType().equals("CBI_address")) {
+                            assertThat(entry.getReason()).isEqualTo("No vertebrate found");
+                        }
+                    }
+                });
     }
 
 
@@ -1005,18 +1030,18 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
         String manualAddId = UUID.randomUUID().toString();
 
         manualRedactions.setIdsToRemove(Set.of(IdRemoval.builder()
-                .annotationId("5b940b2cb401ed9f5be6fc24f6e77bcf")
-                .fileId("fileId")
-                .processedDate(OffsetDateTime.now())
-                .requestDate(OffsetDateTime.now())
-                .build()));
+                                                       .annotationId("5b940b2cb401ed9f5be6fc24f6e77bcf")
+                                                       .fileId("fileId")
+                                                       .processedDate(OffsetDateTime.now())
+                                                       .requestDate(OffsetDateTime.now())
+                                                       .build()));
         manualRedactions.setForceRedactions(Set.of(ManualForceRedaction.builder()
-                .annotationId("675eba69b0c2917de55462c817adaa05")
-                .fileId("fileId")
-                .legalBasis("Something")
-                                .requestDate(OffsetDateTime.now())
-                .processedDate(OffsetDateTime.now())
-                .build()));
+                                                           .annotationId("675eba69b0c2917de55462c817adaa05")
+                                                           .fileId("fileId")
+                                                           .legalBasis("Something")
+                                                           .requestDate(OffsetDateTime.now())
+                                                           .processedDate(OffsetDateTime.now())
+                                                           .build()));
 
         ManualRedactionEntry manualRedactionEntry = new ManualRedactionEntry();
         manualRedactionEntry.setAnnotationId(manualAddId);
@@ -1027,7 +1052,7 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
         manualRedactionEntry.setProcessedDate(OffsetDateTime.now());
         manualRedactionEntry.setRequestDate(OffsetDateTime.now());
         manualRedactionEntry.setPositions(List.of(Rectangle.builder().topLeftX(375.61096f).topLeftY(241.282f).width(7.648041f).height(43.72262f).page(1).build(),
-                Rectangle.builder().topLeftX(384.83517f).topLeftY(241.282f).width(7.648041f).height(17.043358f).page(1).build()));
+                                                  Rectangle.builder().topLeftX(384.83517f).topLeftY(241.282f).width(7.648041f).height(17.043358f).page(1).build()));
 
 //        manualRedactions.getEntriesToAdd().add(manualRedactionEntry);
 
@@ -1038,39 +1063,63 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
 
         manualRedactions.getEntriesToAdd().add(manualRedactionEntry);
         manualRedactions.setIdsToRemove(Set.of(IdRemoval.builder()
-                .annotationId("5b940b2cb401ed9f5be6fc24f6e77bcf")
-                .fileId("fileId")
-                                .requestDate(OffsetDateTime.now())
-                .processedDate(OffsetDateTime.now())
-                .build()));
+                                                       .annotationId("5b940b2cb401ed9f5be6fc24f6e77bcf")
+                                                       .fileId("fileId")
+                                                       .requestDate(OffsetDateTime.now())
+                                                       .processedDate(OffsetDateTime.now())
+                                                       .build()));
         manualRedactions.setLegalBasisChanges((Set.of(ManualLegalBasisChange.builder()
-                .annotationId("675eba69b0c2917de55462c817adaa05")
-                .fileId("fileId")
-                .legalBasis("Manual Legal Basis Change")
-                                .processedDate(OffsetDateTime.now())
-                .requestDate(OffsetDateTime.now())
-                .build())));
+                                                              .annotationId("675eba69b0c2917de55462c817adaa05")
+                                                              .fileId("fileId")
+                                                              .legalBasis("Manual Legal Basis Change")
+                                                              .processedDate(OffsetDateTime.now())
+                                                              .requestDate(OffsetDateTime.now())
+                                                              .build())));
         manualRedactions.setResizeRedactions(Set.of(ManualResizeRedaction.builder()
-                .annotationId("fc287b74be2421156ab2895c7474ccdd")
-                .fileId("fileId")
-                .processedDate(OffsetDateTime.now())
-                .requestDate(OffsetDateTime.now())
-                .value("Syngenta Crop Protection AG, Basel, Switzerland RCC Ltd., Itingen, Switzerland")
-                .positions(List.of(Rectangle.builder().topLeftX(289.44595f).topLeftY(327.567f).width(7.648041f).height(82.51475f).page(1).build(),
-                        Rectangle.builder().topLeftX(298.67056f).topLeftY(327.567f).width(7.648041f).height(75.32377f).page(1).build(),
-                        Rectangle.builder().topLeftX(307.89517f).topLeftY(327.567f).width(7.648041f).height(61.670967f).page(1).build(),
-                        Rectangle.builder().topLeftX(316.99985f).topLeftY(327.567f).width(7.648041f).height(38.104286f).page(1).build()))
-                                .updateDictionary(false)
-                .build()));
+                                                            .annotationId("fc287b74be2421156ab2895c7474ccdd")
+                                                            .fileId("fileId")
+                                                            .processedDate(OffsetDateTime.now())
+                                                            .requestDate(OffsetDateTime.now())
+                                                            .value("Syngenta Crop Protection AG, Basel, Switzerland RCC Ltd., Itingen, Switzerland")
+                                                            .positions(List.of(Rectangle.builder()
+                                                                                       .topLeftX(289.44595f)
+                                                                                       .topLeftY(327.567f)
+                                                                                       .width(7.648041f)
+                                                                                       .height(82.51475f)
+                                                                                       .page(1)
+                                                                                       .build(),
+                                                                               Rectangle.builder()
+                                                                                       .topLeftX(298.67056f)
+                                                                                       .topLeftY(327.567f)
+                                                                                       .width(7.648041f)
+                                                                                       .height(75.32377f)
+                                                                                       .page(1)
+                                                                                       .build(),
+                                                                               Rectangle.builder()
+                                                                                       .topLeftX(307.89517f)
+                                                                                       .topLeftY(327.567f)
+                                                                                       .width(7.648041f)
+                                                                                       .height(61.670967f)
+                                                                                       .page(1)
+                                                                                       .build(),
+                                                                               Rectangle.builder()
+                                                                                       .topLeftX(316.99985f)
+                                                                                       .topLeftY(327.567f)
+                                                                                       .width(7.648041f)
+                                                                                       .height(38.104286f)
+                                                                                       .page(1)
+                                                                                       .build()))
+                                                            .updateDictionary(false)
+                                                            .build()));
 
         analyzeService.reanalyze(request);
 
         AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder()
-                .dossierId(TEST_DOSSIER_ID)
-                .fileId(TEST_FILE_ID)
-                .dossierTemplateId(TEST_DOSSIER_TEMPLATE_ID)
-                .manualRedactions(manualRedactions)
-                .build());
+                                                                               .dossierId(TEST_DOSSIER_ID)
+                                                                               .fileId(TEST_FILE_ID)
+                                                                               .dossierTemplateId(TEST_DOSSIER_TEMPLATE_ID)
+                                                                               .manualRedactions(manualRedactions)
+                                                                               .build());
 
         try (FileOutputStream fileOutputStream = new FileOutputStream(OsUtils.getTemporaryDirectory() + "/Annotated.pdf")) {
             fileOutputStream.write(annotateResponse.getDocument());
@@ -1110,8 +1159,8 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
 
         AnalyzeRequest request = uploadFileToStorage("files/ImportedRedactions/RotateTestFile_without_highlights.pdf");
         storageService.storeObject(TenantContext.getTenantId(),
-                RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.IMPORTED_REDACTIONS),
-                importedRedactions.getInputStream());
+                                   RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.IMPORTED_REDACTIONS),
+                                   importedRedactions.getInputStream());
 
         analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request);
         AnalyzeResult result = analyzeService.analyze(request);
@@ -1124,17 +1173,18 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
             fileOutputStream.write(annotateResponse.getDocument());
         }
 
-        entityLog.getEntityLogEntry().forEach(entry -> {
-            if (entry.getValue() == null) {
-                return;
-            }
-            if (entry.getValue().equals("David")) {
-                assertThat(entry.getImportedRedactionIntersections()).hasSize(1);
-            }
-            if (entry.getValue().equals("annotation")) {
-                assertThat(entry.getImportedRedactionIntersections()).isEmpty();
-            }
-        });
+        entityLog.getEntityLogEntry()
+                .forEach(entry -> {
+                    if (entry.getValue() == null) {
+                        return;
+                    }
+                    if (entry.getValue().equals("David")) {
+                        assertThat(entry.getImportedRedactionIntersections()).hasSize(1);
+                    }
+                    if (entry.getValue().equals("annotation")) {
+                        assertThat(entry.getImportedRedactionIntersections()).isEmpty();
+                    }
+                });
     }
 
 
@@ -1163,7 +1213,10 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
         }
 
         var entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
-        var values = entityLog.getEntityLogEntry().stream().map(EntityLogEntry::getValue).collect(Collectors.toList());
+        var values = entityLog.getEntityLogEntry()
+                .stream()
+                .map(EntityLogEntry::getValue)
+                .collect(Collectors.toList());
 
         assertThat(values).contains("Mrs. Robinson");
         assertThat(values).contains("Mr. Bojangles");
@@ -1178,8 +1231,8 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
         ClassPathResource imageServiceResponseFileResource = new ClassPathResource("files/new/SYNGENTA_EFSA_sanitisation_GFL_v1 (1).IMAGE_INFO.json");
 
         storageService.storeObject(TenantContext.getTenantId(),
-                RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.IMAGE_INFO),
-                imageServiceResponseFileResource.getInputStream());
+                                   RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.IMAGE_INFO),
+                                   imageServiceResponseFileResource.getInputStream());
 
         System.out.println("Start Full integration test");
         analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request);
@@ -1188,23 +1241,26 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
         System.out.println("Finished analysis");
 
         request.setManualRedactions(ManualRedactions.builder()
-                .legalBasisChanges(Set.of(ManualLegalBasisChange.builder()
-                        .annotationId("3029651d0842a625f2d23f8375c23600")
-                        .section("[19, 2]: Paragraph: Contact point: LexCo Contact:")
-                        .value("0049 331 441 551 14")
-                        .requestDate(OffsetDateTime.now())
-                                                .fileId(TEST_FILE_ID)
-                        .legalBasis("Article 39(e)(2) of Regulation (EC) No 178/2002")
-                        .build()))
-                .build());
+                                            .legalBasisChanges(Set.of(ManualLegalBasisChange.builder()
+                                                                              .annotationId("3029651d0842a625f2d23f8375c23600")
+                                                                              .section("[19, 2]: Paragraph: Contact point: LexCo Contact:")
+                                                                              .value("0049 331 441 551 14")
+                                                                              .requestDate(OffsetDateTime.now())
+                                                                              .fileId(TEST_FILE_ID)
+                                                                              .legalBasis("Article 39(e)(2) of Regulation (EC) No 178/2002")
+                                                                              .build()))
+                                            .build());
 
         analyzeService.reanalyze(request);
         System.out.println("Finished reanalysis");
 
         var entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
-        entityLog.getEntityLogEntry().stream().filter(entityLogEntry -> entityLogEntry.getType().equals("signature")).forEach(entityLogEntry -> {
-            assertThat(entityLogEntry.getState() == EntryState.APPLIED).isTrue();
-        });
+        entityLog.getEntityLogEntry()
+                .stream()
+                .filter(entityLogEntry -> entityLogEntry.getType().equals("signature"))
+                .forEach(entityLogEntry -> {
+                    assertThat(entityLogEntry.getState() == EntryState.APPLIED).isTrue();
+                });
     }
 
 
@@ -1215,8 +1271,8 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
         ClassPathResource imageServiceResponseFileResource = new ClassPathResource("files/new/SYNGENTA_EFSA_sanitisation_GFL_v1 (1).IMAGE_INFO.json");
 
         storageService.storeObject(TenantContext.getTenantId(),
-                RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.IMAGE_INFO),
-                imageServiceResponseFileResource.getInputStream());
+                                   RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.IMAGE_INFO),
+                                   imageServiceResponseFileResource.getInputStream());
 
         System.out.println("Start Full integration test");
         analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request);
@@ -1225,21 +1281,22 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
         System.out.println("Finished analysis");
 
         request.setManualRedactions(ManualRedactions.builder()
-                .legalBasisChanges(Set.of(ManualLegalBasisChange.builder()
-                        .annotationId("3029651d0842a625f2d23f8375c23600")
-                        .section("[19, 2]: Paragraph: Contact point: LexCo Contact:")
-                        .value("0049 331 441 551 14")
-                        .requestDate(OffsetDateTime.now())
-                                                .fileId(TEST_FILE_ID)
-                        .legalBasis("Article 39(e)(2) of Regulation (EC) No 178/2002")
-                        .build()))
-                .recategorizations(Set.of(ManualRecategorization.builder()
-                        .annotationId("3029651d0842a625f2d23f8375c23600")
-                        .type("CBI_author")
-                        .requestDate(OffsetDateTime.now())
-                                                .fileId(TEST_FILE_ID)
-                        .build()))
-                .build());
+                                            .legalBasisChanges(Set.of(ManualLegalBasisChange.builder()
+                                                                              .annotationId("3029651d0842a625f2d23f8375c23600")
+                                                                              .section("[19, 2]: Paragraph: Contact point: LexCo Contact:")
+                                                                              .value("0049 331 441 551 14")
+                                                                              .requestDate(OffsetDateTime.now())
+                                                                              .fileId(TEST_FILE_ID)
+                                                                              .legalBasis("Article 39(e)(2) of Regulation (EC) No 178/2002")
+                                                                              .build()))
+                                            .recategorizations(Set.of(ManualRecategorization.builder()
+                                                                              .annotationId("3029651d0842a625f2d23f8375c23600")
+                                                                              .type("CBI_author")
+                                                                              .legalBasis("")
+                                                                              .requestDate(OffsetDateTime.now())
+                                                                              .fileId(TEST_FILE_ID)
+                                                                              .build()))
+                                            .build());
 
         analyzeService.reanalyze(request);
         System.out.println("Finished reanalysis");
@@ -1266,11 +1323,11 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
         String manualAddId2 = UUID.randomUUID().toString();
         List<Rectangle> positions = List.of(Rectangle.builder().topLeftX(305.35f).topLeftY(332.5033f).width(71.40744f).height(13.645125f).page(1).build());
         ManualRedactionEntry manualRedactionEntry = getManualRedactionEntry(manualAddId,
-                positions,
-                "the manufacturing or production process, including the method and innovative aspects thereof, as well as other technical and industrial specifications inherent to that process or method, except for information which is relevant to the assessment of safety");
+                                                                            positions,
+                                                                            "the manufacturing or production process, including the method and innovative aspects thereof, as well as other technical and industrial specifications inherent to that process or method, except for information which is relevant to the assessment of safety");
         ManualRedactionEntry manualRedactionEntry2 = getManualRedactionEntry(manualAddId2,
-                positions,
-                "commercial information revealing sourcing, market shares or business strategy of the applicant");
+                                                                             positions,
+                                                                             "commercial information revealing sourcing, market shares or business strategy of the applicant");
 
         IdRemoval idRemoval = getIdRemoval(manualAddId);
         IdRemoval idRemoval2 = getIdRemoval(manualAddId2);
@@ -1282,55 +1339,101 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
 
         var entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
 
-        assertTrue(entityLog.getEntityLogEntry().stream().anyMatch(entityLogEntry -> entityLogEntry.getId().equals(manualAddId)));
-        assertEquals(entityLog.getEntityLogEntry().stream().filter(entityLogEntry -> entityLogEntry.getId().equals(manualAddId)).findFirst().get().getState(), EntryState.APPLIED);
+        assertTrue(entityLog.getEntityLogEntry()
+                           .stream()
+                           .anyMatch(entityLogEntry -> entityLogEntry.getId().equals(manualAddId)));
+        assertEquals(entityLog.getEntityLogEntry()
+                             .stream()
+                             .filter(entityLogEntry -> entityLogEntry.getId().equals(manualAddId))
+                             .findFirst()
+                             .get().getState(), EntryState.APPLIED);
 
         request.setManualRedactions(ManualRedactions.builder().entriesToAdd(Set.of(manualRedactionEntry)).idsToRemove(Set.of(idRemoval)).build());
         analyzeService.reanalyze(request);
 
         entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
 
-        assertTrue(entityLog.getEntityLogEntry().stream().anyMatch(entityLogEntry -> entityLogEntry.getId().equals(manualAddId)));
-        assertEquals(entityLog.getEntityLogEntry().stream().filter(entityLogEntry -> entityLogEntry.getId().equals(manualAddId)).findFirst().get().getState(), EntryState.REMOVED);
+        assertTrue(entityLog.getEntityLogEntry()
+                           .stream()
+                           .anyMatch(entityLogEntry -> entityLogEntry.getId().equals(manualAddId)));
+        assertEquals(entityLog.getEntityLogEntry()
+                             .stream()
+                             .filter(entityLogEntry -> entityLogEntry.getId().equals(manualAddId))
+                             .findFirst()
+                             .get().getState(), EntryState.REMOVED);
 
         request.setManualRedactions(ManualRedactions.builder().entriesToAdd(Set.of(manualRedactionEntry, manualRedactionEntry2)).idsToRemove(Set.of(idRemoval)).build());
         analyzeService.reanalyze(request);
 
         entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
 
-        assertTrue(entityLog.getEntityLogEntry().stream().anyMatch(entityLogEntry -> entityLogEntry.getId().equals(manualAddId)));
-        assertEquals(entityLog.getEntityLogEntry().stream().filter(entityLogEntry -> entityLogEntry.getId().equals(manualAddId)).findFirst().get().getState(), EntryState.REMOVED);
-        assertTrue(entityLog.getEntityLogEntry().stream().anyMatch(entityLogEntry -> entityLogEntry.getId().equals(manualAddId2)));
-        assertEquals(entityLog.getEntityLogEntry().stream().filter(entityLogEntry -> entityLogEntry.getId().equals(manualAddId2)).findFirst().get().getState(), EntryState.APPLIED);
+        assertTrue(entityLog.getEntityLogEntry()
+                           .stream()
+                           .anyMatch(entityLogEntry -> entityLogEntry.getId().equals(manualAddId)));
+        assertEquals(entityLog.getEntityLogEntry()
+                             .stream()
+                             .filter(entityLogEntry -> entityLogEntry.getId().equals(manualAddId))
+                             .findFirst()
+                             .get().getState(), EntryState.REMOVED);
+        assertTrue(entityLog.getEntityLogEntry()
+                           .stream()
+                           .anyMatch(entityLogEntry -> entityLogEntry.getId().equals(manualAddId2)));
+        assertEquals(entityLog.getEntityLogEntry()
+                             .stream()
+                             .filter(entityLogEntry -> entityLogEntry.getId().equals(manualAddId2))
+                             .findFirst()
+                             .get().getState(), EntryState.APPLIED);
 
         request.setManualRedactions(ManualRedactions.builder()
-                .entriesToAdd(Set.of(manualRedactionEntry, manualRedactionEntry2))
-                .idsToRemove(Set.of(idRemoval, idRemoval2))
-                .build());
+                                            .entriesToAdd(Set.of(manualRedactionEntry, manualRedactionEntry2))
+                                            .idsToRemove(Set.of(idRemoval, idRemoval2))
+                                            .build());
         analyzeService.reanalyze(request);
 
         entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
 
-        assertTrue(entityLog.getEntityLogEntry().stream().anyMatch(entityLogEntry -> entityLogEntry.getId().equals(manualAddId)));
-        assertEquals(entityLog.getEntityLogEntry().stream().filter(entityLogEntry -> entityLogEntry.getId().equals(manualAddId)).findFirst().get().getState(), EntryState.REMOVED);
-        assertTrue(entityLog.getEntityLogEntry().stream().anyMatch(entityLogEntry -> entityLogEntry.getId().equals(manualAddId2)));
-        assertEquals(entityLog.getEntityLogEntry().stream().filter(entityLogEntry -> entityLogEntry.getId().equals(manualAddId2)).findFirst().get().getState(), EntryState.REMOVED);
+        assertTrue(entityLog.getEntityLogEntry()
+                           .stream()
+                           .anyMatch(entityLogEntry -> entityLogEntry.getId().equals(manualAddId)));
+        assertEquals(entityLog.getEntityLogEntry()
+                             .stream()
+                             .filter(entityLogEntry -> entityLogEntry.getId().equals(manualAddId))
+                             .findFirst()
+                             .get().getState(), EntryState.REMOVED);
+        assertTrue(entityLog.getEntityLogEntry()
+                           .stream()
+                           .anyMatch(entityLogEntry -> entityLogEntry.getId().equals(manualAddId2)));
+        assertEquals(entityLog.getEntityLogEntry()
+                             .stream()
+                             .filter(entityLogEntry -> entityLogEntry.getId().equals(manualAddId2))
+                             .findFirst()
+                             .get().getState(), EntryState.REMOVED);
 
         manualRedactionEntry.setRequestDate(OffsetDateTime.now());
-        request.setManualRedactions(ManualRedactions.builder()
-                .entriesToAdd(Set.of(manualRedactionEntry, manualRedactionEntry2))
-                .idsToRemove(Set.of(idRemoval, idRemoval2))
-                .build());
+        request.setManualRedactions(ManualRedactions.builder().entriesToAdd(Set.of(manualRedactionEntry, manualRedactionEntry2)).idsToRemove(Set.of(idRemoval2)).build());
         analyzeService.reanalyze(request);
 
         entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
 
-        assertTrue(entityLog.getEntityLogEntry().stream().anyMatch(entityLogEntry -> entityLogEntry.getId().equals(manualAddId)));
-        assertEquals(entityLog.getEntityLogEntry().stream().filter(entityLogEntry -> entityLogEntry.getId().equals(manualAddId)).findFirst().get().getState(), EntryState.APPLIED);
-        assertTrue(entityLog.getEntityLogEntry().stream().anyMatch(entityLogEntry -> entityLogEntry.getId().equals(manualAddId2)));
-        assertEquals(entityLog.getEntityLogEntry().stream().filter(entityLogEntry -> entityLogEntry.getId().equals(manualAddId2)).findFirst().get().getState(), EntryState.REMOVED);
+        assertTrue(entityLog.getEntityLogEntry()
+                           .stream()
+                           .anyMatch(entityLogEntry -> entityLogEntry.getId().equals(manualAddId)));
+        assertEquals(entityLog.getEntityLogEntry()
+                             .stream()
+                             .filter(entityLogEntry -> entityLogEntry.getId().equals(manualAddId))
+                             .findFirst()
+                             .get().getState(), EntryState.APPLIED);
+        assertTrue(entityLog.getEntityLogEntry()
+                           .stream()
+                           .anyMatch(entityLogEntry -> entityLogEntry.getId().equals(manualAddId2)));
+        assertEquals(entityLog.getEntityLogEntry()
+                             .stream()
+                             .filter(entityLogEntry -> entityLogEntry.getId().equals(manualAddId2))
+                             .findFirst()
+                             .get().getState(), EntryState.REMOVED);
     }
 
+
     @Test
     @SneakyThrows
     public void testResizeWithUpdateDictionaryTrue() {
@@ -1342,34 +1445,44 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
         analyzeService.analyze(request);
 
         var entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
-        var david = entityLog.getEntityLogEntry().stream().filter(e -> e.getValue().equals("David")).findFirst().get();
+        var david = entityLog.getEntityLogEntry()
+                .stream()
+                .filter(e -> e.getValue().equals("David"))
+                .findFirst()
+                .get();
 
         request.setManualRedactions(ManualRedactions.builder()
-                        .resizeRedactions(Set.of(ManualResizeRedaction.builder()
-                                .updateDictionary(true)
-                                .annotationId(david.getId())
-                                .requestDate(OffsetDateTime.now())
-                                .value("David Ksenia")
-                                .positions(List.of(Rectangle.builder().topLeftX(56.8f).topLeftY(293.564f).width(65.592f).height(15.408f).page(1).build()))
-                                        .addToAllDossiers(false)
-                                .build()))
-                .build());
+                                            .resizeRedactions(Set.of(ManualResizeRedaction.builder()
+                                                                             .updateDictionary(true)
+                                                                             .annotationId(david.getId())
+                                                                             .requestDate(OffsetDateTime.now())
+                                                                             .value("David Ksenia")
+                                                                             .positions(List.of(Rectangle.builder()
+                                                                                                        .topLeftX(56.8f)
+                                                                                                        .topLeftY(293.564f)
+                                                                                                        .width(65.592f)
+                                                                                                        .height(15.408f)
+                                                                                                        .page(1)
+                                                                                                        .build()))
+                                                                             .addToAllDossiers(false)
+                                                                             .build()))
+                                            .build());
         analyzeService.reanalyze(request);
         entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
-        var resizedEntity = entityLog.getEntityLogEntry().stream().filter(e -> e.getId().equals(david.getId())).findFirst().get();
+        var resizedEntity = entityLog.getEntityLogEntry()
+                .stream()
+                .filter(e -> e.getId().equals(david.getId()))
+                .findFirst()
+                .get();
         assertEquals(resizedEntity.getState(), EntryState.APPLIED);
-        assertEquals(resizedEntity.getValue(), "David Ksenia");
+        assertEquals(resizedEntity.getValue(), "David");
+        assertEquals(0, resizedEntity.getManualChanges().size());
     }
 
 
     private IdRemoval getIdRemoval(String id) {
 
-        return IdRemoval.builder()
-                .annotationId(id)
-                .removeFromAllDossiers(false)
-                .removeFromDictionary(false)
-                                .requestDate(OffsetDateTime.now())
-                .build();
+        return IdRemoval.builder().annotationId(id).removeFromAllDossiers(false).removeFromDictionary(false).requestDate(OffsetDateTime.now()).build();
     }
 
 

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/TableTest.java

@@ -0,0 +1,162 @@
+package com.iqser.red.service.redaction.v1.server.document.graph;
+
+import static com.iqser.red.service.redaction.v1.server.utils.EntityVisualizationUtility.ENTITY_LAYER;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+import java.awt.Color;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.nio.file.Path;
+import java.util.List;
+import java.util.Set;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+
+import org.drools.io.ClassPathResource;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.springframework.beans.factory.annotation.Autowired;
+
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
+import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
+import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.NodeType;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.Table;
+import com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService;
+import com.iqser.red.service.redaction.v1.server.service.document.EntityEnrichmentService;
+import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
+import com.iqser.red.service.redaction.v1.server.utils.EntityVisualizationUtility;
+import com.knecon.fforesight.service.viewerdoc.model.Visualizations;
+import com.knecon.fforesight.service.viewerdoc.service.ViewerDocumentService;
+import com.knecon.fforesight.tenantcommons.TenantContext;
+
+import lombok.SneakyThrows;
+
+public class TableTest extends BuildDocumentIntegrationTest {
+
+    private static final boolean DRAW_FILE = false;
+
+    @Autowired
+    private EntityEnrichmentService entityEnrichmentService;
+
+    private EntityCreationService entityCreationService;
+
+    private static final String TYPE_1 = "type1";
+    private static final String TYPE_2 = "type2";
+    private static final String TYPE_3 = "type3";
+    private static final String TYPE_4 = "type4";
+
+    private Table table;
+
+    private Set<TextEntity> entities;
+
+
+    @SneakyThrows
+    @BeforeEach
+    public void createTable() {
+
+        entityCreationService = new EntityCreationService(entityEnrichmentService);
+
+        String fileName = "files/Minimal Examples/BasicTable.pdf";
+
+        Document document = buildGraph(fileName);
+
+        table = (Table) document.streamAllSubNodesOfType(NodeType.TABLE)
+                .findAny()
+                .orElseThrow();
+
+        entities = List.of(//
+                           entityCreationService.byString("Cell11", TYPE_1, EntityType.ENTITY, document),
+                           entityCreationService.byString("Cell21", TYPE_1, EntityType.ENTITY, document),
+                           entityCreationService.byString("Cell31", TYPE_1, EntityType.ENTITY, document),
+                           entityCreationService.byString("Cell41", TYPE_1, EntityType.ENTITY, document),
+                           entityCreationService.byString("Cell51", TYPE_1, EntityType.ENTITY, document),
+
+                           entityCreationService.byString("Cell12", TYPE_2, EntityType.ENTITY, document),
+                           entityCreationService.byString("Cell32", TYPE_2, EntityType.ENTITY, document),
+                           entityCreationService.byString("Cell42", TYPE_2, EntityType.ENTITY, document),
+
+                           entityCreationService.byString("Cell23", TYPE_3, EntityType.ENTITY, document),
+                           entityCreationService.byString("Cell53", TYPE_3, EntityType.ENTITY, document),
+
+                           entityCreationService.byString("Cell14", TYPE_4, EntityType.ENTITY, document),
+                           entityCreationService.byString("Cell34", TYPE_4, EntityType.ENTITY, document))
+                .stream()
+                .flatMap(Function.identity())
+                .collect(Collectors.toSet());
+
+        if (DRAW_FILE) {
+            File file = new File("/tmp/" + Path.of(fileName).getFileName().toString());
+            storageService.downloadTo(TenantContext.getTenantId(),
+                                      RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.VIEWER_DOCUMENT),
+                                      file);
+            ViewerDocumentService viewerDocumentService = new ViewerDocumentService(null);
+
+            var visualizationsOnPage = EntityVisualizationUtility.createVisualizationsOnPage(document.getEntities(), Color.MAGENTA);
+
+            viewerDocumentService.addVisualizationsOnPage(file,
+                                                          file,
+                                                          Visualizations.builder()
+                                                                  .layer(ENTITY_LAYER)
+                                                                  .visualizationsOnPages(visualizationsOnPage)
+                                                                  .layerVisibilityDefaultValue(true)
+                                                                  .build());
+        }
+
+    }
+
+
+    @Test
+    public void testStreamEntitiesWhereRowContainsEntitiesOfType() {
+
+        int type_2_count = table.getEntitiesOfType(TYPE_2).size();
+
+        assertEquals(type_2_count,
+                     table.streamEntitiesWhereRowContainsEntitiesOfType(List.of(TYPE_1))
+                             .filter(textEntity -> textEntity.type().equals(TYPE_2))
+                             .count());
+
+        assertEquals(type_2_count,
+                     table.streamEntitiesWhereRowContainsEntitiesOfType(List.of(TYPE_1, TYPE_4))
+                             .filter(textEntity -> textEntity.type().equals(TYPE_2))
+                             .count());
+
+        assertEquals(2,
+                     table.streamEntitiesWhereRowContainsEntitiesOfEachType(List.of(TYPE_1, TYPE_4))
+                             .filter(textEntity -> textEntity.type().equals(TYPE_2))
+                             .count());
+
+        assertEquals(0,
+                     table.streamEntitiesWhereRowContainsEntitiesOfEachType(List.of(TYPE_1, TYPE_3))
+                             .filter(textEntity -> textEntity.type().equals(TYPE_2))
+                             .count());
+
+        assertEquals(0,
+                     table.streamEntitiesWhereRowContainsEntitiesOfEachType(List.of(TYPE_1, TYPE_3, TYPE_4))
+                             .filter(textEntity -> textEntity.type().equals(TYPE_2))
+                             .count());
+
+        assertEquals(type_2_count,
+                     table.streamEntitiesWhereRowContainsEntitiesOfEachType(List.of())
+                             .filter(textEntity -> textEntity.type().equals(TYPE_2))
+                             .count());
+
+        assertEquals(3,
+                     table.streamTextEntitiesInRow(1)
+                             .count());
+
+        assertEquals(2,
+                     table.streamTextEntitiesInRow(4)
+                             .count());
+
+        assertEquals(5,
+                     table.streamTextEntitiesInCol(1)
+                             .count());
+
+        assertEquals(3,
+                     table.streamTextEntitiesInRow(3)
+                             .count());
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/drools/files/management/services/DroolsSyntaxValidationServiceTest.java

@@ -1,5 +1,6 @@
 package com.iqser.red.service.redaction.v1.server.drools.files.management.services;
 
+import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertFalse;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 
@@ -18,6 +19,7 @@ import org.springframework.core.io.ClassPathResource;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType;
 import com.iqser.red.service.redaction.v1.model.DroolsSyntaxValidation;
 import com.iqser.red.service.redaction.v1.model.RuleValidationModel;
+import com.iqser.red.service.redaction.v1.server.DeprecatedElementsFinder;
 import com.iqser.red.service.redaction.v1.server.client.RulesClient;
 import com.iqser.red.service.redaction.v1.server.model.drools.RuleFileBluePrint;
 import com.iqser.red.service.redaction.v1.server.service.document.EntityEnrichmentService;
@@ -47,13 +49,15 @@ class DroolsSyntaxValidationServiceTest {
     @SneakyThrows
     void testRules() {
 
-        DroolsSyntaxValidationService droolsSyntaxValidationService = new DroolsSyntaxValidationService(new KieContainerCreationService(rulesClient));
+        DroolsSyntaxValidationService droolsSyntaxValidationService = new DroolsSyntaxValidationService(new KieContainerCreationService(rulesClient),
+                                                                                                        new DeprecatedElementsFinder());
         var rulesFile = new ClassPathResource("drools/rules.drl");
 
         String rulesString = new String(rulesFile.getInputStream().readAllBytes());
 
         DroolsSyntaxValidation droolsSyntaxValidation = droolsSyntaxValidationService.testRules(new RuleValidationModel(RuleFileType.ENTITY.name(), rulesString));
-        droolsSyntaxValidation.getDroolsSyntaxErrorMessages().forEach(System.out::println);
+        droolsSyntaxValidation.getDroolsSyntaxErrorMessages()
+                .forEach(System.out::println);
         assertTrue(droolsSyntaxValidation.isCompiled());
     }
 
@@ -62,14 +66,16 @@ class DroolsSyntaxValidationServiceTest {
     @SneakyThrows
     void testRulesWithRemovedImportsAndChangedName() {
 
-        DroolsSyntaxValidationService droolsSyntaxValidationService = new DroolsSyntaxValidationService(new KieContainerCreationService(rulesClient));
+        DroolsSyntaxValidationService droolsSyntaxValidationService = new DroolsSyntaxValidationService(new KieContainerCreationService(rulesClient),
+                                                                                                        new DeprecatedElementsFinder());
         var rulesFile = new ClassPathResource("drools/rules.drl");
 
         String rulesString = new String(rulesFile.getInputStream().readAllBytes());
         rulesString = rulesString.replaceAll("import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell;", "");
         rulesString = rulesString.replaceAll("rule \"LDS.0.0: Run local dictionary search\"", "rule \"LDS.0.0: run local dictionary search\"");
         DroolsSyntaxValidation droolsSyntaxValidation = droolsSyntaxValidationService.testRules(new RuleValidationModel(RuleFileType.ENTITY.name(), rulesString));
-        droolsSyntaxValidation.getDroolsSyntaxErrorMessages().forEach(System.out::println);
+        droolsSyntaxValidation.getDroolsSyntaxErrorMessages()
+                .forEach(System.out::println);
         assertTrue(droolsSyntaxValidation.isCompiled());
     }
 
@@ -77,102 +83,32 @@ class DroolsSyntaxValidationServiceTest {
     @SneakyThrows
     void testRulesWithAddedImports() {
 
-        DroolsSyntaxValidationService droolsSyntaxValidationService = new DroolsSyntaxValidationService(new KieContainerCreationService(rulesClient));
+        DroolsSyntaxValidationService droolsSyntaxValidationService = new DroolsSyntaxValidationService(new KieContainerCreationService(rulesClient),
+                                                                                                        new DeprecatedElementsFinder());
         var rulesFile = new ClassPathResource("drools/rules.drl");
 
         String rulesString = new String(rulesFile.getInputStream().readAllBytes());
-        rulesString = rulesString.replaceAll("import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell;", "import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell;\nimport com.iqser.red.service.redaction.v1.server.service.document.EntityComparators;");
+        rulesString = rulesString.replaceAll("import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell;",
+                                             "import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell;\nimport com.iqser.red.service.redaction.v1.server.service.document.EntityComparators;");
         DroolsSyntaxValidation droolsSyntaxValidation = droolsSyntaxValidationService.testRules(new RuleValidationModel(RuleFileType.ENTITY.name(), rulesString));
-        droolsSyntaxValidation.getDroolsSyntaxErrorMessages().forEach(System.out::println);
+        droolsSyntaxValidation.getDroolsSyntaxErrorMessages()
+                .forEach(System.out::println);
         assertTrue(droolsSyntaxValidation.isCompiled());
     }
 
-    @Test
-    @SneakyThrows
-    void testRulesWithUnallowedAddedImports() {
-
-        DroolsSyntaxValidationService droolsSyntaxValidationService = new DroolsSyntaxValidationService(new KieContainerCreationService(rulesClient));
-        var rulesFile = new ClassPathResource("drools/rules.drl");
-
-        String rulesString = new String(rulesFile.getInputStream().readAllBytes());
-        rulesString = rulesString.replaceAll("import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell;", "import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell;\nimport com.google.common.collect.Sets;");
-        DroolsSyntaxValidation droolsSyntaxValidation = droolsSyntaxValidationService.testRules(new RuleValidationModel(RuleFileType.ENTITY.name(), rulesString));
-        droolsSyntaxValidation.getDroolsSyntaxErrorMessages().forEach(System.out::println);
-        assertFalse(droolsSyntaxValidation.isCompiled());
-    }
-
-
-    @Test
-    @SneakyThrows
-    void testAllRules() {
-
-        DroolsSyntaxValidationService droolsSyntaxValidationService = new DroolsSyntaxValidationService(new KieContainerCreationService(rulesClient));
-        var rulesFile = new ClassPathResource("drools/all_redact_manager_rules.drl");
-
-        String rulesString = new String(rulesFile.getInputStream().readAllBytes());
-
-        DroolsSyntaxValidation droolsSyntaxValidation = droolsSyntaxValidationService.testRules(new RuleValidationModel(RuleFileType.ENTITY.name(), rulesString));
-        droolsSyntaxValidation.getDroolsSyntaxErrorMessages().forEach(System.out::println);
-        assertTrue(droolsSyntaxValidation.isCompiled());
-    }
-
-
-    @Test
-    @SneakyThrows
-    void testAcceptanceRules() {
-
-        DroolsSyntaxValidationService droolsSyntaxValidationService = new DroolsSyntaxValidationService(new KieContainerCreationService(rulesClient));
-        var rulesFile = new ClassPathResource("drools/acceptance_rules.drl");
-
-        String rulesString = new String(rulesFile.getInputStream().readAllBytes());
-
-        DroolsSyntaxValidation droolsSyntaxValidation = droolsSyntaxValidationService.testRules(new RuleValidationModel(RuleFileType.ENTITY.name(), rulesString));
-        droolsSyntaxValidation.getDroolsSyntaxErrorMessages().forEach(System.out::println);
-        assertTrue(droolsSyntaxValidation.isCompiled());
-    }
-
-
-    @Test
-    @SneakyThrows
-    void testManualRedactionRules() {
-
-        DroolsSyntaxValidationService droolsSyntaxValidationService = new DroolsSyntaxValidationService(new KieContainerCreationService(rulesClient));
-        var rulesFile = new ClassPathResource("drools/manual_redaction_rules.drl");
-
-        String rulesString = new String(rulesFile.getInputStream().readAllBytes());
-
-        DroolsSyntaxValidation droolsSyntaxValidation = droolsSyntaxValidationService.testRules(new RuleValidationModel(RuleFileType.ENTITY.name(), rulesString));
-        droolsSyntaxValidation.getDroolsSyntaxErrorMessages().forEach(System.out::println);
-        assertTrue(droolsSyntaxValidation.isCompiled());
-    }
-
-
-    @Test
-    @SneakyThrows
-    void testRulesV2() {
-
-        DroolsSyntaxValidationService droolsSyntaxValidationService = new DroolsSyntaxValidationService(new KieContainerCreationService(rulesClient));
-        var rulesFile = new ClassPathResource("drools/rules_v2.drl");
-
-        String rulesString = new String(rulesFile.getInputStream().readAllBytes());
-
-        DroolsSyntaxValidation droolsSyntaxValidation = droolsSyntaxValidationService.testRules(new RuleValidationModel(RuleFileType.ENTITY.name(), rulesString));
-        droolsSyntaxValidation.getDroolsSyntaxErrorMessages().forEach(System.out::println);
-        assertTrue(droolsSyntaxValidation.isCompiled());
-    }
-
-
     @Test
     @SneakyThrows
     void testDocumineRules() {
 
-        DroolsSyntaxValidationService droolsSyntaxValidationService = new DroolsSyntaxValidationService(new KieContainerCreationService(rulesClient));
+        DroolsSyntaxValidationService droolsSyntaxValidationService = new DroolsSyntaxValidationService(new KieContainerCreationService(rulesClient),
+                                                                                                        new DeprecatedElementsFinder());
         var rulesFile = new ClassPathResource("drools/documine_flora.drl");
 
         String rulesString = new String(rulesFile.getInputStream().readAllBytes());
 
         DroolsSyntaxValidation droolsSyntaxValidation = droolsSyntaxValidationService.testRules(new RuleValidationModel(RuleFileType.ENTITY.name(), rulesString));
-        droolsSyntaxValidation.getDroolsSyntaxErrorMessages().forEach(System.out::println);
+        droolsSyntaxValidation.getDroolsSyntaxErrorMessages()
+                .forEach(System.out::println);
         assertTrue(droolsSyntaxValidation.isCompiled());
     }
 
@@ -181,7 +117,8 @@ class DroolsSyntaxValidationServiceTest {
     @SneakyThrows
     void testCorruptedRules() {
 
-        DroolsSyntaxValidationService droolsSyntaxValidationService = new DroolsSyntaxValidationService(new KieContainerCreationService(rulesClient));
+        DroolsSyntaxValidationService droolsSyntaxValidationService = new DroolsSyntaxValidationService(new KieContainerCreationService(rulesClient),
+                                                                                                        new DeprecatedElementsFinder());
         var rulesFile = new ClassPathResource("drools/rules.drl");
 
         String rulesString = new String(rulesFile.getInputStream().readAllBytes());
@@ -196,12 +133,13 @@ class DroolsSyntaxValidationServiceTest {
     @SneakyThrows
     void testCorruptedImports() {
 
-        DroolsSyntaxValidationService droolsSyntaxValidationService = new DroolsSyntaxValidationService(new KieContainerCreationService(rulesClient));
+        DroolsSyntaxValidationService droolsSyntaxValidationService = new DroolsSyntaxValidationService(new KieContainerCreationService(rulesClient),
+                                                                                                        new DeprecatedElementsFinder());
         var rulesFile = new ClassPathResource("drools/rules.drl");
 
         String rulesString = new String(rulesFile.getInputStream().readAllBytes());
         String corruptedRules = rulesString.replaceAll("import com.iqser.red.service.redaction.v1.server.model.document.nodes.ImageType;",
-                "import com.iqser.red.service.redaction.v1.server.model.document.nodes.ImageType;\nimport com.iqser.red.service.redaction.v1.server.model.document.nodes.SomethingElse;");
+                                                       "import com.iqser.red.service.redaction.v1.server.model.document.nodes.ImageType;\nimport com.iqser.red.service.redaction.v1.server.model.document.nodes.SomethingElse;");
 
         DroolsSyntaxValidation droolsSyntaxValidation = droolsSyntaxValidationService.testRules(new RuleValidationModel(RuleFileType.ENTITY.name(), corruptedRules));
         assertFalse(droolsSyntaxValidation.isCompiled());
@@ -212,14 +150,16 @@ class DroolsSyntaxValidationServiceTest {
     @SneakyThrows
     void testRemoveQuery() {
 
-        DroolsSyntaxValidationService droolsSyntaxValidationService = new DroolsSyntaxValidationService(new KieContainerCreationService(rulesClient));
+        DroolsSyntaxValidationService droolsSyntaxValidationService = new DroolsSyntaxValidationService(new KieContainerCreationService(rulesClient),
+                                                                                                        new DeprecatedElementsFinder());
         var rulesFile = new ClassPathResource("drools/rules.drl");
 
         String rulesString = new String(rulesFile.getInputStream().readAllBytes());
         String corruptedRules = rulesString.replaceAll("query \"getFileAttributes\"\n.*\n *end", "");
 
         DroolsSyntaxValidation droolsSyntaxValidation = droolsSyntaxValidationService.testRules(new RuleValidationModel(RuleFileType.ENTITY.name(), corruptedRules));
-        droolsSyntaxValidation.getDroolsSyntaxErrorMessages().forEach(System.out::println);
+        droolsSyntaxValidation.getDroolsSyntaxErrorMessages()
+                .forEach(System.out::println);
         assertFalse(droolsSyntaxValidation.isCompiled());
     }
 
@@ -228,13 +168,15 @@ class DroolsSyntaxValidationServiceTest {
     @SneakyThrows
     void testComponentDrools() {
 
-        DroolsSyntaxValidationService droolsSyntaxValidationService = new DroolsSyntaxValidationService(new KieContainerCreationService(rulesClient));
+        DroolsSyntaxValidationService droolsSyntaxValidationService = new DroolsSyntaxValidationService(new KieContainerCreationService(rulesClient),
+                                                                                                        new DeprecatedElementsFinder());
         var rulesFile = new ClassPathResource("drools/documine_flora_components.drl");
 
         String rulesString = new String(rulesFile.getInputStream().readAllBytes());
 
         DroolsSyntaxValidation droolsSyntaxValidation = droolsSyntaxValidationService.testRules(new RuleValidationModel(RuleFileType.COMPONENT.name(), rulesString));
-        droolsSyntaxValidation.getDroolsSyntaxErrorMessages().forEach(System.out::println);
+        droolsSyntaxValidation.getDroolsSyntaxErrorMessages()
+                .forEach(System.out::println);
         assertTrue(droolsSyntaxValidation.isCompiled());
     }
 
@@ -243,13 +185,15 @@ class DroolsSyntaxValidationServiceTest {
     @SneakyThrows
     void testComponentDroolsAsEntityDrools() {
 
-        DroolsSyntaxValidationService droolsSyntaxValidationService = new DroolsSyntaxValidationService(new KieContainerCreationService(rulesClient));
+        DroolsSyntaxValidationService droolsSyntaxValidationService = new DroolsSyntaxValidationService(new KieContainerCreationService(rulesClient),
+                                                                                                        new DeprecatedElementsFinder());
         var rulesFile = new ClassPathResource("drools/documine_flora_components.drl");
 
         String rulesString = new String(rulesFile.getInputStream().readAllBytes());
 
         DroolsSyntaxValidation droolsSyntaxValidation = droolsSyntaxValidationService.testRules(new RuleValidationModel(RuleFileType.ENTITY.name(), rulesString));
-        droolsSyntaxValidation.getDroolsSyntaxErrorMessages().forEach(System.out::println);
+        droolsSyntaxValidation.getDroolsSyntaxErrorMessages()
+                .forEach(System.out::println);
         assertFalse(droolsSyntaxValidation.isCompiled());
     }
 
@@ -259,13 +203,15 @@ class DroolsSyntaxValidationServiceTest {
     @Disabled
     void attemptImportsFixToAllRuleFiles() {
 
-        DroolsSyntaxValidationService droolsSyntaxValidationService = new DroolsSyntaxValidationService(new KieContainerCreationService(rulesClient));
+        DroolsSyntaxValidationService droolsSyntaxValidationService = new DroolsSyntaxValidationService(new KieContainerCreationService(rulesClient),
+                                                                                                        new DeprecatedElementsFinder());
 
-        List<String> ruleFiles = List.of("drools/rules.drl", "drools/all_redact_manager_rules.drl",
-                "drools/documine_flora.drl",
-                "drools/manual_redaction_rules.drl",
-                "drools/acceptance_rules.drl",
-                "drools/rules_v2.drl");
+        List<String> ruleFiles = List.of("drools/rules.drl",
+                                         "drools/all_redact_manager_rules.drl",
+                                         "drools/documine_flora.drl",
+                                         "drools/manual_redaction_rules.drl",
+                                         "drools/acceptance_rules.drl",
+                                         "drools/rules_v2.drl");
 
         for (String ruleFile : ruleFiles) {
             var rulesFile = new ClassPathResource(ruleFile);
@@ -288,4 +234,109 @@ class DroolsSyntaxValidationServiceTest {
 
     }
 
+
+    @Test
+    @SneakyThrows
+    void testRulesWithUnallowedAddedImports() {
+
+        DroolsSyntaxValidationService droolsSyntaxValidationService = new DroolsSyntaxValidationService(new KieContainerCreationService(rulesClient),
+                                                                                                        new DeprecatedElementsFinder());
+        var rulesFile = new ClassPathResource("drools/rules.drl");
+
+        String rulesString = new String(rulesFile.getInputStream().readAllBytes());
+        rulesString = rulesString.replaceAll("import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell;",
+                                             "import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell;\nimport com.google.common.collect.Sets;");
+        DroolsSyntaxValidation droolsSyntaxValidation = droolsSyntaxValidationService.testRules(new RuleValidationModel(RuleFileType.ENTITY.name(), rulesString));
+        droolsSyntaxValidation.getDroolsSyntaxErrorMessages()
+                .forEach(System.out::println);
+        assertFalse(droolsSyntaxValidation.isCompiled());
+    }
+
+
+    @Test
+    @SneakyThrows
+    void testAllRules() {
+
+        DroolsSyntaxValidationService droolsSyntaxValidationService = new DroolsSyntaxValidationService(new KieContainerCreationService(rulesClient),
+                                                                                                        new DeprecatedElementsFinder());
+        var rulesFile = new ClassPathResource("drools/all_redact_manager_rules.drl");
+
+        String rulesString = new String(rulesFile.getInputStream().readAllBytes());
+
+        DroolsSyntaxValidation droolsSyntaxValidation = droolsSyntaxValidationService.testRules(new RuleValidationModel(RuleFileType.ENTITY.name(), rulesString));
+        droolsSyntaxValidation.getDroolsSyntaxErrorMessages()
+                .forEach(System.out::println);
+        assertTrue(droolsSyntaxValidation.isCompiled());
+    }
+
+
+    @Test
+    @SneakyThrows
+    void testAcceptanceRules() {
+
+        DroolsSyntaxValidationService droolsSyntaxValidationService = new DroolsSyntaxValidationService(new KieContainerCreationService(rulesClient),
+                                                                                                        new DeprecatedElementsFinder());
+        var rulesFile = new ClassPathResource("drools/acceptance_rules.drl");
+
+        String rulesString = new String(rulesFile.getInputStream().readAllBytes());
+
+        DroolsSyntaxValidation droolsSyntaxValidation = droolsSyntaxValidationService.testRules(new RuleValidationModel(RuleFileType.ENTITY.name(), rulesString));
+        droolsSyntaxValidation.getDroolsSyntaxErrorMessages()
+                .forEach(System.out::println);
+        assertTrue(droolsSyntaxValidation.isCompiled());
+    }
+
+
+    @Test
+    @SneakyThrows
+    void testManualRedactionRules() {
+
+        DroolsSyntaxValidationService droolsSyntaxValidationService = new DroolsSyntaxValidationService(new KieContainerCreationService(rulesClient),
+                                                                                                        new DeprecatedElementsFinder());
+        var rulesFile = new ClassPathResource("drools/manual_redaction_rules.drl");
+
+        String rulesString = new String(rulesFile.getInputStream().readAllBytes());
+
+        DroolsSyntaxValidation droolsSyntaxValidation = droolsSyntaxValidationService.testRules(new RuleValidationModel(RuleFileType.ENTITY.name(), rulesString));
+        droolsSyntaxValidation.getDroolsSyntaxErrorMessages()
+                .forEach(System.out::println);
+        assertTrue(droolsSyntaxValidation.isCompiled());
+    }
+
+
+    @Test
+    @SneakyThrows
+    void testRulesV2() {
+
+        DroolsSyntaxValidationService droolsSyntaxValidationService = new DroolsSyntaxValidationService(new KieContainerCreationService(rulesClient),
+                                                                                                        new DeprecatedElementsFinder());
+        var rulesFile = new ClassPathResource("drools/rules_v2.drl");
+
+        String rulesString = new String(rulesFile.getInputStream().readAllBytes());
+
+        DroolsSyntaxValidation droolsSyntaxValidation = droolsSyntaxValidationService.testRules(new RuleValidationModel(RuleFileType.ENTITY.name(), rulesString));
+        droolsSyntaxValidation.getDroolsSyntaxErrorMessages()
+                .forEach(System.out::println);
+        assertTrue(droolsSyntaxValidation.isCompiled());
+    }
+
+
+    @Test
+    @SneakyThrows
+    void testRulesWithDeprecatedFunctions() {
+
+        DroolsSyntaxValidationService droolsSyntaxValidationService = new DroolsSyntaxValidationService(new KieContainerCreationService(rulesClient),
+                                                                                                        new DeprecatedElementsFinder());
+        var rulesFile = new ClassPathResource("drools/rules.drl");
+
+        String rulesString = new String(rulesFile.getInputStream().readAllBytes());
+        rulesString = rulesString.replaceAll(".resize\\(", ".resizeEntityAndReinsert(");
+
+        DroolsSyntaxValidation droolsSyntaxValidation = droolsSyntaxValidationService.testRules(new RuleValidationModel(RuleFileType.ENTITY.name(), rulesString));
+        droolsSyntaxValidation.getDroolsSyntaxErrorMessages()
+                .forEach(System.out::println);
+        assertTrue(droolsSyntaxValidation.isCompiled());
+        assertEquals(droolsSyntaxValidation.getDroolsSyntaxDeprecatedWarnings().size(), 2);
+    }
+
 }

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesEnd2EndTest.java

@@ -38,6 +38,7 @@ import com.iqser.red.commons.jackson.ObjectMapperFactory;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeResult;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState;
@@ -58,7 +59,6 @@ import com.iqser.red.service.redaction.v1.server.Application;
 import com.iqser.red.service.redaction.v1.server.annotate.AnnotateRequest;
 import com.iqser.red.service.redaction.v1.server.annotate.AnnotateResponse;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
-import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.OsUtils;
 import com.iqser.red.service.redaction.v1.server.service.document.DocumentGraphMapper;
@@ -318,6 +318,7 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest {
                 .filter(entry -> entry.getValue().equals("Oxford University Press"))
                 .findFirst()
                 .get();
+        assertFalse(oxfordUniversityPress.getEngines().contains(Engine.MANUAL));
 
         var asyaLyon = redactionLog.getEntityLogEntry()
                 .stream()
@@ -364,6 +365,7 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest {
 
         assertEquals(asyaLyon2.getState(), EntryState.APPLIED);
         assertEquals(1, oxfordUniversityPressRecategorized.getManualChanges().size());
+        assertTrue(oxfordUniversityPressRecategorized.getEngines().contains(Engine.MANUAL));
     }
 
 
@@ -401,10 +403,58 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest {
         analyzeService.reanalyze(request);
 
         EntityLog entityLog = redactionStorageService.getEntityLog(request.getDossierId(), request.getFileId());
-        EntityLogEntry entityLogEntry = entityLog.getEntityLogEntry().stream().filter(entry -> entry.getId().equals(annotationId)).findFirst().orElseThrow();
+        EntityLogEntry entityLogEntry = entityLog.getEntityLogEntry()
+                .stream()
+                .filter(entry -> entry.getId().equals(annotationId))
+                .findFirst()
+                .orElseThrow();
         assertEquals("Expand to Hint", entityLogEntry.getValue());
         assertEquals(1, entityLogEntry.getPositions().size());
-        assertEquals(ManualRedactionType.RESIZE, entityLogEntry.getManualChanges().get(entityLogEntry.getManualChanges().size() - 1).getManualRedactionType());
+        assertEquals(ManualRedactionType.RESIZE,
+                     entityLogEntry.getManualChanges()
+                             .get(entityLogEntry.getManualChanges().size() - 1).getManualRedactionType());
+        assertTrue(entityLogEntry.getEngines().contains(Engine.MANUAL));
+    }
+
+
+    @Test
+    @SneakyThrows
+    public void testAddEngineManualToResizeDictionaryEntry() {
+
+        String filePath = "files/new/crafted document.pdf";
+        AnalyzeRequest request = uploadFileToStorage(filePath);
+        analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request);
+        AnalyzeResult result = analyzeService.analyze(request);
+        ManualRedactions manualRedactions = new ManualRedactions();
+
+        EntityLog entityLog = redactionStorageService.getEntityLog(request.getDossierId(), request.getFileId());
+        var dictionaryEntry = entityLog.getEntityLogEntry()
+                .stream()
+                .filter(entry -> entry.isDictionaryEntry() || entry.isDossierDictionaryEntry())
+                .findFirst()
+                .get();
+        ManualResizeRedaction manualResizeRedaction = ManualResizeRedaction.builder()
+                .annotationId(dictionaryEntry.getId())
+                .requestDate(OffsetDateTime.now())
+                .value("Image")
+                .positions(List.of(new Rectangle(new Point(56.8f, 496.27f), 61.25f, 12.83f, 1)))
+                .updateDictionary(true)
+                .build();
+        manualRedactions.setResizeRedactions(Set.of(manualResizeRedaction));
+        request.setManualRedactions(manualRedactions);
+
+        analyzeService.reanalyze(request);
+
+        entityLog = redactionStorageService.getEntityLog(request.getDossierId(), request.getFileId());
+        EntityLogEntry entityLogEntry = entityLog.getEntityLogEntry()
+                .stream()
+                .filter(entry -> entry.getId().equals(dictionaryEntry.getId()))
+                .findFirst()
+                .orElseThrow();
+        assertEquals(ManualRedactionType.RESIZE_IN_DICTIONARY,
+                     entityLogEntry.getManualChanges()
+                             .get(entityLogEntry.getManualChanges().size() - 1).getManualRedactionType());
+        assertTrue(entityLogEntry.getEngines().contains(Engine.MANUAL));
     }
 
 }

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesUnitTest.java

@@ -115,7 +115,12 @@ public class ManualChangesUnitTest extends BuildDocumentIntegrationTest {
         assertEquals(legalBasis, entity.getManualOverwrite().getLegalBasis().orElse(entity.getMatchedRule().getLegalBasis()));
         assertEquals(section, entity.getManualOverwrite().getSection().orElse(entity.getDeepestFullyContainingNode().toString()));
 
-        ManualRecategorization imageRecategorizationRequest = ManualRecategorization.builder().type("type").requestDate(start.plusSeconds(5)).annotationId(annotationId).build();
+        ManualRecategorization imageRecategorizationRequest = ManualRecategorization.builder()
+                .type("type")
+                .requestDate(start.plusSeconds(5))
+                .annotationId(annotationId)
+                .legalBasis("")
+                .build();
         entity.getManualOverwrite().addChange(imageRecategorizationRequest);
         assertTrue(entity.getManualOverwrite().getRecategorized().isPresent());
         assertTrue(entity.getManualOverwrite().getRecategorized().get());

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/service/document/SemanticNodeComparatorsTest.java

@@ -16,8 +16,8 @@ class SemanticNodeComparatorsTest {
     @Test
     public void testFirstSemanticNode() {
 
-        var node = new Section(List.of(0, 1), null, null, null, null);
-        var otherNode = new Section(List.of(0, 2), null, null, null, null);
+        var node = new Section(List.of(0, 1), null, null, null, null,null);
+        var otherNode = new Section(List.of(0, 2), null, null, null, null,null);
         List<SemanticNode> list = new ArrayList<>();
         list.add(otherNode);
         list.add(node);
@@ -29,8 +29,8 @@ class SemanticNodeComparatorsTest {
     @Test
     public void testFirstSemanticNode2() {
 
-        var node = new Section(Collections.emptyList(), null, null, null, null);
-        var otherNode = new Section(List.of(0, 2), null, null, null, null);
+        var node = new Section(Collections.emptyList(), null, null, null, null, null);
+        var otherNode = new Section(List.of(0, 2), null, null, null, null, null);
         List<SemanticNode> list = new ArrayList<>();
         list.add(otherNode);
         list.add(node);
@@ -42,8 +42,8 @@ class SemanticNodeComparatorsTest {
     @Test
     public void testFirstSemanticNode3() {
 
-        var node = new Section(List.of(1, 5, 8), null, null, null, null);
-        var otherNode = new Section(List.of(0, 2), null, null, null, null);
+        var node = new Section(List.of(1, 5, 8), null, null, null, null, null);
+        var otherNode = new Section(List.of(0, 2), null, null, null, null, null);
         List<SemanticNode> list = new ArrayList<>();
         list.add(otherNode);
         list.add(node);
@@ -55,8 +55,8 @@ class SemanticNodeComparatorsTest {
     @Test
     public void testFirstSemanticNode4() {
 
-        var node = new Section(List.of(1, 5, 8), null, null, null, null);
-        var otherNode = new Section(List.of(1, 5, 9), null, null, null, null);
+        var node = new Section(List.of(1, 5, 8), null, null, null, null,null);
+        var otherNode = new Section(List.of(1, 5, 9), null, null, null, null,null);
         List<SemanticNode> list = new ArrayList<>();
         list.add(otherNode);
         list.add(node);

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/utils/EntityVisualizationUtility.java

@@ -0,0 +1,61 @@
+package com.iqser.red.service.redaction.v1.server.utils;
+
+import java.awt.Color;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import org.apache.pdfbox.cos.COSName;
+
+import com.iqser.red.service.redaction.v1.server.model.document.entity.PositionOnPage;
+import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.Page;
+import com.knecon.fforesight.service.viewerdoc.ContentStreams;
+import com.knecon.fforesight.service.viewerdoc.model.ColoredRectangle;
+import com.knecon.fforesight.service.viewerdoc.model.VisualizationsOnPage;
+
+import lombok.experimental.UtilityClass;
+
+@UtilityClass
+public class EntityVisualizationUtility {
+
+    public static final ContentStreams.Identifier ENTITY_LAYER = new ContentStreams.Identifier("Entities", COSName.getPDFName("KNECON_ENTITIES"), true);
+
+
+    public Map<Integer, VisualizationsOnPage> createVisualizationsOnPage(Collection<TextEntity> entity, Color color) {
+
+        Map<Integer, VisualizationsOnPage> visualizations = new HashMap<>();
+        Set<Page> pages = entity.stream()
+                .map(TextEntity::getPages)
+                .flatMap(Collection::stream)
+                .collect(Collectors.toSet());
+
+        pages.forEach(page -> visualizations.put(page.getNumber() - 1, buildVisualizationsOnPage(color, page)));
+
+        return visualizations;
+    }
+
+
+    private static VisualizationsOnPage buildVisualizationsOnPage(Color color, Page page) {
+
+        return VisualizationsOnPage.builder().coloredRectangles(getEntityRectangles(color, page)).build();
+    }
+
+
+    private static List<ColoredRectangle> getEntityRectangles(Color color, Page page) {
+
+        return page.getEntities()
+                .stream()
+                .map(TextEntity::getPositionsOnPagePerPage)
+                .flatMap(Collection::stream)
+                .filter(p -> p.getPage().equals(page))
+                .map(PositionOnPage::getRectanglePerLine)
+                .flatMap(Collection::stream)
+                .map(r -> new ColoredRectangle(r, color, 1))
+                .toList();
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/utils/LayoutParsingRequestProvider.java

@@ -29,11 +29,14 @@ public class LayoutParsingRequestProvider {
                 .originFileStorageId(originFileStorageId)
                 .tablesFileStorageId(Optional.of(tablesFileStorageId))
                 .imagesFileStorageId(Optional.of(imagesFileStorageId))
+                .visualLayoutParsingFileId(Optional.empty())
                 .structureFileStorageId(structureFileStorageId)
                 .textBlockFileStorageId(textBlockFileStorageId)
                 .positionBlockFileStorageId(positionBlockFileStorageId)
                 .pageFileStorageId(pageFileStorageId)
-                .simplifiedTextStorageId(simplifiedTextStorageId).viewerDocumentStorageId(viewerDocumentStorageId)
+                .simplifiedTextStorageId(simplifiedTextStorageId)
+                .viewerDocumentStorageId(viewerDocumentStorageId)
+                .visualLayoutParsingFileId(Optional.empty())
                 .build();
     }
 

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/acceptance_rules.drl

@@ -54,6 +54,8 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange;
 
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
+
 global Document document
 global EntityCreationService entityCreationService
 global ManualChangesApplicationService manualChangesApplicationService
@@ -690,13 +692,22 @@ rule "ETC.3.1: Redact logos (vertebrate study)"
 
 
 // Rule unit: ETC.5
-rule "ETC.5.0: Ignore dossier_redaction entries if confidentiality is not 'confidential'"
+rule "ETC.5.0: Skip dossier_redaction entries if confidentiality is 'confidential'"
+    when
+        FileAttribute(label == "Confidentiality", value == "confidential")
+        $dossierRedaction: TextEntity(type() == "dossier_redaction")
+    then
+        $dossierRedaction.skip("ETC.5.0", "Ignore dossier_redaction when confidential");
+        $dossierRedaction.getIntersectingNodes().forEach(node -> update(node));
+    end
+
+rule "ETC.5.1: Remove dossier_redaction entries if confidentiality is not 'confidential'"
     when
         not FileAttribute(label == "Confidentiality", value == "confidential")
         $dossierRedaction: TextEntity(type() == "dossier_redaction")
     then
-        $dossierRedaction.ignore("ETC.5.0", "Ignore dossier redactions, when not confidential");
-        $dossierRedaction.getIntersectingNodes().forEach(node -> update(node));
+        $dossierRedaction.remove("ETC.5.1", "Remove dossier_redaction when not confidential");
+        retract($dossierRedaction);
     end
 
 
@@ -846,7 +857,7 @@ rule "MAN.3.2: Apply image recategorization"
 rule "MAN.3.3: Apply recategorization entities by default"
     salience 128
     when
-        $entity: IEntity(getManualOverwrite().getRecategorized().orElse(false), !dictionary.isHint(type()))
+        $entity: TextEntity(getManualOverwrite().getRecategorized().orElse(false), !dictionary.isHint(type()))
     then
         $entity.apply("MAN.3.3", "Recategorized entities are applied by default.", $entity.legalBasis());
     end
@@ -882,8 +893,8 @@ rule "MAN.4.1: Apply legal basis change"
 rule "X.0.0: Remove Entity contained by Entity of same type"
     salience 65
     when
-        $larger: TextEntity($type: type(), $entityType: entityType, active() || skipped())
-        $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges(), active())
+        $larger: TextEntity($type: type(), $entityType: entityType, active())
+        $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges())
     then
         $contained.remove("X.0.0", "remove Entity contained by Entity of same type");
         retract($contained);
@@ -911,7 +922,7 @@ rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
     salience 64
     when
         $falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active())
-        $entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges(), active())
+        $entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges())
     then
         $entity.getIntersectingNodes().forEach(node -> update(node));
         $entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE");
@@ -924,7 +935,7 @@ rule "X.3.0: Remove Entity of type RECOMMENDATION when contained by FALSE_RECOMM
     salience 64
     when
         $falseRecommendation: TextEntity($type: type(), entityType == EntityType.FALSE_RECOMMENDATION, active())
-        $recommendation: TextEntity(containedBy($falseRecommendation), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
+        $recommendation: TextEntity(containedBy($falseRecommendation), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges())
     then
         $recommendation.remove("X.3.0", "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION");
         retract($recommendation);
@@ -936,7 +947,7 @@ rule "X.4.0: Remove Entity of type RECOMMENDATION when text range equals ENTITY
     salience 256
     when
         $entity: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
-        $recommendation: TextEntity(getTextRange().equals($entity.getTextRange()), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
+        $recommendation: TextEntity(getTextRange().equals($entity.getTextRange()), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges())
     then
         $entity.addEngines($recommendation.getEngines());
         $recommendation.remove("X.4.0", "remove Entity of type RECOMMENDATION when text range equals ENTITY with same type");
@@ -949,7 +960,7 @@ rule "X.5.0: Remove Entity of type RECOMMENDATION when intersected by ENTITY"
     salience 256
     when
         $entity: TextEntity((entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
-        $recommendation: TextEntity(intersects($entity), entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
+        $recommendation: TextEntity(intersects($entity), entityType == EntityType.RECOMMENDATION, !hasManualChanges())
     then
         $recommendation.remove("X.5.0", "remove Entity of type RECOMMENDATION when intersected by ENTITY");
         retract($recommendation);
@@ -959,7 +970,7 @@ rule "X.5.1: Remove Entity of type RECOMMENDATION when contained by RECOMMENDATI
     salience 256
     when
         $entity: TextEntity($type: type(), entityType == EntityType.RECOMMENDATION, active())
-        $recommendation: TextEntity(containedBy($entity), type() != $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
+        $recommendation: TextEntity(containedBy($entity), type() != $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges())
     then
         $recommendation.remove("X.5.1", "remove Entity of type RECOMMENDATION when contained by RECOMMENDATION");
         retract($recommendation);
@@ -967,26 +978,26 @@ rule "X.5.1: Remove Entity of type RECOMMENDATION when contained by RECOMMENDATI
 
 
 // Rule unit: X.6
-rule "X.6.0: Remove Entity of lower rank, when contained by by entity of type ENTITY"
+rule "X.6.0: Remove Entity of lower rank, when contained by entity of type ENTITY or HINT"
     salience 32
     when
         $higherRank: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
-        $lowerRank: TextEntity(containedBy($higherRank), type() != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active())
+        $lowerRank: TextEntity(containedBy($higherRank), type() != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges())
     then
         $lowerRank.getIntersectingNodes().forEach(node -> update(node));
-        $lowerRank.remove("X.6.0", "remove Entity of lower rank, when contained by entity of type ENTITY");
+        $lowerRank.remove("X.6.0", "remove Entity of lower rank, when contained by entity of type ENTITY or HINT");
         retract($lowerRank);
     end
 
-rule "X.6.1: remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity"
+rule "X.6.1: remove Entity, when contained in another entity of type ENTITY or HINT with larger text range"
     salience 32
     when
-        $higherRank: TextEntity($type: type(), $value: value, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active(), !hasManualChanges())
-        $lowerRank: TextEntity(intersects($higherRank), type() != $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), active(), $lowerRank.getValue().length() > $value.length())
+         $outer: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
+         $inner: TextEntity(containedBy($outer), type() != $type, $outer.getTextRange().length > getTextRange().length(), !hasManualChanges())
     then
-        $higherRank.getIntersectingNodes().forEach(node -> update(node));
-        $higherRank.remove("X.6.1", "remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity");
-        retract($higherRank);
+        $inner.getIntersectingNodes().forEach(node -> update(node));
+        $inner.remove("X.6.1", "remove Entity, when contained in another entity of type ENTITY or HINT with larger text range");
+        retract($inner);
     end
 
 
@@ -1013,6 +1024,27 @@ rule "X.8.1: Remove Entity when intersected by imported Entity"
     end
 
 
+// Rule unit: X.9
+rule "X.9.0: Merge mostly contained signatures"
+    when
+        $aiSignature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.AI)
+        $signature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.ALGORITHM, mostlyContains($aiSignature, 0.8))
+    then
+        $aiSignature.remove("X.9.0", "removed because already contained by alogrithm signature");
+        $signature.addEngine(LayoutEngine.AI);
+    end
+
+
+// Rule unit: X.10
+rule "X.10.0: remove false positives of ai"
+    when
+        $anyImage: Image(engines contains LayoutEngine.ALGORITHM)
+        $aiSignature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.AI, !mostlyContainedBy($anyImage, 0.8))
+    then
+        $aiSignature.remove("X.10.0", "Removed because false positive");
+    end
+
+
 //------------------------------------ File attributes rules ------------------------------------
 
 // Rule unit: FA.1

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/all_redact_manager_rules.drl

@@ -54,6 +54,8 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange;
 
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
+
 global Document document
 global EntityCreationService entityCreationService
 global ManualChangesApplicationService manualChangesApplicationService
@@ -157,18 +159,17 @@ rule "CBI.3.0: Redacted because Section contains a vertebrate"
 
 rule "CBI.3.1: Redacted because table row contains a vertebrate"
     when
-        $table: Table(hasEntitiesOfType("vertebrate"), (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
+        $table: Table(hasEntitiesOfType("vertebrate"), hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address"))
+        $cellsWithvertebrate: TableCell() from $table.streamTableCellsWhichContainType("vertebrate").toList()
+        $tableCell: TableCell(row == $cellsWithvertebrate.row) from $table.streamTableCells().toList()
+        $authorOrAddress: TextEntity(type() == "CBI_author" || type() == "CBI_address", active()) from $tableCell.getEntities()
     then
-        $table.streamEntitiesWhereRowContainsEntitiesOfType(List.of("vertebrate"))
-            .filter(entity -> entity.getType().equals("CBI_author") || entity.getType().equals("CBI_address"))
-            .forEach(entity -> {
-                entity.applyWithReferences(
+        $authorOrAddress.applyWithReferences(
                         "CBI.3.1",
                         "Vertebrate found",
                         "Reg (EC) No 1107/2009 Art. 63 (2g)",
-                        $table.getEntitiesOfTypeInSameRow("vertebrate", entity)
+                        $table.getEntitiesOfTypeInSameRow("vertebrate", $authorOrAddress)
                 );
-            });
     end
 
 rule "CBI.3.2: Do not redact because Section does not contain a vertebrate"
@@ -207,23 +208,21 @@ rule "CBI.4.0: Do not redact Names and Addresses if no_redaction_indicator is fo
             });
     end
 
-rule "CBI.4.1: Do not redact Names and Addresses if no_redaction_indicator is found in table row"
+rule "CBI.4.1: Redacted because table row contains a vertebrate"
     when
-        $table: Table(hasEntitiesOfType("no_redaction_indicator"),
-                      hasEntitiesOfType("vertebrate"),
-                      (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
+        $table: Table(hasEntitiesOfType("no_redaction_indicator"), hasEntitiesOfType("vertebrate"), hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address"))
+        TableCell($row: row) from $table.streamTableCellsWhichContainType("vertebrate").toList()
+        TableCell(row == $row) from $table.streamTableCellsWhichContainType("no_redaction_indicator").toList()
+        $tableCell: TableCell(row == $row) from $table.streamTableCells().toList()
+        $authorOrAddress: TextEntity(type() == "CBI_author" || type() == "CBI_address", active()) from $tableCell.getEntities()
     then
-        $table.streamEntitiesWhereRowContainsEntitiesOfType(List.of("vertebrate", "no-redaction_indicator"))
-            .filter(entity -> entity.getType().equals("CBI_author") || entity.getType().equals("CBI_address"))
-            .forEach(entity -> {
-                entity.skipWithReferences(
+        $authorOrAddress.skipWithReferences(
                         "CBI.4.1",
                         "Vertebrate but a no redaction indicator found",
                         Stream.concat(
-                            $table.getEntitiesOfTypeInSameRow("vertebrate", entity).stream(),
-                            $table.getEntitiesOfTypeInSameRow("no_redaction_indicator", entity).stream()).toList()
+                            $table.getEntitiesOfTypeInSameRow("vertebrate", $authorOrAddress).stream(),
+                            $table.getEntitiesOfTypeInSameRow("no_redaction_indicator", $authorOrAddress).stream()).toList()
                 );
-            });
     end
 
 
@@ -250,22 +249,20 @@ rule "CBI.5.0: Redact Names and Addresses if no_redaction_indicator but also red
 
 rule "CBI.5.1: Redact Names and Addresses if no_redaction_indicator but also redaction_indicator is found in table row"
     when
-        $table: Table(hasEntitiesOfType("no_redaction_indicator"),
-                      hasEntitiesOfType("redaction_indicator"),
-                      (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
+        $table: Table(hasEntitiesOfType("no_redaction_indicator"), hasEntitiesOfType("redaction_indicator"), hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address"))
+        TableCell($row: row) from $table.streamTableCellsWhichContainType("redaction_indicator").toList()
+        TableCell(row == $row) from $table.streamTableCellsWhichContainType("no_redaction_indicator").toList()
+        $tableCell: TableCell(row == $row) from $table.streamTableCells().toList()
+        $entity: TextEntity(type() == "CBI_author" || type() == "CBI_address", active()) from $tableCell.getEntities()
     then
-        $table.streamEntitiesWhereRowContainsEntitiesOfType(List.of("redaction_indicator", "no_redaction_indicator"))
-            .filter(entity -> entity.getType().equals("CBI_author") || entity.getType().equals("CBI_address"))
-            .forEach(entity -> {
-                entity.applyWithReferences(
+        $entity.applyWithReferences(
                         "CBI.5.1",
                         "no_redaction_indicator but also redaction_indicator found",
                         "Reg (EC) No 1107/2009 Art. 63 (2g)",
                         Stream.concat(
-                            $table.getEntitiesOfTypeInSameRow("vertebrate", entity).stream(),
-                            $table.getEntitiesOfTypeInSameRow("no_redaction_indicator", entity).stream()).toList()
+                            $table.getEntitiesOfTypeInSameRow("redaction_indicator", $entity).stream(),
+                            $table.getEntitiesOfTypeInSameRow("no_redaction_indicator", $entity).stream()).toList()
                 );
-            });
     end
 
 
@@ -355,18 +352,17 @@ rule "CBI.8.0: Redacted because Section contains must_redact entity"
 
 rule "CBI.8.1: Redacted because table row contains must_redact entity"
     when
-        $table: Table(hasEntitiesOfType("must_redact"), (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
+        $table: Table(hasEntitiesOfType("must_redact"), hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address"))
+        $cellsWithMustRedact: TableCell() from $table.streamTableCellsWhichContainType("must_redact").toList()
+        $tableCell: TableCell(row == $cellsWithMustRedact.row) from $table.streamTableCells().toList()
+        $authorOrAddress: TextEntity(type() == "CBI_author" || type() == "CBI_address", active()) from $tableCell.getEntities()
     then
-        $table.streamEntitiesWhereRowContainsEntitiesOfType(List.of("must_redact"))
-            .filter(entity -> entity.getType().equals("CBI_author") || entity.getType().equals("CBI_address"))
-            .forEach(entity -> {
-                entity.applyWithReferences(
+        $authorOrAddress.applyWithReferences(
                         "CBI.8.1",
-                        "must_redact entity found",
+                        "Must_redact found",
                         "Reg (EC) No 1107/2009 Art. 63 (2g)",
-                        $table.getEntitiesOfTypeInSameRow("must_redact", entity)
+                        $table.getEntitiesOfTypeInSameRow("must_redact", $authorOrAddress)
                 );
-            });
     end
 
 
@@ -448,7 +444,6 @@ rule "CBI.12.0: Redact and recommend TableCell with header 'Author' or 'Author(s
         TableCell(!header, containsAnyString("Yes", "Y"), $rowWithYes: row) from $table.streamCol($vertebrateCol).toList()
         $authorCell: TableCell(row == $rowWithYes) from $table.streamCol($authorCol).toList()
     then
-
         entityCreationService.bySemanticNode($authorCell, "CBI_author", EntityType.ENTITY)
           .ifPresent(authorEntity -> {
               authorEntity.redact("CBI.12.0", "Redacted because it's row belongs to a vertebrate study", "Article 39(e)(3) of Regulation (EC) No 178/2002");
@@ -744,7 +739,7 @@ rule "PII.1.2: Redact typoed Emails with indicator"
     when
         $section: Section(containsString("@") || containsStringIgnoreCase("mail"))
     then
-        entityCreationService.byRegexIgnoreCase("mail[:\\.\\s]{1,2}([\\w\\/\\-\\{\\(\\. ]{3,20}(@|a|f)\\s?[\\w\\/\\-\\{\\(\\. ]{3,20}(\\. \\w{2,4}\\b|\\.\\B|\\.\\w{1,4}\\b))", "PII", EntityType.ENTITY, $section)
+        entityCreationService.byRegexIgnoreCase("mail[:\\.\\s]{1,2}([\\w\\/\\-\\{\\(\\. ]{3,20}(@|a|f)\\s?[\\w\\/\\-\\{\\(\\. ]{3,20}(\\. \\w{2,4}\\b|\\.\\B|\\.\\w{1,4}\\b))", "PII", EntityType.ENTITY, 1, $section)
             .forEach(emailEntity -> emailEntity.redact("PII.1.2", "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
     end
 
@@ -1155,13 +1150,22 @@ rule "ETC.4.2: Redact dossier dictionary entries"
 
 
 // Rule unit: ETC.5
-rule "ETC.5.0: Ignore dossier_redaction entries if confidentiality is not 'confidential'"
+rule "ETC.5.0: Skip dossier_redaction entries if confidentiality is 'confidential'"
+    when
+        FileAttribute(label == "Confidentiality", value == "confidential")
+        $dossierRedaction: TextEntity(type() == "dossier_redaction")
+    then
+        $dossierRedaction.skip("ETC.5.0", "Ignore dossier_redaction when confidential");
+        $dossierRedaction.getIntersectingNodes().forEach(node -> update(node));
+    end
+
+rule "ETC.5.1: Remove dossier_redaction entries if confidentiality is not 'confidential'"
     when
         not FileAttribute(label == "Confidentiality", value == "confidential")
         $dossierRedaction: TextEntity(type() == "dossier_redaction")
     then
-        $dossierRedaction.ignore("ETC.5.0", "Ignore dossier redactions, when not confidential");
-        $dossierRedaction.getIntersectingNodes().forEach(node -> update(node));
+        $dossierRedaction.remove("ETC.5.1", "Remove dossier_redaction when not confidential");
+        retract($dossierRedaction);
     end
 
 
@@ -1416,7 +1420,7 @@ rule "MAN.3.2: Apply image recategorization"
 rule "MAN.3.3: Apply recategorization entities by default"
     salience 128
     when
-        $entity: IEntity(getManualOverwrite().getRecategorized().orElse(false), !dictionary.isHint(type()))
+        $entity: TextEntity(getManualOverwrite().getRecategorized().orElse(false), !dictionary.isHint(type()))
     then
         $entity.apply("MAN.3.3", "Recategorized entities are applied by default.", $entity.legalBasis());
     end
@@ -1452,8 +1456,8 @@ rule "MAN.4.1: Apply legal basis change"
 rule "X.0.0: Remove Entity contained by Entity of same type"
     salience 65
     when
-        $larger: TextEntity($type: type(), $entityType: entityType, active() || skipped())
-        $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges(), active())
+        $larger: TextEntity($type: type(), $entityType: entityType, active())
+        $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges())
     then
         $contained.remove("X.0.0", "remove Entity contained by Entity of same type");
         retract($contained);
@@ -1481,7 +1485,7 @@ rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
     salience 64
     when
         $falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active())
-        $entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges(), active())
+        $entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges())
     then
         $entity.getIntersectingNodes().forEach(node -> update(node));
         $entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE");
@@ -1494,7 +1498,7 @@ rule "X.3.0: Remove Entity of type RECOMMENDATION when contained by FALSE_RECOMM
     salience 64
     when
         $falseRecommendation: TextEntity($type: type(), entityType == EntityType.FALSE_RECOMMENDATION, active())
-        $recommendation: TextEntity(containedBy($falseRecommendation), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
+        $recommendation: TextEntity(containedBy($falseRecommendation), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges())
     then
         $recommendation.remove("X.3.0", "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION");
         retract($recommendation);
@@ -1506,7 +1510,7 @@ rule "X.4.0: Remove Entity of type RECOMMENDATION when text range equals ENTITY
     salience 256
     when
         $entity: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
-        $recommendation: TextEntity(getTextRange().equals($entity.getTextRange()), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
+        $recommendation: TextEntity(getTextRange().equals($entity.getTextRange()), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges())
     then
         $entity.addEngines($recommendation.getEngines());
         $recommendation.remove("X.4.0", "remove Entity of type RECOMMENDATION when text range equals ENTITY with same type");
@@ -1519,7 +1523,7 @@ rule "X.5.0: Remove Entity of type RECOMMENDATION when intersected by ENTITY"
     salience 256
     when
         $entity: TextEntity((entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
-        $recommendation: TextEntity(intersects($entity), entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
+        $recommendation: TextEntity(intersects($entity), entityType == EntityType.RECOMMENDATION, !hasManualChanges())
     then
         $recommendation.remove("X.5.0", "remove Entity of type RECOMMENDATION when intersected by ENTITY");
         retract($recommendation);
@@ -1529,7 +1533,7 @@ rule "X.5.1: Remove Entity of type RECOMMENDATION when contained by RECOMMENDATI
     salience 256
     when
         $entity: TextEntity($type: type(), entityType == EntityType.RECOMMENDATION, active())
-        $recommendation: TextEntity(containedBy($entity), type() != $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
+        $recommendation: TextEntity(containedBy($entity), type() != $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges())
     then
         $recommendation.remove("X.5.1", "remove Entity of type RECOMMENDATION when contained by RECOMMENDATION");
         retract($recommendation);
@@ -1537,26 +1541,26 @@ rule "X.5.1: Remove Entity of type RECOMMENDATION when contained by RECOMMENDATI
 
 
 // Rule unit: X.6
-rule "X.6.0: Remove Entity of lower rank, when contained by by entity of type ENTITY"
+rule "X.6.0: Remove Entity of lower rank, when contained by entity of type ENTITY or HINT"
     salience 32
     when
         $higherRank: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
-        $lowerRank: TextEntity(containedBy($higherRank), type() != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active())
+        $lowerRank: TextEntity(containedBy($higherRank), type() != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges())
     then
         $lowerRank.getIntersectingNodes().forEach(node -> update(node));
-        $lowerRank.remove("X.6.0", "remove Entity of lower rank, when contained by entity of type ENTITY");
+        $lowerRank.remove("X.6.0", "remove Entity of lower rank, when contained by entity of type ENTITY or HINT");
         retract($lowerRank);
     end
 
-rule "X.6.1: remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity"
+rule "X.6.1: remove Entity, when contained in another entity of type ENTITY or HINT with larger text range"
     salience 32
     when
-        $higherRank: TextEntity($type: type(), $value: value, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active(), !hasManualChanges())
-        $lowerRank: TextEntity(intersects($higherRank), type() != $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), active(), $lowerRank.getValue().length() > $value.length())
+         $outer: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
+         $inner: TextEntity(containedBy($outer), type() != $type, $outer.getTextRange().length > getTextRange().length(), !hasManualChanges())
     then
-        $higherRank.getIntersectingNodes().forEach(node -> update(node));
-        $higherRank.remove("X.6.1", "remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity");
-        retract($higherRank);
+        $inner.getIntersectingNodes().forEach(node -> update(node));
+        $inner.remove("X.6.1", "remove Entity, when contained in another entity of type ENTITY or HINT with larger text range");
+        retract($inner);
     end
 
 
@@ -1583,6 +1587,27 @@ rule "X.8.1: Remove Entity when intersected by imported Entity"
     end
 
 
+// Rule unit: X.9
+rule "X.9.0: Merge mostly contained signatures"
+    when
+        $aiSignature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.AI)
+        $signature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.ALGORITHM, mostlyContains($aiSignature, 0.8))
+    then
+        $aiSignature.remove("X.9.0", "removed because already contained by alogrithm signature");
+        $signature.addEngine(LayoutEngine.AI);
+    end
+
+
+// Rule unit: X.10
+rule "X.10.0: remove false positives of ai"
+    when
+        $anyImage: Image(engines contains LayoutEngine.ALGORITHM)
+        $aiSignature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.AI, !mostlyContainedBy($anyImage, 0.8))
+    then
+        $aiSignature.remove("X.10.0", "Removed because false positive");
+    end
+
+
 //------------------------------------ File attributes rules ------------------------------------
 
 // Rule unit: FA.1

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora.drl

@@ -54,6 +54,8 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange;
 
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
+
 global Document document
 global EntityCreationService entityCreationService
 global ManualChangesApplicationService manualChangesApplicationService
@@ -1276,7 +1278,7 @@ rule "MAN.3.2: Apply image recategorization"
 rule "MAN.3.3: Apply recategorization entities by default"
     salience 128
     when
-        $entity: IEntity(getManualOverwrite().getRecategorized().orElse(false), !dictionary.isHint(type()))
+        $entity: TextEntity(getManualOverwrite().getRecategorized().orElse(false), !dictionary.isHint(type()))
     then
         $entity.apply("MAN.3.3", "Recategorized entities are applied by default.", $entity.legalBasis());
     end
@@ -1312,8 +1314,8 @@ rule "MAN.4.1: Apply legal basis change"
 rule "X.0.0: Remove Entity contained by Entity of same type"
     salience 65
     when
-        $larger: TextEntity($type: type(), $entityType: entityType, active() || skipped())
-        $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges(), active())
+        $larger: TextEntity($type: type(), $entityType: entityType, active())
+        $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges())
     then
         $contained.remove("X.0.0", "remove Entity contained by Entity of same type");
         retract($contained);
@@ -1325,7 +1327,7 @@ rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
     salience 64
     when
         $falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active())
-        $entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges(), active())
+        $entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges())
     then
         $entity.getIntersectingNodes().forEach(node -> update(node));
         $entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE");
@@ -1338,7 +1340,7 @@ rule "X.3.0: Remove Entity of type RECOMMENDATION when contained by FALSE_RECOMM
     salience 64
     when
         $falseRecommendation: TextEntity($type: type(), entityType == EntityType.FALSE_RECOMMENDATION, active())
-        $recommendation: TextEntity(containedBy($falseRecommendation), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
+        $recommendation: TextEntity(containedBy($falseRecommendation), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges())
     then
         $recommendation.remove("X.3.0", "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION");
         retract($recommendation);
@@ -1350,7 +1352,7 @@ rule "X.4.0: Remove Entity of type RECOMMENDATION when text range equals ENTITY
     salience 256
     when
         $entity: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
-        $recommendation: TextEntity(getTextRange().equals($entity.getTextRange()), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
+        $recommendation: TextEntity(getTextRange().equals($entity.getTextRange()), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges())
     then
         $entity.addEngines($recommendation.getEngines());
         $recommendation.remove("X.4.0", "remove Entity of type RECOMMENDATION when text range equals ENTITY with same type");
@@ -1363,7 +1365,7 @@ rule "X.5.0: Remove Entity of type RECOMMENDATION when intersected by ENTITY"
     salience 256
     when
         $entity: TextEntity((entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
-        $recommendation: TextEntity(intersects($entity), entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
+        $recommendation: TextEntity(intersects($entity), entityType == EntityType.RECOMMENDATION, !hasManualChanges())
     then
         $recommendation.remove("X.5.0", "remove Entity of type RECOMMENDATION when intersected by ENTITY");
         retract($recommendation);
@@ -1373,7 +1375,7 @@ rule "X.5.1: Remove Entity of type RECOMMENDATION when contained by RECOMMENDATI
     salience 256
     when
         $entity: TextEntity($type: type(), entityType == EntityType.RECOMMENDATION, active())
-        $recommendation: TextEntity(containedBy($entity), type() != $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
+        $recommendation: TextEntity(containedBy($entity), type() != $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges())
     then
         $recommendation.remove("X.5.1", "remove Entity of type RECOMMENDATION when contained by RECOMMENDATION");
         retract($recommendation);
@@ -1414,6 +1416,27 @@ rule "X.8.1: Remove Entity when intersected by imported Entity"
     end
 
 
+// Rule unit: X.9
+rule "X.9.0: Merge mostly contained signatures"
+    when
+        $aiSignature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.AI)
+        $signature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.ALGORITHM, mostlyContains($aiSignature, 0.8))
+    then
+        $aiSignature.remove("X.9.0", "removed because already contained by alogrithm signature");
+        $signature.addEngine(LayoutEngine.AI);
+    end
+
+
+// Rule unit: X.10
+rule "X.10.0: remove false positives of ai"
+    when
+        $anyImage: Image(engines contains LayoutEngine.ALGORITHM)
+        $aiSignature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.AI, !mostlyContainedBy($anyImage, 0.8))
+    then
+        $aiSignature.remove("X.10.0", "Removed because false positive");
+    end
+
+
 //------------------------------------ File attributes rules ------------------------------------
 
 // Rule unit: FA.1

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/manual_redaction_rules.drl

@@ -54,6 +54,8 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange;
 
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
+
 global Document document
 global EntityCreationService entityCreationService
 global ManualChangesApplicationService manualChangesApplicationService
@@ -188,7 +190,7 @@ rule "MAN.3.2: Apply image recategorization"
 rule "MAN.3.3: Apply recategorization entities by default"
     salience 128
     when
-        $entity: IEntity(getManualOverwrite().getRecategorized().orElse(false), !dictionary.isHint(type()))
+        $entity: TextEntity(getManualOverwrite().getRecategorized().orElse(false), !dictionary.isHint(type()))
     then
         $entity.apply("MAN.3.3", "Recategorized entities are applied by default.", $entity.legalBasis());
     end
@@ -225,7 +227,7 @@ rule "X.5.1: Remove Entity of type RECOMMENDATION when contained by RECOMMENDATI
     salience 256
     when
         $entity: TextEntity($type: type(), entityType == EntityType.RECOMMENDATION, active())
-        $recommendation: TextEntity(containedBy($entity), type() != $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
+        $recommendation: TextEntity(containedBy($entity), type() != $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges())
     then
         $recommendation.remove("X.5.1", "remove Entity of type RECOMMENDATION when contained by RECOMMENDATION");
         retract($recommendation);
@@ -255,6 +257,27 @@ rule "X.8.1: Remove Entity when intersected by imported Entity"
     end
 
 
+// Rule unit: X.9
+rule "X.9.0: Merge mostly contained signatures"
+    when
+        $aiSignature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.AI)
+        $signature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.ALGORITHM, mostlyContains($aiSignature, 0.8))
+    then
+        $aiSignature.remove("X.9.0", "removed because already contained by alogrithm signature");
+        $signature.addEngine(LayoutEngine.AI);
+    end
+
+
+// Rule unit: X.10
+rule "X.10.0: remove false positives of ai"
+    when
+        $anyImage: Image(engines contains LayoutEngine.ALGORITHM)
+        $aiSignature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.AI, !mostlyContainedBy($anyImage, 0.8))
+    then
+        $aiSignature.remove("X.10.0", "Removed because false positive");
+    end
+
+
 //------------------------------------ Local dictionary search rules ------------------------------------
 
 // Rule unit: LDS.0

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl

@@ -54,6 +54,8 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange;
 
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
+
 global Document document
 global EntityCreationService entityCreationService
 global ManualChangesApplicationService manualChangesApplicationService
@@ -100,18 +102,17 @@ rule "CBI.3.0: Redacted because Section contains a vertebrate"
 
 rule "CBI.3.1: Redacted because table row contains a vertebrate"
     when
-        $table: Table(hasEntitiesOfType("vertebrate"), (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
+        $table: Table(hasEntitiesOfType("vertebrate"), hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address"))
+        $cellsWithvertebrate: TableCell() from $table.streamTableCellsWhichContainType("vertebrate").toList()
+        $tableCell: TableCell(row == $cellsWithvertebrate.row) from $table.streamTableCells().toList()
+        $authorOrAddress: TextEntity(type() == "CBI_author" || type() == "CBI_address", active()) from $tableCell.getEntities()
     then
-        $table.streamEntitiesWhereRowContainsEntitiesOfType(List.of("vertebrate"))
-            .filter(entity -> entity.getType().equals("CBI_author") || entity.getType().equals("CBI_address"))
-            .forEach(entity -> {
-                entity.applyWithReferences(
+        $authorOrAddress.applyWithReferences(
                         "CBI.3.1",
                         "Vertebrate found",
                         "Reg (EC) No 1107/2009 Art. 63 (2g)",
-                        $table.getEntitiesOfTypeInSameRow("vertebrate", entity)
+                        $table.getEntitiesOfTypeInSameRow("vertebrate", $authorOrAddress)
                 );
-            });
     end
 
 rule "CBI.3.2: Do not redact because Section does not contain a vertebrate"
@@ -150,23 +151,21 @@ rule "CBI.4.0: Do not redact Names and Addresses if no_redaction_indicator is fo
             });
     end
 
-rule "CBI.4.1: Do not redact Names and Addresses if no_redaction_indicator is found in table row"
+rule "CBI.4.1: Redacted because table row contains a vertebrate"
     when
-        $table: Table(hasEntitiesOfType("no_redaction_indicator"),
-                      hasEntitiesOfType("vertebrate"),
-                      (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
+        $table: Table(hasEntitiesOfType("no_redaction_indicator"), hasEntitiesOfType("vertebrate"), hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address"))
+        TableCell($row: row) from $table.streamTableCellsWhichContainType("vertebrate").toList()
+        TableCell(row == $row) from $table.streamTableCellsWhichContainType("no_redaction_indicator").toList()
+        $tableCell: TableCell(row == $row) from $table.streamTableCells().toList()
+        $authorOrAddress: TextEntity(type() == "CBI_author" || type() == "CBI_address", active()) from $tableCell.getEntities()
     then
-        $table.streamEntitiesWhereRowContainsEntitiesOfType(List.of("vertebrate", "no-redaction_indicator"))
-            .filter(entity -> entity.getType().equals("CBI_author") || entity.getType().equals("CBI_address"))
-            .forEach(entity -> {
-                entity.skipWithReferences(
+        $authorOrAddress.skipWithReferences(
                         "CBI.4.1",
                         "Vertebrate but a no redaction indicator found",
                         Stream.concat(
-                            $table.getEntitiesOfTypeInSameRow("vertebrate", entity).stream(),
-                            $table.getEntitiesOfTypeInSameRow("no_redaction_indicator", entity).stream()).toList()
+                            $table.getEntitiesOfTypeInSameRow("vertebrate", $authorOrAddress).stream(),
+                            $table.getEntitiesOfTypeInSameRow("no_redaction_indicator", $authorOrAddress).stream()).toList()
                 );
-            });
     end
 
 
@@ -193,22 +192,20 @@ rule "CBI.5.0: Redact Names and Addresses if no_redaction_indicator but also red
 
 rule "CBI.5.1: Redact Names and Addresses if no_redaction_indicator but also redaction_indicator is found in table row"
     when
-        $table: Table(hasEntitiesOfType("no_redaction_indicator"),
-                      hasEntitiesOfType("redaction_indicator"),
-                      (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
+        $table: Table(hasEntitiesOfType("no_redaction_indicator"), hasEntitiesOfType("redaction_indicator"), hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address"))
+        TableCell($row: row) from $table.streamTableCellsWhichContainType("redaction_indicator").toList()
+        TableCell(row == $row) from $table.streamTableCellsWhichContainType("no_redaction_indicator").toList()
+        $tableCell: TableCell(row == $row) from $table.streamTableCells().toList()
+        $entity: TextEntity(type() == "CBI_author" || type() == "CBI_address", active()) from $tableCell.getEntities()
     then
-        $table.streamEntitiesWhereRowContainsEntitiesOfType(List.of("redaction_indicator", "no_redaction_indicator"))
-            .filter(entity -> entity.getType().equals("CBI_author") || entity.getType().equals("CBI_address"))
-            .forEach(entity -> {
-                entity.applyWithReferences(
+        $entity.applyWithReferences(
                         "CBI.5.1",
                         "no_redaction_indicator but also redaction_indicator found",
                         "Reg (EC) No 1107/2009 Art. 63 (2g)",
                         Stream.concat(
-                            $table.getEntitiesOfTypeInSameRow("vertebrate", entity).stream(),
-                            $table.getEntitiesOfTypeInSameRow("no_redaction_indicator", entity).stream()).toList()
+                            $table.getEntitiesOfTypeInSameRow("redaction_indicator", $entity).stream(),
+                            $table.getEntitiesOfTypeInSameRow("no_redaction_indicator", $entity).stream()).toList()
                 );
-            });
     end
 
 
@@ -230,18 +227,17 @@ rule "CBI.8.0: Redacted because Section contains must_redact entity"
 
 rule "CBI.8.1: Redacted because table row contains must_redact entity"
     when
-        $table: Table(hasEntitiesOfType("must_redact"), (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
+        $table: Table(hasEntitiesOfType("must_redact"), hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address"))
+        $cellsWithMustRedact: TableCell() from $table.streamTableCellsWhichContainType("must_redact").toList()
+        $tableCell: TableCell(row == $cellsWithMustRedact.row) from $table.streamTableCells().toList()
+        $authorOrAddress: TextEntity(type() == "CBI_author" || type() == "CBI_address", active()) from $tableCell.getEntities()
     then
-        $table.streamEntitiesWhereRowContainsEntitiesOfType(List.of("must_redact"))
-            .filter(entity -> entity.getType().equals("CBI_author") || entity.getType().equals("CBI_address"))
-            .forEach(entity -> {
-                entity.applyWithReferences(
+        $authorOrAddress.applyWithReferences(
                         "CBI.8.1",
-                        "must_redact entity found",
+                        "Must_redact found",
                         "Reg (EC) No 1107/2009 Art. 63 (2g)",
-                        $table.getEntitiesOfTypeInSameRow("must_redact", entity)
+                        $table.getEntitiesOfTypeInSameRow("must_redact", $authorOrAddress)
                 );
-            });
     end
 
 
@@ -295,7 +291,6 @@ rule "CBI.12.0: Redact and recommend TableCell with header 'Author' or 'Author(s
         TableCell(!header, containsAnyString("Yes", "Y"), $rowWithYes: row) from $table.streamCol($vertebrateCol).toList()
         $authorCell: TableCell(row == $rowWithYes) from $table.streamCol($authorCol).toList()
     then
-
         entityCreationService.bySemanticNode($authorCell, "CBI_author", EntityType.ENTITY)
           .ifPresent(authorEntity -> {
               authorEntity.redact("CBI.12.0", "Redacted because it's row belongs to a vertebrate study", "Article 39(e)(3) of Regulation (EC) No 178/2002");
@@ -807,13 +802,22 @@ rule "ETC.4.0: Redact dossier dictionary entries"
 
 
 // Rule unit: ETC.5
-rule "ETC.5.0: Ignore dossier_redaction entries if confidentiality is not 'confidential'"
+rule "ETC.5.0: Skip dossier_redaction entries if confidentiality is 'confidential'"
+    when
+        FileAttribute(label == "Confidentiality", value == "confidential")
+        $dossierRedaction: TextEntity(type() == "dossier_redaction")
+    then
+        $dossierRedaction.skip("ETC.5.0", "Ignore dossier_redaction when confidential");
+        $dossierRedaction.getIntersectingNodes().forEach(node -> update(node));
+    end
+
+rule "ETC.5.1: Remove dossier_redaction entries if confidentiality is not 'confidential'"
     when
         not FileAttribute(label == "Confidentiality", value == "confidential")
         $dossierRedaction: TextEntity(type() == "dossier_redaction")
     then
-        $dossierRedaction.ignore("ETC.5.0", "Ignore dossier redactions, when not confidential");
-        $dossierRedaction.getIntersectingNodes().forEach(node -> update(node));
+        $dossierRedaction.remove("ETC.5.1", "Remove dossier_redaction when not confidential");
+        retract($dossierRedaction);
     end
 
 
@@ -1006,7 +1010,7 @@ rule "MAN.3.2: Apply image recategorization"
 rule "MAN.3.3: Apply recategorization entities by default"
     salience 128
     when
-        $entity: IEntity(getManualOverwrite().getRecategorized().orElse(false), !dictionary.isHint(type()))
+        $entity: TextEntity(getManualOverwrite().getRecategorized().orElse(false), !dictionary.isHint(type()))
     then
         $entity.apply("MAN.3.3", "Recategorized entities are applied by default.", $entity.legalBasis());
     end
@@ -1042,8 +1046,8 @@ rule "MAN.4.1: Apply legal basis change"
 rule "X.0.0: Remove Entity contained by Entity of same type"
     salience 65
     when
-        $larger: TextEntity($type: type(), $entityType: entityType, active() || skipped())
-        $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges(), active())
+        $larger: TextEntity($type: type(), $entityType: entityType, active())
+        $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges())
     then
         $contained.remove("X.0.0", "remove Entity contained by Entity of same type");
         retract($contained);
@@ -1071,7 +1075,7 @@ rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
     salience 64
     when
         $falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active())
-        $entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges(), active())
+        $entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges())
     then
         $entity.getIntersectingNodes().forEach(node -> update(node));
         $entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE");
@@ -1084,7 +1088,7 @@ rule "X.3.0: Remove Entity of type RECOMMENDATION when contained by FALSE_RECOMM
     salience 64
     when
         $falseRecommendation: TextEntity($type: type(), entityType == EntityType.FALSE_RECOMMENDATION, active())
-        $recommendation: TextEntity(containedBy($falseRecommendation), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
+        $recommendation: TextEntity(containedBy($falseRecommendation), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges())
     then
         $recommendation.remove("X.3.0", "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION");
         retract($recommendation);
@@ -1096,7 +1100,7 @@ rule "X.4.0: Remove Entity of type RECOMMENDATION when text range equals ENTITY
     salience 256
     when
         $entity: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
-        $recommendation: TextEntity(getTextRange().equals($entity.getTextRange()), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
+        $recommendation: TextEntity(getTextRange().equals($entity.getTextRange()), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges())
     then
         $entity.addEngines($recommendation.getEngines());
         $recommendation.remove("X.4.0", "remove Entity of type RECOMMENDATION when text range equals ENTITY with same type");
@@ -1109,7 +1113,7 @@ rule "X.5.0: Remove Entity of type RECOMMENDATION when intersected by ENTITY"
     salience 256
     when
         $entity: TextEntity((entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
-        $recommendation: TextEntity(intersects($entity), entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
+        $recommendation: TextEntity(intersects($entity), entityType == EntityType.RECOMMENDATION, !hasManualChanges())
     then
         $recommendation.remove("X.5.0", "remove Entity of type RECOMMENDATION when intersected by ENTITY");
         retract($recommendation);
@@ -1119,7 +1123,7 @@ rule "X.5.1: Remove Entity of type RECOMMENDATION when contained by RECOMMENDATI
     salience 256
     when
         $entity: TextEntity($type: type(), entityType == EntityType.RECOMMENDATION, active())
-        $recommendation: TextEntity(containedBy($entity), type() != $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
+        $recommendation: TextEntity(containedBy($entity), type() != $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges())
     then
         $recommendation.remove("X.5.1", "remove Entity of type RECOMMENDATION when contained by RECOMMENDATION");
         retract($recommendation);
@@ -1127,26 +1131,26 @@ rule "X.5.1: Remove Entity of type RECOMMENDATION when contained by RECOMMENDATI
 
 
 // Rule unit: X.6
-rule "X.6.0: Remove Entity of lower rank, when contained by by entity of type ENTITY"
+rule "X.6.0: Remove Entity of lower rank, when contained by entity of type ENTITY or HINT"
     salience 32
     when
         $higherRank: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
-        $lowerRank: TextEntity(containedBy($higherRank), type() != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active())
+        $lowerRank: TextEntity(containedBy($higherRank), type() != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges())
     then
         $lowerRank.getIntersectingNodes().forEach(node -> update(node));
-        $lowerRank.remove("X.6.0", "remove Entity of lower rank, when contained by entity of type ENTITY");
+        $lowerRank.remove("X.6.0", "remove Entity of lower rank, when contained by entity of type ENTITY or HINT");
         retract($lowerRank);
     end
 
-rule "X.6.1: remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity"
+rule "X.6.1: remove Entity, when contained in another entity of type ENTITY or HINT with larger text range"
     salience 32
     when
-        $higherRank: TextEntity($type: type(), $value: value, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active(), !hasManualChanges())
-        $lowerRank: TextEntity(intersects($higherRank), type() != $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), active(), $lowerRank.getValue().length() > $value.length())
+         $outer: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
+         $inner: TextEntity(containedBy($outer), type() != $type, $outer.getTextRange().length > getTextRange().length(), !hasManualChanges())
     then
-        $higherRank.getIntersectingNodes().forEach(node -> update(node));
-        $higherRank.remove("X.6.1", "remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity");
-        retract($higherRank);
+        $inner.getIntersectingNodes().forEach(node -> update(node));
+        $inner.remove("X.6.1", "remove Entity, when contained in another entity of type ENTITY or HINT with larger text range");
+        retract($inner);
     end
 
 
@@ -1173,6 +1177,27 @@ rule "X.8.1: Remove Entity when intersected by imported Entity"
     end
 
 
+// Rule unit: X.9
+rule "X.9.0: Merge mostly contained signatures"
+    when
+        $aiSignature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.AI)
+        $signature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.ALGORITHM, mostlyContains($aiSignature, 0.8))
+    then
+        $aiSignature.remove("X.9.0", "removed because already contained by alogrithm signature");
+        $signature.addEngine(LayoutEngine.AI);
+    end
+
+
+// Rule unit: X.10
+rule "X.10.0: remove false positives of ai"
+    when
+        $anyImage: Image(engines contains LayoutEngine.ALGORITHM)
+        $aiSignature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.AI, !mostlyContainedBy($anyImage, 0.8))
+    then
+        $aiSignature.remove("X.10.0", "Removed because false positive");
+    end
+
+
 //------------------------------------ File attributes rules ------------------------------------
 
 // Rule unit: FA.1

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_v2.drl

@@ -54,6 +54,8 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange;
 
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
+
 global Document document
 global EntityCreationService entityCreationService
 global ManualChangesApplicationService manualChangesApplicationService
@@ -212,7 +214,7 @@ rule "MAN.3.1: Apply entity recategorization of same type"
 rule "MAN.3.3: Apply recategorization entities by default"
     salience 128
     when
-        $entity: IEntity(getManualOverwrite().getRecategorized().orElse(false), !dictionary.isHint(type()))
+        $entity: TextEntity(getManualOverwrite().getRecategorized().orElse(false), !dictionary.isHint(type()))
     then
         $entity.apply("MAN.3.3", "Recategorized entities are applied by default.", $entity.legalBasis());
     end
@@ -248,8 +250,8 @@ rule "MAN.4.1: Apply legal basis change"
 rule "X.0.0: Remove Entity contained by Entity of same type"
     salience 65
     when
-        $larger: TextEntity($type: type(), $entityType: entityType, active() || skipped())
-        $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges(), active())
+        $larger: TextEntity($type: type(), $entityType: entityType, active())
+        $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges())
     then
         $contained.remove("X.0.0", "remove Entity contained by Entity of same type");
         retract($contained);
@@ -277,7 +279,7 @@ rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
     salience 64
     when
         $falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active())
-        $entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges(), active())
+        $entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges())
     then
         $entity.getIntersectingNodes().forEach(node -> update(node));
         $entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE");
@@ -290,7 +292,7 @@ rule "X.3.0: Remove Entity of type RECOMMENDATION when contained by FALSE_RECOMM
     salience 64
     when
         $falseRecommendation: TextEntity($type: type(), entityType == EntityType.FALSE_RECOMMENDATION, active())
-        $recommendation: TextEntity(containedBy($falseRecommendation), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
+        $recommendation: TextEntity(containedBy($falseRecommendation), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges())
     then
         $recommendation.remove("X.3.0", "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION");
         retract($recommendation);
@@ -302,7 +304,7 @@ rule "X.4.0: Remove Entity of type RECOMMENDATION when text range equals ENTITY
     salience 256
     when
         $entity: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
-        $recommendation: TextEntity(getTextRange().equals($entity.getTextRange()), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
+        $recommendation: TextEntity(getTextRange().equals($entity.getTextRange()), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges())
     then
         $entity.addEngines($recommendation.getEngines());
         $recommendation.remove("X.4.0", "remove Entity of type RECOMMENDATION when text range equals ENTITY with same type");
@@ -315,7 +317,7 @@ rule "X.5.0: Remove Entity of type RECOMMENDATION when intersected by ENTITY"
     salience 256
     when
         $entity: TextEntity((entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
-        $recommendation: TextEntity(intersects($entity), entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
+        $recommendation: TextEntity(intersects($entity), entityType == EntityType.RECOMMENDATION, !hasManualChanges())
     then
         $recommendation.remove("X.5.0", "remove Entity of type RECOMMENDATION when intersected by ENTITY");
         retract($recommendation);
@@ -325,7 +327,7 @@ rule "X.5.1: Remove Entity of type RECOMMENDATION when contained by RECOMMENDATI
     salience 256
     when
         $entity: TextEntity($type: type(), entityType == EntityType.RECOMMENDATION, active())
-        $recommendation: TextEntity(containedBy($entity), type() != $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
+        $recommendation: TextEntity(containedBy($entity), type() != $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges())
     then
         $recommendation.remove("X.5.1", "remove Entity of type RECOMMENDATION when contained by RECOMMENDATION");
         retract($recommendation);
@@ -333,26 +335,26 @@ rule "X.5.1: Remove Entity of type RECOMMENDATION when contained by RECOMMENDATI
 
 
 // Rule unit: X.6
-rule "X.6.0: Remove Entity of lower rank, when contained by by entity of type ENTITY"
+rule "X.6.0: Remove Entity of lower rank, when contained by entity of type ENTITY or HINT"
     salience 32
     when
         $higherRank: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
-        $lowerRank: TextEntity(containedBy($higherRank), type() != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active())
+        $lowerRank: TextEntity(containedBy($higherRank), type() != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges())
     then
         $lowerRank.getIntersectingNodes().forEach(node -> update(node));
-        $lowerRank.remove("X.6.0", "remove Entity of lower rank, when contained by entity of type ENTITY");
+        $lowerRank.remove("X.6.0", "remove Entity of lower rank, when contained by entity of type ENTITY or HINT");
         retract($lowerRank);
     end
 
-rule "X.6.1: remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity"
+rule "X.6.1: remove Entity, when contained in another entity of type ENTITY or HINT with larger text range"
     salience 32
     when
-        $higherRank: TextEntity($type: type(), $value: value, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active(), !hasManualChanges())
-        $lowerRank: TextEntity(intersects($higherRank), type() != $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), active(), $lowerRank.getValue().length() > $value.length())
+         $outer: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
+         $inner: TextEntity(containedBy($outer), type() != $type, $outer.getTextRange().length > getTextRange().length(), !hasManualChanges())
     then
-        $higherRank.getIntersectingNodes().forEach(node -> update(node));
-        $higherRank.remove("X.6.1", "remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity");
-        retract($higherRank);
+        $inner.getIntersectingNodes().forEach(node -> update(node));
+        $inner.remove("X.6.1", "remove Entity, when contained in another entity of type ENTITY or HINT with larger text range");
+        retract($inner);
     end
 
 
@@ -379,6 +381,27 @@ rule "X.8.1: Remove Entity when intersected by imported Entity"
     end
 
 
+// Rule unit: X.9
+rule "X.9.0: Merge mostly contained signatures"
+    when
+        $aiSignature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.AI)
+        $signature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.ALGORITHM, mostlyContains($aiSignature, 0.8))
+    then
+        $aiSignature.remove("X.9.0", "removed because already contained by alogrithm signature");
+        $signature.addEngine(LayoutEngine.AI);
+    end
+
+
+// Rule unit: X.10
+rule "X.10.0: remove false positives of ai"
+    when
+        $anyImage: Image(engines contains LayoutEngine.ALGORITHM)
+        $aiSignature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.AI, !mostlyContainedBy($anyImage, 0.8))
+    then
+        $aiSignature.remove("X.10.0", "Removed because false positive");
+    end
+
+
 //------------------------------------ File attributes rules ------------------------------------
 
 // Rule unit: FA.1

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/table_demo.drl

@@ -54,6 +54,8 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange;
 
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
+
 global Document document
 global EntityCreationService entityCreationService
 global ManualChangesApplicationService manualChangesApplicationService
@@ -338,7 +340,7 @@ rule "MAN.3.2: Apply image recategorization"
 rule "MAN.3.3: Apply recategorization entities by default"
     salience 128
     when
-        $entity: IEntity(getManualOverwrite().getRecategorized().orElse(false), !dictionary.isHint(type()))
+        $entity: TextEntity(getManualOverwrite().getRecategorized().orElse(false), !dictionary.isHint(type()))
     then
         $entity.apply("MAN.3.3", "Recategorized entities are applied by default.", $entity.legalBasis());
     end
@@ -374,8 +376,8 @@ rule "MAN.4.1: Apply legal basis change"
 rule "X.0.0: Remove Entity contained by Entity of same type"
     salience 65
     when
-        $larger: TextEntity($type: type(), $entityType: entityType, active() || skipped())
-        $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges(), active())
+        $larger: TextEntity($type: type(), $entityType: entityType, active())
+        $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges())
     then
         $contained.remove("X.0.0", "remove Entity contained by Entity of same type");
         retract($contained);
@@ -387,7 +389,7 @@ rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
     salience 64
     when
         $falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active())
-        $entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges(), active())
+        $entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges())
     then
         $entity.getIntersectingNodes().forEach(node -> update(node));
         $entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE");
@@ -400,7 +402,7 @@ rule "X.3.0: Remove Entity of type RECOMMENDATION when contained by FALSE_RECOMM
     salience 64
     when
         $falseRecommendation: TextEntity($type: type(), entityType == EntityType.FALSE_RECOMMENDATION, active())
-        $recommendation: TextEntity(containedBy($falseRecommendation), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
+        $recommendation: TextEntity(containedBy($falseRecommendation), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges())
     then
         $recommendation.remove("X.3.0", "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION");
         retract($recommendation);
@@ -412,7 +414,7 @@ rule "X.4.0: Remove Entity of type RECOMMENDATION when text range equals ENTITY
     salience 256
     when
         $entity: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
-        $recommendation: TextEntity(getTextRange().equals($entity.getTextRange()), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
+        $recommendation: TextEntity(getTextRange().equals($entity.getTextRange()), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges())
     then
         $entity.addEngines($recommendation.getEngines());
         $recommendation.remove("X.4.0", "remove Entity of type RECOMMENDATION when text range equals ENTITY with same type");
@@ -425,7 +427,7 @@ rule "X.5.0: Remove Entity of type RECOMMENDATION when intersected by ENTITY"
     salience 256
     when
         $entity: TextEntity((entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
-        $recommendation: TextEntity(intersects($entity), entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
+        $recommendation: TextEntity(intersects($entity), entityType == EntityType.RECOMMENDATION, !hasManualChanges())
     then
         $recommendation.remove("X.5.0", "remove Entity of type RECOMMENDATION when intersected by ENTITY");
         retract($recommendation);
@@ -435,7 +437,7 @@ rule "X.5.1: Remove Entity of type RECOMMENDATION when contained by RECOMMENDATI
     salience 256
     when
         $entity: TextEntity($type: type(), entityType == EntityType.RECOMMENDATION, active())
-        $recommendation: TextEntity(containedBy($entity), type() != $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
+        $recommendation: TextEntity(containedBy($entity), type() != $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges())
     then
         $recommendation.remove("X.5.1", "remove Entity of type RECOMMENDATION when contained by RECOMMENDATION");
         retract($recommendation);
@@ -443,26 +445,26 @@ rule "X.5.1: Remove Entity of type RECOMMENDATION when contained by RECOMMENDATI
 
 
 // Rule unit: X.6
-rule "X.6.0: Remove Entity of lower rank, when contained by by entity of type ENTITY"
+rule "X.6.0: Remove Entity of lower rank, when contained by entity of type ENTITY or HINT"
     salience 32
     when
         $higherRank: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
-        $lowerRank: TextEntity(containedBy($higherRank), type() != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active())
+        $lowerRank: TextEntity(containedBy($higherRank), type() != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges())
     then
         $lowerRank.getIntersectingNodes().forEach(node -> update(node));
-        $lowerRank.remove("X.6.0", "remove Entity of lower rank, when contained by entity of type ENTITY");
+        $lowerRank.remove("X.6.0", "remove Entity of lower rank, when contained by entity of type ENTITY or HINT");
         retract($lowerRank);
     end
 
-rule "X.6.1: remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity"
+rule "X.6.1: remove Entity, when contained in another entity of type ENTITY or HINT with larger text range"
     salience 32
     when
-        $higherRank: TextEntity($type: type(), $value: value, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active(), !hasManualChanges())
-        $lowerRank: TextEntity(intersects($higherRank), type() != $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), active(), $lowerRank.getValue().length() > $value.length())
+         $outer: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
+         $inner: TextEntity(containedBy($outer), type() != $type, $outer.getTextRange().length > getTextRange().length(), !hasManualChanges())
     then
-        $higherRank.getIntersectingNodes().forEach(node -> update(node));
-        $higherRank.remove("X.6.1", "remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity");
-        retract($higherRank);
+        $inner.getIntersectingNodes().forEach(node -> update(node));
+        $inner.remove("X.6.1", "remove Entity, when contained in another entity of type ENTITY or HINT with larger text range");
+        retract($inner);
     end
 
 
@@ -500,6 +502,27 @@ rule "X.8.1: Remove Entity when intersected by imported Entity"
     end
 
 
+// Rule unit: X.9
+rule "X.9.0: Merge mostly contained signatures"
+    when
+        $aiSignature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.AI)
+        $signature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.ALGORITHM, mostlyContains($aiSignature, 0.8))
+    then
+        $aiSignature.remove("X.9.0", "removed because already contained by alogrithm signature");
+        $signature.addEngine(LayoutEngine.AI);
+    end
+
+
+// Rule unit: X.10
+rule "X.10.0: remove false positives of ai"
+    when
+        $anyImage: Image(engines contains LayoutEngine.ALGORITHM)
+        $aiSignature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.AI, !mostlyContainedBy($anyImage, 0.8))
+    then
+        $aiSignature.remove("X.10.0", "Removed because false positive");
+    end
+
+
 //------------------------------------ File attributes rules ------------------------------------
 
 // Rule unit: FA.1

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/test_rules.drl

@@ -54,6 +54,8 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange;
 
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
+
 global Document document
 global EntityCreationService entityCreationService
 global ManualChangesApplicationService manualChangesApplicationService
@@ -238,7 +240,7 @@ rule "MAN.3.2: Apply image recategorization"
 rule "MAN.3.3: Apply recategorization entities by default"
     salience 128
     when
-        $entity: IEntity(getManualOverwrite().getRecategorized().orElse(false), !dictionary.isHint(type()))
+        $entity: TextEntity(getManualOverwrite().getRecategorized().orElse(false), !dictionary.isHint(type()))
     then
         $entity.apply("MAN.3.3", "Recategorized entities are applied by default.", $entity.legalBasis());
     end
@@ -274,8 +276,8 @@ rule "MAN.4.1: Apply legal basis change"
 rule "X.0.0: Remove Entity contained by Entity of same type"
     salience 65
     when
-        $larger: TextEntity($type: type(), $entityType: entityType, active() || skipped())
-        $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges(), active())
+        $larger: TextEntity($type: type(), $entityType: entityType, active())
+        $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges())
     then
         $contained.remove("X.0.0", "remove Entity contained by Entity of same type");
         retract($contained);
@@ -303,7 +305,7 @@ rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
     salience 64
     when
         $falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active())
-        $entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges(), active())
+        $entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges())
     then
         $entity.getIntersectingNodes().forEach(node -> update(node));
         $entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE");
@@ -316,7 +318,7 @@ rule "X.3.0: Remove Entity of type RECOMMENDATION when contained by FALSE_RECOMM
     salience 64
     when
         $falseRecommendation: TextEntity($type: type(), entityType == EntityType.FALSE_RECOMMENDATION, active())
-        $recommendation: TextEntity(containedBy($falseRecommendation), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
+        $recommendation: TextEntity(containedBy($falseRecommendation), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges())
     then
         $recommendation.remove("X.3.0", "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION");
         retract($recommendation);
@@ -328,7 +330,7 @@ rule "X.4.0: Remove Entity of type RECOMMENDATION when text range equals ENTITY
     salience 256
     when
         $entity: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
-        $recommendation: TextEntity(getTextRange().equals($entity.getTextRange()), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
+        $recommendation: TextEntity(getTextRange().equals($entity.getTextRange()), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges())
     then
         $entity.addEngines($recommendation.getEngines());
         $recommendation.remove("X.4.0", "remove Entity of type RECOMMENDATION when text range equals ENTITY with same type");
@@ -341,7 +343,7 @@ rule "X.5.0: Remove Entity of type RECOMMENDATION when intersected by ENTITY"
     salience 256
     when
         $entity: TextEntity((entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
-        $recommendation: TextEntity(intersects($entity), entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
+        $recommendation: TextEntity(intersects($entity), entityType == EntityType.RECOMMENDATION, !hasManualChanges())
     then
         $recommendation.remove("X.5.0", "remove Entity of type RECOMMENDATION when intersected by ENTITY");
         retract($recommendation);
@@ -351,7 +353,7 @@ rule "X.5.1: Remove Entity of type RECOMMENDATION when contained by RECOMMENDATI
     salience 256
     when
         $entity: TextEntity($type: type(), entityType == EntityType.RECOMMENDATION, active())
-        $recommendation: TextEntity(containedBy($entity), type() != $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
+        $recommendation: TextEntity(containedBy($entity), type() != $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges())
     then
         $recommendation.remove("X.5.1", "remove Entity of type RECOMMENDATION when contained by RECOMMENDATION");
         retract($recommendation);
@@ -359,26 +361,26 @@ rule "X.5.1: Remove Entity of type RECOMMENDATION when contained by RECOMMENDATI
 
 
 // Rule unit: X.6
-rule "X.6.0: Remove Entity of lower rank, when contained by by entity of type ENTITY"
+rule "X.6.0: Remove Entity of lower rank, when contained by entity of type ENTITY or HINT"
     salience 32
     when
         $higherRank: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
-        $lowerRank: TextEntity(containedBy($higherRank), type() != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active())
+        $lowerRank: TextEntity(containedBy($higherRank), type() != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges())
     then
         $lowerRank.getIntersectingNodes().forEach(node -> update(node));
-        $lowerRank.remove("X.6.0", "remove Entity of lower rank, when contained by entity of type ENTITY");
+        $lowerRank.remove("X.6.0", "remove Entity of lower rank, when contained by entity of type ENTITY or HINT");
         retract($lowerRank);
     end
 
-rule "X.6.1: remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity"
+rule "X.6.1: remove Entity, when contained in another entity of type ENTITY or HINT with larger text range"
     salience 32
     when
-        $higherRank: TextEntity($type: type(), $value: value, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active(), !hasManualChanges())
-        $lowerRank: TextEntity(intersects($higherRank), type() != $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), active(), $lowerRank.getValue().length() > $value.length())
+         $outer: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
+         $inner: TextEntity(containedBy($outer), type() != $type, $outer.getTextRange().length > getTextRange().length(), !hasManualChanges())
     then
-        $higherRank.getIntersectingNodes().forEach(node -> update(node));
-        $higherRank.remove("X.6.1", "remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity");
-        retract($higherRank);
+        $inner.getIntersectingNodes().forEach(node -> update(node));
+        $inner.remove("X.6.1", "remove Entity, when contained in another entity of type ENTITY or HINT with larger text range");
+        retract($inner);
     end
 
 
@@ -416,6 +418,27 @@ rule "X.8.1: Remove Entity when intersected by imported Entity"
     end
 
 
+// Rule unit: X.9
+rule "X.9.0: Merge mostly contained signatures"
+    when
+        $aiSignature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.AI)
+        $signature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.ALGORITHM, mostlyContains($aiSignature, 0.8))
+    then
+        $aiSignature.remove("X.9.0", "removed because already contained by alogrithm signature");
+        $signature.addEngine(LayoutEngine.AI);
+    end
+
+
+// Rule unit: X.10
+rule "X.10.0: remove false positives of ai"
+    when
+        $anyImage: Image(engines contains LayoutEngine.ALGORITHM)
+        $aiSignature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.AI, !mostlyContainedBy($anyImage, 0.8))
+    then
+        $aiSignature.remove("X.10.0", "Removed because false positive");
+    end
+
+
 //------------------------------------ File attributes rules ------------------------------------
 
 // Rule unit: FA.1

redaction-service-v1/redaction-service-server-v1/src/test/resources/performance/dictionaries/EFSA_sanitisation_GFL_v1/rules.drl

@@ -449,14 +449,22 @@ rule "ETC.3.1: Redact logos (non vertebrate study)"
 
 
 // Rule unit: ETC.5
-rule "ETC.5.0: Ignore dossier_redaction entries if confidentiality is not 'confidential'"
+rule "ETC.5.0: Skip dossier_redaction entries if confidentiality is 'confidential'"
+    when
+        FileAttribute(label == "Confidentiality", value == "confidential")
+        $dossierRedaction: TextEntity(type() == "dossier_redaction")
+    then
+        $dossierRedaction.skip("ETC.5.0", "Ignore dossier_redaction when confidential");
+        $dossierRedaction.getIntersectingNodes().forEach(node -> update(node));
+    end
+
+rule "ETC.5.1: Remove dossier_redaction entries if confidentiality is not 'confidential'"
     when
         not FileAttribute(label == "Confidentiality", value == "confidential")
         $dossierRedaction: TextEntity(type() == "dossier_redaction")
     then
-        $dossierRedaction.ignore("ETC.5.0", "Ignore dossier redactions, when not confidential");
-        update($dossierRedaction);
-        $dossierRedaction.getIntersectingNodes().forEach(node -> update(node));
+        $dossierRedaction.remove("ETC.5.1", "Remove dossier_redaction when not confidential");
+        retract($dossierRedaction);
     end
 
 

redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/migration/RuleFileMigrator.java

@@ -33,7 +33,11 @@ public class RuleFileMigrator {
             rulesToAdd.forEach(ruleFileBluePrint::addRule);
         }
 
-        String migratedRulesString = RuleFileFactory.buildRuleString(ruleFileBluePrint);
+        RuleFileBluePrint newBluePrint = new RuleFileBluePrint(combinedBluePrint.imports(),
+                                                               combinedBluePrint.globals(),
+                                                               combinedBluePrint.queries(),
+                                                               ruleFileBluePrint.ruleClasses());
+        String migratedRulesString = RuleFileFactory.buildRuleString(newBluePrint);
         String migratedFilePath = ruleFile.getAbsolutePath();
         try (var out = new FileOutputStream(migratedFilePath)) {
             out.write(migratedRulesString.getBytes(StandardCharsets.UTF_8));

redaction-service-v1/rules-management/src/main/resources/all_redact_manager_rules.drl

@@ -54,6 +54,8 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange;
 
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
+
 global Document document
 global EntityCreationService entityCreationService
 global ManualChangesApplicationService manualChangesApplicationService
@@ -157,20 +159,20 @@ rule "CBI.3.0: Redacted because Section contains a vertebrate"
 
 rule "CBI.3.1: Redacted because table row contains a vertebrate"
     when
-        $table: Table(hasEntitiesOfType("vertebrate"), (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
+        $table: Table(hasEntitiesOfType("vertebrate"), hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address"))
+        $cellsWithvertebrate: TableCell() from $table.streamTableCellsWhichContainType("vertebrate").toList()
+        $tableCell: TableCell(row == $cellsWithvertebrate.row) from $table.streamTableCells().toList()
+        $authorOrAddress: TextEntity(type() == "CBI_author" || type() == "CBI_address", active()) from $tableCell.getEntities()
     then
-        $table.streamEntitiesWhereRowContainsEntitiesOfType(List.of("vertebrate"))
-            .filter(entity -> entity.getType().equals("CBI_author") || entity.getType().equals("CBI_address"))
-            .forEach(entity -> {
-                entity.applyWithReferences(
+        $authorOrAddress.applyWithReferences(
                         "CBI.3.1",
                         "Vertebrate found",
                         "Reg (EC) No 1107/2009 Art. 63 (2g)",
-                        $table.getEntitiesOfTypeInSameRow("vertebrate", entity)
+                        $table.getEntitiesOfTypeInSameRow("vertebrate", $authorOrAddress)
                 );
-            });
     end
 
+
 rule "CBI.3.2: Do not redact because Section does not contain a vertebrate"
     when
         $section: Section(!hasTables(), !hasEntitiesOfType("vertebrate"), (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
@@ -207,23 +209,21 @@ rule "CBI.4.0: Do not redact Names and Addresses if no_redaction_indicator is fo
             });
     end
 
-rule "CBI.4.1: Do not redact Names and Addresses if no_redaction_indicator is found in table row"
+rule "CBI.4.1: Redacted because table row contains a vertebrate"
     when
-        $table: Table(hasEntitiesOfType("no_redaction_indicator"),
-                      hasEntitiesOfType("vertebrate"),
-                      (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
+        $table: Table(hasEntitiesOfType("no_redaction_indicator"), hasEntitiesOfType("vertebrate"), hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address"))
+        TableCell($row: row) from $table.streamTableCellsWhichContainType("vertebrate").toList()
+        TableCell(row == $row) from $table.streamTableCellsWhichContainType("no_redaction_indicator").toList()
+        $tableCell: TableCell(row == $row) from $table.streamTableCells().toList()
+        $authorOrAddress: TextEntity(type() == "CBI_author" || type() == "CBI_address", active()) from $tableCell.getEntities()
     then
-        $table.streamEntitiesWhereRowContainsEntitiesOfType(List.of("vertebrate", "no-redaction_indicator"))
-            .filter(entity -> entity.getType().equals("CBI_author") || entity.getType().equals("CBI_address"))
-            .forEach(entity -> {
-                entity.skipWithReferences(
+        $authorOrAddress.skipWithReferences(
                         "CBI.4.1",
                         "Vertebrate but a no redaction indicator found",
                         Stream.concat(
-                            $table.getEntitiesOfTypeInSameRow("vertebrate", entity).stream(),
-                            $table.getEntitiesOfTypeInSameRow("no_redaction_indicator", entity).stream()).toList()
+                            $table.getEntitiesOfTypeInSameRow("vertebrate", $authorOrAddress).stream(),
+                            $table.getEntitiesOfTypeInSameRow("no_redaction_indicator", $authorOrAddress).stream()).toList()
                 );
-            });
     end
 
 
@@ -250,22 +250,20 @@ rule "CBI.5.0: Redact Names and Addresses if no_redaction_indicator but also red
 
 rule "CBI.5.1: Redact Names and Addresses if no_redaction_indicator but also redaction_indicator is found in table row"
     when
-        $table: Table(hasEntitiesOfType("no_redaction_indicator"),
-                      hasEntitiesOfType("redaction_indicator"),
-                      (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
+        $table: Table(hasEntitiesOfType("no_redaction_indicator"), hasEntitiesOfType("redaction_indicator"), hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address"))
+        TableCell($row: row) from $table.streamTableCellsWhichContainType("redaction_indicator").toList()
+        TableCell(row == $row) from $table.streamTableCellsWhichContainType("no_redaction_indicator").toList()
+        $tableCell: TableCell(row == $row) from $table.streamTableCells().toList()
+        $entity: TextEntity(type() == "CBI_author" || type() == "CBI_address", active()) from $tableCell.getEntities()
     then
-        $table.streamEntitiesWhereRowContainsEntitiesOfType(List.of("redaction_indicator", "no_redaction_indicator"))
-            .filter(entity -> entity.getType().equals("CBI_author") || entity.getType().equals("CBI_address"))
-            .forEach(entity -> {
-                entity.applyWithReferences(
+        $entity.applyWithReferences(
                         "CBI.5.1",
                         "no_redaction_indicator but also redaction_indicator found",
                         "Reg (EC) No 1107/2009 Art. 63 (2g)",
                         Stream.concat(
-                            $table.getEntitiesOfTypeInSameRow("vertebrate", entity).stream(),
-                            $table.getEntitiesOfTypeInSameRow("no_redaction_indicator", entity).stream()).toList()
+                            $table.getEntitiesOfTypeInSameRow("redaction_indicator", $entity).stream(),
+                            $table.getEntitiesOfTypeInSameRow("no_redaction_indicator", $entity).stream()).toList()
                 );
-            });
     end
 
 
@@ -355,18 +353,17 @@ rule "CBI.8.0: Redacted because Section contains must_redact entity"
 
 rule "CBI.8.1: Redacted because table row contains must_redact entity"
     when
-        $table: Table(hasEntitiesOfType("must_redact"), (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
+        $table: Table(hasEntitiesOfType("must_redact"), hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address"))
+        $cellsWithMustRedact: TableCell() from $table.streamTableCellsWhichContainType("must_redact").toList()
+        $tableCell: TableCell(row == $cellsWithMustRedact.row) from $table.streamTableCells().toList()
+        $authorOrAddress: TextEntity(type() == "CBI_author" || type() == "CBI_address", active()) from $tableCell.getEntities()
     then
-        $table.streamEntitiesWhereRowContainsEntitiesOfType(List.of("must_redact"))
-            .filter(entity -> entity.getType().equals("CBI_author") || entity.getType().equals("CBI_address"))
-            .forEach(entity -> {
-                entity.applyWithReferences(
+        $authorOrAddress.applyWithReferences(
                         "CBI.8.1",
-                        "must_redact entity found",
+                        "Must_redact found",
                         "Reg (EC) No 1107/2009 Art. 63 (2g)",
-                        $table.getEntitiesOfTypeInSameRow("must_redact", entity)
+                        $table.getEntitiesOfTypeInSameRow("must_redact", $authorOrAddress)
                 );
-            });
     end
 
 
@@ -448,7 +445,6 @@ rule "CBI.12.0: Redact and recommend TableCell with header 'Author' or 'Author(s
         TableCell(!header, containsAnyString("Yes", "Y"), $rowWithYes: row) from $table.streamCol($vertebrateCol).toList()
         $authorCell: TableCell(row == $rowWithYes) from $table.streamCol($authorCol).toList()
     then
-
         entityCreationService.bySemanticNode($authorCell, "CBI_author", EntityType.ENTITY)
           .ifPresent(authorEntity -> {
               authorEntity.redact("CBI.12.0", "Redacted because it's row belongs to a vertebrate study", "Article 39(e)(3) of Regulation (EC) No 178/2002");
@@ -743,7 +739,7 @@ rule "PII.1.2: Redact typoed Emails with indicator"
     when
         $section: Section(containsString("@") || containsStringIgnoreCase("mail"))
     then
-        entityCreationService.byRegexIgnoreCase("mail[:\\.\\s]{1,2}([\\w\\/\\-\\{\\(\\. ]{3,20}(@|a|f)\\s?[\\w\\/\\-\\{\\(\\. ]{3,20}(\\. \\w{2,4}\\b|\\.\\B|\\.\\w{1,4}\\b))", "PII", EntityType.ENTITY, $section)
+        entityCreationService.byRegexIgnoreCase("mail[:\\.\\s]{1,2}([\\w\\/\\-\\{\\(\\. ]{3,20}(@|a|f)\\s?[\\w\\/\\-\\{\\(\\. ]{3,20}(\\. \\w{2,4}\\b|\\.\\B|\\.\\w{1,4}\\b))", "PII", EntityType.ENTITY, 1, $section)
             .forEach(emailEntity -> emailEntity.redact("PII.1.2", "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
     end
 
@@ -1163,13 +1159,22 @@ rule "ETC.4.2: Redact dossier dictionary entries"
 
 
 // Rule unit: ETC.5
-rule "ETC.5.0: Ignore dossier_redaction entries if confidentiality is not 'confidential'"
+rule "ETC.5.0: Skip dossier_redaction entries if confidentiality is 'confidential'"
+    when
+        FileAttribute(label == "Confidentiality", value == "confidential")
+        $dossierRedaction: TextEntity(type() == "dossier_redaction")
+    then
+        $dossierRedaction.skip("ETC.5.0", "Ignore dossier_redaction when confidential");
+        $dossierRedaction.getIntersectingNodes().forEach(node -> update(node));
+    end
+
+rule "ETC.5.1: Remove dossier_redaction entries if confidentiality is not 'confidential'"
     when
         not FileAttribute(label == "Confidentiality", value == "confidential")
         $dossierRedaction: TextEntity(type() == "dossier_redaction")
     then
-        $dossierRedaction.ignore("ETC.5.0", "Ignore dossier redactions, when not confidential");
-        $dossierRedaction.getIntersectingNodes().forEach(node -> update(node));
+        $dossierRedaction.remove("ETC.5.1", "Remove dossier_redaction when not confidential");
+        retract($dossierRedaction);
     end
 
 
@@ -1438,7 +1443,7 @@ rule "MAN.3.2: Apply image recategorization"
 rule "MAN.3.3: Apply recategorization entities by default"
     salience 128
     when
-        $entity: IEntity(getManualOverwrite().getRecategorized().orElse(false), !dictionary.isHint(type()))
+        $entity: TextEntity(getManualOverwrite().getRecategorized().orElse(false), !dictionary.isHint(type()))
     then
         $entity.apply("MAN.3.3", "Recategorized entities are applied by default.", $entity.legalBasis());
     end
@@ -1473,8 +1478,8 @@ rule "MAN.4.1: Apply legal basis change"
 rule "X.0.0: Remove Entity contained by Entity of same type"
     salience 65
     when
-        $larger: TextEntity($type: type(), $entityType: entityType, active() || skipped())
-        $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges(), active())
+        $larger: TextEntity($type: type(), $entityType: entityType, active())
+        $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges())
     then
         $contained.remove("X.0.0", "remove Entity contained by Entity of same type");
         retract($contained);
@@ -1502,7 +1507,7 @@ rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
     salience 64
     when
         $falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active())
-        $entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges(), active())
+        $entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges())
     then
         $entity.getIntersectingNodes().forEach(node -> update(node));
         $entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE");
@@ -1515,7 +1520,7 @@ rule "X.3.0: Remove Entity of type RECOMMENDATION when contained by FALSE_RECOMM
     salience 64
     when
         $falseRecommendation: TextEntity($type: type(), entityType == EntityType.FALSE_RECOMMENDATION, active())
-        $recommendation: TextEntity(containedBy($falseRecommendation), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
+        $recommendation: TextEntity(containedBy($falseRecommendation), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges())
     then
         $recommendation.remove("X.3.0", "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION");
         retract($recommendation);
@@ -1527,7 +1532,7 @@ rule "X.4.0: Remove Entity of type RECOMMENDATION when text range equals ENTITY
     salience 256
     when
         $entity: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
-        $recommendation: TextEntity(getTextRange().equals($entity.getTextRange()), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
+        $recommendation: TextEntity(getTextRange().equals($entity.getTextRange()), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges())
     then
         $entity.addEngines($recommendation.getEngines());
         $recommendation.remove("X.4.0", "remove Entity of type RECOMMENDATION when text range equals ENTITY with same type");
@@ -1540,19 +1545,17 @@ rule "X.5.0: Remove Entity of type RECOMMENDATION when intersected by ENTITY"
     salience 256
     when
         $entity: TextEntity((entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
-        $recommendation: TextEntity(intersects($entity), entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
+        $recommendation: TextEntity(intersects($entity), entityType == EntityType.RECOMMENDATION, !hasManualChanges())
     then
         $recommendation.remove("X.5.0", "remove Entity of type RECOMMENDATION when intersected by ENTITY");
         retract($recommendation);
     end
 
-
-// Rule unit: X.5
 rule "X.5.1: Remove Entity of type RECOMMENDATION when contained by RECOMMENDATION"
     salience 256
     when
         $entity: TextEntity($type: type(), entityType == EntityType.RECOMMENDATION, active())
-        $recommendation: TextEntity(containedBy($entity), type() != $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
+        $recommendation: TextEntity(containedBy($entity), type() != $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges())
     then
         $recommendation.remove("X.5.1", "remove Entity of type RECOMMENDATION when contained by RECOMMENDATION");
         retract($recommendation);
@@ -1560,27 +1563,26 @@ rule "X.5.1: Remove Entity of type RECOMMENDATION when contained by RECOMMENDATI
 
 
 // Rule unit: X.6
-rule "X.6.0: Remove Entity of lower rank, when contained by by entity of type ENTITY"
+rule "X.6.0: Remove Entity of lower rank, when contained by entity of type ENTITY or HINT"
     salience 32
     when
         $higherRank: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
-        $lowerRank: TextEntity(containedBy($higherRank), type() != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active())
+        $lowerRank: TextEntity(containedBy($higherRank), type() != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges())
     then
         $lowerRank.getIntersectingNodes().forEach(node -> update(node));
-        $lowerRank.remove("X.6.0", "remove Entity of lower rank, when contained by entity of type ENTITY");
+        $lowerRank.remove("X.6.0", "remove Entity of lower rank, when contained by entity of type ENTITY or HINT");
         retract($lowerRank);
     end
 
-
-rule "X.6.1: remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity"
+rule "X.6.1: remove Entity, when contained in another entity of type ENTITY or HINT with larger text range"
     salience 32
     when
-        $higherRank: TextEntity($type: type(), $value: value, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active(), !hasManualChanges())
-        $lowerRank: TextEntity(intersects($higherRank), type() != $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), active(), $lowerRank.getValue().length() > $value.length())
+         $outer: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
+         $inner: TextEntity(containedBy($outer), type() != $type, $outer.getTextRange().length > getTextRange().length(), !hasManualChanges())
     then
-        $higherRank.getIntersectingNodes().forEach(node -> update(node));
-        $higherRank.remove("X.6.1", "remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity");
-        retract($higherRank);
+        $inner.getIntersectingNodes().forEach(node -> update(node));
+        $inner.remove("X.6.1", "remove Entity, when contained in another entity of type ENTITY or HINT with larger text range");
+        retract($inner);
     end
 
 
@@ -1606,6 +1608,24 @@ rule "X.8.1: Remove Entity when intersected by imported Entity"
         retract($other);
     end
 
+// Rule unit: X.9.0
+rule "X.9.0: Merge mostly contained signatures"
+    when
+        $aiSignature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.AI)
+        $signature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.ALGORITHM, mostlyContains($aiSignature, 0.8))
+    then
+        $aiSignature.remove("X.9.0", "removed because already contained by alogrithm signature");
+        $signature.addEngine(LayoutEngine.AI);
+    end
+
+// Rule unit: X.10.0
+rule "X.10.0: remove false positives of ai"
+    when
+        $anyImage: Image(engines contains LayoutEngine.ALGORITHM)
+        $aiSignature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.AI, !mostlyContainedBy($anyImage, 0.8))
+    then
+        $aiSignature.remove("X.10.0", "Removed because false positive");
+    end
 
 //------------------------------------ File attributes rules ------------------------------------
 

redaction-service-v1/rules-management/src/main/resources/all_rules_documine.drl

@@ -54,6 +54,8 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange;
 
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
+
 global Document document
 global EntityCreationService entityCreationService
 global ManualChangesApplicationService manualChangesApplicationService
@@ -1299,7 +1301,6 @@ rule "TAB.7.0: Indicator (Species)"
             .ifPresent(redactionEntity -> redactionEntity.apply("TAB.7.0", "Vertebrate study found"));
     end
 
-
 //------------------------------------ Manual redaction rules ------------------------------------
 
 // Rule unit: MAN.0
@@ -1423,7 +1424,7 @@ rule "MAN.3.2: Apply image recategorization"
 rule "MAN.3.3: Apply recategorization entities by default"
     salience 128
     when
-        $entity: IEntity(getManualOverwrite().getRecategorized().orElse(false), !dictionary.isHint(type()))
+        $entity: TextEntity(getManualOverwrite().getRecategorized().orElse(false), !dictionary.isHint(type()))
     then
         $entity.apply("MAN.3.3", "Recategorized entities are applied by default.", $entity.legalBasis());
     end
@@ -1458,8 +1459,8 @@ rule "MAN.4.1: Apply legal basis change"
 rule "X.0.0: Remove Entity contained by Entity of same type"
     salience 65
     when
-        $larger: TextEntity($type: type(), $entityType: entityType, active() || skipped())
-        $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges(), active())
+        $larger: TextEntity($type: type(), $entityType: entityType, active())
+        $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges())
     then
         $contained.remove("X.0.0", "remove Entity contained by Entity of same type");
         retract($contained);
@@ -1471,7 +1472,7 @@ rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
     salience 64
     when
         $falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active())
-        $entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges(), active())
+        $entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges())
     then
         $entity.getIntersectingNodes().forEach(node -> update(node));
         $entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE");
@@ -1484,7 +1485,7 @@ rule "X.3.0: Remove Entity of type RECOMMENDATION when contained by FALSE_RECOMM
     salience 64
     when
         $falseRecommendation: TextEntity($type: type(), entityType == EntityType.FALSE_RECOMMENDATION, active())
-        $recommendation: TextEntity(containedBy($falseRecommendation), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
+        $recommendation: TextEntity(containedBy($falseRecommendation), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges())
     then
         $recommendation.remove("X.3.0", "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION");
         retract($recommendation);
@@ -1496,7 +1497,7 @@ rule "X.4.0: Remove Entity of type RECOMMENDATION when text range equals ENTITY
     salience 256
     when
         $entity: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
-        $recommendation: TextEntity(getTextRange().equals($entity.getTextRange()), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
+        $recommendation: TextEntity(getTextRange().equals($entity.getTextRange()), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges())
     then
         $entity.addEngines($recommendation.getEngines());
         $recommendation.remove("X.4.0", "remove Entity of type RECOMMENDATION when text range equals ENTITY with same type");
@@ -1509,7 +1510,7 @@ rule "X.5.0: Remove Entity of type RECOMMENDATION when intersected by ENTITY"
     salience 256
     when
         $entity: TextEntity((entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
-        $recommendation: TextEntity(intersects($entity), entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
+        $recommendation: TextEntity(intersects($entity), entityType == EntityType.RECOMMENDATION, !hasManualChanges())
     then
         $recommendation.remove("X.5.0", "remove Entity of type RECOMMENDATION when intersected by ENTITY");
         retract($recommendation);
@@ -1521,7 +1522,7 @@ rule "X.5.1: Remove Entity of type RECOMMENDATION when contained by RECOMMENDATI
     salience 256
     when
         $entity: TextEntity($type: type(), entityType == EntityType.RECOMMENDATION, active())
-        $recommendation: TextEntity(containedBy($entity), type() != $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
+        $recommendation: TextEntity(containedBy($entity), type() != $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges())
     then
         $recommendation.remove("X.5.1", "remove Entity of type RECOMMENDATION when contained by RECOMMENDATION");
         retract($recommendation);
@@ -1561,6 +1562,26 @@ rule "X.8.1: Remove Entity when intersected by imported Entity"
         retract($other);
     end
 
+// Rule unit: X.9.0
+rule "X.9.0: Merge mostly contained signatures"
+    when
+        $aiSignature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.AI)
+        $signature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.ALGORITHM, mostlyContains($aiSignature, 0.8))
+    then
+        $aiSignature.remove("X.9.0", "removed because already contained by alogrithm signature");
+        $signature.addEngine(LayoutEngine.AI);
+    end
+
+// Rule unit: X.10.0
+rule "X.10.0: remove false positives of ai"
+    when
+        $anyImage: Image(engines contains LayoutEngine.ALGORITHM)
+        $aiSignature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.AI, !mostlyContainedBy($anyImage, 0.8))
+    then
+        $aiSignature.remove("X.10.0", "Removed because false positive");
+    end
+
+
 
 //------------------------------------ File attributes rules ------------------------------------
 

redaction-service-v1/rules-management/src/test/java/com/knecon/fforesight/utility/rules/management/RuleFileMigrationTest.java

@@ -25,9 +25,9 @@ public class RuleFileMigrationTest {
 
     // Put your redaction service drools paths and dossier-templates paths both RM and DM here
     static final List<String> ruleFileDirs = List.of(
-            "/home/kschuettler/iqser/redaction/redaction-service/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools",
-            "/home/kschuettler/iqser/fforesight/dossier-templates-v2/",
-            "/home/kschuettler/iqser/redaction/dossier-templates-v2/");
+            "/Users/maverickstuder/Documents/RedactManager/redaction-service/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools",
+            "/Users/maverickstuder/Documents/DocuMine/dossier-templates-v2/",
+            "/Users/maverickstuder/Documents/RedactManager/dossier-templates-v2/");
 
 
     @Test