test publish stuff

RED-8590 - Missing reason for added CBI Address, recategorized LOGO and signature

Closes RED-8590

See merge request redactmanager/redaction-service!293

.gitlab-ci.yml

@@ -21,3 +21,16 @@ deploy:
     - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
     - if: $CI_COMMIT_BRANCH =~ /^release/
     - if: $CI_COMMIT_TAG
+
+pages:
+  stage: build
+  tags:
+    - dind
+  script:
+    - echo "Generating Javadoc..."
+    - gradle generateJavaDoc -PjavadocDestinationDir="javadoc"
+    - mkdir public
+    - mv redaction-service-v1/redaction-service-server-v1/javadoc/* public/
+  artifacts:
+    paths:
+      - public

buildSrc/src/main/kotlin/com.iqser.red.service.java-conventions.gradle.kts

@@ -11,6 +11,10 @@ group = "com.iqser.red.service"
 java.sourceCompatibility = JavaVersion.VERSION_17
 java.targetCompatibility = JavaVersion.VERSION_17
 
+pmd {
+    isConsoleOutput = true
+}
+
 tasks.pmdMain {
     pmd.ruleSetFiles = files("${rootDir}/config/pmd/pmd.xml")
 }

config/pmd/pmd.xml

@@ -9,13 +9,12 @@
     </description>
 
     <rule ref="category/java/errorprone.xml">
-        <exclude name="DataflowAnomalyAnalysis"/>
         <exclude name="MissingSerialVersionUID"/>
-        <exclude name="NullAssignment"/>
         <exclude name="AvoidLiteralsInIfCondition"/>
         <exclude name="AvoidDuplicateLiterals"/>
-        <exclude name="AvoidFieldNameMatchingMethodName"/>
+        <exclude name="NullAssignment"/>
         <exclude name="AssignmentInOperand"/>
+        <exclude name="BeanMembersShouldSerialize"/>
     </rule>
 
-</ruleset>
+</ruleset>

config/pmd/test_pmd.xml

@@ -10,15 +10,13 @@
 
 
     <rule ref="category/java/errorprone.xml">
-        <exclude name="DataflowAnomalyAnalysis"/>
         <exclude name="MissingSerialVersionUID"/>
-        <exclude name="NullAssignment"/>
         <exclude name="AvoidLiteralsInIfCondition"/>
         <exclude name="AvoidDuplicateLiterals"/>
-        <exclude name="AvoidFieldNameMatchingMethodName"/>
-        <exclude name="AvoidFieldNameMatchingTypeName"/>
+        <exclude name="NullAssignment"/>
         <exclude name="AssignmentInOperand"/>
         <exclude name="TestClassWithoutTestCases"/>
+        <exclude name="BeanMembersShouldSerialize"/>
     </rule>
 
-</ruleset>
+</ruleset>

redaction-service-v1/redaction-service-api-v1/build.gradle.kts

@@ -7,7 +7,7 @@ description = "redaction-service-api-v1"
 
 dependencies {
     implementation("org.springframework:spring-web:6.0.12")
-    implementation("com.iqser.red.service:persistence-service-internal-api-v1:2.279.0")
+    implementation("com.iqser.red.service:persistence-service-internal-api-v1:2.351.0")
 }
 
 publishing {

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/DroolsSyntaxDeprecatedWarnings.java

@@ -0,0 +1,20 @@
+package com.iqser.red.service.redaction.v1.model;
+
+import lombok.AccessLevel;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+import lombok.experimental.FieldDefaults;
+
+@Data
+@Builder
+@AllArgsConstructor
+@NoArgsConstructor
+@FieldDefaults(level = AccessLevel.PRIVATE)
+public class DroolsSyntaxDeprecatedWarnings {
+
+    Integer line;
+    Integer column;
+    String message;
+}

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/DroolsSyntaxValidation.java

@@ -19,6 +19,8 @@ public class DroolsSyntaxValidation {
 
     @Builder.Default
     List<DroolsSyntaxErrorMessage> droolsSyntaxErrorMessages = new LinkedList<>();
+    @Builder.Default
+    List<DroolsSyntaxDeprecatedWarnings> droolsSyntaxDeprecatedWarnings = new LinkedList<>();
 
 
     public void addErrorMessage(int line, int column, String message) {

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/MigrationRequest.java

@@ -1,7 +1,6 @@
 package com.iqser.red.service.redaction.v1.model;
 
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog;
 
 import lombok.AllArgsConstructor;
 import lombok.Builder;

redaction-service-v1/redaction-service-server-v1/build.gradle.kts

@@ -12,11 +12,11 @@ plugins {
 description = "redaction-service-server-v1"
 
 
-val layoutParserVersion = "0.86.0"
+val layoutParserVersion = "0.94.0"
 val jacksonVersion = "2.15.2"
 val droolsVersion = "9.44.0.Final"
 val pdfBoxVersion = "3.0.0"
-val persistenceServiceVersion = "2.279.0"
+val persistenceServiceVersion = "2.360.0"
 val springBootStarterVersion = "3.1.5"
 
 configurations {
@@ -38,8 +38,8 @@ dependencies {
 
     implementation("com.iqser.red.commons:dictionary-merge-commons:1.5.0")
     implementation("com.iqser.red.commons:storage-commons:2.45.0")
-    implementation("com.knecon.fforesight:tenant-commons:0.19.0")
-    implementation("com.knecon.fforesight:tracing-commons:0.3.0")
+    implementation("com.knecon.fforesight:tenant-commons:0.21.0")
+    implementation("com.knecon.fforesight:tracing-commons:0.5.0")
 
     implementation("com.fasterxml.jackson.module:jackson-module-afterburner:${jacksonVersion}")
     implementation("com.fasterxml.jackson.datatype:jackson-datatype-jsr310:${jacksonVersion}")
@@ -60,11 +60,14 @@ dependencies {
     implementation("net.logstash.logback:logstash-logback-encoder:7.4")
     implementation("ch.qos.logback:logback-classic")
 
+    implementation("org.reflections:reflections:0.10.2")
+
     testImplementation(project(":rules-management"))
     testImplementation("org.apache.pdfbox:pdfbox:${pdfBoxVersion}")
     testImplementation("org.apache.pdfbox:pdfbox-tools:${pdfBoxVersion}")
 
     testImplementation("org.springframework.boot:spring-boot-starter-test:${springBootStarterVersion}")
+    testImplementation("com.knecon.fforesight:viewer-doc-processor:${layoutParserVersion}")
     testImplementation("com.knecon.fforesight:layoutparser-service-processor:${layoutParserVersion}") {
         exclude(
             group = "com.iqser.red.service",
@@ -77,9 +80,10 @@ tasks.test {
     configure<JacocoTaskExtension> {
         excludes = listOf("org/drools/**/*")
     }
+    minHeapSize = "512m"
+    maxHeapSize = "1024m"
 }
 
-
 tasks.named<BootBuildImage>("bootBuildImage") {
 
     environment.put("BPE_DELIM_JAVA_TOOL_OPTIONS", " ")
@@ -106,3 +110,40 @@ tasks.named<BootBuildImage>("bootBuildImage") {
         tags.set(listOf(dockerTag))
     }
 }
+
+fun parseDroolsImports(droolsFilePath: String): List<String> {
+    val imports = mutableListOf<String>()
+    val importPattern = Regex("^import\\s+(com\\.iqser\\.red\\.service\\.redaction\\.v1\\.[\\w.]+);")
+    val desiredPrefix = "com.iqser.red.service.redaction.v1"
+
+    File(droolsFilePath).forEachLine { line ->
+        importPattern.find(line)?.let { matchResult ->
+            val importPath = matchResult.groupValues[1].trim()
+            if (importPath.startsWith(desiredPrefix)) {
+                val formattedPath = importPath.replace('.', '/')
+                imports.add("$formattedPath.java")
+            }
+        }
+    }
+
+    return imports
+}
+
+val droolsImports = parseDroolsImports("redaction-service-v1/redaction-service-server-v1/src/main/resources/drools/all_rules_documine.drl")
+
+tasks.register("generateJavaDoc", Javadoc::class) {
+    dependsOn("compileJava")
+    dependsOn("delombok")
+    classpath = project.sourceSets["main"].runtimeClasspath
+    source = fileTree("${buildDir}/generated/sources/delombok/java/main") {
+        include(droolsImports)
+    }
+    destinationDir = file(project.findProperty("javadocDestinationDir")?.toString() ?: "")
+
+    options.memberLevel = JavadocMemberLevel.PUBLIC
+    (options as StandardJavadocDocletOptions).apply {
+        header = "Redaction Service ${project.version}"
+        footer = "Redaction Service ${project.version}"
+        title = "API Documentation for Redaction Service ${project.version}"
+    }
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/Application.java

@@ -46,10 +46,18 @@ public class Application {
         return new TimedAspect(registry);
     }
 
+
     @Bean
     public DictionaryMergeService dictionaryMergeService() {
 
         return new DictionaryMergeService();
     }
 
+
+    @Bean(initMethod = "findDeprecatedMethods")
+    public DeprecatedElementsFinder deprecatedElementsFinder() {
+
+        return new DeprecatedElementsFinder();
+    }
+
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/DeprecatedElementsFinder.java

@@ -0,0 +1,108 @@
+package com.iqser.red.service.redaction.v1.server;
+
+import static java.util.stream.Collectors.groupingBy;
+
+import java.lang.reflect.Method;
+import java.lang.reflect.Type;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import org.reflections.Reflections;
+import org.reflections.scanners.Scanners;
+import org.reflections.util.ConfigurationBuilder;
+
+import com.iqser.red.service.redaction.v1.server.model.dictionary.SearchImplementation;
+
+import lombok.Getter;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+public class DeprecatedElementsFinder {
+
+    public static final String PACKAGE_NAME = "com.iqser.red.service.redaction.v1.server";
+    private Set<Method> deprecatedMethods;
+    @Getter
+    private Map<String, String> deprecatedMethodsSignaturesMap;
+    private Set<Class<?>> deprecatedClasses;
+    @Getter
+    private SearchImplementation classesSearchImplementation;
+    @Getter
+    private SearchImplementation methodsSearchImplementation;
+
+
+    public void findDeprecatedMethods() {
+
+        deprecatedMethods = new HashSet<>();
+        deprecatedClasses = new HashSet<>();
+        deprecatedMethodsSignaturesMap = new HashMap<>();
+
+        Reflections reflections = new Reflections(new ConfigurationBuilder().forPackage(PACKAGE_NAME)
+                                                          .setExpandSuperTypes(true)
+                                                          .setScanners(Scanners.MethodsAnnotated, Scanners.TypesAnnotated, Scanners.SubTypes));
+
+        deprecatedMethods = reflections.get(Scanners.MethodsAnnotated.with(Deprecated.class).as(Method.class));
+
+        deprecatedClasses = reflections.get(Scanners.TypesAnnotated.with(Deprecated.class).asClass());
+
+        if (!deprecatedClasses.isEmpty()) {
+            deprecatedClasses.forEach(deprecatedType -> deprecatedMethods.addAll(Arrays.stream(deprecatedType.getDeclaredMethods())
+                                                                                         .collect(Collectors.toSet())));
+            classesSearchImplementation = new SearchImplementation(deprecatedClasses.stream()
+                                                                           .map(Class::getName)
+                                                                           .collect(Collectors.toSet()), false);
+        }
+
+        if (!deprecatedMethods.isEmpty()) {
+            methodsSearchImplementation = new SearchImplementation(deprecatedMethods.stream()
+                                                                           .map(Method::getName)
+                                                                           .collect(Collectors.toSet()), false);
+            deprecatedMethodsSignaturesMap = new HashMap<>();
+            Map<String, List<Method>> deprecatedMethodsNameMap = deprecatedMethods.stream()
+                    .collect(groupingBy(Method::getName));
+            for (String deprecatedMethod : deprecatedMethodsNameMap.keySet()) {
+                deprecatedMethodsSignaturesMap.computeIfAbsent(deprecatedMethod,
+                                                               key -> deprecatedMethodsNameMap.get(deprecatedMethod)
+                                                                       .stream()
+                                                                       .map(this::getMethodSignature)
+                                                                       .collect(Collectors.joining("\n")));
+            }
+
+        }
+
+        log.info("For package: {} found deprecated classes {} deprecated methods {}", PACKAGE_NAME, deprecatedClasses.size(), deprecatedMethods.size());
+    }
+
+
+    public Set<Method> getDeprecatedMethods() {
+
+        if (this.deprecatedMethods == null) {
+            findDeprecatedMethods();
+        }
+        return this.deprecatedMethods;
+    }
+
+
+    public Set<Class<?>> getDeprecatedClasses() {
+
+        if (this.deprecatedClasses == null) {
+            findDeprecatedMethods();
+        }
+        return this.deprecatedClasses;
+    }
+
+    private String getMethodSignature(Method method) {
+
+        String methodName = method.getName();
+        String parameterTypes = Arrays.stream(method.getGenericParameterTypes())
+                .map(Type::getTypeName)
+                .collect(Collectors.joining(", "));
+
+        return method.getGenericReturnType() + " " + methodName + "(" + parameterTypes + ")";
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/migration/LegacyRedactionLogMergeService.java

@@ -12,7 +12,6 @@ import org.springframework.stereotype.Service;
 
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.BaseAnnotation;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange;
@@ -43,10 +42,10 @@ public class LegacyRedactionLogMergeService {
     private final DictionaryService dictionaryService;
 
 
-    public RedactionLog mergeManualChanges(RedactionLog redactionLog, ManualRedactions manualRedactions, String dossierTemplateId) {
+    public RedactionLog addManualAddEntriesAndRemoveSkippedImported(RedactionLog redactionLog, ManualRedactions manualRedactions, String dossierTemplateId) {
 
-        var skippedImportedRedactions = new HashSet<>();
-        log.info("Merging Redaction log with manual redactions");
+        Set<String> skippedImportedRedactions = new HashSet<>();
+        log.info("Adding manual add Entries and removing skipped or imported entries");
         if (manualRedactions != null) {
 
             var manualRedactionLogEntries = addManualAddEntries(manualRedactions.getEntriesToAdd(), redactionLog.getAnalysisNumber());
@@ -57,10 +56,14 @@ public class LegacyRedactionLogMergeService {
 
             for (RedactionLogEntry entry : redactionLog.getRedactionLogEntry()) {
 
-                processRedactionLogEntry(manualRedactionWrappers.stream().filter(mr -> entry.getId().equals(mr.getId())).collect(Collectors.toList()), entry, dossierTemplateId);
-
-                if (entry.isImported() && !entry.isRedacted()) {
-                    skippedImportedRedactions.add(entry.getId());
+                if (entry.isImported()) {
+                    processRedactionLogEntry(manualRedactionWrappers.stream()
+                                                     .filter(ManualRedactionWrapper::isApproved)
+                                                     .filter(mr -> entry.getId().equals(mr.getId()))
+                                                     .collect(Collectors.toList()), entry, dossierTemplateId);
+                    if (!entry.isRedacted()) {
+                        skippedImportedRedactions.add(entry.getId());
+                    }
                 }
 
             }
@@ -69,7 +72,9 @@ public class LegacyRedactionLogMergeService {
 
         Set<String> processedIds = new HashSet<>();
         redactionLog.getRedactionLogEntry().removeIf(entry -> {
-
+            if (entry.isFalsePositive()) {
+                return true;
+            }
             if (entry.getImportedRedactionIntersections() != null) {
                 entry.getImportedRedactionIntersections().removeAll(skippedImportedRedactions);
                 if (!entry.getImportedRedactionIntersections().isEmpty() && (!entry.isImage() || entry.isImage() && !(entry.getType().equals("image") || entry.getType()
@@ -87,40 +92,49 @@ public class LegacyRedactionLogMergeService {
         return redactionLog;
     }
 
+    public long getNumberOfAffectedAnnotations(ManualRedactions manualRedactions) {
+
+        return createManualRedactionWrappers(manualRedactions).stream().map(ManualRedactionWrapper::getId).distinct().count();
+    }
 
     private List<ManualRedactionWrapper> createManualRedactionWrappers(ManualRedactions manualRedactions) {
 
         List<ManualRedactionWrapper> manualRedactionWrappers = new ArrayList<>();
 
-        manualRedactions.getRecategorizations().stream().filter(BaseAnnotation::isApproved).forEach(item -> {
-            if (item.getSoftDeletedTime() == null) {
-                manualRedactionWrappers.add(new ManualRedactionWrapper(item.getAnnotationId(), item.getRequestDate(), item));
-            }
-        });
+        manualRedactions.getRecategorizations()
+                .forEach(item -> {
+                    if (item.getSoftDeletedTime() == null) {
+                        manualRedactionWrappers.add(new ManualRedactionWrapper(item.getAnnotationId(), item.getRequestDate(), item, item.isApproved()));
+                    }
+                });
 
-        manualRedactions.getIdsToRemove().stream().filter(BaseAnnotation::isApproved).forEach(item -> {
-            if (item.getSoftDeletedTime() == null) {
-                manualRedactionWrappers.add(new ManualRedactionWrapper(item.getAnnotationId(), item.getRequestDate(), item));
-            }
-        });
+        manualRedactions.getIdsToRemove()
+                .forEach(item -> {
+                    if (item.getSoftDeletedTime() == null) {
+                        manualRedactionWrappers.add(new ManualRedactionWrapper(item.getAnnotationId(), item.getRequestDate(), item, item.isApproved()));
+                    }
+                });
 
-        manualRedactions.getForceRedactions().stream().filter(BaseAnnotation::isApproved).forEach(item -> {
-            if (item.getSoftDeletedTime() == null) {
-                manualRedactionWrappers.add(new ManualRedactionWrapper(item.getAnnotationId(), item.getRequestDate(), item));
-            }
-        });
+        manualRedactions.getForceRedactions()
+                .forEach(item -> {
+                    if (item.getSoftDeletedTime() == null) {
+                        manualRedactionWrappers.add(new ManualRedactionWrapper(item.getAnnotationId(), item.getRequestDate(), item, item.isApproved()));
+                    }
+                });
 
-        manualRedactions.getLegalBasisChanges().stream().filter(BaseAnnotation::isApproved).forEach(item -> {
-            if (item.getSoftDeletedTime() == null) {
-                manualRedactionWrappers.add(new ManualRedactionWrapper(item.getAnnotationId(), item.getRequestDate(), item));
-            }
-        });
+        manualRedactions.getLegalBasisChanges()
+                .forEach(item -> {
+                    if (item.getSoftDeletedTime() == null) {
+                        manualRedactionWrappers.add(new ManualRedactionWrapper(item.getAnnotationId(), item.getRequestDate(), item, item.isApproved()));
+                    }
+                });
 
-        manualRedactions.getResizeRedactions().stream().filter(BaseAnnotation::isApproved).forEach(item -> {
-            if (item.getSoftDeletedTime() == null) {
-                manualRedactionWrappers.add(new ManualRedactionWrapper(item.getAnnotationId(), item.getRequestDate(), item));
-            }
-        });
+        manualRedactions.getResizeRedactions()
+                .forEach(item -> {
+                    if (item.getSoftDeletedTime() == null) {
+                        manualRedactionWrappers.add(new ManualRedactionWrapper(item.getAnnotationId(), item.getRequestDate(), item, item.isApproved()));
+                    }
+                });
 
         Collections.sort(manualRedactionWrappers);
 
@@ -222,7 +236,7 @@ public class LegacyRedactionLogMergeService {
 
         redactionLogEntry.getManualChanges()
                 .add(ManualChange.from(manualRemoval)
-                        .withManualRedactionType(manualRemoval.isRemoveFromDictionary() ? ManualRedactionType.REMOVE_FROM_DICTIONARY : ManualRedactionType.REMOVE_LOCALLY));
+                             .withManualRedactionType(manualRemoval.isRemoveFromDictionary() ? ManualRedactionType.REMOVE_FROM_DICTIONARY : ManualRedactionType.REMOVE_LOCALLY));
     }
 
 
@@ -355,7 +369,13 @@ public class LegacyRedactionLogMergeService {
     @SuppressWarnings("PMD.UselessParentheses")
     private boolean shouldCreateManualEntry(ManualRedactionEntry manualRedactionEntry) {
 
-        return (!manualRedactionEntry.isAddToDictionary() && !manualRedactionEntry.isAddToDossierDictionary()) || ((manualRedactionEntry.isAddToDictionary() || manualRedactionEntry.isAddToDossierDictionary()) && manualRedactionEntry.getProcessedDate() == null);
+        if (!manualRedactionEntry.isApproved()) {
+            return false;
+        }
+
+        return (!manualRedactionEntry.isAddToDictionary() && !manualRedactionEntry.isAddToDossierDictionary()) || ((manualRedactionEntry.isAddToDictionary()
+                                                                                                                    || manualRedactionEntry.isAddToDossierDictionary())
+                                                                                                                   && manualRedactionEntry.getProcessedDate() == null);
     }
 
 
@@ -394,6 +414,7 @@ public class LegacyRedactionLogMergeService {
         private String id;
         private OffsetDateTime date;
         private Object item;
+        private boolean approved;
 
 
         @Override

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/migration/MigrationMapper.java

@@ -0,0 +1,93 @@
+package com.iqser.red.service.redaction.v1.server.migration;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ChangeType;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualChange;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Change;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ManualRedactionType;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
+
+public class MigrationMapper {
+
+    public static com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Change toEntityLogChanges(Change change) {
+
+        return new com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Change(change.getAnalysisNumber(),
+                                                                                                              toEntityLogType(change.getType()),
+                                                                                                              change.getDateTime());
+    }
+
+
+    public static com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualChange toEntityLogManualChanges(com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ManualChange manualChange) {
+
+        return new com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualChange(toManualRedactionType(manualChange.getManualRedactionType()),
+                                                                                                                    manualChange.getProcessedDate(),
+                                                                                                                    manualChange.getRequestedDate(),
+                                                                                                                    manualChange.getUserId(),
+                                                                                                                    manualChange.getPropertyChanges());
+    }
+
+
+    public static ChangeType toEntityLogType(com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ChangeType type) {
+
+        return switch (type) {
+            case ADDED -> ChangeType.ADDED;
+            case REMOVED -> ChangeType.REMOVED;
+            case CHANGED -> ChangeType.CHANGED;
+        };
+    }
+
+
+    public static com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType toManualRedactionType(ManualRedactionType manualRedactionType) {
+
+        return switch (manualRedactionType) {
+            case ADD_LOCALLY -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.ADD;
+            case ADD_TO_DICTIONARY -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.ADD_TO_DICTIONARY;
+            case REMOVE_LOCALLY -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.REMOVE;
+            case REMOVE_FROM_DICTIONARY -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.REMOVE_FROM_DICTIONARY;
+            case FORCE_REDACT -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.FORCE;
+            case FORCE_HINT -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.FORCE;
+            case RECATEGORIZE -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.RECATEGORIZE;
+            case LEGAL_BASIS_CHANGE -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.LEGAL_BASIS_CHANGE;
+            case RESIZE -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.RESIZE;
+        };
+    }
+
+
+    public static com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine toEntityLogEngine(Engine engine) {
+
+        return switch (engine) {
+            case DICTIONARY -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine.DICTIONARY;
+            case NER -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine.NER;
+            case RULE -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine.RULE;
+        };
+    }
+
+
+    public static Set<com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine> getMigratedEngines(RedactionLogEntry entry) {
+
+        if (entry.getEngines() == null) {
+            return Collections.emptySet();
+        }
+        return entry.getEngines()
+                .stream()
+                .map(MigrationMapper::toEntityLogEngine)
+                .collect(Collectors.toSet());
+    }
+
+
+    public List<ManualChange> migrateManualChanges(List<com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ManualChange> manualChanges) {
+
+        if (manualChanges == null) {
+            return Collections.emptyList();
+        }
+        return manualChanges.stream()
+                .map(MigrationMapper::toEntityLogManualChanges)
+                .toList();
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/migration/MigrationMessageReceiver.java

@@ -58,19 +58,28 @@ public class MigrationMessageReceiver {
         if (redactionLog.getAnalysisVersion() == 0) {
             redactionLog = legacyVersion0MigrationService.mergeDuplicateAnnotationIds(redactionLog);
         } else if (migrationRequest.getManualRedactions() != null) {
-            redactionLog = legacyRedactionLogMergeService.mergeManualChanges(redactionLog, migrationRequest.getManualRedactions(), migrationRequest.getDossierTemplateId());
+            redactionLog = legacyRedactionLogMergeService.addManualAddEntriesAndRemoveSkippedImported(redactionLog,
+                                                                                                      migrationRequest.getManualRedactions(),
+                                                                                                      migrationRequest.getDossierTemplateId());
         }
 
-        MigratedEntityLog migratedEntityLog = redactionLogToEntityLogMigrationService.migrate(redactionLog, document);
+        MigratedEntityLog migratedEntityLog = redactionLogToEntityLogMigrationService.migrate(redactionLog,
+                                                                                              document,
+                                                                                              migrationRequest.getDossierTemplateId(),
+                                                                                              migrationRequest.getManualRedactions(),
+                                                                                              migrationRequest.getFileId());
 
+        log.info("Storing migrated entityLog and ids to migrate in DB for file {}", migrationRequest.getFileId());
         redactionStorageService.storeObject(migrationRequest.getDossierId(), migrationRequest.getFileId(), FileType.ENTITY_LOG, migratedEntityLog.getEntityLog());
         redactionStorageService.storeObject(migrationRequest.getDossierId(), migrationRequest.getFileId(), FileType.MIGRATED_IDS, migratedEntityLog.getMigratedIds());
 
         sendFinished(MigrationResponse.builder().dossierId(migrationRequest.getDossierId()).fileId(migrationRequest.getFileId()).build());
-        log.info("Migrated {} redactionLog entries for dossierId {} and fileId {}",
-                migratedEntityLog.getEntityLog().getEntityLogEntry().size(),
-                migrationRequest.getDossierId(),
-                migrationRequest.getFileId());
+        log.info("Migrated {} redactionLog entries, found {} annotation ids for migration in the db, {} new manual entries, for dossierId {} and fileId {}",
+                 migratedEntityLog.getEntityLog().getEntityLogEntry().size(),
+                 migratedEntityLog.getMigratedIds().getMappings().size(),
+                 migratedEntityLog.getMigratedIds().getManualRedactionEntriesToAdd().size(),
+                 migrationRequest.getDossierId(),
+                 migrationRequest.getFileId());
         log.info("");
     }
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/migration/RedactionLogToEntityLogMigrationService.java

@@ -2,6 +2,7 @@ package com.iqser.red.service.redaction.v1.server.migration;
 
 import java.awt.geom.Rectangle2D;
 import java.util.Collection;
+import java.util.Collections;
 import java.util.Comparator;
 import java.util.LinkedList;
 import java.util.List;
@@ -16,25 +17,26 @@ import org.springframework.stereotype.Service;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogLegalBasis;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.migration.MigratedIds;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.BaseAnnotation;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogLegalBasis;
-import com.iqser.red.service.redaction.v1.server.model.ManualEntity;
 import com.iqser.red.service.redaction.v1.server.model.MigratedEntityLog;
 import com.iqser.red.service.redaction.v1.server.model.MigrationEntity;
+import com.iqser.red.service.redaction.v1.server.model.PrecursorEntity;
 import com.iqser.red.service.redaction.v1.server.model.RectangleWithPage;
-import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
-import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Image;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.ImageType;
-import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
-import com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService;
+import com.iqser.red.service.redaction.v1.server.service.DictionaryService;
+import com.iqser.red.service.redaction.v1.server.service.ManualChangesApplicationService;
 import com.iqser.red.service.redaction.v1.server.service.document.EntityEnrichmentService;
 import com.iqser.red.service.redaction.v1.server.service.document.EntityFindingUtility;
+import com.iqser.red.service.redaction.v1.server.service.document.EntityFromPrecursorCreationService;
 import com.iqser.red.service.redaction.v1.server.utils.IdBuilder;
 import com.iqser.red.service.redaction.v1.server.utils.MigratedIdsCollector;
 
@@ -53,14 +55,20 @@ public class RedactionLogToEntityLogMigrationService {
     private static final double MATCH_THRESHOLD = 10;
     EntityFindingUtility entityFindingUtility;
     EntityEnrichmentService entityEnrichmentService;
+    DictionaryService dictionaryService;
+    ManualChangesApplicationService manualChangesApplicationService;
 
 
-    public MigratedEntityLog migrate(RedactionLog redactionLog, Document document) {
+    public MigratedEntityLog migrate(RedactionLog redactionLog, Document document, String dossierTemplateId, ManualRedactions manualRedactions, String fileId) {
 
-        List<MigrationEntity> entitiesToMigrate = calculateMigrationEntitiesFromRedactionLog(redactionLog, document);
-        MigratedIds migratedIds = entitiesToMigrate.stream().collect(new MigratedIdsCollector());
-        MigratedIds idsToMigrateInDb = entitiesToMigrate.stream().filter(MigrationEntity::hasManualChangesOrComments).collect(new MigratedIdsCollector());
+        log.info("Migrating entities for file {}", fileId);
+        List<MigrationEntity> entitiesToMigrate = calculateMigrationEntitiesFromRedactionLog(redactionLog, document, dossierTemplateId, fileId);
 
+        MigratedIds migratedIds = entitiesToMigrate.stream()
+                .collect(new MigratedIdsCollector());
+
+        applyManualChanges(entitiesToMigrate, manualRedactions);
+        log.info("applying manual changes to migrated entities for file {}", fileId);
 
         EntityLog entityLog = new EntityLog();
         entityLog.setAnalysisNumber(redactionLog.getAnalysisNumber());
@@ -69,34 +77,77 @@ public class RedactionLogToEntityLogMigrationService {
         entityLog.setDossierDictionaryVersion(redactionLog.getDossierDictionaryVersion());
         entityLog.setLegalBasisVersion(redactionLog.getLegalBasisVersion());
         entityLog.setAnalysisVersion(redactionLog.getAnalysisVersion());
-        entityLog.setLegalBasis(redactionLog.getLegalBasis().stream().map(RedactionLogToEntityLogMigrationService::toEntityLogLegalBasis).toList());
+        entityLog.setLegalBasis(redactionLog.getLegalBasis()
+                                        .stream()
+                                        .map(RedactionLogToEntityLogMigrationService::toEntityLogLegalBasis)
+                                        .toList());
 
         Map<String, String> oldToNewIDMapping = migratedIds.buildOldToNewMapping();
-        entityLog.setEntityLogEntry(entitiesToMigrate.stream().map(migrationEntity -> migrationEntity.toEntityLogEntry(oldToNewIDMapping)).toList());
+
+        log.info("Writing migrated entities to entityLog for file {}", fileId);
+        entityLog.setEntityLogEntry(entitiesToMigrate.stream()
+                                            .map(migrationEntity -> migrationEntity.toEntityLogEntry(oldToNewIDMapping))
+                                            .toList());
 
         if (getNumberOfApprovedEntries(redactionLog) != entityLog.getEntityLogEntry().size()) {
             String message = String.format("Not all entities have been found during the migration redactionLog has %d entries and new entityLog %d",
-                    redactionLog.getRedactionLogEntry().size(),
-                    entityLog.getEntityLogEntry().size());
+                                           redactionLog.getRedactionLogEntry().size(),
+                                           entityLog.getEntityLogEntry().size());
             log.error(message);
             throw new AssertionError(message);
         }
 
+        MigratedIds idsToMigrateInDb = entitiesToMigrate.stream()
+                .filter(MigrationEntity::hasManualChangesOrComments)
+                .filter(m -> !m.getOldId().equals(m.getNewId()))
+                .collect(new MigratedIdsCollector());
+
+        List<ManualRedactionEntry> manualRedactionEntriesToAdd = entitiesToMigrate.stream()
+                .filter(MigrationEntity::needsManualEntry)
+                .map(MigrationEntity::buildManualRedactionEntry)
+                .toList();
+
+        idsToMigrateInDb.setManualRedactionEntriesToAdd(manualRedactionEntriesToAdd);
+
         return new MigratedEntityLog(idsToMigrateInDb, entityLog);
     }
 
 
+    private void applyManualChanges(List<MigrationEntity> entitiesToMigrate, ManualRedactions manualRedactions) {
+
+        if (manualRedactions == null) {
+            return;
+        }
+
+        Map<String, List<BaseAnnotation>> manualChangesPerAnnotationId = Stream.of(manualRedactions.getIdsToRemove(),
+                                                                                   manualRedactions.getEntriesToAdd(),
+                                                                                   manualRedactions.getForceRedactions(),
+                                                                                   manualRedactions.getResizeRedactions(),
+                                                                                   manualRedactions.getLegalBasisChanges(),
+                                                                                   manualRedactions.getRecategorizations())
+                .flatMap(Collection::stream)
+                .collect(Collectors.groupingBy(BaseAnnotation::getAnnotationId));
+
+        entitiesToMigrate.forEach(migrationEntity -> migrationEntity.applyManualChanges(manualChangesPerAnnotationId.getOrDefault(migrationEntity.getOldId(),
+                                                                                                                                  Collections.emptyList()),
+                                                                                        manualChangesApplicationService));
+
+    }
+
+
     private static long getNumberOfApprovedEntries(RedactionLog redactionLog) {
 
         return redactionLog.getRedactionLogEntry().size();
     }
 
 
-    private List<MigrationEntity> calculateMigrationEntitiesFromRedactionLog(RedactionLog redactionLog, Document document) {
+    private List<MigrationEntity> calculateMigrationEntitiesFromRedactionLog(RedactionLog redactionLog, Document document, String dossierTemplateId, String fileId) {
 
-        List<MigrationEntity> images = getImageBasedMigrationEntities(redactionLog, document);
-        List<MigrationEntity> textMigrationEntities = getTextBasedMigrationEntities(redactionLog, document);
-        return Stream.of(textMigrationEntities.stream(), images.stream()).flatMap(Function.identity()).toList();
+        List<MigrationEntity> images = getImageBasedMigrationEntities(redactionLog, document, fileId);
+        List<MigrationEntity> textMigrationEntities = getTextBasedMigrationEntities(redactionLog, document, dossierTemplateId, fileId);
+        return Stream.of(textMigrationEntities.stream(), images.stream())
+                .flatMap(Function.identity())
+                .toList();
     }
 
 
@@ -106,9 +157,10 @@ public class RedactionLogToEntityLogMigrationService {
     }
 
 
-    private List<MigrationEntity> getImageBasedMigrationEntities(RedactionLog redactionLog, Document document) {
+    private List<MigrationEntity> getImageBasedMigrationEntities(RedactionLog redactionLog, Document document, String fileId) {
 
-        List<Image> images = document.streamAllImages().collect(Collectors.toList());
+        List<Image> images = document.streamAllImages()
+                .collect(Collectors.toList());
 
         List<RedactionLogEntry> redactionLogImages = redactionLog.getRedactionLogEntry()
                 .stream()
@@ -117,12 +169,16 @@ public class RedactionLogToEntityLogMigrationService {
 
         List<MigrationEntity> migrationEntities = new LinkedList<>();
         for (RedactionLogEntry redactionLogImage : redactionLogImages) {
-            List<RectangleWithPage> imagePositions = redactionLogImage.getPositions().stream().map(RectangleWithPage::fromRedactionLogRectangle).toList();
+            List<RectangleWithPage> imagePositions = redactionLogImage.getPositions()
+                    .stream()
+                    .map(RectangleWithPage::fromRedactionLogRectangle)
+                    .toList();
             assert imagePositions.size() == 1;
             Optional<Image> optionalClosestImage = images.stream()
-                    .filter(image -> image.onPage(redactionLogImage.getPositions().get(0).getPage()))
-                    .min(Comparator.comparingDouble(image -> entityFindingUtility.calculateDistance(image.getPosition(), imagePositions.get(0).rectangle2D())))
-                    .filter(image -> entityFindingUtility.calculateDistance(image.getPosition(), imagePositions.get(0).rectangle2D()) <= MATCH_THRESHOLD);
+                    .filter(image -> image.onPage(redactionLogImage.getPositions()
+                                                          .get(0).getPage()))
+                    .min(Comparator.comparingDouble(image -> EntityFindingUtility.calculateDistance(image.getPosition(), imagePositions.get(0).rectangle2D())))
+                    .filter(image -> EntityFindingUtility.calculateDistance(image.getPosition(), imagePositions.get(0).rectangle2D()) <= MATCH_THRESHOLD);
 
             Image closestImage;
             if (optionalClosestImage.isEmpty()) { // if no fitting image can be found create a new one with the previous values!
@@ -132,15 +188,23 @@ public class RedactionLogToEntityLogMigrationService {
                 images.remove(closestImage);
             }
 
-            String ruleIdentifier = "OLDIMG." + redactionLogImage.getMatchedRule() + ".0";
-            if (redactionLogImage.lastChangeIsRemoved()) {
-                closestImage.remove(ruleIdentifier, redactionLogImage.getReason());
-            } else if (redactionLogImage.isRedacted()) {
-                closestImage.apply(ruleIdentifier, redactionLogImage.getReason(), redactionLogImage.getLegalBasis());
+            String ruleIdentifier;
+            String reason = Optional.ofNullable(redactionLogImage.getReason())
+                    .orElse("");
+            if (redactionLogImage.getMatchedRule().isBlank() || redactionLogImage.getMatchedRule() == null) {
+                ruleIdentifier = "OLDIMG.0.0";
             } else {
-                closestImage.skip(ruleIdentifier, redactionLogImage.getReason());
+                ruleIdentifier = "OLDIMG." + redactionLogImage.getMatchedRule() + ".0";
             }
-            migrationEntities.add(new MigrationEntity(null, redactionLogImage, closestImage, redactionLogImage.getId(), closestImage.getId()));
+
+            if (redactionLogImage.lastChangeIsRemoved()) {
+                closestImage.remove(ruleIdentifier, reason);
+            } else if (redactionLogImage.isRedacted()) {
+                closestImage.apply(ruleIdentifier, reason, redactionLogImage.getLegalBasis());
+            } else {
+                closestImage.skip(ruleIdentifier, reason);
+            }
+            migrationEntities.add(MigrationEntity.fromRedactionLogImage(redactionLogImage, closestImage, fileId));
         }
         return migrationEntities;
     }
@@ -152,13 +216,24 @@ public class RedactionLogToEntityLogMigrationService {
                 .documentTree(document.getDocumentTree())
                 .imageType(ImageType.fromString(redactionLogImage.getType()))
                 .transparent(redactionLogImage.isImageHasTransparency())
-                .page(document.getPages().stream().filter(p -> p.getNumber() == redactionLogImage.getPositions().get(0).getPage()).findFirst().orElseThrow())
-                .position(toRectangle2D(redactionLogImage.getPositions().get(0)))
+                .page(document.getPages()
+                              .stream()
+                              .filter(p -> p.getNumber() == redactionLogImage.getPositions()
+                                      .get(0).getPage())
+                              .findFirst()
+                              .orElseThrow())
+                .position(toRectangle2D(redactionLogImage.getPositions()
+                                                .get(0)))
                 .build();
 
         List<Integer> treeId = document.getDocumentTree().createNewMainEntryAndReturnId(image);
         image.setTreeId(treeId);
-        image.setId(IdBuilder.buildId(image.getPages(), image.getBBox().values().stream().toList(), "", ""));
+        image.setId(IdBuilder.buildId(image.getPages(),
+                                      image.getBBox().values()
+                                              .stream()
+                                              .toList(),
+                                      "",
+                                      ""));
 
         return image;
     }
@@ -170,66 +245,46 @@ public class RedactionLogToEntityLogMigrationService {
     }
 
 
-    private List<MigrationEntity> getTextBasedMigrationEntities(RedactionLog redactionLog, Document document) {
+    private List<MigrationEntity> getTextBasedMigrationEntities(RedactionLog redactionLog, Document document, String dossierTemplateId, String fileId) {
 
         List<MigrationEntity> entitiesToMigrate = redactionLog.getRedactionLogEntry()
                 .stream()
                 .filter(redactionLogEntry -> !redactionLogEntry.isImage())
-                .map(MigrationEntity::fromRedactionLogEntry)
-                .peek(migrationEntity -> {
-                    if (migrationEntity.getRedactionLogEntry().lastChangeIsRemoved()) {
-                        if (migrationEntity.getManualEntity().getEntityType().equals(EntityType.HINT)) {
-                            migrationEntity.getManualEntity().ignore(migrationEntity.getManualEntity().getRuleIdentifier(), migrationEntity.getRedactionLogEntry().getReason());
-                        } else {
-                            migrationEntity.getManualEntity().remove(migrationEntity.getManualEntity().getRuleIdentifier(), migrationEntity.getRedactionLogEntry().getReason());
-                        }
-                    } else if (migrationEntity.getManualEntity().isApplied()) {
-                        migrationEntity.getManualEntity()
-                                .apply(migrationEntity.getManualEntity().getRuleIdentifier(),
-                                        migrationEntity.getManualEntity().getReason(),
-                                        migrationEntity.getManualEntity().getLegalBasis());
-                    } else {
-                        migrationEntity.getManualEntity().skip(migrationEntity.getManualEntity().getRuleIdentifier(), migrationEntity.getManualEntity().getReason());
-                    }
-                })
+                .map(entry -> MigrationEntity.fromRedactionLogEntry(entry, dictionaryService.isHint(entry.getType(), dossierTemplateId), fileId))
                 .toList();
 
-        Map<String, List<TextEntity>> tempEntitiesByValue = entityFindingUtility.findAllPossibleEntitiesAndGroupByValue(document,
-                entitiesToMigrate.stream().map(MigrationEntity::getManualEntity).toList());
+        List<PrecursorEntity> precursorEntities = entitiesToMigrate.stream()
+                .map(MigrationEntity::getPrecursorEntity)
+                .toList();
+
+        log.info("Finding all possible entities");
+        Map<String, List<TextEntity>> tempEntitiesByValue = entityFindingUtility.findAllPossibleEntitiesAndGroupByValue(document, precursorEntities);
 
         for (MigrationEntity migrationEntity : entitiesToMigrate) {
-            Optional<TextEntity> optionalTextEntity = entityFindingUtility.findClosestEntityAndReturnEmptyIfNotFound(migrationEntity.getManualEntity(),
-                    tempEntitiesByValue,
-                    MATCH_THRESHOLD);
+            Optional<TextEntity> optionalTextEntity = entityFindingUtility.findClosestEntityAndReturnEmptyIfNotFound(migrationEntity.getPrecursorEntity(),
+                                                                                                                     tempEntitiesByValue,
+                                                                                                                     MATCH_THRESHOLD);
 
             if (optionalTextEntity.isEmpty()) {
-                migrationEntity.setMigratedEntity(migrationEntity.getManualEntity());
-                migrationEntity.setOldId(migrationEntity.getManualEntity().getId());
-                migrationEntity.setNewId(migrationEntity.getManualEntity().getId());
+                migrationEntity.setMigratedEntity(migrationEntity.getPrecursorEntity());
+                migrationEntity.setOldId(migrationEntity.getPrecursorEntity().getId());
+                migrationEntity.setNewId(migrationEntity.getPrecursorEntity().getId());
                 continue;
             }
 
-            TextEntity entity = createCorrectEntity(migrationEntity.getManualEntity(), document, optionalTextEntity.get().getTextRange());
-            migrationEntity.setMigratedEntity(entity);
-            migrationEntity.setOldId(migrationEntity.getManualEntity().getId());
-            migrationEntity.setNewId(entity.getPositionsOnPagePerPage().get(0).getId()); // Can only be on one page, since redactionLogEntries can only be on one page
+            TextEntity migratedEntity = EntityFromPrecursorCreationService.createCorrectEntity(migrationEntity.getPrecursorEntity(), optionalTextEntity.get(), true);
+
+            migrationEntity.setMigratedEntity(migratedEntity);
+            migrationEntity.setOldId(migrationEntity.getPrecursorEntity().getId());
+            migrationEntity.setNewId(migratedEntity.getId());
         }
 
-        tempEntitiesByValue.values().stream().flatMap(Collection::stream).forEach(TextEntity::removeFromGraph);
+        tempEntitiesByValue.values()
+                .stream()
+                .flatMap(Collection::stream)
+                .forEach(TextEntity::removeFromGraph);
+
         return entitiesToMigrate;
     }
 
-
-    private TextEntity createCorrectEntity(ManualEntity manualEntity, SemanticNode node, TextRange closestTextRange) {
-
-        EntityCreationService entityCreationService = new EntityCreationService(entityEnrichmentService);
-        TextEntity correctEntity = entityCreationService.forceByTextRange(closestTextRange, manualEntity.getType(), manualEntity.getEntityType(), node);
-
-        correctEntity.addMatchedRules(manualEntity.getMatchedRuleList());
-        correctEntity.setDictionaryEntry(manualEntity.isDictionaryEntry());
-        correctEntity.setDossierDictionaryEntry(manualEntity.isDossierDictionaryEntry());
-        correctEntity.getManualOverwrite().addChanges(manualEntity.getManualOverwrite().getManualChangeLog());
-        return correctEntity;
-    }
-
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/ClosestEntity.java

@@ -1,6 +1,6 @@
 package com.iqser.red.service.redaction.v1.server.model;
 
-import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
+import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
 
 import lombok.Builder;
 import lombok.Getter;
@@ -12,6 +12,6 @@ import lombok.Setter;
 public class ClosestEntity {
 
     private double distance;
-    private TextRange textRange;
+    private TextEntity textEntity;
 
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/MigratedEntityLog.java

@@ -1,7 +1,10 @@
 package com.iqser.red.service.redaction.v1.server.model;
 
+import java.util.List;
+
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.migration.MigratedIds;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
 
 import lombok.AllArgsConstructor;
 import lombok.Builder;
@@ -16,5 +19,4 @@ public class MigratedEntityLog {
 
     MigratedIds migratedIds;
     EntityLog entityLog;
-
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/MigrationEntity.java

@@ -1,67 +1,121 @@
 package com.iqser.red.service.redaction.v1.server.model;
 
+import static com.iqser.red.service.redaction.v1.server.service.EntityLogCreatorService.buildEntryState;
+import static com.iqser.red.service.redaction.v1.server.service.EntityLogCreatorService.buildEntryType;
+
+import java.awt.geom.Rectangle2D;
 import java.util.Collections;
+import java.util.LinkedList;
 import java.util.List;
-import java.util.Locale;
 import java.util.Map;
+import java.util.Optional;
 import java.util.Set;
 import java.util.stream.Collectors;
 
-import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ChangeType;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryType;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Change;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ManualChange;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.BaseAnnotation;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ManualRedactionType;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
+import com.iqser.red.service.redaction.v1.server.migration.MigrationMapper;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.IEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.ManualChangeOverwrite;
-import com.iqser.red.service.redaction.v1.server.model.document.entity.PositionOnPage;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Image;
-import com.iqser.red.service.redaction.v1.server.model.document.nodes.ImageType;
+import com.iqser.red.service.redaction.v1.server.service.ManualChangeFactory;
+import com.iqser.red.service.redaction.v1.server.service.ManualChangesApplicationService;
 
 import lombok.AllArgsConstructor;
+import lombok.Builder;
 import lombok.Data;
 import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
 
+@Slf4j
 @Data
+@Builder
 @AllArgsConstructor
 @RequiredArgsConstructor
 public final class MigrationEntity {
 
-    private final ManualEntity manualEntity;
+    private final PrecursorEntity precursorEntity;
     private final RedactionLogEntry redactionLogEntry;
     private IEntity migratedEntity;
     private String oldId;
     private String newId;
+    private String fileId;
+
+    @Builder.Default
+    List<BaseAnnotation> manualChanges = new LinkedList<>();
 
 
-    public static MigrationEntity fromRedactionLogEntry(RedactionLogEntry redactionLogEntry) {
+    public static MigrationEntity fromRedactionLogEntry(RedactionLogEntry redactionLogEntry, boolean hint, String fileId) {
 
-        return new MigrationEntity(createManualEntity(redactionLogEntry), redactionLogEntry);
+        PrecursorEntity precursorEntity = createPrecursorEntity(redactionLogEntry, hint);
+
+        if (precursorEntity.getEntityType().equals(EntityType.HINT) && !redactionLogEntry.isHint() && !redactionLogEntry.isRedacted()) {
+            precursorEntity.ignore(precursorEntity.getRuleIdentifier(), precursorEntity.getReason());
+        } else if (redactionLogEntry.lastChangeIsRemoved()) {
+            precursorEntity.remove(precursorEntity.getRuleIdentifier(), precursorEntity.getReason());
+        } else if (lastManualChangeIsRemove(redactionLogEntry)) {
+            precursorEntity.ignore(precursorEntity.getRuleIdentifier(), precursorEntity.getReason());
+        } else if (precursorEntity.isApplied() && redactionLogEntry.isRecommendation()) {
+            precursorEntity.skip(precursorEntity.getRuleIdentifier(), precursorEntity.getReason());
+        } else if (precursorEntity.isApplied()) {
+            precursorEntity.apply(precursorEntity.getRuleIdentifier(), precursorEntity.getReason(), precursorEntity.getLegalBasis());
+        } else {
+            precursorEntity.skip(precursorEntity.getRuleIdentifier(), precursorEntity.getReason());
+        }
+
+        return MigrationEntity.builder().precursorEntity(precursorEntity).redactionLogEntry(redactionLogEntry).oldId(redactionLogEntry.getId()).fileId(fileId).build();
     }
 
 
-    public static ManualEntity createManualEntity(RedactionLogEntry redactionLogEntry) {
+    public static MigrationEntity fromRedactionLogImage(RedactionLogEntry redactionLogImage, Image image, String fileId) {
+
+        return MigrationEntity.builder().redactionLogEntry(redactionLogImage).migratedEntity(image).oldId(redactionLogImage.getId()).newId(image.getId()).fileId(fileId).build();
+    }
+
+
+    private static boolean lastManualChangeIsRemove(RedactionLogEntry redactionLogEntry) {
+
+        if (redactionLogEntry.getManualChanges() == null) {
+            return false;
+        }
+
+        return redactionLogEntry.getManualChanges()
+                .stream()
+                .reduce((a, b) -> b)
+                .map(m -> m.getManualRedactionType().equals(ManualRedactionType.REMOVE_LOCALLY))
+                .orElse(false);
+    }
+
+
+    public static PrecursorEntity createPrecursorEntity(RedactionLogEntry redactionLogEntry, boolean hint) {
 
         String ruleIdentifier = buildRuleIdentifier(redactionLogEntry);
-        List<RectangleWithPage> rectangleWithPages = redactionLogEntry.getPositions().stream().map(RectangleWithPage::fromRedactionLogRectangle).toList();
-        EntityType entityType = getEntityType(redactionLogEntry);
-        return ManualEntity.builder()
+        List<RectangleWithPage> rectangleWithPages = redactionLogEntry.getPositions()
+                .stream()
+                .map(RectangleWithPage::fromRedactionLogRectangle)
+                .toList();
+        EntityType entityType = getEntityType(redactionLogEntry, hint);
+        return PrecursorEntity.builder()
                 .id(redactionLogEntry.getId())
                 .value(redactionLogEntry.getValue())
                 .entityPosition(rectangleWithPages)
                 .ruleIdentifier(ruleIdentifier)
-                .reason(redactionLogEntry.getReason())
+                .reason(Optional.ofNullable(redactionLogEntry.getReason())
+                                .orElse(""))
                 .legalBasis(redactionLogEntry.getLegalBasis())
                 .type(redactionLogEntry.getType())
                 .section(redactionLogEntry.getSection())
+                .engines(MigrationMapper.getMigratedEngines(redactionLogEntry))
                 .entityType(entityType)
                 .applied(redactionLogEntry.isRedacted())
                 .isDictionaryEntry(redactionLogEntry.isDictionaryEntry())
@@ -84,26 +138,24 @@ public final class MigrationEntity {
     }
 
 
-    private static EntityType getEntityType(RedactionLogEntry redactionLogEntry) {
+    private static EntityType getEntityType(RedactionLogEntry redactionLogEntry, boolean hint) {
 
-        if (redactionLogEntry.isRecommendation()) {
-            return EntityType.RECOMMENDATION;
+        if (hint) {
+            return EntityType.HINT;
+        }
+        if (redactionLogEntry.isFalsePositive()) {
+            return EntityType.FALSE_POSITIVE;
         }
         if (redactionLogEntry.isHint()) {
             return EntityType.HINT;
         }
+        if (redactionLogEntry.isRecommendation()) {
+            return EntityType.RECOMMENDATION;
+        }
         return EntityType.ENTITY;
     }
 
 
-    private static com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Change toEntityLogChanges(Change change) {
-
-        return new com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Change(change.getAnalysisNumber(),
-                toEntityLogType(change.getType()),
-                change.getDateTime());
-    }
-
-
     private static EntryType getEntryType(EntityType entityType) {
 
         return switch (entityType) {
@@ -116,42 +168,6 @@ public final class MigrationEntity {
     }
 
 
-    private static com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualChange toEntityLogManualChanges(ManualChange manualChange) {
-
-        return new com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualChange(toManualRedactionType(manualChange.getManualRedactionType()),
-                manualChange.getProcessedDate(),
-                manualChange.getRequestedDate(),
-                manualChange.getUserId(),
-                manualChange.getPropertyChanges());
-    }
-
-
-    private static ChangeType toEntityLogType(com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ChangeType type) {
-
-        return switch (type) {
-            case ADDED -> ChangeType.ADDED;
-            case REMOVED -> ChangeType.REMOVED;
-            case CHANGED -> ChangeType.CHANGED;
-        };
-    }
-
-
-    private static com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType toManualRedactionType(ManualRedactionType manualRedactionType) {
-
-        return switch (manualRedactionType) {
-            case ADD_LOCALLY -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.ADD_LOCALLY;
-            case ADD_TO_DICTIONARY -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.ADD_TO_DICTIONARY;
-            case REMOVE_LOCALLY -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.REMOVE_LOCALLY;
-            case REMOVE_FROM_DICTIONARY -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.REMOVE_FROM_DICTIONARY;
-            case FORCE_REDACT -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.FORCE_REDACT;
-            case FORCE_HINT -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.FORCE_HINT;
-            case RECATEGORIZE -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.RECATEGORIZE;
-            case LEGAL_BASIS_CHANGE -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.LEGAL_BASIS_CHANGE;
-            case RESIZE -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.RESIZE;
-        };
-    }
-
-
     public EntityLogEntry toEntityLogEntry(Map<String, String> oldToNewIdMapping) {
 
         EntityLogEntry entityLogEntry;
@@ -159,17 +175,21 @@ public final class MigrationEntity {
             entityLogEntry = createEntityLogEntry(image);
         } else if (migratedEntity instanceof TextEntity textEntity) {
             entityLogEntry = createEntityLogEntry(textEntity);
-        } else if (migratedEntity instanceof ManualEntity entity) {
+        } else if (migratedEntity instanceof PrecursorEntity entity) {
             entityLogEntry = createEntityLogEntry(entity);
         } else {
             throw new UnsupportedOperationException("Unknown subclass " + migratedEntity.getClass());
         }
-        entityLogEntry.setChanges(redactionLogEntry.getChanges().stream().map(MigrationEntity::toEntityLogChanges).toList());
-        entityLogEntry.setManualChanges(migrateManualChanges(redactionLogEntry.getManualChanges()));
+
+        entityLogEntry.setManualChanges(ManualChangeFactory.toManualChangeList(migratedEntity.getManualOverwrite().getManualChangeLog(), redactionLogEntry.isHint()));
         entityLogEntry.setColor(redactionLogEntry.getColor());
+        entityLogEntry.setChanges(redactionLogEntry.getChanges()
+                                          .stream()
+                                          .map(MigrationMapper::toEntityLogChanges)
+                                          .toList());
         entityLogEntry.setReference(migrateSetOfIds(redactionLogEntry.getReference(), oldToNewIdMapping));
         entityLogEntry.setImportedRedactionIntersections(migrateSetOfIds(redactionLogEntry.getImportedRedactionIntersections(), oldToNewIdMapping));
-        entityLogEntry.setEngines(getMigratedEngines(redactionLogEntry));
+        entityLogEntry.setEngines(MigrationMapper.getMigratedEngines(redactionLogEntry));
         if (redactionLogEntry.getLegalBasis() != null) {
             entityLogEntry.setLegalBasis(redactionLogEntry.getLegalBasis());
         }
@@ -187,66 +207,38 @@ public final class MigrationEntity {
         return entityLogEntry.getManualChanges()
                 .stream()
                 .reduce((a, b) -> b)
-                .filter(mc -> mc.getManualRedactionType()
-                        .equals(com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.REMOVE_LOCALLY))
+                .filter(mc -> mc.getManualRedactionType().equals(com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.REMOVE))
                 .isPresent();
     }
 
-    private List<com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualChange> migrateManualChanges(List<ManualChange> manualChanges) {
-
-        if (manualChanges == null) {
-            return Collections.emptyList();
-        }
-        return manualChanges.stream()
-                .filter(manualChange -> manualChange.getAnnotationStatus().equals(AnnotationStatus.APPROVED))
-                .map(MigrationEntity::toEntityLogManualChanges)
-                .toList();
-    }
-
-
-    private static Set<com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine> getMigratedEngines(RedactionLogEntry entry) {
-
-        if (entry.getEngines() == null) {
-            return Collections.emptySet();
-        }
-        return entry.getEngines().stream().map(MigrationEntity::toEntityLogEngine).collect(Collectors.toSet());
-    }
-
 
     private Set<String> migrateSetOfIds(Set<String> ids, Map<String, String> oldToNewIdMapping) {
 
         if (ids == null) {
             return Collections.emptySet();
         }
-        return ids.stream().map(oldToNewIdMapping::get).collect(Collectors.toSet());
-    }
-
-
-    private static com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine toEntityLogEngine(Engine engine) {
-
-        return switch (engine) {
-            case DICTIONARY -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine.DICTIONARY;
-            case NER -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine.NER;
-            case RULE -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine.RULE;
-        };
+        return ids.stream()
+                .map(oldToNewIdMapping::get)
+                .collect(Collectors.toSet());
     }
 
 
     public EntityLogEntry createEntityLogEntry(Image image) {
 
-        String imageType = image.getImageType().equals(ImageType.OTHER) ? "image" : image.getImageType().toString().toLowerCase(Locale.ENGLISH);
+        List<Position> positions = getPositionsFromOverride(image).orElse(List.of(new Position(image.getPosition(), image.getPage().getNumber())));
         return EntityLogEntry.builder()
                 .id(image.getId())
                 .value(image.value())
-                .type(imageType)
+                .type(image.type())
                 .reason(image.buildReasonWithManualChangeDescriptions())
                 .legalBasis(image.legalBasis())
                 .matchedRule(image.getMatchedRule().getRuleIdentifier().toString())
                 .dictionaryEntry(false)
-                .positions(List.of(new Position(image.getPosition(), image.getPage().getNumber())))
+                .positions(positions)
                 .containingNodeId(image.getTreeId())
                 .closestHeadline(image.getHeadline().getTextBlock().getSearchText())
-                .section(redactionLogEntry.getSection())
+                .section(image.getManualOverwrite().getSection()
+                                 .orElse(redactionLogEntry.getSection()))
                 .textAfter(redactionLogEntry.getTextAfter())
                 .textBefore(redactionLogEntry.getTextBefore())
                 .imageHasTransparency(image.isTransparent())
@@ -257,33 +249,32 @@ public final class MigrationEntity {
     }
 
 
-    public EntityLogEntry createEntityLogEntry(ManualEntity manualEntity) {
+    public EntityLogEntry createEntityLogEntry(PrecursorEntity precursorEntity) {
 
-        String type = manualEntity.getManualOverwrite().getType().orElse(manualEntity.getType());
         return EntityLogEntry.builder()
-                .id(manualEntity.getId())
-                .reason(manualEntity.buildReasonWithManualChangeDescriptions())
-                .legalBasis(manualEntity.legalBasis())
-                .value(manualEntity.value())
-                .type(type)
-                .state(buildEntryState(manualEntity))
-                .entryType(buildEntryType(manualEntity))
-                .section(redactionLogEntry.getSection())
+                .id(precursorEntity.getId())
+                .reason(precursorEntity.buildReasonWithManualChangeDescriptions())
+                .legalBasis(precursorEntity.legalBasis())
+                .value(precursorEntity.value())
+                .type(precursorEntity.type())
+                .state(buildEntryState(precursorEntity))
+                .entryType(buildEntryType(precursorEntity))
+                .section(precursorEntity.getManualOverwrite().getSection()
+                                 .orElse(redactionLogEntry.getSection()))
                 .textAfter(redactionLogEntry.getTextAfter())
                 .textBefore(redactionLogEntry.getTextBefore())
                 .containingNodeId(Collections.emptyList())
                 .closestHeadline("")
-                .matchedRule(manualEntity.getMatchedRule().getRuleIdentifier().toString())
-                .dictionaryEntry(manualEntity.isDictionaryEntry())
-                .dossierDictionaryEntry(manualEntity.isDossierDictionaryEntry())
+                .matchedRule(precursorEntity.getMatchedRule().getRuleIdentifier().toString())
+                .dictionaryEntry(precursorEntity.isDictionaryEntry())
+                .dossierDictionaryEntry(precursorEntity.isDossierDictionaryEntry())
                 .startOffset(-1)
                 .endOffset(-1)
-                .positions(manualEntity.getManualOverwrite()
-                        .getPositions()
-                        .orElse(manualEntity.getEntityPosition())
-                        .stream()
-                        .map(entityPosition -> new Position(entityPosition.rectangle2D(), entityPosition.pageNumber()))
-                        .toList())
+                .positions(precursorEntity.getManualOverwrite().getPositions()
+                                   .orElse(precursorEntity.getEntityPosition())
+                                   .stream()
+                                   .map(entityPosition -> new Position(entityPosition.rectangle2D(), entityPosition.pageNumber()))
+                                   .toList())
                 .engines(Collections.emptySet())
                 .build();
     }
@@ -292,21 +283,19 @@ public final class MigrationEntity {
     public EntityLogEntry createEntityLogEntry(TextEntity entity) {
 
         assert entity.getPositionsOnPagePerPage().size() == 1;
-        PositionOnPage positionOnPage = entity.getPositionsOnPagePerPage().get(0);
-        List<Position> rectanglesPerLine = positionOnPage.getRectanglePerLine()
-                .stream()
-                .map(rectangle2D -> new Position(rectangle2D, positionOnPage.getPage().getNumber()))
-                .toList();
+        List<Position> rectanglesPerLine = getRectanglesPerLine(entity);
         return EntityLogEntry.builder()
-                .id(positionOnPage.getId())
+                .id(entity.getId())
                 .positions(rectanglesPerLine)
                 .reason(entity.buildReasonWithManualChangeDescriptions())
                 .legalBasis(entity.legalBasis())
-                .value(entity.getManualOverwrite().getValue().orElse(entity.getMatchedRule().isWriteValueWithLineBreaks() ? entity.getValueWithLineBreaks() : entity.getValue()))
-                .type(entity.getType())
-                .section(redactionLogEntry.getSection())
-                .textAfter(redactionLogEntry.getTextAfter())
-                .textBefore(redactionLogEntry.getTextBefore())
+                .value(entity.getManualOverwrite().getValue()
+                               .orElse(entity.getMatchedRule().isWriteValueWithLineBreaks() ? entity.getValueWithLineBreaks() : entity.getValue()))
+                .type(entity.type())
+                .section(entity.getManualOverwrite().getSection()
+                                 .orElse(redactionLogEntry.getSection()))
+                .textAfter(entity.getTextAfter())
+                .textBefore(entity.getTextBefore())
                 .containingNodeId(entity.getDeepestFullyContainingNode().getTreeId())
                 .closestHeadline(entity.getDeepestFullyContainingNode().getHeadline().getTextBlock().getSearchText())
                 .matchedRule(entity.getMatchedRule().getRuleIdentifier().toString())
@@ -321,40 +310,132 @@ public final class MigrationEntity {
     }
 
 
-    private EntryState buildEntryState(IEntity entity) {
+    private static List<Position> getRectanglesPerLine(TextEntity entity) {
 
-        if (entity.applied() && entity.active()) {
-            return EntryState.APPLIED;
-        } else if (entity.skipped() && entity.active()) {
-            return EntryState.SKIPPED;
-        } else if (entity.ignored()) {
-            return EntryState.IGNORED;
-        } else {
-            return EntryState.REMOVED;
-        }
+        return getPositionsFromOverride(entity).orElse(entity.getPositionsOnPagePerPage()
+                                                               .get(0).getRectanglePerLine()
+                                                               .stream()
+                                                               .map(rectangle2D -> new Position(rectangle2D,
+                                                                                                entity.getPositionsOnPagePerPage()
+                                                                                                        .get(0).getPage().getNumber()))
+                                                               .toList());
     }
 
 
-    private EntryType buildEntryType(IEntity entity) {
+    private static Optional<List<Position>> getPositionsFromOverride(IEntity entity) {
 
-        if (entity instanceof TextEntity textEntity) {
-            return getEntryType(textEntity.getEntityType());
-        } else if (entity instanceof ManualEntity manualEntity) {
-            if (manualEntity.isRectangle()) {
-                return EntryType.AREA;
-            }
-            return getEntryType(manualEntity.getEntityType());
-        } else if (entity instanceof Image) {
-            return EntryType.IMAGE;
-        }
-        throw new UnsupportedOperationException(String.format("Entity subclass %s is not implemented!", entity.getClass()));
+        return entity.getManualOverwrite().getPositions()
+                .map(rects -> rects.stream()
+                        .map(r -> new Position(r.rectangle2D(), r.pageNumber()))
+                        .toList());
     }
 
 
     public boolean hasManualChangesOrComments() {
 
         return !(redactionLogEntry.getManualChanges() == null || redactionLogEntry.getManualChanges().isEmpty()) || //
-                !(redactionLogEntry.getComments() == null || redactionLogEntry.getComments().isEmpty());
+               !(redactionLogEntry.getComments() == null || redactionLogEntry.getComments().isEmpty()) //
+               || hasManualChanges();
+    }
+
+
+    public boolean hasManualChanges() {
+
+        return !manualChanges.isEmpty();
+    }
+
+
+    public void applyManualChanges(List<BaseAnnotation> manualChangesToApply, ManualChangesApplicationService manualChangesApplicationService) {
+
+        manualChanges.addAll(manualChangesToApply);
+        manualChangesToApply.forEach(manualChange -> {
+            if (manualChange instanceof ManualResizeRedaction manualResizeRedaction && migratedEntity instanceof TextEntity textEntity) {
+                // Due to the value in the old redaction log already being resized, there is no way to find the original entity ID and therefore to migrate the resize annotation correctly.
+                // Instead, we add an add_locally change to the db.
+                ManualResizeRedaction migratedManualResizeRedaction = ManualResizeRedaction.builder()
+                        .positions(manualResizeRedaction.getPositions())
+                        .annotationId(getNewId())
+                        .updateDictionary(manualResizeRedaction.getUpdateDictionary())
+                        .addToAllDossiers(manualResizeRedaction.isAddToAllDossiers())
+                        .textAfter(manualResizeRedaction.getTextAfter())
+                        .textBefore(manualResizeRedaction.getTextBefore())
+                        .build();
+                manualChangesApplicationService.resize(textEntity, migratedManualResizeRedaction);
+            } else {
+                migratedEntity.getManualOverwrite().addChange(manualChange);
+            }
+        });
+    }
+
+
+    public ManualRedactionEntry buildManualRedactionEntry() {
+
+        assert hasManualChanges();
+
+        // currently we need to insert a manual redaction entry, whenever an entity has been resized.
+        String user = manualChanges.stream()
+                .filter(mc -> mc instanceof ManualResizeRedaction)
+                .findFirst()
+                .orElse(manualChanges.get(0)).getUser();
+
+        return ManualRedactionEntry.builder()
+                .annotationId(newId)
+                .fileId(fileId)
+                .type(redactionLogEntry.getType())
+                .value(redactionLogEntry.getValue())
+                .reason(redactionLogEntry.getReason())
+                .legalBasis(redactionLogEntry.getLegalBasis())
+                .section(redactionLogEntry.getSection())
+                .addToDictionary(false)
+                .addToDossierDictionary(false)
+                .rectangle(false)
+                .positions(buildPositions(migratedEntity))
+                .user(user)
+                .build();
+    }
+
+
+    private List<Rectangle> buildPositions(IEntity entity) {
+
+        if (entity instanceof TextEntity textEntity) {
+
+            var positionsOnPage = textEntity.getPositionsOnPagePerPage()
+                    .get(0);
+            return positionsOnPage.getRectanglePerLine()
+                    .stream()
+                    .map(p -> new Rectangle((float) p.getX(), (float) p.getY(), (float) p.getWidth(), (float) p.getHeight(), positionsOnPage.getPage().getNumber()))
+                    .toList();
+        }
+        if (entity instanceof PrecursorEntity pEntity) {
+
+            return pEntity.getManualOverwrite().getPositions()
+                    .orElse(pEntity.getEntityPosition())
+                    .stream()
+                    .map(p -> new Rectangle((float) p.rectangle2D().getX(),
+                                            (float) p.rectangle2D().getY(),
+                                            (float) p.rectangle2D().getWidth(),
+                                            (float) p.rectangle2D().getHeight(),
+                                            p.pageNumber()))
+                    .toList();
+        }
+        if (entity instanceof Image image) {
+
+            Rectangle2D position = image.getManualOverwrite().getPositions()
+                    .map(p -> p.get(0).rectangle2D())
+                    .orElse(image.getPosition());
+
+            return List.of(new Rectangle((float) position.getX(), (float) position.getY(), (float) position.getWidth(), (float) position.getHeight(), image.getPage().getNumber()));
+
+        } else {
+            throw new UnsupportedOperationException();
+        }
+    }
+
+
+    public boolean needsManualEntry() {
+
+        return manualChanges.stream()
+                       .anyMatch(mc -> mc instanceof ManualResizeRedaction && !((ManualResizeRedaction) mc).getUpdateDictionary()) && !(migratedEntity instanceof Image);
     }
 
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/PrecursorEntity.java

@@ -1,11 +1,19 @@
 package com.iqser.red.service.redaction.v1.server.model;
 
-import java.util.List;
-import java.util.PriorityQueue;
+import static com.iqser.red.service.redaction.v1.server.service.NotFoundImportedEntitiesService.IMPORTED_REDACTION_TYPE;
 
+import java.util.List;
+import java.util.Optional;
+import java.util.PriorityQueue;
+import java.util.Set;
+import java.util.UUID;
+
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryType;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedaction;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
 import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.IEntity;
@@ -22,7 +30,8 @@ import lombok.experimental.FieldDefaults;
 @Builder
 @AllArgsConstructor
 @FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
-public class ManualEntity implements IEntity {
+@SuppressWarnings("PMD.AvoidFieldNameMatchingMethodName")
+public class PrecursorEntity implements IEntity {
 
     // The id must be mapped into a TextEntity as is for comments to work correctly
     String id;
@@ -38,19 +47,24 @@ public class ManualEntity implements IEntity {
     boolean isDictionaryEntry;
     boolean isDossierDictionaryEntry;
     boolean rectangle;
+    Set<Engine> engines;
 
     @Builder.Default
     PriorityQueue<MatchedRule> matchedRuleList = new PriorityQueue<>();
     ManualChangeOverwrite manualOverwrite;
 
 
-    public static ManualEntity fromManualRedactionEntry(ManualRedactionEntry manualRedactionEntry, boolean hint) {
+    public static PrecursorEntity fromManualRedactionEntry(ManualRedactionEntry manualRedactionEntry, boolean hint) {
+
+        List<RectangleWithPage> rectangleWithPages = manualRedactionEntry.getPositions()
+                .stream()
+                .map(RectangleWithPage::fromAnnotationRectangle)
+                .toList();
 
-        List<RectangleWithPage> rectangleWithPages = manualRedactionEntry.getPositions().stream().map(RectangleWithPage::fromAnnotationRectangle).toList();
         var entityType = hint ? EntityType.HINT : EntityType.ENTITY;
         ManualChangeOverwrite manualChangeOverwrite = new ManualChangeOverwrite(entityType);
         manualChangeOverwrite.addChange(manualRedactionEntry);
-        return ManualEntity.builder()
+        return PrecursorEntity.builder()
                 .id(manualRedactionEntry.getAnnotationId())
                 .value(manualRedactionEntry.getValue())
                 .entityPosition(rectangleWithPages)
@@ -65,14 +79,19 @@ public class ManualEntity implements IEntity {
                 .isDossierDictionaryEntry(false)
                 .rectangle(manualRedactionEntry.isRectangle())
                 .manualOverwrite(manualChangeOverwrite)
+                .engines(Set.of(Engine.MANUAL))
                 .build();
     }
 
-    public static ManualEntity fromEntityLogEntry(EntityLogEntry entityLogEntry) {
 
-        List<RectangleWithPage> rectangleWithPages = entityLogEntry.getPositions().stream().map(RectangleWithPage::fromEntityLogPosition).toList();
+    public static PrecursorEntity fromEntityLogEntry(EntityLogEntry entityLogEntry) {
+
+        List<RectangleWithPage> rectangleWithPages = entityLogEntry.getPositions()
+                .stream()
+                .map(RectangleWithPage::fromEntityLogPosition)
+                .toList();
         EntityType entityType = getEntityType(entityLogEntry.getEntryType());
-        return ManualEntity.builder()
+        return PrecursorEntity.builder()
                 .id(entityLogEntry.getId())
                 .value(entityLogEntry.getValue())
                 .entityPosition(rectangleWithPages)
@@ -85,6 +104,54 @@ public class ManualEntity implements IEntity {
                 .isDictionaryEntry(entityLogEntry.isDictionaryEntry())
                 .isDossierDictionaryEntry(entityLogEntry.isDossierDictionaryEntry())
                 .manualOverwrite(new ManualChangeOverwrite(entityType))
+                .engines(entityLogEntry.getEngines())
+                .build();
+    }
+
+
+    public static PrecursorEntity fromImportedEntry(ImportedRedaction importedRedaction) {
+
+        List<RectangleWithPage> rectangleWithPages = importedRedaction.getPositions()
+                .stream()
+                .map(RectangleWithPage::fromEntityLogPosition)
+                .toList();
+        EntryType entryType = Optional.ofNullable(importedRedaction.getEntryType())
+                .orElse(EntryType.ENTITY);
+        EntityType entityType = getEntityType(entryType);
+        String value = Optional.ofNullable(importedRedaction.getValue())
+                .orElse("");
+        return PrecursorEntity.builder()
+                .id(importedRedaction.getId())
+                .value(value)
+                .entityPosition(rectangleWithPages)
+                .reason(Optional.ofNullable(importedRedaction.getReason())
+                                .orElse(""))
+                .legalBasis(Optional.ofNullable(importedRedaction.getLegalBasis())
+                                    .orElse(""))
+                .type(Optional.ofNullable(importedRedaction.getType())
+                              .orElse(IMPORTED_REDACTION_TYPE))
+                .entityType(entityType)
+                .isDictionaryEntry(false)
+                .isDossierDictionaryEntry(false)
+                .rectangle(value.isBlank() || entryType.equals(EntryType.IMAGE) || entryType.equals(EntryType.IMAGE_HINT) || entryType.equals(EntryType.AREA))
+                .manualOverwrite(new ManualChangeOverwrite(entityType))
+                .engines(Set.of(Engine.IMPORTED))
+                .build();
+    }
+
+
+    public static PrecursorEntity fromManualResizeRedaction(ManualResizeRedaction manualResizeRedaction) {
+
+        List<RectangleWithPage> rectangleWithPages = manualResizeRedaction.getPositions()
+                .stream()
+                .map(RectangleWithPage::fromAnnotationRectangle)
+                .toList();
+        return PrecursorEntity.builder()
+                .id(UUID.randomUUID().toString())
+                .value(manualResizeRedaction.getValue())
+                .entityPosition(rectangleWithPages)
+                .entityType(EntityType.ENTITY)
+                .manualOverwrite(new ManualChangeOverwrite(EntityType.ENTITY))
                 .build();
     }
 
@@ -99,9 +166,11 @@ public class ManualEntity implements IEntity {
     @Override
     public String type() {
 
-        return getManualOverwrite().getType().orElse(type);
+        return getManualOverwrite().getType()
+                .orElse(type);
     }
 
+
     private static EntityType getEntityType(EntryType entryType) {
 
         switch (entryType) {
@@ -123,8 +192,4 @@ public class ManualEntity implements IEntity {
         }
     }
 
-    private EntityType getEntityType(boolean isHint) {
-        return isHint ? EntityType.HINT : EntityType.ENTITY;
-    }
-
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/component/Entity.java

@@ -1,6 +1,7 @@
 package com.iqser.red.service.redaction.v1.server.model.component;
 
 import java.util.List;
+import java.util.Optional;
 import java.util.Set;
 
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Change;
@@ -87,7 +88,7 @@ public class Entity {
                 .textAfter(e.getTextAfter())
                 .startOffset(e.getStartOffset())
                 .endOffset(e.getEndOffset())
-                .length(e.getValue().length())
+                .length(Optional.ofNullable(e.getValue()).orElse("").length())
                 .imageHasTransparency(e.isImageHasTransparency())
                 .isDictionaryEntry(e.isDictionaryEntry())
                 .isDossierDictionaryEntry(e.isDossierDictionaryEntry())

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/Dictionary.java

@@ -128,19 +128,19 @@ public class Dictionary {
 
     public void recommendEverywhere(TextEntity textEntity) {
 
-        addLocalDictionaryEntry(textEntity.getType(), textEntity.getValue(), textEntity.getMatchedRuleList(), false);
+        addLocalDictionaryEntry(textEntity.type(), textEntity.getValue(), textEntity.getMatchedRuleList(), false);
     }
 
 
     public void recommendEverywhereWithLastNameSeparately(TextEntity textEntity) {
 
-        addLocalDictionaryEntry(textEntity.getType(), textEntity.getValue(), textEntity.getMatchedRuleList(), true);
+        addLocalDictionaryEntry(textEntity.type(), textEntity.getValue(), textEntity.getMatchedRuleList(), true);
     }
 
 
     public void addMultipleAuthorsAsRecommendation(TextEntity textEntity) {
 
-        splitIntoAuthorNames(textEntity).forEach(authorName -> addLocalDictionaryEntry(textEntity.getType(), authorName, textEntity.getMatchedRuleList(), true));
+        splitIntoAuthorNames(textEntity).forEach(authorName -> addLocalDictionaryEntry(textEntity.type(), authorName, textEntity.getMatchedRuleList(), true));
 
     }
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/TextRange.java

@@ -13,6 +13,7 @@ import lombok.Setter;
 
 @Setter
 @EqualsAndHashCode
+@SuppressWarnings("PMD.AvoidFieldNameMatchingMethodName")
 public class TextRange implements Comparable<TextRange> {
 
     private int start;

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/ManualChangeOverwrite.java

@@ -27,12 +27,12 @@ import lombok.experimental.FieldDefaults;
 public class ManualChangeOverwrite {
 
     private static final Map<Class<? extends BaseAnnotation>, String> MANUAL_CHANGE_DESCRIPTIONS = Map.of(//
-            ManualRedactionEntry.class, "created by manual change", //
-            ManualLegalBasisChange.class, "legal basis was manually changed", //
-            ManualResizeRedaction.class, "resized by manual override", //
-            ManualForceRedaction.class, "forced by manual override", //
-            IdRemoval.class, "removed by manual override", //
-            ManualRecategorization.class, "recategorized by manual override");
+                                                                                                          ManualRedactionEntry.class, "created by manual change", //
+                                                                                                          ManualLegalBasisChange.class, "legal basis was manually changed", //
+                                                                                                          ManualResizeRedaction.class, "resized by manual override", //
+                                                                                                          ManualForceRedaction.class, "forced by manual override", //
+                                                                                                          IdRemoval.class, "removed by manual override", //
+                                                                                                          ManualRecategorization.class, "recategorized by manual override");
 
     List<BaseAnnotation> manualChanges = new LinkedList<>();
     boolean changed;
@@ -80,7 +80,8 @@ public class ManualChangeOverwrite {
         manualChanges.sort(Comparator.comparing(BaseAnnotation::getRequestDate));
         updateFields(manualChanges);
         // make list unmodifiable.
-        return manualChanges.stream().toList();
+        return manualChanges.stream()
+                .toList();
     }
 
 
@@ -121,14 +122,18 @@ public class ManualChangeOverwrite {
                 resized = true;
                 // This is only for not found Manual Entities.
                 value = manualResizeRedaction.getValue();
-                positions = manualResizeRedaction.getPositions().stream().map(RectangleWithPage::fromAnnotationRectangle).toList();
+                positions = manualResizeRedaction.getPositions()
+                        .stream()
+                        .map(RectangleWithPage::fromAnnotationRectangle)
+                        .toList();
             }
 
             if (manualChange instanceof ManualRecategorization recategorization) {
-                // recategorization logic happens in ManualChangesApplicationService.
                 recategorized = true;
-                // this is only relevant for ManualEntities. Image and TextEntity is recategorized in the ManualChangesApplicationService.
                 type = recategorization.getType();
+                if (recategorization.getLegalBasis() != null && !recategorization.getLegalBasis().isEmpty()) {
+                    legalBasis = recategorization.getLegalBasis();
+                }
             }
 
             descriptions.add(MANUAL_CHANGE_DESCRIPTIONS.get(manualChange.getClass()));

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/MatchedRule.java

@@ -24,7 +24,8 @@ public final class MatchedRule implements Comparable<MatchedRule> {
 
     public static final RuleType FINAL_TYPE = RuleType.fromString("FINAL");
     public static final RuleType ELIMINATION_RULE_TYPE = RuleType.fromString("X");
-    private static final List<RuleType> RULE_TYPE_PRIORITIES = List.of(FINAL_TYPE, ELIMINATION_RULE_TYPE);
+    public static final RuleType IMPORTED_TYPE = RuleType.fromString("IMP");
+    private static final List<RuleType> RULE_TYPE_PRIORITIES = List.of(FINAL_TYPE, ELIMINATION_RULE_TYPE, IMPORTED_TYPE);
 
     RuleIdentifier ruleIdentifier;
     @Builder.Default

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/TextEntity.java

@@ -1,6 +1,7 @@
 package com.iqser.red.service.redaction.v1.server.model.document.entity;
 
 import java.awt.geom.Rectangle2D;
+import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Comparator;
 import java.util.HashSet;
@@ -28,17 +29,20 @@ import lombok.experimental.FieldDefaults;
 @AllArgsConstructor
 @FieldDefaults(level = AccessLevel.PRIVATE)
 @EqualsAndHashCode(onlyExplicitlyIncluded = true)
+@SuppressWarnings("PMD.AvoidFieldNameMatchingMethodName")
 public class TextEntity implements IEntity {
 
     // primary key
     @EqualsAndHashCode.Include
-    final TextRange textRange;
-    @EqualsAndHashCode.Include
-    final String type;
-    @EqualsAndHashCode.Include
-    final EntityType entityType;
+    final String id;
     // primary key end
 
+    TextRange textRange;
+    @Builder.Default
+    List<TextRange> duplicateTextRanges = new ArrayList<>();
+    String type; // TODO: make final once ManualChangesApplicatioService recategorize is deleted
+    final EntityType entityType;
+
     @Builder.Default
     final PriorityQueue<MatchedRule> matchedRuleList = new PriorityQueue<>();
     final ManualChangeOverwrite manualOverwrite;
@@ -61,12 +65,30 @@ public class TextEntity implements IEntity {
     SemanticNode deepestFullyContainingNode;
 
 
-    public static TextEntity initialEntityNode(TextRange textRange, String type, EntityType entityType) {
+    public static TextEntity initialEntityNode(TextRange textRange, String type, EntityType entityType, SemanticNode node) {
 
-        return TextEntity.builder().type(type).entityType(entityType).textRange(textRange).manualOverwrite(new ManualChangeOverwrite(entityType)).build();
+        return TextEntity.builder().id(buildId(node, textRange, type, entityType)).type(type).entityType(entityType).textRange(textRange).manualOverwrite(new ManualChangeOverwrite(entityType)).build();
     }
 
 
+    public static TextEntity initialEntityNode(TextRange textRange, String type, EntityType entityType, String id) {
+
+        return TextEntity.builder().id(id).type(type).entityType(entityType).textRange(textRange).manualOverwrite(new ManualChangeOverwrite(entityType)).build();
+    }
+
+
+    private static String buildId(SemanticNode node, TextRange textRange, String type, EntityType entityType) {
+
+        Map<Page, List<Rectangle2D>> rectanglesPerLinePerPage = node.getPositionsPerPage(textRange);
+        return IdBuilder.buildId(rectanglesPerLinePerPage.keySet(), rectanglesPerLinePerPage.values().stream().flatMap(Collection::stream).toList(), type, entityType.name());
+    }
+
+
+    public void addTextRange(TextRange textRange) {
+
+        duplicateTextRanges.add(textRange);
+    }
+
     public boolean occursInNodeOfType(Class<? extends SemanticNode> clazz) {
 
         return intersectingNodes.stream().anyMatch(clazz::isInstance);
@@ -81,13 +103,13 @@ public class TextEntity implements IEntity {
 
     public boolean isType(String type) {
 
-        return this.type.equals(type);
+        return type().equals(type);
     }
 
 
     public boolean isAnyType(List<String> types) {
 
-        return types.contains(type);
+        return types.contains(type());
     }
 
 
@@ -124,7 +146,6 @@ public class TextEntity implements IEntity {
                     .min(Comparator.comparingInt(Page::getNumber))
                     .orElseThrow(() -> new RuntimeException("No Positions found on any page!"));
 
-            String id = IdBuilder.buildId(pages, rectanglesPerLinePerPage.values().stream().flatMap(Collection::stream).toList(), type, entityType.name());
             positionsOnPagePerPage = rectanglesPerLinePerPage.entrySet().stream().map(entry -> buildPositionOnPage(firstPage, id, entry)).toList();
         }
         return positionsOnPagePerPage;
@@ -192,7 +213,7 @@ public class TextEntity implements IEntity {
         });
         sb.delete(sb.length() - 2, sb.length());
         sb.append("], type = \"");
-        sb.append(type);
+        sb.append(type());
         sb.append("\", EntityType.");
         sb.append(entityType);
         sb.append("]");
@@ -206,4 +227,11 @@ public class TextEntity implements IEntity {
         return getManualOverwrite().getType().orElse(type);
     }
 
+
+    @Override
+    public String value() {
+
+        return getManualOverwrite().getValue().orElse(getMatchedRule().isWriteValueWithLineBreaks() ? getValueWithLineBreaks() : value);
+    }
+
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Document.java

@@ -14,11 +14,13 @@ import com.iqser.red.service.redaction.v1.server.model.document.DocumentTree;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlockCollector;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
 
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;
+import lombok.EqualsAndHashCode;
 import lombok.NoArgsConstructor;
 import lombok.experimental.FieldDefaults;
 
@@ -27,8 +29,14 @@ import lombok.experimental.FieldDefaults;
 @AllArgsConstructor
 @NoArgsConstructor
 @FieldDefaults(level = AccessLevel.PRIVATE)
+@EqualsAndHashCode(onlyExplicitlyIncluded = true)
 public class Document implements GenericSemanticNode {
 
+    @EqualsAndHashCode.Include
+    List<Integer> treeId = Collections.emptyList();
+    @Builder.Default
+    Set<LayoutEngine> engines = new HashSet<>(Set.of(LayoutEngine.ALGORITHM));
+
     Set<Page> pages;
     DocumentTree documentTree;
     Integer numberOfPages;
@@ -67,13 +75,6 @@ public class Document implements GenericSemanticNode {
     }
 
 
-    @Override
-    public List<Integer> getTreeId() {
-
-        return Collections.emptyList();
-    }
-
-
     @Override
     public void setTreeId(List<Integer> tocId) {
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Footer.java

@@ -9,6 +9,7 @@ import java.util.Set;
 import com.iqser.red.service.redaction.v1.server.model.document.DocumentTree;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
 
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
@@ -23,22 +24,23 @@ import lombok.experimental.FieldDefaults;
 @AllArgsConstructor
 @NoArgsConstructor
 @FieldDefaults(level = AccessLevel.PRIVATE)
+@EqualsAndHashCode(onlyExplicitlyIncluded = true)
 public class Footer implements GenericSemanticNode {
 
-    @Builder.Default
     final static SectionIdentifier sectionIdentifier = SectionIdentifier.empty();
+    @Builder.Default
+    Set<LayoutEngine> engines = new HashSet<>(Set.of(LayoutEngine.ALGORITHM));
 
+    @EqualsAndHashCode.Include
     List<Integer> treeId;
+
     TextBlock leafTextBlock;
 
-    @EqualsAndHashCode.Exclude
     DocumentTree documentTree;
 
     @Builder.Default
-    @EqualsAndHashCode.Exclude
     Set<TextEntity> entities = new HashSet<>();
 
-    @EqualsAndHashCode.Exclude
     Map<Page, Rectangle2D> bBoxCache;
 
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Header.java

@@ -9,6 +9,7 @@ import java.util.Set;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.DocumentTree;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
 
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
@@ -23,22 +24,23 @@ import lombok.experimental.FieldDefaults;
 @AllArgsConstructor
 @NoArgsConstructor
 @FieldDefaults(level = AccessLevel.PRIVATE)
+@EqualsAndHashCode(onlyExplicitlyIncluded = true)
 public class Header implements GenericSemanticNode {
 
-    @Builder.Default
     final static SectionIdentifier sectionIdentifier = SectionIdentifier.empty();
+    @Builder.Default
+    Set<LayoutEngine> engines = new HashSet<>(Set.of(LayoutEngine.ALGORITHM));
 
+    @EqualsAndHashCode.Include
     List<Integer> treeId;
+
     TextBlock leafTextBlock;
 
-    @EqualsAndHashCode.Exclude
     DocumentTree documentTree;
 
     @Builder.Default
-    @EqualsAndHashCode.Exclude
     Set<TextEntity> entities = new HashSet<>();
 
-    @EqualsAndHashCode.Exclude
     Map<Page, Rectangle2D> bBoxCache;
 
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Headline.java

@@ -10,6 +10,7 @@ import com.iqser.red.service.redaction.v1.server.model.document.DocumentTree;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.AtomicTextBlock;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
 
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
@@ -24,20 +25,21 @@ import lombok.experimental.FieldDefaults;
 @AllArgsConstructor
 @NoArgsConstructor
 @FieldDefaults(level = AccessLevel.PRIVATE)
+@EqualsAndHashCode(onlyExplicitlyIncluded = true)
 public class Headline implements GenericSemanticNode {
 
+    @Builder.Default
+    Set<LayoutEngine> engines = new HashSet<>(Set.of(LayoutEngine.ALGORITHM));
+    @EqualsAndHashCode.Include
     List<Integer> treeId;
     TextBlock leafTextBlock;
     SectionIdentifier sectionIdentifier;
 
-    @EqualsAndHashCode.Exclude
     DocumentTree documentTree;
 
     @Builder.Default
-    @EqualsAndHashCode.Exclude
     Set<TextEntity> entities = new HashSet<>();
 
-    @EqualsAndHashCode.Exclude
     Map<Page, Rectangle2D> bBoxCache;
 
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Image.java

@@ -18,6 +18,7 @@ import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRu
 import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlockCollector;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
 
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
@@ -32,8 +33,13 @@ import lombok.experimental.FieldDefaults;
 @AllArgsConstructor
 @NoArgsConstructor
 @FieldDefaults(level = AccessLevel.PRIVATE)
+@EqualsAndHashCode(onlyExplicitlyIncluded = true)
 public class Image implements GenericSemanticNode, IEntity {
 
+    @Builder.Default
+    Set<LayoutEngine> engines = new HashSet<>(Set.of(LayoutEngine.ALGORITHM));
+
+    @EqualsAndHashCode.Include
     List<Integer> treeId;
     String id;
 
@@ -68,7 +74,9 @@ public class Image implements GenericSemanticNode, IEntity {
     @Override
     public TextBlock getTextBlock() {
 
-        return streamAllSubNodes().filter(SemanticNode::isLeaf).map(SemanticNode::getLeafTextBlock).collect(new TextBlockCollector());
+        return streamAllSubNodes().filter(SemanticNode::isLeaf)
+                .map(SemanticNode::getLeafTextBlock)
+                .collect(new TextBlockCollector());
     }
 
 
@@ -89,7 +97,8 @@ public class Image implements GenericSemanticNode, IEntity {
     @Override
     public String type() {
 
-        return getManualOverwrite().getType().orElse(imageType.toString());
+        return getManualOverwrite().getType()
+                .orElse(imageType.toString());
     }
 
 
@@ -122,6 +131,32 @@ public class Image implements GenericSemanticNode, IEntity {
     }
 
 
+    public boolean mostlyContainedBy(Image image, double containmentThreshold) {
+
+        Map<Page, Rectangle2D> bboxImage = image.getBBox();
+        Map<Page, Rectangle2D> bbox = this.getBBox();
+        //image needs to be on the same page
+        if(bboxImage.get(this.page) != null) {
+            Rectangle2D intersection = bboxImage.get(this.page).createIntersection(bbox.get(this.page));
+            double calculatedIntersection = intersection.getWidth() * intersection.getHeight();
+            double area = bbox.get(this.page).getWidth() * bbox.get(this.page).getHeight();
+            return (calculatedIntersection / area) > containmentThreshold;
+        }
+        return false;
+    }
+
+
+    public boolean mostlyContains(Image image, double containmentThreshold) {
+
+        Map<Page, Rectangle2D> bboxImage = image.getBBox();
+        Map<Page, Rectangle2D> bbox = this.getBBox();
+        Rectangle2D intersection = bbox.get(this.page).createIntersection(bboxImage.get(this.page));
+        double calculatedIntersection = intersection.getWidth() * intersection.getHeight();
+        double area = bbox.get(this.page).getWidth() * bbox.get(this.page).getHeight();
+        return (area / calculatedIntersection) > containmentThreshold;
+    }
+
+
     public int length() {
 
         return 0;

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/ImageType.java

@@ -6,10 +6,22 @@ public enum ImageType {
     LOGO,
     FORMULA,
     SIGNATURE,
-    OTHER,
+    OTHER {
+        @Override
+        public String toString() {
+
+            return "image";
+        }
+    },
     OCR;
 
 
+    public String toString() {
+
+        return name().toLowerCase(Locale.ENGLISH);
+    }
+
+
     public static ImageType fromString(String imageType) {
 
         return switch (imageType.toLowerCase(Locale.ROOT)) {

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Page.java

@@ -23,26 +23,23 @@ import lombok.experimental.FieldDefaults;
 @NoArgsConstructor
 @AllArgsConstructor
 @FieldDefaults(level = AccessLevel.PRIVATE)
+@EqualsAndHashCode(onlyExplicitlyIncluded = true)
 public class Page {
 
+    @EqualsAndHashCode.Include
     Integer number;
     Integer height;
     Integer width;
     Integer rotation;
 
-    @EqualsAndHashCode.Exclude
     List<SemanticNode> mainBody;
-    @EqualsAndHashCode.Exclude
     Header header;
-    @EqualsAndHashCode.Exclude
     Footer footer;
 
     @Builder.Default
-    @EqualsAndHashCode.Exclude
     Set<TextEntity> entities = new HashSet<>();
 
     @Builder.Default
-    @EqualsAndHashCode.Exclude
     Set<Image> images = new HashSet<>();
 
 
@@ -57,19 +54,4 @@ public class Page {
 
         return String.valueOf(number);
     }
-
-
-    @Override
-    public int hashCode() {
-
-        return number;
-    }
-
-
-    @Override
-    public boolean equals(Object o) {
-
-        return o instanceof Page && o.hashCode() == this.hashCode();
-    }
-
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Paragraph.java

@@ -9,6 +9,7 @@ import java.util.Set;
 import com.iqser.red.service.redaction.v1.server.model.document.DocumentTree;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
 
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
@@ -21,19 +22,21 @@ import lombok.experimental.FieldDefaults;
 @Builder
 @AllArgsConstructor
 @FieldDefaults(level = AccessLevel.PRIVATE)
+@EqualsAndHashCode(onlyExplicitlyIncluded = true)
 public class Paragraph implements GenericSemanticNode {
 
+    @Builder.Default
+    Set<LayoutEngine> engines = new HashSet<>(Set.of(LayoutEngine.ALGORITHM));
+
+    @EqualsAndHashCode.Include
     List<Integer> treeId;
     TextBlock leafTextBlock;
 
-    @EqualsAndHashCode.Exclude
     DocumentTree documentTree;
 
     @Builder.Default
-    @EqualsAndHashCode.Exclude
     Set<TextEntity> entities = new HashSet<>();
 
-    @EqualsAndHashCode.Exclude
     Map<Page, Rectangle2D> bBoxCache;
 
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Section.java

@@ -10,12 +10,14 @@ import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntit
 import com.iqser.red.service.redaction.v1.server.model.document.DocumentTree;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlockCollector;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
 
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;
 import lombok.EqualsAndHashCode;
+import lombok.RequiredArgsConstructor;
 import lombok.experimental.FieldDefaults;
 import lombok.extern.slf4j.Slf4j;
 
@@ -24,21 +26,23 @@ import lombok.extern.slf4j.Slf4j;
 @Builder
 @AllArgsConstructor
 @FieldDefaults(level = AccessLevel.PRIVATE)
+@EqualsAndHashCode(onlyExplicitlyIncluded = true)
 public class Section implements GenericSemanticNode {
 
+    @EqualsAndHashCode.Include
     List<Integer> treeId;
 
     TextBlock textBlock;
-    @EqualsAndHashCode.Exclude
     DocumentTree documentTree;
 
     @Builder.Default
-    @EqualsAndHashCode.Exclude
     Set<TextEntity> entities = new HashSet<>();
 
-    @EqualsAndHashCode.Exclude
     Map<Page, Rectangle2D> bBoxCache;
 
+    @Builder.Default
+    Set<LayoutEngine> engines = new HashSet<>(Set.of(LayoutEngine.ALGORITHM));
+
 
     @Override
     public NodeType getType() {

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/SemanticNode.java

@@ -19,8 +19,10 @@ import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.AtomicTextBlock;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
+import com.iqser.red.service.redaction.v1.server.service.document.NodeVisitor;
 import com.iqser.red.service.redaction.v1.server.utils.RectangleTransformations;
 import com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
 
 public interface SemanticNode {
 
@@ -80,8 +82,8 @@ public interface SemanticNode {
      */
     default Set<Page> getPages(TextRange textRange) {
 
-        if (!getTextRange().contains(textRange)) {
-            throw new IllegalArgumentException(format("%s which was used to query for pages is not contained in the %s of this node!", textRange, getTextRange()));
+        if (!getTextRange().intersects(textRange)) {
+            throw new IllegalArgumentException(format("%s which was used to query for pages is not intersected in the %s of this node!", textRange, getTextRange()));
         }
         return getTextBlock().getPages(textRange);
     }
@@ -247,7 +249,7 @@ public interface SemanticNode {
      */
     default boolean hasEntitiesOfType(String type) {
 
-        return getEntities().stream().filter(TextEntity::active).anyMatch(redactionEntity -> redactionEntity.getType().equals(type));
+        return getEntities().stream().filter(TextEntity::active).anyMatch(redactionEntity -> redactionEntity.type().equals(type));
     }
 
 
@@ -260,7 +262,7 @@ public interface SemanticNode {
      */
     default boolean hasEntitiesOfAnyType(String... types) {
 
-        return getEntities().stream().filter(TextEntity::active).anyMatch(redactionEntity -> Arrays.stream(types).anyMatch(type -> redactionEntity.getType().equals(type)));
+        return getEntities().stream().filter(TextEntity::active).anyMatch(redactionEntity -> Arrays.stream(types).anyMatch(type -> redactionEntity.type().equals(type)));
     }
 
 
@@ -273,7 +275,7 @@ public interface SemanticNode {
      */
     default boolean hasEntitiesOfAllTypes(String... types) {
 
-        return getEntities().stream().filter(TextEntity::active).map(TextEntity::getType).collect(Collectors.toUnmodifiableSet()).containsAll(Arrays.stream(types).toList());
+        return getEntities().stream().filter(TextEntity::active).map(TextEntity::type).collect(Collectors.toUnmodifiableSet()).containsAll(Arrays.stream(types).toList());
     }
 
 
@@ -286,7 +288,7 @@ public interface SemanticNode {
      */
     default List<TextEntity> getEntitiesOfType(String type) {
 
-        return getEntities().stream().filter(TextEntity::active).filter(redactionEntity -> redactionEntity.getType().equals(type)).toList();
+        return getEntities().stream().filter(TextEntity::active).filter(redactionEntity -> redactionEntity.type().equals(type)).toList();
     }
 
 
@@ -355,6 +357,13 @@ public interface SemanticNode {
         return getTextBlock().getSearchText().contains(string);
     }
 
+    Set<LayoutEngine> getEngines();
+
+    default void addEngine(LayoutEngine engine) {
+        getEngines().add(engine);
+    }
+
+
 
     /**
      * Checks whether this SemanticNode contains all the provided Strings.
@@ -553,7 +562,6 @@ public interface SemanticNode {
             if (textBlock.containsTextRange(textEntity.getTextRange())) {
                 textEntity.setDeepestFullyContainingNode(this);
             }
-
             textEntity.addIntersectingNode(this);
             streamChildren().filter(semanticNode -> semanticNode.getTextRange().intersects(textEntity.getTextRange()))
                     .forEach(node -> node.addThisToEntityIfIntersects(textEntity));
@@ -627,6 +635,27 @@ public interface SemanticNode {
     }
 
 
+    /**
+     * For a given TextRange this function returns a List of rectangle around the text in the range.
+     * These Rectangles are split either by a new line or by a large gap in the current line.
+     * This is mainly used to find the positions of TextEntities
+     *
+     * @param textRange A TextRange to calculate the positions for.
+     * @return A Map, where the keys are the pages and the values are a list of rectangles describing the position of words
+     */
+    default Map<Page, List<Rectangle2D>> getPositionsPerPage(TextRange textRange) {
+
+        if (isLeaf()) {
+            return getTextBlock().getPositionsPerPage(textRange);
+        }
+        Optional<SemanticNode> containingChildNode = streamChildren().filter(child -> child.getTextRange().contains(textRange)).findFirst();
+        if (containingChildNode.isEmpty()) {
+            return getTextBlock().getPositionsPerPage(textRange);
+        }
+        return containingChildNode.get().getPositionsPerPage(textRange);
+    }
+
+
     /**
      * If this Node is a Leaf it will calculate the boundingBox of its LeafTextBlock, otherwise it will calculate the Union of the BoundingBoxes of all its Children.
      * If called on the Document, it will return the cropbox of each page
@@ -693,4 +722,18 @@ public interface SemanticNode {
         return bBoxPerPage;
     }
 
+
+    /**
+     * Accepts a {@link NodeVisitor} and initiates a depth-first traversal of the semantic tree rooted at this node.
+     * The visitor's {@link NodeVisitor#visit(SemanticNode)} method is invoked for each node encountered during the traversal.
+     *
+     * @param visitor The {@link NodeVisitor} to accept and apply during the traversal.
+     * @see NodeVisitor
+     */
+    default void accept(NodeVisitor visitor) {
+
+        visitor.visit(this);
+        streamChildren().forEach(childNode -> childNode.accept(visitor));
+    }
+
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Table.java

@@ -9,6 +9,7 @@ import java.util.List;
 import java.util.Locale;
 import java.util.Map;
 import java.util.Set;
+import java.util.stream.Collectors;
 import java.util.stream.IntStream;
 import java.util.stream.Stream;
 
@@ -16,6 +17,7 @@ import com.iqser.red.service.redaction.v1.server.model.document.DocumentTree;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlockCollector;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
 
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
@@ -28,8 +30,12 @@ import lombok.experimental.FieldDefaults;
 @Builder
 @AllArgsConstructor
 @FieldDefaults(level = AccessLevel.PRIVATE)
+@EqualsAndHashCode(onlyExplicitlyIncluded = true)
 public class Table implements SemanticNode {
 
+    @Builder.Default
+    Set<LayoutEngine> engines = new HashSet<>(Set.of(LayoutEngine.ALGORITHM));
+    @EqualsAndHashCode.Include
     List<Integer> treeId;
     DocumentTree documentTree;
 
@@ -39,10 +45,8 @@ public class Table implements SemanticNode {
     TextBlock textBlock;
 
     @Builder.Default
-    @EqualsAndHashCode.Exclude
     Set<TextEntity> entities = new HashSet<>();
 
-    @EqualsAndHashCode.Exclude
     Map<Page, Rectangle2D> bBoxCache;
 
 
@@ -64,8 +68,7 @@ public class Table implements SemanticNode {
      */
     public Stream<TextEntity> streamEntitiesWhereRowContainsStringsIgnoreCase(List<String> strings) {
 
-        return IntStream.range(0, numberOfRows)
-                .boxed()
+        return IntStream.range(0, numberOfRows).boxed()
                 .filter(row -> rowContainsStringsIgnoreCase(row, strings))
                 .flatMap(this::streamRow)
                 .map(TableCell::getEntities)
@@ -82,8 +85,11 @@ public class Table implements SemanticNode {
      */
     public boolean rowContainsStringsIgnoreCase(Integer row, List<String> strings) {
 
-        String rowText = streamRow(row).map(TableCell::getTextBlock).collect(new TextBlockCollector()).getSearchText().toLowerCase(Locale.ROOT);
-        return strings.stream().map(String::toLowerCase).allMatch(rowText::contains);
+        String rowText = streamRow(row).map(TableCell::getTextBlock)
+                .collect(new TextBlockCollector()).getSearchText().toLowerCase(Locale.ROOT);
+        return strings.stream()
+                .map(String::toLowerCase)
+                .allMatch(rowText::contains);
     }
 
 
@@ -96,9 +102,13 @@ public class Table implements SemanticNode {
      */
     public Stream<TextEntity> streamEntitiesWhereRowHasHeaderAndValue(String header, String value) {
 
-        List<Integer> vertebrateStudyCols = streamHeaders().filter(headerNode -> headerNode.containsString(header)).map(TableCell::getCol).toList();
+        List<Integer> vertebrateStudyCols = streamHeaders().filter(headerNode -> headerNode.containsString(header))
+                .map(TableCell::getCol)
+                .toList();
         return streamTableCells().filter(tableCellNode -> vertebrateStudyCols.stream()
-                .anyMatch(vertebrateStudyCol -> getCell(tableCellNode.getRow(), vertebrateStudyCol).containsString(value))).map(TableCell::getEntities).flatMap(Collection::stream);
+                        .anyMatch(vertebrateStudyCol -> getCell(tableCellNode.getRow(), vertebrateStudyCol).containsString(value)))
+                .map(TableCell::getEntities)
+                .flatMap(Collection::stream);
     }
 
 
@@ -111,9 +121,13 @@ public class Table implements SemanticNode {
      */
     public Stream<TextEntity> streamEntitiesWhereRowHasHeaderAndAnyValue(String header, List<String> values) {
 
-        List<Integer> colsWithHeader = streamHeaders().filter(headerNode -> headerNode.containsString(header)).map(TableCell::getCol).toList();
+        List<Integer> colsWithHeader = streamHeaders().filter(headerNode -> headerNode.containsString(header))
+                .map(TableCell::getCol)
+                .toList();
         return streamTableCells().filter(tableCellNode -> colsWithHeader.stream()
-                .anyMatch(colWithHeader -> getCell(tableCellNode.getRow(), colWithHeader).containsAnyString(values))).map(TableCell::getEntities).flatMap(Collection::stream);
+                        .anyMatch(colWithHeader -> getCell(tableCellNode.getRow(), colWithHeader).containsAnyString(values)))
+                .map(TableCell::getEntities)
+                .flatMap(Collection::stream);
     }
 
 
@@ -126,16 +140,33 @@ public class Table implements SemanticNode {
      */
     public Stream<TextEntity> streamEntitiesWhereRowContainsEntitiesOfType(List<String> types) {
 
-        List<Integer> rowsWithEntityOfType = getEntities().stream()
-                .filter(TextEntity::active)
-                .filter(redactionEntity -> types.stream().anyMatch(type -> type.equals(redactionEntity.getType())))
-                .map(TextEntity::getIntersectingNodes)
-                .filter(node -> node instanceof TableCell)
-                .map(node -> (TableCell) node)
-                .map(TableCell::getRow)
-                .toList();
+        return IntStream.range(0, numberOfRows).boxed()
+                .filter(rowNumber -> streamTextEntitiesInRow(rowNumber).map(TextEntity::type)
+                        .anyMatch(types::contains))
+                .flatMap(this::streamRow)
+                .map(TableCell::getEntities)
+                .flatMap(Collection::stream);
+    }
 
-        return rowsWithEntityOfType.stream().flatMap(this::streamRow).map(TableCell::getEntities).flatMap(Collection::stream);
+
+    /**
+     * Streams all entities in this table, that appear in a row, which contains at least one entity of each of the provided types.
+     * Ignores Entity with ignored == true or removed == true.
+     *
+     * @param types type strings to check whether a row contains an entity like them
+     * @return Stream of all entities in this table, that appear in a row, which contains at least one entity of each of the provided types.
+     */
+    public Stream<TextEntity> streamEntitiesWhereRowContainsEntitiesOfEachType(List<String> types) {
+
+        return IntStream.range(0, numberOfRows).boxed()
+                .filter(rowNumber -> {
+                    Set<String> entityTypes = streamTextEntitiesInRow(rowNumber).map(TextEntity::type)
+                            .collect(Collectors.toSet());
+                    return entityTypes.containsAll(types);
+                })
+                .flatMap(this::streamRow)
+                .map(TableCell::getEntities)
+                .flatMap(Collection::stream);
     }
 
 
@@ -148,18 +179,43 @@ public class Table implements SemanticNode {
      */
     public Stream<TextEntity> streamEntitiesWhereRowContainsNoEntitiesOfType(List<String> types) {
 
-        return IntStream.range(0, numberOfRows)
-                .boxed()
-                .filter(rowNumber -> streamRow(rowNumber).map(TableCell::getEntities)
-                        .flatMap(Collection::stream)
-                        .filter(TextEntity::active)
-                        .noneMatch(entity -> types.contains(entity.getType())))
+        return IntStream.range(0, numberOfRows).boxed()
+                .filter(rowNumber -> streamTextEntitiesInRow(rowNumber).map(TextEntity::type)
+                        .noneMatch(types::contains))
                 .flatMap(this::streamRow)
                 .map(TableCell::getEntities)
                 .flatMap(Collection::stream);
     }
 
 
+    /**
+     * Streams all Entities in the given row.
+     *
+     * @param rowNumber the row number to look for
+     * @return stream of TextEntities occurring in row
+     */
+    public Stream<TextEntity> streamTextEntitiesInRow(int rowNumber) {
+
+        return streamRow(rowNumber).map(TableCell::getEntities)
+                .flatMap(Collection::stream)
+                .filter(TextEntity::active);
+    }
+
+
+    /**
+     * Streams all Entities in the given col.
+     *
+     * @param colNumber the column number to look for
+     * @return stream of TextEntities occurring in row
+     */
+    public Stream<TextEntity> streamTextEntitiesInCol(int colNumber) {
+
+        return streamCol(colNumber).map(TableCell::getEntities)
+                .flatMap(Collection::stream)
+                .filter(TextEntity::active);
+    }
+
+
     /**
      * Returns a TableCell at the provided row and column location.
      *
@@ -173,7 +229,8 @@ public class Table implements SemanticNode {
             throw new IllegalArgumentException(format("row %d, col %d is out of bounds for number of rows of %d and number of cols %d", row, col, numberOfRows, numberOfCols));
         }
         int idx = row * numberOfCols + col;
-        return (TableCell) documentTree.getEntryById(treeId).getChildren().get(idx).getNode();
+        return (TableCell) documentTree.getEntryById(treeId).getChildren()
+                .get(idx).getNode();
     }
 
 
@@ -196,7 +253,7 @@ public class Table implements SemanticNode {
      */
     public Stream<TableCell> streamTableCellsWhichContainType(String type) {
 
-        return streamTableCells().filter(tableCell -> tableCell.getEntities().stream().filter(TextEntity::active).anyMatch(entity -> entity.getType().equals(type)));
+        return streamTableCells().filter(tableCell -> tableCell.hasEntitiesOfType(type));
     }
 
 
@@ -222,7 +279,8 @@ public class Table implements SemanticNode {
      */
     public Stream<TableCell> streamCol(int col) {
 
-        return IntStream.range(0, numberOfRows).boxed().map(row -> getCell(row, col));
+        return IntStream.range(0, numberOfRows).boxed()
+                .map(row -> getCell(row, col));
     }
 
 
@@ -234,7 +292,8 @@ public class Table implements SemanticNode {
      */
     public Stream<TableCell> streamRow(int row) {
 
-        return IntStream.range(0, numberOfCols).boxed().map(col -> getCell(row, col));
+        return IntStream.range(0, numberOfCols).boxed()
+                .map(col -> getCell(row, col));
     }
 
 
@@ -258,7 +317,8 @@ public class Table implements SemanticNode {
      */
     public Stream<TableCell> streamHeadersForCell(int row, int col) {
 
-        return Stream.concat(streamRow(row), streamCol(col)).filter(TableCell::isHeader);
+        return Stream.concat(streamRow(row), streamCol(col))
+                .filter(TableCell::isHeader);
     }
 
 
@@ -348,7 +408,9 @@ public class Table implements SemanticNode {
     public TextBlock getTextBlock() {
 
         if (textBlock == null) {
-            textBlock = streamAllSubNodes().filter(SemanticNode::isLeaf).map(SemanticNode::getLeafTextBlock).collect(new TextBlockCollector());
+            textBlock = streamAllSubNodes().filter(SemanticNode::isLeaf)
+                    .map(SemanticNode::getLeafTextBlock)
+                    .collect(new TextBlockCollector());
         }
         return textBlock;
     }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/TableCell.java

@@ -11,6 +11,7 @@ import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntit
 import com.iqser.red.service.redaction.v1.server.model.document.DocumentTree;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlockCollector;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
 
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
@@ -23,8 +24,13 @@ import lombok.experimental.FieldDefaults;
 @Builder
 @AllArgsConstructor
 @FieldDefaults(level = AccessLevel.PRIVATE)
+@EqualsAndHashCode(onlyExplicitlyIncluded = true)
 public class TableCell implements GenericSemanticNode {
 
+    @Builder.Default
+    Set<LayoutEngine> engines = new HashSet<>(Set.of(LayoutEngine.ALGORITHM));
+
+    @EqualsAndHashCode.Include
     List<Integer> treeId;
     int row;
     int col;
@@ -36,11 +42,9 @@ public class TableCell implements GenericSemanticNode {
 
     TextBlock textBlock;
 
-    @EqualsAndHashCode.Exclude
     DocumentTree documentTree;
 
     @Builder.Default
-    @EqualsAndHashCode.Exclude
     Set<TextEntity> entities = new HashSet<>();
 
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/textblock/ConcatenatedTextBlock.java

@@ -71,7 +71,16 @@ public class ConcatenatedTextBlock implements TextBlock {
 
     private List<AtomicTextBlock> getAllAtomicTextBlocksPartiallyInStringBoundary(TextRange textRange) {
 
-        return atomicTextBlocks.stream().filter(tb -> tb.getTextRange().intersects(textRange)).toList();
+        List<AtomicTextBlock> intersectingAtomicTextBlocks = new LinkedList<>();
+        for (AtomicTextBlock atomicTextBlock : atomicTextBlocks) {
+            if (atomicTextBlock.getTextRange().start() > textRange.end()) {
+                break; // early stop, following TextBlocks will never intersect
+            }
+            if (atomicTextBlock.getTextRange().intersects(textRange)) {
+                intersectingAtomicTextBlocks.add(atomicTextBlock);
+            }
+        }
+        return intersectingAtomicTextBlocks;
     }
 
 
@@ -122,13 +131,13 @@ public class ConcatenatedTextBlock implements TextBlock {
     }
 
 
-
     @Override
     public Rectangle2D getPosition(int stringIdx) {
 
         return getAtomicTextBlockByStringIndex(stringIdx).getPosition(stringIdx);
     }
 
+
     public TextRange getLineTextRange(int lineNumber) {
 
         if (atomicTextBlocks.size() == 1) {
@@ -144,10 +153,10 @@ public class ConcatenatedTextBlock implements TextBlock {
         return new TextRange(textRange.start(), textRange.start());
     }
 
+
     @Override
     public List<Rectangle2D> getPositions(TextRange stringTextRange) {
 
-
         List<AtomicTextBlock> textBlocks = getAllAtomicTextBlocksPartiallyInStringBoundary(stringTextRange);
 
         if (textBlocks.isEmpty()) {

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalyzeService.java

@@ -1,11 +1,13 @@
 package com.iqser.red.service.redaction.v1.server.service;
 
+import java.util.Collection;
 import java.util.Collections;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
 import org.springframework.stereotype.Service;
 import org.springframework.web.bind.annotation.RequestBody;
@@ -18,6 +20,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileTyp
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentLog;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogChanges;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedactions;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.legalbasis.LegalBasis;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog;
@@ -28,7 +31,7 @@ import com.iqser.red.service.redaction.v1.server.RedactionServiceSettings;
 import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient;
 import com.iqser.red.service.redaction.v1.server.client.model.NerEntitiesModel;
 import com.iqser.red.service.redaction.v1.server.model.KieWrapper;
-import com.iqser.red.service.redaction.v1.server.model.ManualEntity;
+import com.iqser.red.service.redaction.v1.server.model.PrecursorEntity;
 import com.iqser.red.service.redaction.v1.server.model.NerEntities;
 import com.iqser.red.service.redaction.v1.server.model.component.Component;
 import com.iqser.red.service.redaction.v1.server.model.dictionary.Dictionary;
@@ -37,6 +40,7 @@ import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryVers
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
 import com.iqser.red.service.redaction.v1.server.service.document.DocumentGraphMapper;
+import com.iqser.red.service.redaction.v1.server.service.document.ImportedRedactionEntryService;
 import com.iqser.red.service.redaction.v1.server.service.document.ManualRedactionEntryService;
 import com.iqser.red.service.redaction.v1.server.service.document.NerEntitiesAdapter;
 import com.iqser.red.service.redaction.v1.server.service.document.SectionFinderService;
@@ -65,18 +69,15 @@ public class AnalyzeService {
     ComponentDroolsExecutionService componentDroolsExecutionService;
     KieContainerCreationService kieContainerCreationService;
     DictionarySearchService dictionarySearchService;
-    RedactionLogCreatorService redactionLogCreatorService;
     EntityLogCreatorService entityLogCreatorService;
     ComponentLogCreatorService componentLogCreatorService;
     RedactionStorageService redactionStorageService;
-    RedactionChangeLogService redactionChangeLogService;
-    LegalBasisClient legalBasisClient;
     RedactionServiceSettings redactionServiceSettings;
-    ImportedRedactionService importedRedactionService;
+    NotFoundImportedEntitiesService notFoundImportedEntitiesService;
     SectionFinderService sectionFinderService;
     ManualRedactionEntryService manualRedactionEntryService;
+    ImportedRedactionEntryService importedRedactionEntryService;
     ObservedStorageService observedStorageService;
-
     FunctionTimerValues redactmanagerAnalyzePagewiseValues;
 
 
@@ -86,51 +87,60 @@ public class AnalyzeService {
     public AnalyzeResult reanalyze(@RequestBody AnalyzeRequest analyzeRequest) {
 
         long startTime = System.currentTimeMillis();
-        RedactionLog previousRedactionLog = redactionStorageService.getRedactionLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
         EntityLog previousEntityLog = redactionStorageService.getEntityLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
         log.info("Loaded previous entity log for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
 
         Document document = DocumentGraphMapper.toDocumentGraph(observedStorageService.getDocumentData(analyzeRequest.getDossierId(), analyzeRequest.getFileId()));
         log.info("Loaded Document Graph for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
 
+        ImportedRedactions importedRedactions = redactionStorageService.getImportedRedactions(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
+        log.info("Loaded Imported Redactions for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
+
         // not yet ready for reanalysis
         if (previousEntityLog == null || document == null || document.getNumberOfPages() == 0) {
             return analyze(analyzeRequest);
         }
 
         DictionaryIncrement dictionaryIncrement = dictionaryService.getDictionaryIncrements(analyzeRequest.getDossierTemplateId(),
-                new DictionaryVersion(previousEntityLog.getDictionaryVersion(), previousEntityLog.getDossierDictionaryVersion()),
-                analyzeRequest.getDossierId());
+                                                                                            new DictionaryVersion(previousEntityLog.getDictionaryVersion(),
+                                                                                                                  previousEntityLog.getDossierDictionaryVersion()),
+                                                                                            analyzeRequest.getDossierId());
 
-        Set<Integer> sectionsToReanalyseIds = getSectionsToReanalyseIds(analyzeRequest, previousEntityLog, document, dictionaryIncrement);
+        Set<Integer> sectionsToReanalyseIds = getSectionsToReanalyseIds(analyzeRequest, previousEntityLog, document, dictionaryIncrement, importedRedactions);
         List<SemanticNode> sectionsToReAnalyse = getSectionsToReAnalyse(document, sectionsToReanalyseIds);
         log.info("{} Sections to reanalyze found for file {} in dossier {}", sectionsToReanalyseIds.size(), analyzeRequest.getFileId(), analyzeRequest.getDossierId());
 
         if (sectionsToReAnalyse.isEmpty()) {
 
+            EntityLogChanges entityLogChanges = entityLogCreatorService.updateVersionsAndReturnChanges(previousEntityLog,
+                                                                                                       dictionaryIncrement.getDictionaryVersion(),
+                                                                                                       analyzeRequest,
+                                                                                                       false);
+
             return finalizeAnalysis(analyzeRequest,
-                    startTime,
-                    kieContainerCreationService.getLatestKieContainer(analyzeRequest.getDossierTemplateId(), RuleFileType.COMPONENT),
-                    entityLogCreatorService.updateVersionsAndReturnChanges(previousEntityLog,
-                            dictionaryIncrement.getDictionaryVersion(),
-                            analyzeRequest.getDossierTemplateId(),
-                            false),
-                    document,
-                    previousRedactionLog,
-                    document.getNumberOfPages(),
-                    dictionaryIncrement.getDictionaryVersion(),
-                    true,
-                    Collections.emptySet());
+                                    startTime,
+                                    kieContainerCreationService.getLatestKieContainer(analyzeRequest.getDossierTemplateId(), RuleFileType.COMPONENT),
+                                    entityLogChanges,
+                                    document,
+                                    document.getNumberOfPages(),
+                                    dictionaryIncrement.getDictionaryVersion(),
+                                    true,
+                                    Collections.emptySet());
         }
+
         KieWrapper kieWrapperEntityRules = kieContainerCreationService.getLatestKieContainer(analyzeRequest.getDossierTemplateId(), RuleFileType.ENTITY);
         log.info("Updated entity rules to version {} for file {} in dossier {}", kieWrapperEntityRules.rulesVersion(), analyzeRequest.getFileId(), analyzeRequest.getDossierId());
 
         NerEntities nerEntities = getEntityRecognitionEntitiesFilteredBySectionIds(analyzeRequest, document, sectionsToReanalyseIds);
         log.info("Loaded Ner Entities for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
 
-        List<ManualEntity> notFoundManualRedactionEntries = manualRedactionEntryService.addManualRedactionEntriesAndReturnNotFoundEntries(analyzeRequest,
-                document,
-                analyzeRequest.getDossierTemplateId());
+        var notFoundManualRedactionEntries = manualRedactionEntryService.addManualRedactionEntriesAndReturnNotFoundEntries(analyzeRequest,
+                                                                                                                           document,
+                                                                                                                           analyzeRequest.getDossierTemplateId());
+        var notFoundImportedEntries = importedRedactionEntryService.addImportedEntriesAndReturnNotFoundEntries(analyzeRequest, importedRedactions, document);
+        var notFoundManualOrImportedEntries = Stream.of(notFoundManualRedactionEntries, notFoundImportedEntries)
+                .flatMap(Collection::stream)
+                .collect(Collectors.toList());
 
         Dictionary dictionary = dictionaryService.getDeepCopyDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId());
         log.info("Updated Dictionaries for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
@@ -138,33 +148,34 @@ public class AnalyzeService {
         dictionarySearchService.addDictionaryEntities(dictionary, sectionsToReAnalyse);
         log.info("Finished Dictionary Search for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
 
+        // we could add the imported redactions similar to the manual redactions here as well for additional processing
         List<FileAttribute> allFileAttributes = entityDroolsExecutionService.executeRules(kieWrapperEntityRules.container(),
-                document,
-                sectionsToReAnalyse,
-                dictionary,
-                analyzeRequest.getFileAttributes(),
-                analyzeRequest.getManualRedactions(),
-                nerEntities);
+                                                                                          document,
+                                                                                          sectionsToReAnalyse,
+                                                                                          dictionary,
+                                                                                          analyzeRequest.getFileAttributes(),
+                                                                                          analyzeRequest.getManualRedactions(),
+                                                                                          nerEntities);
         log.info("Finished entity rule execution for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
 
-        RedactionLog redactionLog = updatePreviousRedactionLog(analyzeRequest, document, notFoundManualRedactionEntries, previousRedactionLog, sectionsToReanalyseIds);
         EntityLogChanges entityLogChanges = entityLogCreatorService.updatePreviousEntityLog(analyzeRequest,
-                document,
-                notFoundManualRedactionEntries,
-                previousEntityLog,
-                sectionsToReanalyseIds,
-                dictionary.getVersion());
+                                                                                            document,
+                                                                                            notFoundManualOrImportedEntries,
+                                                                                            previousEntityLog,
+                                                                                            sectionsToReanalyseIds,
+                                                                                            dictionary.getVersion());
+
+        notFoundImportedEntitiesService.processEntityLog(entityLogChanges.getEntityLog(), analyzeRequest, notFoundImportedEntries);
 
         return finalizeAnalysis(analyzeRequest,
-                startTime,
-                kieContainerCreationService.getLatestKieContainer(analyzeRequest.getDossierTemplateId(), RuleFileType.COMPONENT),
-                entityLogChanges,
-                document,
-                redactionLog,
-                document.getNumberOfPages(),
-                dictionaryIncrement.getDictionaryVersion(),
-                true,
-                new HashSet<>(allFileAttributes));
+                                startTime,
+                                kieContainerCreationService.getLatestKieContainer(analyzeRequest.getDossierTemplateId(), RuleFileType.COMPONENT),
+                                entityLogChanges,
+                                document,
+                                document.getNumberOfPages(),
+                                dictionaryIncrement.getDictionaryVersion(),
+                                true,
+                                new HashSet<>(allFileAttributes));
     }
 
 
@@ -183,6 +194,9 @@ public class AnalyzeService {
         Document document = DocumentGraphMapper.toDocumentGraph(observedStorageService.getDocumentData(analyzeRequest.getDossierId(), analyzeRequest.getFileId()));
         log.info("Loaded Document Graph for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
 
+        ImportedRedactions importedRedactions = redactionStorageService.getImportedRedactions(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
+        log.info("Loaded Imported Redactions for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
+
         NerEntities nerEntities = getEntityRecognitionEntities(analyzeRequest, document);
         log.info("Loaded Ner Entities for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
 
@@ -190,64 +204,43 @@ public class AnalyzeService {
         Dictionary dictionary = dictionaryService.getDeepCopyDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId());
         log.info("Updated Dictionaries for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
 
-        List<ManualEntity> notFoundManualRedactionEntries = manualRedactionEntryService.addManualRedactionEntriesAndReturnNotFoundEntries(analyzeRequest,
-                document,
-                analyzeRequest.getDossierTemplateId());
+        var notFoundManualRedactionEntries = manualRedactionEntryService.addManualRedactionEntriesAndReturnNotFoundEntries(analyzeRequest,
+                                                                                                                           document,
+                                                                                                                           analyzeRequest.getDossierTemplateId());
+        var notFoundImportedEntries = importedRedactionEntryService.addImportedEntriesAndReturnNotFoundEntries(analyzeRequest, importedRedactions, document);
+        var notFoundManualOrImportedEntries = Stream.of(notFoundManualRedactionEntries, notFoundImportedEntries)
+                .flatMap(Collection::stream)
+                .collect(Collectors.toList());
 
         dictionarySearchService.addDictionaryEntities(dictionary, document);
         log.info("Finished Dictionary Search for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
 
+        // we could add the imported redactions similar to the manual redactions here as well for additional processing
         List<FileAttribute> allFileAttributes = entityDroolsExecutionService.executeRules(kieWrapperEntityRules.container(),
-                document,
-                dictionary,
-                analyzeRequest.getFileAttributes(),
-                analyzeRequest.getManualRedactions(),
-                nerEntities);
+                                                                                          document,
+                                                                                          dictionary,
+                                                                                          analyzeRequest.getFileAttributes(),
+                                                                                          analyzeRequest.getManualRedactions(),
+                                                                                          nerEntities);
         log.info("Finished entity rule execution for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
 
-        RedactionLog redactionLog = createRedactionLog(analyzeRequest, document, notFoundManualRedactionEntries, dictionary, kieWrapperEntityRules);
         EntityLog entityLog = entityLogCreatorService.createInitialEntityLog(analyzeRequest,
-                document,
-                notFoundManualRedactionEntries,
-                dictionary.getVersion(),
-                kieWrapperEntityRules.rulesVersion());
+                                                                             document,
+                                                                             notFoundManualOrImportedEntries,
+                                                                             dictionary.getVersion(),
+                                                                             kieWrapperEntityRules.rulesVersion());
+
+        notFoundImportedEntitiesService.processEntityLog(entityLog, analyzeRequest, notFoundImportedEntries);
 
         return finalizeAnalysis(analyzeRequest,
-                startTime,
-                kieWrapperComponentRules,
-                new EntityLogChanges(entityLog, false),
-                document,
-                redactionLog,
-                document.getNumberOfPages(),
-                dictionary.getVersion(),
-                false,
-                new HashSet<>(allFileAttributes));
-    }
-
-
-    @Deprecated(forRemoval = true)
-    private RedactionLog updatePreviousRedactionLog(AnalyzeRequest analyzeRequest,
-                                                    Document document,
-                                                    List<ManualEntity> notFoundManualRedactionEntries,
-                                                    RedactionLog previousRedactionLog,
-                                                    Set<Integer> sectionsToReanalyseIds) {
-
-        List<RedactionLogEntry> newRedactionLogEntries = redactionLogCreatorService.createRedactionLog(document,
-                analyzeRequest.getDossierTemplateId(),
-                notFoundManualRedactionEntries);
-
-        var importedRedactionFilteredEntries = importedRedactionService.processImportedRedactions(analyzeRequest.getDossierTemplateId(),
-                analyzeRequest.getDossierId(),
-                analyzeRequest.getFileId(),
-                newRedactionLogEntries,
-                false);
-
-        previousRedactionLog.getRedactionLogEntry()
-                .removeIf(entry -> sectionsToReanalyseIds.contains(entry.getSectionNumber()) && !entry.getType().equals(ImportedRedactionService.IMPORTED_REDACTION_TYPE));
-
-        previousRedactionLog.getRedactionLogEntry().addAll(importedRedactionFilteredEntries);
-
-        return previousRedactionLog;
+                                startTime,
+                                kieWrapperComponentRules,
+                                new EntityLogChanges(entityLog, false),
+                                document,
+                                document.getNumberOfPages(),
+                                dictionary.getVersion(),
+                                false,
+                                new HashSet<>(allFileAttributes));
     }
 
 
@@ -256,14 +249,11 @@ public class AnalyzeService {
                                            KieWrapper kieWrapperComponentRules,
                                            EntityLogChanges entityLogChanges,
                                            Document document,
-                                           RedactionLog redactionLog,
                                            int numberOfPages,
                                            DictionaryVersion dictionaryVersion,
                                            boolean isReanalysis,
                                            Set<FileAttribute> addedFileAttributes) {
 
-        finalizeRedactionLog(analyzeRequest, redactionLog, dictionaryVersion);
-
         EntityLog entityLog = entityLogChanges.getEntityLog();
         redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.ENTITY_LOG, entityLogChanges.getEntityLog());
 
@@ -310,9 +300,10 @@ public class AnalyzeService {
         }
 
         List<Component> components = componentDroolsExecutionService.executeRules(kieWrapperComponentRules.container(),
-                entityLogChanges.getEntityLog(),
-                document,
-                addedFileAttributes.stream().toList());
+                                                                                  entityLogChanges.getEntityLog(),
+                                                                                  document,
+                                                                                  addedFileAttributes.stream()
+                                                                                          .toList());
         log.info("Finished component rule execution for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
 
         ComponentLog componentLog = componentLogCreatorService.buildComponentLog(analyzeRequest.getAnalysisNumber(), components, kieWrapperComponentRules.rulesVersion());
@@ -322,33 +313,22 @@ public class AnalyzeService {
     }
 
 
-    private RedactionLogChanges finalizeRedactionLog(AnalyzeRequest analyzeRequest, RedactionLog redactionLog, DictionaryVersion dictionaryVersion) {
-        // TODO: remove redactionLog related stuff
-
-        RedactionLog previousRedactionLog = redactionStorageService.getRedactionLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
-
-        redactionLog.setDictionaryVersion(dictionaryVersion.getDossierTemplateVersion());
-        redactionLog.setDossierDictionaryVersion(dictionaryVersion.getDossierVersion());
-
-        RedactionLogChanges redactionLogChange = redactionChangeLogService.computeChanges(previousRedactionLog, redactionLog, analyzeRequest.getAnalysisNumber());
-
-        redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.REDACTION_LOG, redactionLogChange.getRedactionLog());
-
-        return redactionLogChange;
-    }
-
-
     private static List<SemanticNode> getSectionsToReAnalyse(Document document, Set<Integer> sectionsToReanalyseIds) {
 
-        return document.streamChildren().filter(section -> sectionsToReanalyseIds.contains(section.getTreeId().get(0))).collect(Collectors.toList());
+        return document.streamChildren()
+                .filter(section -> sectionsToReanalyseIds.contains(section.getTreeId()
+                                                                           .get(0)))
+                .collect(Collectors.toList());
     }
 
 
-    private Set<Integer> getSectionsToReanalyseIds(AnalyzeRequest analyzeRequest, EntityLog entityLog, Document document, DictionaryIncrement dictionaryIncrement) {
+    private Set<Integer> getSectionsToReanalyseIds(AnalyzeRequest analyzeRequest,
+                                                   EntityLog entityLog,
+                                                   Document document,
+                                                   DictionaryIncrement dictionaryIncrement,
+                                                   ImportedRedactions importedRedactions) {
 
-        return analyzeRequest.getSectionsToReanalyse().isEmpty() //
-                ? sectionFinderService.findSectionsToReanalyse(dictionaryIncrement, entityLog, document, analyzeRequest) //
-                : analyzeRequest.getSectionsToReanalyse();
+        return sectionFinderService.findSectionsToReanalyse(dictionaryIncrement, entityLog, document, analyzeRequest, importedRedactions);
     }
 
 
@@ -368,9 +348,10 @@ public class AnalyzeService {
 
     private static NerEntitiesModel filterNerEntitiesModelBySectionIds(Set<Integer> sectionsToReanalyseIds, NerEntitiesModel nerEntitiesModel) {
 
-        return new NerEntitiesModel(nerEntitiesModel.getData().entrySet().stream() //
-                .filter(entry -> sectionsToReanalyseIds.contains(entry.getKey())) //
-                .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)));
+        return new NerEntitiesModel(nerEntitiesModel.getData().entrySet()
+                                            .stream() //
+                                            .filter(entry -> sectionsToReanalyseIds.contains(entry.getKey())) //
+                                            .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)));
     }
 
 
@@ -385,41 +366,4 @@ public class AnalyzeService {
         return nerEntities;
     }
 
-
-    @Deprecated(forRemoval = true)
-    private RedactionLog createRedactionLog(AnalyzeRequest analyzeRequest,
-                                            Document document,
-                                            List<ManualEntity> notFoundManualRedactionEntries,
-                                            Dictionary dictionary,
-                                            KieWrapper wrapper) {
-
-        List<RedactionLogEntry> redactionLogEntries = redactionLogCreatorService.createRedactionLog(document,
-                analyzeRequest.getDossierTemplateId(),
-                notFoundManualRedactionEntries);
-
-        List<LegalBasis> legalBasis = legalBasisClient.getLegalBasisMapping(analyzeRequest.getDossierTemplateId());
-        RedactionLog redactionLog = new RedactionLog(redactionServiceSettings.getAnalysisVersion(),
-                analyzeRequest.getAnalysisNumber(),
-                redactionLogEntries,
-                toRedactionLogLegalBasis(legalBasis),
-                dictionary.getVersion().getDossierTemplateVersion(),
-                dictionary.getVersion().getDossierVersion(),
-                wrapper.rulesVersion(),
-                legalBasisClient.getVersion(analyzeRequest.getDossierTemplateId()));
-
-        List<RedactionLogEntry> importedRedactionFilteredEntries = importedRedactionService.processImportedRedactions(analyzeRequest.getDossierTemplateId(),
-                analyzeRequest.getDossierId(),
-                analyzeRequest.getFileId(),
-                redactionLog.getRedactionLogEntry(),
-                true);
-        redactionLog.setRedactionLogEntry(importedRedactionFilteredEntries);
-        return redactionLog;
-    }
-
-
-    public List<RedactionLogLegalBasis> toRedactionLogLegalBasis(List<LegalBasis> legalBasis) {
-
-        return legalBasis.stream().map(l -> new RedactionLogLegalBasis(l.getName(), l.getDescription(), l.getReason())).collect(Collectors.toList());
-    }
-
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/DictionarySearchService.java

@@ -1,6 +1,7 @@
 package com.iqser.red.service.redaction.v1.server.service;
 
 import java.util.List;
+import java.util.Set;
 
 import org.springframework.stereotype.Service;
 
@@ -28,7 +29,8 @@ public class DictionarySearchService {
 
 
     @Observed(name = "DictionarySearchService", contextualName = "add-dictionary-entries")
-    public void addDictionaryEntities(Dictionary dictionary, List<SemanticNode> semanticNodes){
+    public void addDictionaryEntities(Dictionary dictionary, List<SemanticNode> semanticNodes) {
+
         semanticNodes.forEach(node -> addDictionaryEntities(dictionary, node));
     }
 
@@ -52,11 +54,12 @@ public class DictionarySearchService {
 
         EntityCreationService entityCreationService = new EntityCreationService(entityEnrichmentService);
         searchImplementation.getBoundaries(node.getTextBlock(), node.getTextRange())
-                .stream().filter(boundary -> entityCreationService.isValidEntityTextRange(node.getTextBlock(), boundary))
-                .map(bounds -> entityCreationService.forceByTextRange(bounds, type, entityType, node))
-                .peek(entity -> entity.setDictionaryEntry(true))
-                .peek(entity -> entity.setDossierDictionaryEntry(isDossierDictionaryEntry))
-                .forEach(entity -> entity.addEngine(Engine.DICTIONARY));
+                .stream()
+                .filter(boundary -> entityCreationService.isValidEntityTextRange(node.getTextBlock(), boundary))
+                .forEach(bounds -> entityCreationService.byTextRangeWithEngine(bounds, type, entityType, node, Set.of(Engine.DICTIONARY)).ifPresent(entity -> {
+                    entity.setDictionaryEntry(true);
+                    entity.setDossierDictionaryEntry(isDossierDictionaryEntry);
+                }));
     }
 
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/DictionaryService.java

@@ -160,13 +160,19 @@ public class DictionaryService {
 
                 List<DictionaryModel> dictionary = typeResponse.stream().map(t -> {
 
-                    Optional<DictionaryModel> oldModel;
+                    Optional<DictionaryModel> optionalOldModel;
                     if (dossierId == null) {
                         var representation = getDossierTemplateDictionary(dossierTemplateId);
-                        oldModel = representation != null ? representation.getDictionary().stream().filter(f -> f.getType().equals(t.getType())).findAny() : Optional.empty();
+                        optionalOldModel = representation != null ? representation.getDictionary()
+                                .stream()
+                                .filter(f -> f.getType().equals(t.getType()))
+                                .findAny() : Optional.empty();
                     } else {
                         var representation = getDossierDictionary(dossierId);
-                        oldModel = representation != null ? representation.getDictionary().stream().filter(f -> f.getType().equals(t.getType())).findAny() : Optional.empty();
+                        optionalOldModel = representation != null ? representation.getDictionary()
+                                .stream()
+                                .filter(f -> f.getType().equals(t.getType()))
+                                .findAny() : Optional.empty();
                     }
 
                     Set<DictionaryEntryModel> entries = new HashSet<>();
@@ -179,19 +185,38 @@ public class DictionaryService {
                     var newFalsePositivesValues = newEntries.getFalsePositives().stream().map(DictionaryEntry::getValue).collect(Collectors.toSet());
                     var newFalseRecommendationsValues = newEntries.getFalseRecommendations().stream().map(DictionaryEntry::getValue).collect(Collectors.toSet());
 
-                    oldModel.ifPresent(oldDictionaryModel -> {
+                    optionalOldModel.ifPresent(oldDictionaryModel -> {
 
                     });
-                    // add old entries from existing DictionaryModel
-                    oldModel.ifPresent(dictionaryModel -> entries.addAll(dictionaryModel.getEntries().stream().filter(f -> !newValues.contains(f.getValue())).toList()));
-                    oldModel.ifPresent(dictionaryModel -> falsePositives.addAll(dictionaryModel.getFalsePositives()
-                            .stream()
-                            .filter(f -> !newFalsePositivesValues.contains(f.getValue()))
-                            .toList()));
-                    oldModel.ifPresent(dictionaryModel -> falseRecommendations.addAll(dictionaryModel.getFalseRecommendations()
-                            .stream()
-                            .filter(f -> !newFalseRecommendationsValues.contains(f.getValue()))
-                            .toList()));
+                    if (optionalOldModel.isPresent()) {
+                        var oldModel = optionalOldModel.get();
+                        if (oldModel.isCaseInsensitive() && !t.isCaseInsensitive()) {
+                            // add old entries from existing DictionaryModel but exclude lower case representation
+                            entries.addAll(oldModel.getEntries().stream().filter(f -> !newValues.stream().map(s -> s.toLowerCase(Locale.ROOT)).toList().contains(f.getValue())).toList());
+                            falsePositives.addAll(oldModel.getFalsePositives()
+                                    .stream()
+                                    .filter(f -> !newFalsePositivesValues.stream().map(s -> s.toLowerCase(Locale.ROOT)).toList().contains(f.getValue()))
+                                    .toList());
+                            falseRecommendations.addAll(oldModel.getFalseRecommendations()
+                                    .stream()
+                                    .filter(f -> !newFalseRecommendationsValues.stream().map(s -> s.toLowerCase(Locale.ROOT)).toList().contains(f.getValue()))
+                                    .toList());
+                        } else if (!oldModel.isCaseInsensitive() && t.isCaseInsensitive()) {
+                            // add old entries from existing DictionaryModel but exclude upper case representation
+                            entries.addAll(oldModel.getEntries().stream().filter(f -> !newValues.contains(f.getValue().toLowerCase(Locale.ROOT))).toList());
+                            falsePositives.addAll(oldModel.getFalsePositives().stream().filter(f -> !newFalsePositivesValues.contains(f.getValue().toLowerCase(Locale.ROOT))).toList());
+                            falseRecommendations.addAll(oldModel.getFalseRecommendations()
+                                    .stream()
+                                    .filter(f -> !newFalseRecommendationsValues.contains(f.getValue().toLowerCase(Locale.ROOT)))
+                                    .toList());
+
+                        } else {
+                            // add old entries from existing DictionaryModel
+                            entries.addAll(oldModel.getEntries().stream().filter(f -> !newValues.contains(f.getValue())).toList());
+                            falsePositives.addAll(oldModel.getFalsePositives().stream().filter(f -> !newFalsePositivesValues.contains(f.getValue())).toList());
+                            falseRecommendations.addAll(oldModel.getFalseRecommendations().stream().filter(f -> !newFalseRecommendationsValues.contains(f.getValue())).toList());
+                        }
+                    }
 
                     // Add Increments
                     entries.addAll(newEntries.getEntries());

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityChangeLogService.java

@@ -1,6 +1,7 @@
 package com.iqser.red.service.redaction.v1.server.service;
 
 import java.time.OffsetDateTime;
+import java.util.Comparator;
 import java.util.List;
 import java.util.Optional;
 import java.util.Set;
@@ -12,19 +13,24 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ChangeType;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualChange;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;
 
 import io.micrometer.core.annotation.Timed;
+import lombok.AccessLevel;
 import lombok.RequiredArgsConstructor;
+import lombok.experimental.FieldDefaults;
 import lombok.extern.slf4j.Slf4j;
 
 @Slf4j
 @Service
 @RequiredArgsConstructor
+@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
 public class EntityChangeLogService {
 
-
     @Timed("redactmanager_computeChanges")
-    public boolean computeChanges(List<EntityLogEntry> previousEntityLogEntries, List<EntityLogEntry> newEntityLogEntries, int analysisNumber) {
+    public boolean computeChanges(List<EntityLogEntry> previousEntityLogEntries, List<EntityLogEntry> newEntityLogEntries, ManualRedactions manualRedactions, int analysisNumber) {
 
         var now = OffsetDateTime.now();
         if (previousEntityLogEntries.isEmpty()) {
@@ -35,40 +41,67 @@ public class EntityChangeLogService {
         boolean hasChanges = false;
 
         for (EntityLogEntry entityLogEntry : newEntityLogEntries) {
-            Optional<EntityLogEntry> optionalPreviousEntity = previousEntityLogEntries.stream().filter(entry -> entry.getId().equals(entityLogEntry.getId())).findAny();
+            Optional<EntityLogEntry> optionalPreviousEntity = previousEntityLogEntries.stream()
+                    .filter(entry -> entry.getId().equals(entityLogEntry.getId()))
+                    .findAny();
             if (optionalPreviousEntity.isEmpty()) {
                 hasChanges = true;
                 entityLogEntry.getChanges().add(new Change(analysisNumber, ChangeType.ADDED, now));
                 continue;
             }
-            var previousEntity = optionalPreviousEntity.get();
+
+            EntityLogEntry previousEntity = optionalPreviousEntity.get();
             entityLogEntry.getChanges().addAll(previousEntity.getChanges());
             if (!previousEntity.getState().equals(entityLogEntry.getState())) {
                 hasChanges = true;
                 ChangeType changeType = calculateChangeType(entityLogEntry.getState(), previousEntity.getState());
-                if (changeType != null) {
-                    entityLogEntry.getChanges().add(new Change(analysisNumber, changeType, now));
-                }
+                entityLogEntry.getChanges().add(new Change(analysisNumber, changeType, now));
             }
         }
-        addRemovedEntriesAsRemoved(previousEntityLogEntries, newEntityLogEntries, analysisNumber, now);
+        addRemovedEntriesAsRemoved(previousEntityLogEntries, newEntityLogEntries, manualRedactions, analysisNumber, now);
         return hasChanges;
     }
 
 
-    private static void addRemovedEntriesAsRemoved(List<EntityLogEntry> previousEntityLogEntries,
-                                                   List<EntityLogEntry> newEntityLogEntries,
-                                                   int analysisNumber,
-                                                   OffsetDateTime now) {
+    private void addRemovedEntriesAsRemoved(List<EntityLogEntry> previousEntityLogEntries,
+                                            List<EntityLogEntry> newEntityLogEntries,
+                                            ManualRedactions manualRedactions,
+                                            int analysisNumber,
+                                            OffsetDateTime now) {
 
-        Set<String> existingIds = newEntityLogEntries.stream().map(EntityLogEntry::getId).collect(Collectors.toSet());
-        List<EntityLogEntry> removedEntries = previousEntityLogEntries.stream().filter(entry -> !existingIds.contains(entry.getId())).toList();
+        Set<String> existingIds = newEntityLogEntries.stream()
+                .map(EntityLogEntry::getId)
+                .collect(Collectors.toSet());
+        List<EntityLogEntry> removedEntries = previousEntityLogEntries.stream()
+                .filter(entry -> !existingIds.contains(entry.getId()))
+                .collect(Collectors.toList());
+        List<EntityLogEntry> removedDossierRedaction = removedEntries.stream()
+                .filter(e -> e.getState() == EntryState.REMOVED && e.getType().equals("dossier_redaction"))
+                .toList();
+        previousEntityLogEntries.removeAll(removedDossierRedaction);
+        removedEntries.removeAll(removedDossierRedaction);
         removedEntries.forEach(entry -> entry.getChanges().add(new Change(analysisNumber, ChangeType.REMOVED, now)));
         removedEntries.forEach(entry -> entry.setState(EntryState.REMOVED));
+        removedEntries.forEach(entry -> addManualChangeForDictionaryRemovals(entry, manualRedactions));
         newEntityLogEntries.addAll(removedEntries);
     }
 
 
+    private void addManualChangeForDictionaryRemovals(EntityLogEntry entry, ManualRedactions manualRedactions) {
+
+        if (manualRedactions == null || manualRedactions.getIdsToRemove().isEmpty()) {
+            return;
+        }
+
+        manualRedactions.getIdsToRemove()
+                .stream()
+                .filter(IdRemoval::isRemoveFromDictionary)//
+                .filter(removed -> removed.getAnnotationId().equals(entry.getId()))//
+                .findFirst()//
+                .ifPresent(idRemove -> entry.getManualChanges().add(ManualChangeFactory.toManualChange(idRemove, false)));
+    }
+
+
     private ChangeType calculateChangeType(EntryState state, EntryState previousState) {
 
         if (state.equals(previousState)) {
@@ -89,5 +122,4 @@ public class EntityChangeLogService {
         return (state.equals(EntryState.REMOVED) || state.equals(EntryState.IGNORED));
     }
 
-
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java

@@ -1,7 +1,5 @@
 package com.iqser.red.service.redaction.v1.server.service;
 
-import static com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryType.IMAGE;
-
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashSet;
@@ -13,6 +11,7 @@ import java.util.stream.Collectors;
 import org.springframework.stereotype.Service;
 
 import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogChanges;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
@@ -23,10 +22,11 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.legalbasis.LegalBasis;
 import com.iqser.red.service.redaction.v1.server.RedactionServiceSettings;
 import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient;
-import com.iqser.red.service.redaction.v1.server.model.ManualEntity;
+import com.iqser.red.service.redaction.v1.server.model.PrecursorEntity;
 import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryVersion;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.IEntity;
+import com.iqser.red.service.redaction.v1.server.model.document.entity.ManualChangeOverwrite;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.PositionOnPage;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
@@ -46,8 +46,6 @@ import lombok.extern.slf4j.Slf4j;
 public class EntityLogCreatorService {
 
     DictionaryService dictionaryService;
-    ManualChangeFactory manualChangeFactory;
-    ImportedRedactionService importedRedactionService;
     RedactionServiceSettings redactionServiceSettings;
     LegalBasisClient legalBasisClient;
     EntityChangeLogService entityChangeLogService;
@@ -62,32 +60,26 @@ public class EntityLogCreatorService {
 
     public EntityLog createInitialEntityLog(AnalyzeRequest analyzeRequest,
                                             Document document,
-                                            List<ManualEntity> notFoundManualEntities,
+                                            List<PrecursorEntity> notFoundEntities,
                                             DictionaryVersion dictionaryVersion,
                                             long rulesVersion) {
 
-        List<EntityLogEntry> entityLogEntries = createEntityLogEntries(document, analyzeRequest.getDossierTemplateId(), notFoundManualEntities);
+        List<EntityLogEntry> entityLogEntries = createEntityLogEntries(document, analyzeRequest, notFoundEntities);
 
         List<LegalBasis> legalBasis = legalBasisClient.getLegalBasisMapping(analyzeRequest.getDossierTemplateId());
-        EntityLog entityLog = new EntityLog(redactionServiceSettings.getAnalysisVersion(),
-                analyzeRequest.getAnalysisNumber(),
-                entityLogEntries,
-                toEntityLogLegalBasis(legalBasis),
-                dictionaryVersion.getDossierTemplateVersion(),
-                dictionaryVersion.getDossierVersion(),
-                rulesVersion,
-                legalBasisClient.getVersion(analyzeRequest.getDossierTemplateId()));
 
-        List<EntityLogEntry> importedRedactionFilteredEntries = importedRedactionService.processImportedEntities(analyzeRequest.getDossierTemplateId(),
-                analyzeRequest.getDossierId(),
-                analyzeRequest.getFileId(),
-                entityLog.getEntityLogEntry(),
-                true);
-
-        entityLog.setEntityLogEntry(importedRedactionFilteredEntries);
         List<EntityLogEntry> previousExistingEntityLogEntries = getPreviousEntityLogEntries(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
-        entityChangeLogService.computeChanges(previousExistingEntityLogEntries, entityLogEntries, analyzeRequest.getAnalysisNumber());
-        return entityLog;
+
+        entityChangeLogService.computeChanges(previousExistingEntityLogEntries, entityLogEntries, analyzeRequest.getManualRedactions(), analyzeRequest.getAnalysisNumber());
+
+        return new EntityLog(redactionServiceSettings.getAnalysisVersion(),
+                             analyzeRequest.getAnalysisNumber(),
+                             entityLogEntries,
+                             toEntityLogLegalBasis(legalBasis),
+                             dictionaryVersion.getDossierTemplateVersion(),
+                             dictionaryVersion.getDossierVersion(),
+                             rulesVersion,
+                             legalBasisClient.getVersion(analyzeRequest.getDossierTemplateId()));
     }
 
 
@@ -101,13 +93,14 @@ public class EntityLogCreatorService {
     }
 
 
-    public EntityLogChanges updateVersionsAndReturnChanges(EntityLog entityLog, DictionaryVersion dictionaryVersion, String dossierTemplateId, boolean hasChanges) {
+    public EntityLogChanges updateVersionsAndReturnChanges(EntityLog entityLog, DictionaryVersion dictionaryVersion, AnalyzeRequest analyzeRequest, boolean hasChanges) {
 
-        List<LegalBasis> legalBasis = legalBasisClient.getLegalBasisMapping(dossierTemplateId);
-        entityLog.setLegalBasisVersion(legalBasisClient.getVersion(dossierTemplateId));
+        List<LegalBasis> legalBasis = legalBasisClient.getLegalBasisMapping(analyzeRequest.getDossierTemplateId());
+        entityLog.setLegalBasisVersion(legalBasisClient.getVersion(analyzeRequest.getDossierTemplateId()));
         entityLog.setLegalBasis(toEntityLogLegalBasis(legalBasis));
         entityLog.setDictionaryVersion(dictionaryVersion.getDossierTemplateVersion());
         entityLog.setDossierDictionaryVersion(dictionaryVersion.getDossierVersion());
+        entityLog.setAnalysisNumber(analyzeRequest.getAnalysisNumber());
 
         return new EntityLogChanges(entityLog, hasChanges);
     }
@@ -115,89 +108,89 @@ public class EntityLogCreatorService {
 
     public EntityLogChanges updatePreviousEntityLog(AnalyzeRequest analyzeRequest,
                                                     Document document,
-                                                    List<ManualEntity> notFoundManualRedactionEntries,
+                                                    List<PrecursorEntity> notFoundEntries,
                                                     EntityLog previousEntityLog,
                                                     Set<Integer> sectionsToReanalyseIds,
                                                     DictionaryVersion dictionaryVersion) {
 
-        List<EntityLogEntry> newEntityLogEntries = createEntityLogEntries(document, analyzeRequest.getDossierTemplateId(), notFoundManualRedactionEntries);
-        Set<String> newEntityIds = newEntityLogEntries.stream().map(EntityLogEntry::getId).collect(Collectors.toSet());
+        List<EntityLogEntry> newEntityLogEntries = createEntityLogEntries(document, analyzeRequest, notFoundEntries).stream()
+                .filter(entry -> entry.getContainingNodeId().isEmpty() || sectionsToReanalyseIds.contains(entry.getContainingNodeId()
+                                                                                                                  .get(0)))
+                .collect(Collectors.toList());
+        Set<String> newEntityIds = newEntityLogEntries.stream()
+                .map(EntityLogEntry::getId)
+                .collect(Collectors.toSet());
 
         List<EntityLogEntry> previousEntriesFromReAnalyzedSections = previousEntityLog.getEntityLogEntry()
                 .stream()
-                .filter(entry -> !entry.getType().equals(ImportedRedactionService.IMPORTED_REDACTION_TYPE) && (newEntityIds.contains(entry.getId()) || entry.getContainingNodeId()
-                        .isEmpty() || sectionsToReanalyseIds.contains(entry.getContainingNodeId().get(0))))
-                .toList();
+                .filter(entry -> (newEntityIds.contains(entry.getId()) || entry.getContainingNodeId().isEmpty() || sectionsToReanalyseIds.contains(entry.getContainingNodeId()
+                                                                                                                                                           .get(0))))
+                .collect(Collectors.toList());
         previousEntityLog.getEntityLogEntry().removeAll(previousEntriesFromReAnalyzedSections);
 
-        boolean hasChanges = entityChangeLogService.computeChanges(previousEntriesFromReAnalyzedSections, newEntityLogEntries, analyzeRequest.getAnalysisNumber());
+        boolean hasChanges = entityChangeLogService.computeChanges(previousEntriesFromReAnalyzedSections,
+                                                                   newEntityLogEntries,
+                                                                   analyzeRequest.getManualRedactions(),
+                                                                   analyzeRequest.getAnalysisNumber());
+        previousEntityLog.getEntityLogEntry().addAll(newEntityLogEntries);
 
-        var newEntityLogWithImportedEntities = importedRedactionService.processImportedEntities(analyzeRequest.getDossierTemplateId(),
-                analyzeRequest.getDossierId(),
-                analyzeRequest.getFileId(),
-                newEntityLogEntries,
-                false);
-
-        previousEntityLog.getEntityLogEntry().addAll(newEntityLogWithImportedEntities);
-
-        return updateVersionsAndReturnChanges(previousEntityLog, dictionaryVersion, analyzeRequest.getDossierTemplateId(), hasChanges);
+        return updateVersionsAndReturnChanges(previousEntityLog, dictionaryVersion, analyzeRequest, hasChanges);
     }
 
 
-    private List<EntityLogEntry> createEntityLogEntries(Document document, String dossierTemplateId, List<ManualEntity> notFoundManualRedactionEntries) {
+    private List<EntityLogEntry> createEntityLogEntries(Document document, AnalyzeRequest analyzeRequest, List<PrecursorEntity> notFoundPrecursorEntries) {
+
+        String dossierTemplateId = analyzeRequest.getDossierTemplateId();
 
         List<EntityLogEntry> entries = new ArrayList<>();
-        Set<String> processedIds = new HashSet<>();
+
         document.getEntities()
                 .stream()
                 .filter(entity -> !entity.getValue().isEmpty())
                 .filter(EntityLogCreatorService::notFalsePositiveOrFalseRecommendation)
                 .filter(entity -> !entity.removed())
-                .forEach(entityNode -> entries.addAll(toEntityLogEntries(entityNode, dossierTemplateId, processedIds)));
-        document.streamAllImages().filter(entity -> !entity.removed()).forEach(imageNode -> entries.add(createEntityLogEntry(imageNode, dossierTemplateId)));
-        notFoundManualRedactionEntries.stream().filter(entity -> !entity.removed()).forEach(manualEntity -> entries.add(createEntityLogEntry(manualEntity, dossierTemplateId)));
+                .forEach(entityNode -> entries.addAll(toEntityLogEntries(entityNode)));
+        document.streamAllImages()
+                .filter(entity -> !entity.removed())
+                .forEach(imageNode -> entries.add(createEntityLogEntry(imageNode, dossierTemplateId)));
+        notFoundPrecursorEntries.stream()
+                .filter(entity -> !entity.removed())
+                .forEach(precursorEntity -> entries.add(createEntityLogEntry(precursorEntity, dossierTemplateId)));
         return entries;
     }
 
 
-    private List<EntityLogEntry> toEntityLogEntries(TextEntity textEntity, String dossierTemplateId, Set<String> processedIds) {
+    private List<EntityLogEntry> toEntityLogEntries(TextEntity textEntity) {
 
-        List<EntityLogEntry> redactionLogEntities = new ArrayList<>();
+        List<EntityLogEntry> entityLogEntries = new ArrayList<>();
 
+        // split entity into multiple entries if it occurs on multiple pages, since FE can't handle multi page entities
         for (PositionOnPage positionOnPage : textEntity.getPositionsOnPagePerPage()) {
 
-            // Duplicates should be removed. They might exist due to table extraction duplicating cells spanning multiple columns/rows.
-            if (processedIds.contains(positionOnPage.getId())) {
-                continue;
-            }
-            processedIds.add(positionOnPage.getId());
-
-            EntityLogEntry entityLogEntries = createEntityLogEntry(textEntity, dossierTemplateId);
-            entityLogEntries.setId(positionOnPage.getId());
+            EntityLogEntry entityLogEntry = createEntityLogEntry(textEntity);
 
             List<Position> rectanglesPerLine = positionOnPage.getRectanglePerLine()
                     .stream()
                     .map(rectangle2D -> new Position(rectangle2D, positionOnPage.getPage().getNumber()))
                     .toList();
 
-            entityLogEntries.setPositions(rectanglesPerLine);
-            redactionLogEntities.add(entityLogEntries);
+            // set the ID from the positions, since it might contain a "-" with the page number if the entity is split across multiple pages
+            entityLogEntry.setId(positionOnPage.getId());
+            entityLogEntry.setPositions(rectanglesPerLine);
+            entityLogEntries.add(entityLogEntry);
         }
 
-        return redactionLogEntities;
+        return entityLogEntries;
     }
 
 
     private EntityLogEntry createEntityLogEntry(Image image, String dossierTemplateId) {
 
-        String imageType = image.getImageType().equals(ImageType.OTHER) ? "image" : image.getImageType().toString().toLowerCase(Locale.ENGLISH);
-        boolean isHint = dictionaryService.isHint(imageType, dossierTemplateId);
+        boolean isHint = dictionaryService.isHint(image.type(), dossierTemplateId);
         return EntityLogEntry.builder()
                 .id(image.getId())
                 .value(image.value())
-                .color(getColor(imageType, dossierTemplateId, image.applied(), isHint))
-                .value(image.value())
-                .type(imageType)
+                .type(image.type())
                 .reason(image.buildReasonWithManualChangeDescriptions())
                 .legalBasis(image.legalBasis())
                 .matchedRule(image.getMatchedRule().getRuleIdentifier().toString())
@@ -205,64 +198,74 @@ public class EntityLogCreatorService {
                 .positions(List.of(new Position(image.getPosition(), image.getPage().getNumber())))
                 .containingNodeId(image.getTreeId())
                 .closestHeadline(image.getHeadline().getTextBlock().getSearchText())
-                .section(image.getManualOverwrite().getSection().orElse(image.getParent().toString()))
+                .section(image.getManualOverwrite().getSection()
+                                 .orElse(image.getParent().toString()))
                 .imageHasTransparency(image.isTransparent())
-                .manualChanges(manualChangeFactory.toManualChangeList(image.getManualOverwrite().getManualChangeLog(), isHint))
+                .manualChanges(ManualChangeFactory.toManualChangeList(image.getManualOverwrite().getManualChangeLog(), isHint))
                 .state(buildEntryState(image))
                 .entryType(isHint ? EntryType.IMAGE_HINT : EntryType.IMAGE)
+                .engines(getEngines(null, image.getManualOverwrite()))
                 .build();
 
     }
 
 
-    private EntityLogEntry createEntityLogEntry(ManualEntity manualEntity, String dossierTemplateId) {
+    private EntityLogEntry createEntityLogEntry(PrecursorEntity precursorEntity, String dossierTemplateId) {
 
-        String type = manualEntity.getManualOverwrite().getType().orElse(manualEntity.getType());
-        boolean isHint = isHint(manualEntity.getEntityType());
+        String type = precursorEntity.getManualOverwrite().getType()
+                .orElse(precursorEntity.getType());
+        boolean isHint = isHint(precursorEntity.getEntityType());
         return EntityLogEntry.builder()
-                .id(manualEntity.getId())
-                .color(getColor(type, dossierTemplateId, manualEntity.applied(), isHint))
-                .reason(manualEntity.buildReasonWithManualChangeDescriptions())
-                .legalBasis(manualEntity.legalBasis())
-                .value(manualEntity.value())
+                .id(precursorEntity.getId())
+                .reason(precursorEntity.buildReasonWithManualChangeDescriptions())
+                .legalBasis(precursorEntity.legalBasis())
+                .value(precursorEntity.value())
                 .type(type)
-                .state(buildEntryState(manualEntity))
-                .entryType(buildEntryType(manualEntity))
-                .section(manualEntity.getManualOverwrite().getSection().orElse(manualEntity.getSection()))
+                .state(buildEntryState(precursorEntity))
+                .entryType(buildEntryType(precursorEntity))
+                .section(precursorEntity.getManualOverwrite().getSection()
+                                 .orElse(precursorEntity.getSection()))
                 .containingNodeId(Collections.emptyList())
                 .closestHeadline("")
-                .matchedRule(manualEntity.getMatchedRule().getRuleIdentifier().toString())
-                .dictionaryEntry(manualEntity.isDictionaryEntry())
-                .dossierDictionaryEntry(manualEntity.isDossierDictionaryEntry())
+                .matchedRule(precursorEntity.getMatchedRule().getRuleIdentifier().toString())
+                .dictionaryEntry(precursorEntity.isDictionaryEntry())
+                .dossierDictionaryEntry(precursorEntity.isDossierDictionaryEntry())
                 .textAfter("")
                 .textBefore("")
                 .startOffset(-1)
                 .endOffset(-1)
-                .positions(manualEntity.getManualOverwrite()
-                        .getPositions()
-                        .orElse(manualEntity.getEntityPosition())
-                        .stream()
-                        .map(entityPosition -> new Position(entityPosition.rectangle2D(), entityPosition.pageNumber()))
-                        .toList())
-                .engines(Collections.emptySet())
+                .positions(precursorEntity.getManualOverwrite().getPositions()
+                                   .orElse(precursorEntity.getEntityPosition())
+                                   .stream()
+                                   .map(entityPosition -> new Position(entityPosition.rectangle2D(), entityPosition.pageNumber()))
+                                   .toList())
+                .engines(getEngines(precursorEntity.getEngines(), precursorEntity.getManualOverwrite()))
+                //imported is no longer used, frontend should check engines
+                //(was .imported(precursorEntity.getEngines() != null && precursorEntity.getEngines().contains(Engine.IMPORTED)))
+                .imported(false)
                 .reference(Collections.emptySet())
-                .manualChanges(manualChangeFactory.toManualChangeList(manualEntity.getManualOverwrite().getManualChangeLog(), isHint))
+                .manualChanges(ManualChangeFactory.toManualChangeList(precursorEntity.getManualOverwrite().getManualChangeLog(), isHint))
                 .build();
     }
 
 
-    private EntityLogEntry createEntityLogEntry(TextEntity entity, String dossierTemplateId) {
+    private EntityLogEntry createEntityLogEntry(TextEntity entity) {
 
         Set<String> referenceIds = new HashSet<>();
-        entity.references().stream().filter(TextEntity::active).forEach(ref -> ref.getPositionsOnPagePerPage().forEach(pos -> referenceIds.add(pos.getId())));
+        entity.references()
+                .stream()
+                .filter(TextEntity::active)
+                .forEach(ref -> ref.getPositionsOnPagePerPage()
+                        .forEach(pos -> referenceIds.add(pos.getId())));
         boolean isHint = isHint(entity.getEntityType());
         return EntityLogEntry.builder()
-                .color(getColor(entity.getType(), dossierTemplateId, entity.applied(), isHint))
                 .reason(entity.buildReasonWithManualChangeDescriptions())
                 .legalBasis(entity.legalBasis())
-                .value(entity.getManualOverwrite().getValue().orElse(entity.getMatchedRule().isWriteValueWithLineBreaks() ? entity.getValueWithLineBreaks() : entity.getValue()))
-                .type(entity.getType())
-                .section(entity.getManualOverwrite().getSection().orElse(entity.getDeepestFullyContainingNode().toString()))
+                .value(entity.getManualOverwrite().getValue()
+                               .orElse(entity.getMatchedRule().isWriteValueWithLineBreaks() ? entity.getValueWithLineBreaks() : entity.getValue()))
+                .type(entity.type())
+                .section(entity.getManualOverwrite().getSection()
+                                 .orElse(entity.getDeepestFullyContainingNode().toString()))
                 .containingNodeId(entity.getDeepestFullyContainingNode().getTreeId())
                 .closestHeadline(entity.getDeepestFullyContainingNode().getHeadline().getTextBlock().getSearchText())
                 .matchedRule(entity.getMatchedRule().getRuleIdentifier().toString())
@@ -272,32 +275,36 @@ public class EntityLogCreatorService {
                 .startOffset(entity.getTextRange().start())
                 .endOffset(entity.getTextRange().end())
                 .dossierDictionaryEntry(entity.isDossierDictionaryEntry())
-                .engines(entity.getEngines() != null ? entity.getEngines() : Collections.emptySet())
+                .engines(getEngines(entity.getEngines(), entity.getManualOverwrite()))
+                //imported is no longer used, frontend should check engines
+                //(was .imported(entity.getEngines() != null && entity.getEngines().contains(Engine.IMPORTED)))
+                .imported(false)
                 .reference(referenceIds)
-                .manualChanges(manualChangeFactory.toManualChangeList(entity.getManualOverwrite().getManualChangeLog(), isHint))
+                .manualChanges(ManualChangeFactory.toManualChangeList(entity.getManualOverwrite().getManualChangeLog(), isHint))
                 .state(buildEntryState(entity))
                 .entryType(buildEntryType(entity))
                 .build();
     }
 
 
+    private Set<Engine> getEngines(Set<Engine> currentEngines, ManualChangeOverwrite manualChangeOverwrite) {
+
+        Set<Engine> engines = currentEngines != null ? new HashSet<>(currentEngines) : new HashSet<>();
+
+        if (manualChangeOverwrite != null && !manualChangeOverwrite.getManualChangeLog().isEmpty()) {
+            engines.add(Engine.MANUAL);
+        }
+        return engines;
+    }
+
+
     private boolean isHint(EntityType entityType) {
 
         return entityType.equals(EntityType.HINT);
     }
 
 
-    private float[] getColor(String type, String dossierTemplateId, boolean isApplied, boolean isHint) {
-
-        if (!isApplied && !isHint) {
-            return dictionaryService.getNotRedactedColor(dossierTemplateId);
-        }
-        return dictionaryService.getColor(type, dossierTemplateId);
-
-    }
-
-
-    private EntryState buildEntryState(IEntity entity) {
+    public static EntryState buildEntryState(IEntity entity) {
 
         if (entity.applied() && entity.active()) {
             return EntryState.APPLIED;
@@ -311,15 +318,17 @@ public class EntityLogCreatorService {
     }
 
 
-    private EntryType buildEntryType(IEntity entity) {
+    public static EntryType buildEntryType(IEntity entity) {
 
         if (entity instanceof TextEntity textEntity) {
             return getEntryType(textEntity.getEntityType());
-        } else if (entity instanceof ManualEntity manualEntity) {
-            if (manualEntity.isRectangle()) {
+        } else if (entity instanceof PrecursorEntity precursorEntity) {
+            if (precursorEntity.isRectangle()) {
                 return EntryType.AREA;
             }
-            return getEntryType(manualEntity.getEntityType());
+            return getEntryType(precursorEntity.getEntityType());
+        } else if (entity instanceof Image) {
+            return EntryType.IMAGE;
         }
         throw new UnsupportedOperationException(String.format("Entity subclass %s is not implemented!", entity.getClass()));
     }
@@ -339,7 +348,9 @@ public class EntityLogCreatorService {
 
     private List<EntityLogLegalBasis> toEntityLogLegalBasis(List<LegalBasis> legalBasis) {
 
-        return legalBasis.stream().map(l -> new EntityLogLegalBasis(l.getName(), l.getDescription(), l.getReason())).collect(Collectors.toList());
+        return legalBasis.stream()
+                .map(l -> new EntityLogLegalBasis(l.getName(), l.getDescription(), l.getReason()))
+                .collect(Collectors.toList());
     }
 
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/ImportedRedactionService.java

@@ -1,272 +0,0 @@
-package com.iqser.red.service.redaction.v1.server.service;
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.List;
-import java.util.stream.Collectors;
-
-import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedaction;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedactions;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Point;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
-import org.springframework.stereotype.Service;
-
-import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryType;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position;
-
-
-import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
-
-import io.micrometer.core.annotation.Timed;
-import lombok.RequiredArgsConstructor;
-
-@Service
-@RequiredArgsConstructor
-public class ImportedRedactionService {
-
-    public static final String IMPORTED_REDACTION_TYPE = "imported_redaction";
-
-    private final DictionaryService dictionaryService;
-    private final RedactionStorageService redactionStorageService;
-
-
-    @Deprecated(forRemoval = true)
-    @Timed("redactmanager_processImportedRedactions")
-    public List<RedactionLogEntry> processImportedRedactions(String dossierTemplateId,
-                                                             String dossierId,
-                                                             String fileId,
-                                                             List<RedactionLogEntry> redactionLogEntries,
-                                                             boolean addImportedRedactions) {
-
-        var importedRedactions = redactionStorageService.getImportedRedactions(dossierId, fileId);
-        if (importedRedactions == null) {
-            return redactionLogEntries;
-        }
-
-        redactionLogEntries.forEach(redactionLogEntry -> addIntersections(redactionLogEntry, importedRedactions));
-
-        if (addImportedRedactions) {
-            return addImportedRedactionsRedactionLogEntries(dossierTemplateId, redactionLogEntries, importedRedactions);
-        }
-
-        return redactionLogEntries;
-    }
-
-
-    @Timed("redactmanager_processImportedRedactions")
-    public List<EntityLogEntry> processImportedEntities(String dossierTemplateId,
-                                                        String dossierId,
-                                                        String fileId,
-                                                        List<EntityLogEntry> entityLogEntries,
-                                                        boolean addImportedRedactions) {
-
-        ImportedRedactions importedRedactions = redactionStorageService.getImportedRedactions(dossierId, fileId);
-        if (importedRedactions == null) {
-            return entityLogEntries;
-        }
-
-        entityLogEntries.forEach(redactionLogEntry -> addIntersections(redactionLogEntry, importedRedactions));
-
-        if (addImportedRedactions) {
-            return addImportedRedactionsEntityLogEntries(dossierTemplateId, entityLogEntries, importedRedactions);
-        }
-
-        return entityLogEntries;
-    }
-
-
-    @Deprecated(forRemoval = true)
-    private List<RedactionLogEntry> addImportedRedactionsRedactionLogEntries(String dossierTemplateId,
-                                                                             List<RedactionLogEntry> redactionLogEntries,
-                                                                             ImportedRedactions importedRedactions) {
-
-        for (var importedRedactionsValues : importedRedactions.getImportedRedactions().values()) {
-            for (var importedRedaction : importedRedactionsValues) {
-                RedactionLogEntry redactionLogEntry = RedactionLogEntry.builder()
-                        .id(importedRedaction.getId())
-                        .type(IMPORTED_REDACTION_TYPE)
-                        .imported(true)
-                        .redacted(true)
-                        .positions(importedRedaction.getPositions().stream().map(this::normalize).collect(Collectors.toList()))
-                        .color(getColor(IMPORTED_REDACTION_TYPE, dossierTemplateId))
-                        .build();
-
-                redactionLogEntries.add(redactionLogEntry);
-            }
-        }
-
-        return redactionLogEntries;
-    }
-
-
-    private List<EntityLogEntry> addImportedRedactionsEntityLogEntries(String dossierTemplateId, List<EntityLogEntry> entityLogEntries, ImportedRedactions importedRedactions) {
-
-        for (var importedRedactionsValues : importedRedactions.getImportedRedactions().values()) {
-            for (var importedRedaction : importedRedactionsValues) {
-                EntityLogEntry redactionLogEntry = EntityLogEntry.builder()
-                        .id(importedRedaction.getId())
-                        .type(IMPORTED_REDACTION_TYPE)
-                        .entryType(EntryType.ENTITY)
-                        .imported(true)
-                        .state(EntryState.APPLIED)
-                        .positions(importedRedaction.getPositions())
-                        .color(getColor(IMPORTED_REDACTION_TYPE, dossierTemplateId))
-                        .containingNodeId(Collections.emptyList())
-                        .value("")
-                        .build();
-
-                entityLogEntries.add(redactionLogEntry);
-            }
-        }
-
-        return entityLogEntries;
-    }
-
-
-    private List<Position> toPositions(List<Rectangle> positions) {
-
-        return positions.stream().map(this::toPosition).toList();
-    }
-
-
-    private Position toPosition(Rectangle rectangle) {
-
-        return new Position(new float[]{rectangle.getTopLeft().getX(), rectangle.getTopLeft().getY(), rectangle.getWidth(), rectangle.getHeight()}, rectangle.getPage());
-    }
-
-
-    @Deprecated(forRemoval = true)
-    private void addIntersections(RedactionLogEntry redactionLogEntry, ImportedRedactions importedRedactions) {
-
-        for (Rectangle rectangle : redactionLogEntry.getPositions()) {
-            var normalizedRectangle = normalize(rectangle);
-            if (importedRedactions.getImportedRedactions().containsKey(rectangle.getPage())) {
-                var importedRedactionsOnPage = importedRedactions.getImportedRedactions().get(rectangle.getPage());
-                for (ImportedRedaction importedRedaction : importedRedactionsOnPage) {
-                    for (Position importedRedactionPosition : importedRedaction.getPositions()) {
-                        if (rectOverlap(normalizedRectangle, normalize(importedRedactionPosition))) {
-                            if (redactionLogEntry.getImportedRedactionIntersections() == null) {
-                                redactionLogEntry.setImportedRedactionIntersections(new HashSet<>());
-                            }
-                            redactionLogEntry.getImportedRedactionIntersections().add(importedRedaction.getId());
-                        }
-                    }
-                }
-            }
-        }
-    }
-
-
-    private void addIntersections(EntityLogEntry entityLogEntry, ImportedRedactions importedRedactions) {
-
-        for (Position rectangle : entityLogEntry.getPositions()) {
-            var normalizedRectangle = normalize(rectangle);
-            if (importedRedactions.getImportedRedactions().containsKey(rectangle.getPageNumber())) {
-                var importedRedactionsOnPage = importedRedactions.getImportedRedactions().get(rectangle.getPageNumber());
-                for (ImportedRedaction importedRedaction : importedRedactionsOnPage) {
-                    for (Position importedRedactionPosition : importedRedaction.getPositions()) {
-                        if (rectOverlap(normalizedRectangle, normalize(importedRedactionPosition))) {
-                            if (entityLogEntry.getImportedRedactionIntersections() == null) {
-                                entityLogEntry.setImportedRedactionIntersections(new HashSet<>());
-                            }
-                            entityLogEntry.getImportedRedactionIntersections().add(importedRedaction.getId());
-                        }
-                    }
-                }
-            }
-        }
-    }
-
-
-    boolean valueInRange(float value, float min, float max) {
-
-        return round(value) >= round(min) && round(value) <= round(max);
-    }
-
-
-    boolean rectOverlap(Rectangle a, Rectangle b) {
-
-        boolean xOverlap = valueInRange(a.getTopLeft().getX(), b.getTopLeft().getX(), b.getTopLeft().getX() + b.getWidth()) || valueInRange(b.getTopLeft().getX(),
-                a.getTopLeft().getX(),
-                a.getTopLeft().getX() + a.getWidth());
-
-        boolean yOverlap = valueInRange(a.getTopLeft().getY(), b.getTopLeft().getY(), b.getTopLeft().getY() + b.getHeight()) || valueInRange(b.getTopLeft().getY(),
-                a.getTopLeft().getY(),
-                a.getTopLeft().getY() + a.getHeight());
-
-        return xOverlap && yOverlap;
-    }
-
-
-    @Deprecated(forRemoval = true)
-    private Rectangle normalize(Rectangle rectangle) {
-
-        Rectangle r = new Rectangle();
-        Point p = new Point();
-
-        if (rectangle.getWidth() < 0) {
-            p.setX(rectangle.getTopLeft().getX() + rectangle.getWidth());
-            r.setWidth(Math.abs(rectangle.getWidth()));
-        } else {
-            p.setX(rectangle.getTopLeft().getX());
-            r.setWidth(rectangle.getWidth());
-        }
-
-        if (rectangle.getHeight() < 0) {
-            p.setY(rectangle.getTopLeft().getY() + rectangle.getHeight());
-            r.setHeight(Math.abs(rectangle.getHeight()));
-        } else {
-            p.setY(rectangle.getTopLeft().getY());
-            r.setHeight(rectangle.getHeight());
-        }
-
-        r.setTopLeft(p);
-        r.setPage(rectangle.getPage());
-        return r;
-    }
-
-
-    private Rectangle normalize(Position position) {
-
-        Rectangle r = new Rectangle();
-        Point p = new Point();
-
-        if (position.getRectangle()[2] < 0) {
-            p.setX(position.getRectangle()[0] + position.getRectangle()[2]);
-            r.setWidth(Math.abs(position.getRectangle()[2]));
-        } else {
-            p.setX(position.getRectangle()[0]);
-            r.setWidth(position.getRectangle()[2]);
-        }
-
-        if (position.getRectangle()[3] < 0) {
-            p.setY(position.getRectangle()[1] + position.getRectangle()[3]);
-            r.setHeight(Math.abs(position.getRectangle()[3]));
-        } else {
-            p.setY(position.getRectangle()[1]);
-            r.setHeight(position.getRectangle()[3]);
-        }
-
-        r.setTopLeft(p);
-        r.setPage(position.getPageNumber());
-        return r;
-    }
-
-
-    private float round(float value) {
-
-        double d = Math.pow(10, 0);
-        return (float) (Math.round(value * d) / d);
-    }
-
-
-    private float[] getColor(String type, String dossierTemplateId) {
-
-        return dictionaryService.getColor(type, dossierTemplateId);
-    }
-
-}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/ManualChangeFactory.java

@@ -2,9 +2,10 @@ package com.iqser.red.service.redaction.v1.server.service;
 
 import java.time.OffsetDateTime;
 import java.util.List;
+import java.util.stream.Collectors;
 
-import org.springframework.stereotype.Service;
-
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualChange;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.BaseAnnotation;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction;
@@ -12,31 +13,34 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualChange;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType;
 
-@Service
+import lombok.experimental.UtilityClass;
+
+@UtilityClass
 public class ManualChangeFactory {
 
     public List<ManualChange> toManualChangeList(List<BaseAnnotation> manualChanges, boolean isHint) {
 
-        return manualChanges.stream().map(baseAnnotation -> toManualChange(baseAnnotation, isHint)).toList();
+        return manualChanges.stream()
+                .map(baseAnnotation -> toManualChange(baseAnnotation, isHint))
+                .collect(Collectors.toList());
     }
 
 
-    private ManualChange toManualChange(BaseAnnotation baseAnnotation, boolean isHint) {
+    public ManualChange toManualChange(BaseAnnotation baseAnnotation, boolean isHint) {
 
         ManualChange manualChange = ManualChange.from(baseAnnotation);
         if (baseAnnotation instanceof ManualRecategorization imageRecategorization) {
             manualChange.withManualRedactionType(ManualRedactionType.RECATEGORIZE).withChange("type", imageRecategorization.getType());
         } else if (baseAnnotation instanceof IdRemoval manualRemoval) {
-            manualChange.withManualRedactionType(manualRemoval.isRemoveFromDictionary() ? ManualRedactionType.REMOVE_FROM_DICTIONARY : ManualRedactionType.REMOVE_LOCALLY);
-        } else if (baseAnnotation instanceof ManualForceRedaction) {
-            manualChange.withManualRedactionType(isHint ? ManualRedactionType.FORCE_HINT : ManualRedactionType.FORCE_REDACT);
+            manualChange.withManualRedactionType(manualRemoval.isRemoveFromDictionary() ? ManualRedactionType.REMOVE_FROM_DICTIONARY : ManualRedactionType.REMOVE);
+        } else if (baseAnnotation instanceof ManualForceRedaction manualForceRedaction) {
+            manualChange.withManualRedactionType(ManualRedactionType.FORCE).withChange("legalBasis", manualForceRedaction.getLegalBasis());
         } else if (baseAnnotation instanceof ManualResizeRedaction manualResizeRedact) {
-            manualChange.withManualRedactionType(ManualRedactionType.RESIZE).withChange("value", manualResizeRedact.getValue());
+            manualChange.withManualRedactionType(manualResizeRedact.getUpdateDictionary() ? ManualRedactionType.RESIZE_IN_DICTIONARY : ManualRedactionType.RESIZE)
+                    .withChange("value", manualResizeRedact.getValue());
         } else if (baseAnnotation instanceof ManualRedactionEntry manualRedactionEntry) {
-            manualChange.withManualRedactionType(manualRedactionEntry.isAddToDictionary() ? ManualRedactionType.ADD_TO_DICTIONARY : ManualRedactionType.ADD_LOCALLY)
+            manualChange.withManualRedactionType(manualRedactionEntry.isAddToDictionary() ? ManualRedactionType.ADD_TO_DICTIONARY : ManualRedactionType.ADD)
                     .withChange("value", manualRedactionEntry.getValue());
         } else if (baseAnnotation instanceof ManualLegalBasisChange manualLegalBasisChange) {
             manualChange.withManualRedactionType(ManualRedactionType.LEGAL_BASIS_CHANGE)

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/ManualChangesApplicationService.java

@@ -1,54 +1,70 @@
 package com.iqser.red.service.redaction.v1.server.service;
 
 import java.awt.geom.Rectangle2D;
+import java.util.ArrayList;
+import java.util.Collection;
 import java.util.HashSet;
-import java.util.LinkedList;
 import java.util.List;
+import java.util.Map;
 import java.util.NoSuchElementException;
+import java.util.Optional;
+import java.util.Set;
 import java.util.stream.Collectors;
 
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle;
+import org.springframework.stereotype.Service;
+
+import com.google.common.collect.Sets;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
-import com.iqser.red.service.redaction.v1.server.model.ClosestEntity;
-import com.iqser.red.service.redaction.v1.server.model.dictionary.SearchImplementation;
-import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
-import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
+import com.iqser.red.service.redaction.v1.server.model.PrecursorEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.IEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.PositionOnPage;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Image;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.ImageType;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.Page;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
-import com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService;
+import com.iqser.red.service.redaction.v1.server.service.document.EntityFindingUtility;
 import com.iqser.red.service.redaction.v1.server.utils.RectangleTransformations;
 
 import lombok.AccessLevel;
 import lombok.RequiredArgsConstructor;
 import lombok.experimental.FieldDefaults;
 
+@Service
 @RequiredArgsConstructor
 @FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
 public class ManualChangesApplicationService {
 
-    EntityCreationService entityCreationService;
+    static double MATCH_THRESHOLD = 10;
 
-    public void recategorize(IEntity IEntityToBeReCategorized, ManualRecategorization manualRecategorization) {
+    EntityFindingUtility entityFindingUtility;
 
-        if (IEntityToBeReCategorized instanceof Image image) {
+
+    @Deprecated
+    public void recategorize(IEntity entityToBeReCategorized, ManualRecategorization manualRecategorization) {
+
+        entityToBeReCategorized.getMatchedRuleList().clear();
+        entityToBeReCategorized.getManualOverwrite().addChange(manualRecategorization);
+
+        if (entityToBeReCategorized instanceof Image image) {
             image.setImageType(ImageType.fromString(manualRecategorization.getType()));
             return;
         }
-        // need to create a new entity and copy over all values, since type is part of the primary key for entities and should never be changed!
-        if (IEntityToBeReCategorized instanceof TextEntity textEntity) {
-            TextEntity recategorizedEntity = entityCreationService.copyEntityWithoutRules(textEntity, manualRecategorization.getType(), textEntity.getEntityType(), textEntity.getDeepestFullyContainingNode());
-            recategorizedEntity.setPositionsOnPagePerPage(textEntity.getPositionsOnPagePerPage());
-            recategorizedEntity.getManualOverwrite().addChange(manualRecategorization);
-            textEntity.remove("FINAL.0.0", "removed by manual recategorization");
+
+        if (entityToBeReCategorized instanceof TextEntity textEntity) {
+            textEntity.setType(manualRecategorization.getType());
         }
     }
 
 
+    public void resize(TextEntity entityToBeResized, ManualResizeRedaction manualResizeRedaction) {
+
+        resizeEntityAndReinsert(entityToBeResized, manualResizeRedaction);
+    }
+
+
+    @Deprecated
     public void resizeEntityAndReinsert(TextEntity entityToBeResized, ManualResizeRedaction manualResizeRedaction) {
 
         PositionOnPage positionOnPageToBeResized = entityToBeResized.getPositionsOnPagePerPage()
@@ -57,93 +73,60 @@ public class ManualChangesApplicationService {
                 .findFirst()
                 .orElseThrow(() -> new NoSuchElementException("No redaction position with matching annotation id found!"));
 
-        positionOnPageToBeResized.setRectanglePerLine(manualResizeRedaction.getPositions().stream().map(ManualChangesApplicationService::toRectangle2D).collect(Collectors.toList()));
+        positionOnPageToBeResized.setRectanglePerLine(manualResizeRedaction.getPositions()
+                .stream()
+                .map(ManualChangesApplicationService::toRectangle2D)
+                .collect(Collectors.toList()));
+
+        entityToBeResized.getManualOverwrite().addChange(manualResizeRedaction);
 
-        String value = manualResizeRedaction.getValue();
-        int newStartOffset = -1;
         SemanticNode node = entityToBeResized.getDeepestFullyContainingNode();
-        ClosestEntity closestEntity = ClosestEntity.builder().distance(100).textRange(null).build();
-
+        PrecursorEntity searchEntity = PrecursorEntity.fromManualResizeRedaction(manualResizeRedaction);
         // Loop through nodes starting from the deepest fully containing node all the way to the document node
         while (node != null) {
-            if (node.containsString(value)) {
-                SearchImplementation searchImplementation = new SearchImplementation(value, false);
-                List<TextRange> textRanges = searchImplementation.getBoundaries(node.getTextBlock(), node.getTextRange());
 
-                for (TextRange textRange : textRanges) {
-                    SemanticNode finalNode = node;
+            Map<String, List<TextEntity>> possibleEntities = entityFindingUtility.findAllPossibleEntitiesAndGroupByValue(node, List.of(searchEntity));
+            Optional<TextEntity> closestEntity = entityFindingUtility.findClosestEntityAndReturnEmptyIfNotFound(searchEntity, possibleEntities, MATCH_THRESHOLD);
 
-                    List<TextEntity> tempEntities = searchImplementation.getBoundaries(node.getTextBlock(), textRange)
-                            .stream()
-                            .map(boundary -> entityCreationService.forceByTextRange(boundary, "temp", EntityType.ENTITY, finalNode))
-                            .collect(Collectors.toList());
-
-                    // If a value appears multiple times in a section after resizing, we need to make sure we select the surrounding text for the correct one.
-                    determineCorrectEntity(manualResizeRedaction, textRange, tempEntities, closestEntity);
-
-                    // Remove all temp entities from the graph
-                    tempEntities.forEach(TextEntity::removeFromGraph);
-                }
-                break;
+            if (closestEntity.isPresent()) {
+                copyValuesFromClosestEntity(entityToBeResized, manualResizeRedaction, closestEntity.get());
+                possibleEntities.values().stream().flatMap(Collection::stream).forEach(TextEntity::removeFromGraph);
+                return;
             }
 
-            // If the current node is the document node then it does not have a parent, meaning we could not find the value anywhere.
+            possibleEntities.values().stream().flatMap(Collection::stream).forEach(TextEntity::removeFromGraph);
+
             if (node.hasParent()) {
                 node = node.getParent();
             } else {
                 break;
             }
         }
-
-        if (closestEntity.getTextRange() != null) {
-            newStartOffset = closestEntity.getTextRange().start();
-        }
-
-        // need to reinsert the entity, due to the boundary having changed.
-        if (newStartOffset > -1) {
-            removeAndUpdateAndReInsertEntity(entityToBeResized, manualResizeRedaction, newStartOffset);
-        }
-        entityToBeResized.getManualOverwrite().addChange(manualResizeRedaction);
     }
 
 
-    public static void determineCorrectEntity(ManualResizeRedaction manualResizeRedaction,
-                                        TextRange textRange,
-                                        List<TextEntity> tempEntities,
-                                        ClosestEntity closestEntity) {
+    private static void copyValuesFromClosestEntity(TextEntity entityToBeResized, ManualResizeRedaction manualResizeRedaction, TextEntity closestEntity) {
 
-        double currentDistance = calculateClosest(manualResizeRedaction.getPositions().get(0), tempEntities.get(0).getPositionsOnPagePerPage());
-        if (currentDistance < closestEntity.getDistance()) {
-            closestEntity.setDistance(currentDistance);
-            closestEntity.setTextRange(textRange);
-        }
-    }
+        Set<SemanticNode> currentIntersectingNodes = new HashSet<>(entityToBeResized.getIntersectingNodes());
+        Set<SemanticNode> newIntersectingNodes = new HashSet<>(closestEntity.getIntersectingNodes());
 
+        Sets.difference(currentIntersectingNodes, newIntersectingNodes).forEach(removedNode -> removedNode.getEntities().remove(entityToBeResized));
+        Sets.difference(newIntersectingNodes, currentIntersectingNodes).forEach(addedNode -> addedNode.getEntities().add(entityToBeResized));
 
-    private static double calculateClosest(Rectangle position, List<PositionOnPage> positionOnPages) {
+        Set<Page> currentIntersectingPages = new HashSet<>(entityToBeResized.getPages());
+        Set<Page> newIntersectingPages = new HashSet<>(closestEntity.getPages());
 
-        Rectangle2D rectangle2D = positionOnPages.get(0).getRectanglePerLine().get(0);
-        double difference = 0;
-        difference +=  position.getTopLeftX() > rectangle2D.getX() ? (position.getTopLeftX() - rectangle2D.getX()) : (rectangle2D.getX() - position.getTopLeftX());
-        difference +=  position.getTopLeftY() > rectangle2D.getY() ? (position.getTopLeftY() - rectangle2D.getY()) : (rectangle2D.getY() - position.getTopLeftY());
-        difference +=  position.getWidth() > rectangle2D.getWidth() ? (position.getWidth() - rectangle2D.getWidth()) : (rectangle2D.getWidth() - position.getWidth());
-        difference +=  position.getHeight() > rectangle2D.getHeight() ? (position.getHeight() - rectangle2D.getHeight()) : (rectangle2D.getHeight() - position.getHeight());
-        return difference;
-    }
+        Sets.difference(currentIntersectingPages, newIntersectingPages).forEach(removedPage -> removedPage.getEntities().remove(entityToBeResized));
+        Sets.difference(newIntersectingPages, currentIntersectingPages).forEach(addedPage -> addedPage.getEntities().add(entityToBeResized));
 
-
-    private void removeAndUpdateAndReInsertEntity(TextEntity entityToBeResized, ManualResizeRedaction manualResizeRedaction, int newStartOffset) {
-
-        SemanticNode nodeToInsertInto = entityToBeResized.getDeepestFullyContainingNode().getDocumentTree().getRoot().getNode();
-        entityToBeResized.getIntersectingNodes().forEach(node -> node.getEntities().remove(this));
-        entityToBeResized.getPages().forEach(page -> page.getEntities().remove(this));
-        entityToBeResized.setIntersectingNodes(new LinkedList<>());
-        entityToBeResized.setDeepestFullyContainingNode(null);
-        entityToBeResized.setPages(new HashSet<>());
-        entityToBeResized.getTextRange().setStart(newStartOffset);
-        entityToBeResized.getTextRange().setEnd(newStartOffset + manualResizeRedaction.getValue().length());
-
-        entityCreationService.addEntityToGraph(entityToBeResized, nodeToInsertInto);
+        entityToBeResized.setDeepestFullyContainingNode(closestEntity.getDeepestFullyContainingNode());
+        entityToBeResized.setIntersectingNodes(new ArrayList<>(newIntersectingNodes));
+        entityToBeResized.setTextRange(closestEntity.getTextRange());
+        entityToBeResized.setTextAfter(closestEntity.getTextAfter());
+        entityToBeResized.setTextBefore(closestEntity.getTextBefore());
+        entityToBeResized.setDuplicateTextRanges(new ArrayList<>(closestEntity.getDuplicateTextRanges()));
+        entityToBeResized.setValue(closestEntity.getValue());
+        entityToBeResized.setPages(newIntersectingPages);
     }
 
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/NotFoundImportedEntitiesService.java

@@ -0,0 +1,102 @@
+package com.iqser.red.service.redaction.v1.server.service;
+
+import java.time.OffsetDateTime;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Change;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ChangeType;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position;
+import com.iqser.red.service.redaction.v1.server.model.PrecursorEntity;
+import com.iqser.red.service.redaction.v1.server.model.RectangleWithPage;
+import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
+
+import org.springframework.stereotype.Service;
+
+import io.micrometer.core.annotation.Timed;
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+
+@Service
+@Slf4j
+@RequiredArgsConstructor
+public class NotFoundImportedEntitiesService {
+
+    public static final String IMPORTED_REDACTION_TYPE = "imported_redaction";
+
+
+    @Timed("redactmanager_processEntityLog")
+    public void processEntityLog(EntityLog entityLog, AnalyzeRequest analyzeRequest, List<PrecursorEntity> notFoundEntities) {
+        // recreate imported redactions with manual changes
+        if (notFoundEntities == null || notFoundEntities.isEmpty()) {
+            log.debug("checkForImportedRedactions - no not found imported entities");
+            return;
+        }
+
+        // remove all  intersections and update imported entities
+        calculateIntersectionsForNotFound(notFoundEntities, entityLog, analyzeRequest.getAnalysisNumber());
+    }
+
+
+    private void calculateIntersectionsForNotFound(List<PrecursorEntity> notFoundEntities, EntityLog entityLog, int analysisNumber) {
+
+        if (!notFoundEntities.isEmpty()) {
+            // imported redactions present, intersections must be added with merged imported redactions
+            Map<Integer, List<PrecursorEntity>> importedRedactionsMap = mapImportedRedactionsOnPage(notFoundEntities);
+            entityLog.getEntityLogEntry().stream().filter(entry -> !entry.getEngines().contains(Engine.IMPORTED)).forEach(redactionLogEntry -> {
+                redactionLogEntry.setImportedRedactionIntersections(new HashSet<>());
+                addIntersections(redactionLogEntry, importedRedactionsMap, analysisNumber);
+            });
+        }
+    }
+
+
+    public Map<Integer, List<PrecursorEntity>> mapImportedRedactionsOnPage(List<PrecursorEntity> importedEntities) {
+
+        Map<Integer, List<PrecursorEntity>> importedRedactionsMap = new HashMap<>();
+        Set<Integer> pageNumbers = importedEntities.stream()
+                .map(PrecursorEntity::getEntityPosition)
+                .flatMap(Collection::stream)
+                .map(RectangleWithPage::pageNumber)
+                .collect(Collectors.toSet());
+        pageNumbers.forEach(pageNumber -> importedRedactionsMap.put(pageNumber,
+                importedEntities.stream().filter(i -> pageNumber == i.getEntityPosition().get(0).pageNumber()).collect(Collectors.toList())));
+        return importedRedactionsMap;
+    }
+
+
+    public void addIntersections(EntityLogEntry entityLogEntry, Map<Integer, List<PrecursorEntity>> importedEntitiesMap, int analysisNumber) {
+
+        for (Position rectangle : entityLogEntry.getPositions()) {
+            if (importedEntitiesMap.containsKey(rectangle.getPageNumber())) {
+                var importedEntitiesOnPage = importedEntitiesMap.get(rectangle.getPageNumber());
+                for (PrecursorEntity precursorEntity : importedEntitiesOnPage) {
+                    for (RectangleWithPage rectangleWithPage : precursorEntity.getEntityPosition()) {
+                        if (rectangle.toRectangle2D().intersects(rectangleWithPage.rectangle2D())) {
+                            if (entityLogEntry.getImportedRedactionIntersections() == null) {
+                                entityLogEntry.setImportedRedactionIntersections(new HashSet<>());
+                            }
+                            entityLogEntry.getImportedRedactionIntersections().add(precursorEntity.getId());
+
+                            if (entityLogEntry.getState() != EntryState.REMOVED) {
+                                entityLogEntry.setState(EntryState.REMOVED);
+                                entityLogEntry.getChanges().add(new Change(analysisNumber, ChangeType.REMOVED, OffsetDateTime.now()));
+                            }
+                        }
+                    }
+                }
+            }
+        }
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RedactionChangeLogService.java

@@ -1,125 +0,0 @@
-package com.iqser.red.service.redaction.v1.server.service;
-
-import java.time.OffsetDateTime;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.stream.Collectors;
-
-import org.springframework.stereotype.Service;
-
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Change;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ChangeType;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ManualRedactionType;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogChanges;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
-import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
-
-import io.micrometer.core.annotation.Timed;
-import lombok.RequiredArgsConstructor;
-import lombok.extern.slf4j.Slf4j;
-
-@Deprecated(forRemoval = true)
-@Slf4j
-@Service
-@RequiredArgsConstructor
-public class RedactionChangeLogService {
-
-    private final RedactionStorageService redactionStorageService;
-
-
-    @Timed("redactmanager_computeChanges")
-    public RedactionLogChanges computeChanges(RedactionLog previousRedactionLog, RedactionLog currentRedactionLog, int analysisNumber) {
-
-        long start = System.currentTimeMillis();
-
-        if (previousRedactionLog == null) {
-            currentRedactionLog.getRedactionLogEntry().forEach(entry -> {
-                entry.getChanges().add(new Change(analysisNumber, ChangeType.ADDED, OffsetDateTime.now()));
-            });
-            return new RedactionLogChanges(currentRedactionLog, false);
-        }
-
-        List<RedactionLogEntry> previouslyExistingEntries = previousRedactionLog.getRedactionLogEntry().stream().filter(entry -> !entry.lastChangeIsRemoved()).toList();
-
-        Map<String, RedactionLogEntry> addedEntryIds = getEntriesThatExistInCurrentButNotInPreviousRedactionLog(currentRedactionLog, previouslyExistingEntries);
-        Set<String> removedIds = getEntryIdsThatExistInPreviousButNotInCurrentRedactionLog(currentRedactionLog, previouslyExistingEntries);
-
-        List<RedactionLogEntry> newRedactionLogEntries = previousRedactionLog.getRedactionLogEntry();
-
-        List<RedactionLogEntry> toRemove = new ArrayList<>();
-        newRedactionLogEntries.forEach(entry -> {
-            if (removedIds.contains(entry.getId()) && addedEntryIds.containsKey(entry.getId())) {
-                List<Change> changes = entry.getChanges();
-                changes.add(new Change(analysisNumber, ChangeType.CHANGED, OffsetDateTime.now()));
-                var newEntry = addedEntryIds.get(entry.getId());
-                newEntry.setChanges(changes);
-                addedEntryIds.put(entry.getId(), newEntry);
-                toRemove.add(entry);
-            } else if (removedIds.contains(entry.getId())) {
-                entry.getChanges().add(new Change(analysisNumber, ChangeType.REMOVED, OffsetDateTime.now()));
-            } else if (addedEntryIds.containsKey(entry.getId())) {
-                List<Change> changes = entry.getChanges();
-                changes.add(new Change(analysisNumber, ChangeType.ADDED, OffsetDateTime.now()));
-                var newEntry = addedEntryIds.get(entry.getId());
-                newEntry.setChanges(changes);
-                addedEntryIds.put(entry.getId(), newEntry);
-                toRemove.add(entry);
-            }
-        });
-
-        newRedactionLogEntries.removeAll(toRemove);
-
-        addedEntryIds.forEach((k, v) -> {
-            if (v.getChanges().isEmpty()) {
-                v.getChanges().add(new Change(analysisNumber, ChangeType.ADDED, OffsetDateTime.now()));
-            }
-            newRedactionLogEntries.add(v);
-        });
-
-        currentRedactionLog.setRedactionLogEntry(newRedactionLogEntries);
-
-        log.debug("Change computation took: {}", System.currentTimeMillis() - start);
-        return new RedactionLogChanges(currentRedactionLog, !addedEntryIds.isEmpty() || !removedIds.isEmpty());
-    }
-
-
-    private static Set<String> getEntryIdsThatExistInPreviousButNotInCurrentRedactionLog(RedactionLog currentRedactionLog, List<RedactionLogEntry> previouslyExistingEntries) {
-
-        Set<RedactionLogEntry> removed = new HashSet<>(previouslyExistingEntries);
-        currentRedactionLog.getRedactionLogEntry().forEach(removed::remove);
-
-        Set<String> removedIds = removed.stream().map(RedactionLogEntry::getId).collect(Collectors.toSet());
-        return removedIds;
-    }
-
-
-    private static Map<String, RedactionLogEntry> getEntriesThatExistInCurrentButNotInPreviousRedactionLog(RedactionLog currentRedactionLog,
-                                                                                                           List<RedactionLogEntry> previouslyExistingEntries) {
-
-        Set<RedactionLogEntry> currentExistingEntries = currentRedactionLog.getRedactionLogEntry()
-                .stream()
-                .filter(entry -> (entry.getChanges().isEmpty() || !entry.lastChangeIsRemoved()) && !isLastManualChangeRemove(entry))
-                .collect(Collectors.toSet());
-
-        previouslyExistingEntries.forEach(currentExistingEntries::remove);
-        Map<String, RedactionLogEntry> addedIds = new HashMap<>();
-        currentExistingEntries.forEach(entry -> {
-            addedIds.put(entry.getId(), entry);
-        });
-        return addedIds;
-    }
-
-
-    private static boolean isLastManualChangeRemove(RedactionLogEntry redactionLogEntry){
-        if(redactionLogEntry.getManualChanges() == null || redactionLogEntry.getManualChanges().isEmpty()){
-            return false;
-        }
-        return redactionLogEntry.getManualChanges().get(redactionLogEntry.getManualChanges().size() -1).getManualRedactionType() == ManualRedactionType.REMOVE_LOCALLY;
-    }
-
-}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RedactionLogCreatorService.java

@@ -1,259 +0,0 @@
-package com.iqser.red.service.redaction.v1.server.service;
-
-import java.awt.geom.Rectangle2D;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Locale;
-import java.util.Objects;
-import java.util.Set;
-import java.util.stream.Collectors;
-
-import org.springframework.beans.BeanUtils;
-import org.springframework.stereotype.Service;
-
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ManualChange;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ManualRedactionType;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Point;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
-import com.iqser.red.service.redaction.v1.server.model.ManualEntity;
-import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
-import com.iqser.red.service.redaction.v1.server.model.document.entity.IEntity;
-import com.iqser.red.service.redaction.v1.server.model.document.entity.ManualChangeOverwrite;
-import com.iqser.red.service.redaction.v1.server.model.document.entity.PositionOnPage;
-import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
-import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
-import com.iqser.red.service.redaction.v1.server.model.document.nodes.Image;
-import com.iqser.red.service.redaction.v1.server.model.document.nodes.ImageType;
-
-import lombok.RequiredArgsConstructor;
-import lombok.extern.slf4j.Slf4j;
-
-@Deprecated(forRemoval = true)
-@Service
-@Slf4j
-@RequiredArgsConstructor
-public class RedactionLogCreatorService {
-
-    private final DictionaryService dictionaryService;
-    private final ManualChangeFactory manualChangeFactory;
-
-
-    public List<RedactionLogEntry> createRedactionLog(Document document, String dossierTemplateId, List<ManualEntity> notFoundManualRedactionEntries) {
-
-        List<RedactionLogEntry> entries = new ArrayList<>();
-        Set<String> processIds = new HashSet<>();
-        document.getEntities()
-                .stream()
-                .filter(entity -> !entity.getValue().isEmpty())
-                .filter(RedactionLogCreatorService::notFalsePositiveOrFalseRecommendation)
-                .filter(IEntity::active)
-                .forEach(entityNode -> entries.addAll(toRedactionLogEntries(entityNode, dossierTemplateId, processIds)));
-        document.streamAllImages().filter(image -> !image.removed()).forEach(imageNode -> entries.add(createRedactionLogEntry(imageNode, dossierTemplateId)));
-        notFoundManualRedactionEntries.forEach(entityIdentifier -> entries.add(createRedactionLogEntry(entityIdentifier, dossierTemplateId)));
-        return entries;
-    }
-
-
-    private static boolean notFalsePositiveOrFalseRecommendation(TextEntity textEntity) {
-
-        return !(textEntity.getEntityType() == EntityType.FALSE_POSITIVE || textEntity.getEntityType() == EntityType.FALSE_RECOMMENDATION);
-    }
-
-
-    private List<RedactionLogEntry> toRedactionLogEntries(TextEntity textEntity, String dossierTemplateId, Set<String> processedIds) {
-
-        List<RedactionLogEntry> redactionLogEntities = new ArrayList<>();
-
-        for (PositionOnPage positionOnPage : textEntity.getPositionsOnPagePerPage()) {
-
-            // Duplicates should be removed. They might exist due to table extraction duplicating cells spanning multiple columns/rows.
-            if (processedIds.contains(positionOnPage.getId())) {
-                continue;
-            }
-            processedIds.add(positionOnPage.getId());
-
-            RedactionLogEntry redactionLogEntry = createRedactionLogEntry(textEntity, dossierTemplateId);
-            redactionLogEntry.setId(positionOnPage.getId());
-
-            List<Rectangle> rectanglesPerLine = positionOnPage.getRectanglePerLine()
-                    .stream()
-                    .map(rectangle2D -> toRedactionLogRectangle(rectangle2D, positionOnPage.getPage().getNumber()))
-                    .toList();
-
-            redactionLogEntry.setPositions(rectanglesPerLine);
-            redactionLogEntities.add(redactionLogEntry);
-        }
-
-        return redactionLogEntities;
-    }
-
-
-    private RedactionLogEntry createRedactionLogEntry(TextEntity entity, String dossierTemplateId) {
-
-        Set<String> referenceIds = new HashSet<>();
-        entity.references().stream().filter(TextEntity::active).forEach(ref -> ref.getPositionsOnPagePerPage().forEach(pos -> referenceIds.add(pos.getId())));
-        int sectionNumber = entity.getDeepestFullyContainingNode().getTreeId().isEmpty() ? 0 : entity.getDeepestFullyContainingNode().getTreeId().get(0);
-        boolean isHint = isHint(entity.getEntityType());
-        return RedactionLogEntry.builder()
-                .color(getColor(entity.getType(), dossierTemplateId, entity.applied(), isHint))
-                .reason(entity.buildReasonWithManualChangeDescriptions())
-                .legalBasis(entity.legalBasis())
-                .value(entity.getManualOverwrite().getValue().orElse(entity.getMatchedRule().isWriteValueWithLineBreaks() ? entity.getValueWithLineBreaks() : entity.getValue()))
-                .type(entity.getType())
-                .redacted(entity.applied())
-                .isHint(isHint)
-                .isRecommendation(entity.getEntityType().equals(EntityType.RECOMMENDATION))
-                .isFalsePositive(entity.getEntityType().equals(EntityType.FALSE_POSITIVE) || entity.getEntityType().equals(EntityType.FALSE_RECOMMENDATION))
-                .section(entity.getManualOverwrite().getSection().orElse(entity.getDeepestFullyContainingNode().toString()))
-                .sectionNumber(sectionNumber)
-                .matchedRule(entity.getMatchedRule().getRuleIdentifier().toString())
-                .isDictionaryEntry(entity.isDictionaryEntry())
-                .textAfter(entity.getTextAfter())
-                .textBefore(entity.getTextBefore())
-                .startOffset(entity.getTextRange().start())
-                .endOffset(entity.getTextRange().end())
-                .isDossierDictionaryEntry(entity.isDossierDictionaryEntry())
-                .engines(getEngines(entity))
-                .reference(referenceIds)
-                .manualChanges(mapManualChanges(entity.getManualOverwrite(), isHint))
-                .build();
-    }
-
-
-    private Set<Engine> getEngines(TextEntity entity) {
-
-        return entity.getEngines() != null ? mapToEngines(entity.getEngines()) : Collections.emptySet();
-    }
-
-
-    private Set<Engine> mapToEngines(Set<com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine> engines) {
-
-        return engines.stream().map(engine -> switch (engine) {
-            case NER -> Engine.NER;
-            case RULE -> Engine.RULE;
-            case DICTIONARY -> Engine.DICTIONARY;
-            default -> null;
-        }).filter(Objects::nonNull).collect(Collectors.toSet());
-    }
-
-
-    private boolean isHint(EntityType entityType) {
-
-        return entityType.equals(EntityType.HINT);
-    }
-
-
-    private List<ManualChange> mapManualChanges(ManualChangeOverwrite manualEntity, boolean isHint) {
-
-        return manualChangeFactory.toManualChangeList(manualEntity.getManualChangeLog(), isHint).stream().map(this::mapManualChange).toList();
-    }
-
-
-    private ManualChange mapManualChange(com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualChange manualChange) {
-
-        ManualChange manualChange1 = new ManualChange();
-        BeanUtils.copyProperties(manualChange, manualChange1);
-        manualChange1.setManualRedactionType(switch (manualChange.getManualRedactionType()) {
-            case ADD_LOCALLY -> ManualRedactionType.ADD_LOCALLY;
-            case ADD_TO_DICTIONARY -> ManualRedactionType.ADD_TO_DICTIONARY;
-            case REMOVE_LOCALLY -> ManualRedactionType.REMOVE_LOCALLY;
-            case REMOVE_FROM_DICTIONARY -> ManualRedactionType.REMOVE_FROM_DICTIONARY;
-            case FORCE_REDACT -> ManualRedactionType.FORCE_REDACT;
-            case FORCE_HINT -> ManualRedactionType.FORCE_HINT;
-            case RECATEGORIZE -> ManualRedactionType.RECATEGORIZE;
-            case LEGAL_BASIS_CHANGE -> ManualRedactionType.LEGAL_BASIS_CHANGE;
-            case RESIZE -> ManualRedactionType.RESIZE;
-        });
-        manualChange1.setProcessedDate(manualChange.getProcessedDate());
-        manualChange1.setRequestedDate(manualChange.getRequestedDate());
-        manualChange1.setPropertyChanges(manualChange.getPropertyChanges());
-        manualChange1.setAnnotationStatus(AnnotationStatus.APPROVED);
-        return manualChange1;
-    }
-
-
-    private float[] getColor(String type, String dossierTemplateId, boolean isRedaction, boolean isHint) {
-
-        if (!isRedaction && isHint) {
-            return dictionaryService.getNotRedactedColor(dossierTemplateId);
-        }
-        return dictionaryService.getColor(type, dossierTemplateId);
-    }
-
-
-    public RedactionLogEntry createRedactionLogEntry(Image image, String dossierTemplateId) {
-
-        String imageType = image.getImageType().equals(ImageType.OTHER) ? "image" : image.getImageType().toString().toLowerCase(Locale.ENGLISH);
-        boolean isHint = dictionaryService.isHint(imageType, dossierTemplateId);
-        return RedactionLogEntry.builder()
-                .id(image.getId())
-                .color(getColor(imageType, dossierTemplateId, image.applied(), isHint))
-                .isImage(true)
-                .value(image.value())
-                .type(imageType)
-                .redacted(image.applied())
-                .reason(image.buildReasonWithManualChangeDescriptions())
-                .legalBasis(image.legalBasis())
-                .matchedRule(image.getMatchedRule().getRuleIdentifier().toString())
-                .isHint(isHint)
-                .isDictionaryEntry(false)
-                .isRecommendation(false)
-                .positions(List.of(toRedactionLogRectangle(image.getPosition(), image.getPage().getNumber())))
-                .sectionNumber(image.getTreeId().get(0))
-                .section(image.getManualOverwrite().getSection().orElse(image.getParent().toString()))
-                .imageHasTransparency(image.isTransparent())
-                .manualChanges(mapManualChanges(image.getManualOverwrite(), isHint))
-                .build();
-
-    }
-
-
-    private Rectangle toRedactionLogRectangle(Rectangle2D rectangle2D, int pageNumber) {
-
-        return new Rectangle(new Point((float) rectangle2D.getMinX(), (float) rectangle2D.getMinY()), (float) rectangle2D.getWidth(), (float) rectangle2D.getHeight(), pageNumber);
-    }
-
-
-    public RedactionLogEntry createRedactionLogEntry(ManualEntity manualEntity, String dossierTemplateId) {
-
-        String type = manualEntity.getManualOverwrite().getType().orElse(manualEntity.getType());
-        boolean isHint = isHint(manualEntity.getEntityType());
-        return RedactionLogEntry.builder()
-                .id(manualEntity.getId())
-                .color(getColor(type, dossierTemplateId, manualEntity.applied(), isHint))
-                .reason(manualEntity.buildReasonWithManualChangeDescriptions())
-                .legalBasis(manualEntity.legalBasis())
-                .value(manualEntity.value())
-                .type(type)
-                .redacted(manualEntity.applied())
-                .isHint(isHint)
-                .isRecommendation(manualEntity.getEntityType().equals(EntityType.RECOMMENDATION))
-                .isFalsePositive(manualEntity.getEntityType().equals(EntityType.FALSE_POSITIVE) || manualEntity.getEntityType().equals(EntityType.FALSE_RECOMMENDATION))
-                .section(manualEntity.getManualOverwrite().getSection().orElse(manualEntity.getSection()))
-                .sectionNumber(0)
-                .matchedRule(manualEntity.getMatchedRule().getRuleIdentifier().toString())
-                .rectangle(manualEntity.isRectangle())
-                .isDictionaryEntry(manualEntity.isDictionaryEntry())
-                .isDossierDictionaryEntry(manualEntity.isDossierDictionaryEntry())
-                .textAfter("")
-                .textBefore("")
-                .startOffset(-1)
-                .endOffset(-1)
-                .positions(manualEntity.getManualOverwrite()
-                        .getPositions()
-                        .orElse(manualEntity.getEntityPosition())
-                        .stream()
-                        .map(entityPosition -> toRedactionLogRectangle(entityPosition.rectangle2D(), entityPosition.pageNumber()))
-                        .toList())
-                .engines(Collections.emptySet())
-                .reference(Collections.emptySet())
-                .manualChanges(mapManualChanges(manualEntity.getManualOverwrite(), isHint))
-                .build();
-    }
-
-}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/UnprocessedChangesService.java

@@ -3,10 +3,8 @@ package com.iqser.red.service.redaction.v1.server.service;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.HashSet;
-import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
-import java.util.NoSuchElementException;
 import java.util.Optional;
 import java.util.Set;
 import java.util.stream.Collectors;
@@ -18,30 +16,22 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequ
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.BaseAnnotation;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
 import com.iqser.red.service.redaction.v1.model.AnalyzeResponse;
 import com.iqser.red.service.redaction.v1.model.QueueNames;
 import com.iqser.red.service.redaction.v1.model.UnprocessedManualEntity;
-import com.iqser.red.service.redaction.v1.server.model.ClosestEntity;
-import com.iqser.red.service.redaction.v1.server.model.ManualEntity;
-import com.iqser.red.service.redaction.v1.server.model.dictionary.SearchImplementation;
-import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
-import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
-import com.iqser.red.service.redaction.v1.server.model.document.entity.PositionOnPage;
+import com.iqser.red.service.redaction.v1.server.model.PrecursorEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
-import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
 import com.iqser.red.service.redaction.v1.server.service.document.DocumentGraphMapper;
-import com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService;
-import com.iqser.red.service.redaction.v1.server.service.document.EntityEnrichmentService;
 import com.iqser.red.service.redaction.v1.server.service.document.EntityFindingUtility;
-import com.iqser.red.service.redaction.v1.server.service.document.ManualEntityCreationService;
+import com.iqser.red.service.redaction.v1.server.service.document.EntityFromPrecursorCreationService;
 import com.iqser.red.service.redaction.v1.server.storage.ObservedStorageService;
 import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
 
 import io.micrometer.observation.annotation.Observed;
-import jakarta.annotation.PostConstruct;
 import lombok.AccessLevel;
 import lombok.RequiredArgsConstructor;
 import lombok.experimental.FieldDefaults;
@@ -59,17 +49,9 @@ public class UnprocessedChangesService {
     final ObservedStorageService observedStorageService;
     final EntityFindingUtility entityFindingUtility;
     final RedactionStorageService redactionStorageService;
-    final EntityEnrichmentService entityEnrichmentService;
-    final ManualEntityCreationService manualEntityCreationService;
+    final EntityFromPrecursorCreationService entityFromPrecursorCreationService;
     final DictionaryService dictionaryService;
-    EntityCreationService entityCreationService;
-
-
-    @PostConstruct
-    public void initEntityCreationService() {
-
-        entityCreationService = new EntityCreationService(entityEnrichmentService);
-    }
+    final ManualChangesApplicationService manualChangesApplicationService;
 
 
     @Observed(name = "UnprocessedChangesService", contextualName = "analyse-surrounding-text")
@@ -82,201 +64,134 @@ public class UnprocessedChangesService {
         EntityLog previousEntityLog = redactionStorageService.getEntityLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
         Document document = DocumentGraphMapper.toDocumentGraph(observedStorageService.getDocumentData(analyzeRequest.getDossierId(), analyzeRequest.getFileId()));
 
-        Set<String> allAnnotationIds = analyzeRequest.getManualRedactions().getEntriesToAdd().stream().map(ManualRedactionEntry::getAnnotationId).collect(Collectors.toSet());
-        Set<String> resizeIds = analyzeRequest.getManualRedactions().getResizeRedactions().stream().map(ManualResizeRedaction::getAnnotationId).collect(Collectors.toSet());
+        Set<String> allAnnotationIds = analyzeRequest.getManualRedactions().getEntriesToAdd()
+                .stream()
+                .map(ManualRedactionEntry::getAnnotationId)
+                .collect(Collectors.toSet());
+        Set<String> resizeIds = analyzeRequest.getManualRedactions().getResizeRedactions()
+                .stream()
+                .map(ManualResizeRedaction::getAnnotationId)
+                .collect(Collectors.toSet());
         allAnnotationIds.addAll(resizeIds);
 
-        List<ManualResizeRedaction> manualResizeRedactions = analyzeRequest.getManualRedactions().getResizeRedactions().stream().toList();
-        List<ManualEntity> manualEntitiesToBeResized = previousEntityLog.getEntityLogEntry()
-                .stream().filter(entityLogEntry -> resizeIds.contains(entityLogEntry.getId())).toList()
-                .stream().map(ManualEntity::fromEntityLogEntry).toList();
+        List<ManualResizeRedaction> manualResizeRedactions = analyzeRequest.getManualRedactions().getResizeRedactions()
+                .stream()
+                .toList();
+        List<PrecursorEntity> manualEntitiesToBeResized = previousEntityLog.getEntityLogEntry()
+                .stream()
+                .filter(entityLogEntry -> resizeIds.contains(entityLogEntry.getId()))
+                .toList()
+                .stream()
+                .map(PrecursorEntity::fromEntityLogEntry)
+                .toList();
 
         if (!manualResizeRedactions.isEmpty()) {
             processManualResizeRedactions(document, manualEntitiesToBeResized, unprocessedManualEntities, manualResizeRedactions);
         }
 
-        List<ManualEntity> notFoundManualEntities = new ArrayList<>();
-        List<ManualEntity> manualEntities = manualEntitiesConverter(analyzeRequest.getManualRedactions(), analyzeRequest.getDossierTemplateId());
+        List<PrecursorEntity> notFoundManualEntities = new ArrayList<>();
+        List<PrecursorEntity> manualEntities = manualEntitiesConverter(analyzeRequest.getManualRedactions(), analyzeRequest.getDossierTemplateId());
         if (!manualEntities.isEmpty()) {
-            notFoundManualEntities = manualEntityCreationService.toTextEntity(manualEntities, document);
+            notFoundManualEntities = entityFromPrecursorCreationService.toTextEntity(manualEntities, document);
         }
 
-        document.getEntities().forEach(textEntity -> {
-            Set<String> processedIds = new HashSet<>();
-            for (var positionsOnPerPage : textEntity.getPositionsOnPagePerPage()) {
-                if (processedIds.contains(positionsOnPerPage.getId())) {
-                    continue;
-                }
-                processedIds.add(positionsOnPerPage.getId());
-                List<Position> positions = positionsOnPerPage.getRectanglePerLine()
-                        .stream()
-                        .map(rectangle2D -> new Position(rectangle2D, positionsOnPerPage.getPage().getNumber()))
-                        .collect(Collectors.toList());
-                unprocessedManualEntities.add(UnprocessedManualEntity.builder()
-                        .annotationId(allAnnotationIds.stream().filter(textEntity::matchesAnnotationId).findFirst().orElse(""))
-                        .textBefore(textEntity.getTextBefore())
-                        .textAfter(textEntity.getTextAfter())
-                        .section(textEntity.getManualOverwrite().getSection().orElse(textEntity.getDeepestFullyContainingNode().toString()))
-                        .positions(positions)
-                        .build());
-            }
-        });
+        document.getEntities()
+                .forEach(textEntity -> {
+                    Set<String> processedIds = new HashSet<>();
+                    for (var positionsOnPerPage : textEntity.getPositionsOnPagePerPage()) {
+                        if (processedIds.contains(positionsOnPerPage.getId())) {
+                            continue;
+                        }
+                        processedIds.add(positionsOnPerPage.getId());
+                        List<Position> positions = positionsOnPerPage.getRectanglePerLine()
+                                .stream()
+                                .map(rectangle2D -> new Position(rectangle2D, positionsOnPerPage.getPage().getNumber()))
+                                .collect(Collectors.toList());
+                        unprocessedManualEntities.add(UnprocessedManualEntity.builder()
+                                                              .annotationId(allAnnotationIds.stream()
+                                                                                    .filter(textEntity::matchesAnnotationId)
+                                                                                    .findFirst()
+                                                                                    .orElse(""))
+                                                              .textBefore(textEntity.getTextBefore())
+                                                              .textAfter(textEntity.getTextAfter())
+                                                              .section(textEntity.getManualOverwrite().getSection()
+                                                                               .orElse(textEntity.getDeepestFullyContainingNode().toString()))
+                                                              .positions(positions)
+                                                              .build());
+                    }
+                });
 
         notFoundManualEntities.forEach(manualEntity -> unprocessedManualEntities.add(builDefaultUnprocessedManualEntity(manualEntity)));
 
         rabbitTemplate.convertAndSend(QueueNames.REDACTION_ANALYSIS_RESPONSE_QUEUE,
-                AnalyzeResponse.builder().fileId(analyzeRequest.getFileId()).unprocessedManualEntities(unprocessedManualEntities).build());
+                                      AnalyzeResponse.builder().fileId(analyzeRequest.getFileId()).unprocessedManualEntities(unprocessedManualEntities).build());
     }
 
 
     private void processManualResizeRedactions(Document document,
-                                               List<ManualEntity> manualEntities,
+                                               List<PrecursorEntity> manualEntities,
                                                List<UnprocessedManualEntity> unprocessedManualEntities,
                                                List<ManualResizeRedaction> manualResizeRedactions) {
 
         Map<String, List<TextEntity>> tempEntities = entityFindingUtility.findAllPossibleEntitiesAndGroupByValue(document, manualEntities);
 
-        for (ManualEntity manualEntity : manualEntities) {
+        for (PrecursorEntity precursorEntity : manualEntities) {
 
-            Optional<TextEntity> optionalTextEntity = entityFindingUtility.findClosestEntityAndReturnEmptyIfNotFound(manualEntity, tempEntities, THRESHOLD);
+            Optional<TextEntity> optionalTextEntity = entityFindingUtility.findClosestEntityAndReturnEmptyIfNotFound(precursorEntity, tempEntities, THRESHOLD);
 
             if (optionalTextEntity.isEmpty()) {
-                unprocessedManualEntities.add(builDefaultUnprocessedManualEntity(manualEntity));
+                unprocessedManualEntities.add(builDefaultUnprocessedManualEntity(precursorEntity));
                 continue;
             }
 
-            TextEntity correctEntity = createCorrectEntity(manualEntity, document, optionalTextEntity.get().getTextRange());
-            resizeEntityAndReinsert(correctEntity, manualResizeRedactions.stream().filter(manualResizeRedaction -> manualResizeRedaction.getAnnotationId().equals(manualEntity.getId())).findFirst());
+            TextEntity correctEntity = EntityFromPrecursorCreationService.createCorrectEntity(precursorEntity, optionalTextEntity.get());
+            Optional<ManualResizeRedaction> optionalManualResizeRedaction = manualResizeRedactions.stream()
+                    .filter(manualResizeRedaction -> manualResizeRedaction.getAnnotationId().equals(precursorEntity.getId()))
+                    .findFirst();
+            if (optionalManualResizeRedaction.isPresent()) {
+                ManualResizeRedaction manualResizeRedaction = optionalManualResizeRedaction.get();
+                manualChangesApplicationService.resize(correctEntity, manualResizeRedaction);
+
+                // If the entity's value is not the same as the manual resize request's value it means we didn't find it anywhere and we want to remove it
+                // from the graph, so it does not get processed and sent back to persistence-service to update its value.
+                if (!correctEntity.getValue().equals(manualResizeRedaction.getValue())) {
+                    correctEntity.removeFromGraph();
+                }
+            }
         }
 
         // remove all temp entities from the graph
-        tempEntities.values().stream().flatMap(Collection::stream).forEach(TextEntity::removeFromGraph);
-    }
-
-
-    public void resizeEntityAndReinsert(TextEntity entityToBeResized, Optional<ManualResizeRedaction> optionalManualResizeRedaction) {
-
-        if (optionalManualResizeRedaction.isEmpty()) {
-            return;
-        }
-
-        ManualResizeRedaction manualResizeRedaction = optionalManualResizeRedaction.get();
-
-        PositionOnPage positionOnPageToBeResized = entityToBeResized.getPositionsOnPagePerPage()
+        tempEntities.values()
                 .stream()
-                .filter(redactionPosition -> redactionPosition.getId().equals(manualResizeRedaction.getAnnotationId()))
-                .findFirst()
-                .orElseThrow(() -> new NoSuchElementException("No redaction position with matching annotation id found!"));
-
-        positionOnPageToBeResized.setRectanglePerLine(manualResizeRedaction.getPositions().stream().map(ManualChangesApplicationService::toRectangle2D).collect(Collectors.toList()));
-
-        String value = manualResizeRedaction.getValue();
-        int newStartOffset = -1;
-        SemanticNode node = entityToBeResized.getDeepestFullyContainingNode();
-        ClosestEntity closestEntity = ClosestEntity.builder().distance(100).textRange(null).build();
-
-        // Loop through nodes starting from the deepest fully containing node all the way to the document node
-        while (node != null) {
-            if (node.containsString(value)) {
-                SearchImplementation searchImplementation = new SearchImplementation(value, false);
-                List<TextRange> textRanges = searchImplementation.getBoundaries(node.getTextBlock(), node.getTextRange());
-
-                for (TextRange textRange : textRanges) {
-                    SemanticNode finalNode = node;
-
-                    List<TextEntity> tempEntities = searchImplementation.getBoundaries(node.getTextBlock(), textRange)
-                            .stream()
-                            .map(boundary -> entityCreationService.forceByTextRange(boundary, "temp", EntityType.ENTITY, finalNode))
-                            .collect(Collectors.toList());
-
-                    // If a value appears multiple times in a section after resizing, we need to make sure we select the surrounding text for the correct one.
-                    ManualChangesApplicationService.determineCorrectEntity(manualResizeRedaction, textRange, tempEntities, closestEntity);
-
-                    // Remove all temp entities from the graph
-                    tempEntities.forEach(TextEntity::removeFromGraph);
-                }
-                break;
-            }
-
-            // If the current node is the document node then it does not have a parent, meaning we could not find the value anywhere.
-            if (node.hasParent()) {
-                node = node.getParent();
-            } else {
-                break;
-            }
-        }
-
-        if (closestEntity.getTextRange() != null) {
-            newStartOffset = closestEntity.getTextRange().start();
-        }
-
-        // need to reinsert the entity, due to the boundary having changed.
-        removeAndUpdateAndReInsertEntity(entityToBeResized, manualResizeRedaction, newStartOffset);
-        entityToBeResized.getManualOverwrite().addChange(manualResizeRedaction);
+                .flatMap(Collection::stream)
+                .forEach(TextEntity::removeFromGraph);
     }
 
 
-    private void removeAndUpdateAndReInsertEntity(TextEntity entityToBeResized, ManualResizeRedaction manualResizeRedaction, int newStartOffset) {
-
-        SemanticNode nodeToInsertInto = entityToBeResized.getDeepestFullyContainingNode().getDocumentTree().getRoot().getNode();
-        entityToBeResized.removeFromGraph();
-        entityToBeResized.getIntersectingNodes().forEach(node -> node.getEntities().remove(this));
-        entityToBeResized.getPages().forEach(page -> page.getEntities().remove(this));
-        entityToBeResized.setIntersectingNodes(new LinkedList<>());
-        entityToBeResized.setDeepestFullyContainingNode(null);
-        entityToBeResized.setPages(new HashSet<>());
-        entityToBeResized.getTextRange().setStart(newStartOffset);
-        entityToBeResized.getTextRange().setEnd(newStartOffset + manualResizeRedaction.getValue().length());
-        // Don't insert into the graph if newStartOffset is -1 because it means nothing was found.
-        if (newStartOffset > -1) {
-            entityCreationService.addEntityToGraph(entityToBeResized, nodeToInsertInto);
-        }
-    }
-
-
-    private TextEntity createCorrectEntity(ManualEntity manualEntity, SemanticNode node, TextRange closestTextRange) {
-
-        TextEntity correctEntity = entityCreationService.forceByTextRange(closestTextRange, manualEntity.getType(), manualEntity.getEntityType(), node);
-
-        correctEntity.addMatchedRules(manualEntity.getMatchedRuleList());
-        correctEntity.setDictionaryEntry(manualEntity.isDictionaryEntry());
-        correctEntity.setDossierDictionaryEntry(manualEntity.isDossierDictionaryEntry());
-        correctEntity.getManualOverwrite().addChanges(manualEntity.getManualOverwrite().getManualChangeLog());
-
-        List<PositionOnPage> redactionPositionsWithIdOfManualOnPage = new ArrayList<>(correctEntity.getPositionsOnPagePerPage().size());
-        for (PositionOnPage positionOnPage : correctEntity.getPositionsOnPagePerPage()) {
-            redactionPositionsWithIdOfManualOnPage.add(new PositionOnPage(manualEntity.getId(), positionOnPage.getPage(), positionOnPage.getRectanglePerLine()));
-        }
-        correctEntity.setPositionsOnPagePerPage(redactionPositionsWithIdOfManualOnPage);
-
-        return correctEntity;
-    }
-
-
-    private UnprocessedManualEntity builDefaultUnprocessedManualEntity(ManualEntity manualEntity) {
+    private UnprocessedManualEntity builDefaultUnprocessedManualEntity(PrecursorEntity precursorEntity) {
 
         return UnprocessedManualEntity.builder()
-                .annotationId(manualEntity.getId())
+                .annotationId(precursorEntity.getId())
                 .textAfter("")
                 .textBefore("")
                 .section("")
-                .positions(manualEntity.getManualOverwrite()
-                        .getPositions()
-                        .orElse(manualEntity.getEntityPosition())
-                        .stream()
-                        .map(entityPosition -> new Position(entityPosition.rectangle2D(), entityPosition.pageNumber()))
-                        .toList())
+                .positions(precursorEntity.getManualOverwrite().getPositions()
+                                   .orElse(precursorEntity.getEntityPosition())
+                                   .stream()
+                                   .map(entityPosition -> new Position(entityPosition.rectangle2D(), entityPosition.pageNumber()))
+                                   .toList())
                 .build();
     }
 
 
-    private List<ManualEntity> manualEntitiesConverter(ManualRedactions manualRedactions, String dossierTemplateId) {
+    private List<PrecursorEntity> manualEntitiesConverter(ManualRedactions manualRedactions, String dossierTemplateId) {
 
         return manualRedactions.getEntriesToAdd()
                 .stream()
                 .filter(manualRedactionEntry -> manualRedactionEntry.getPositions() != null && !manualRedactionEntry.getPositions().isEmpty())
-                .map(manualRedactionEntry -> ManualEntity.fromManualRedactionEntry(manualRedactionEntry,
-                        dictionaryService.isHint(manualRedactionEntry.getType(), dossierTemplateId)))
+                .filter(BaseAnnotation::isLocal)
+                .map(manualRedactionEntry -> PrecursorEntity.fromManualRedactionEntry(manualRedactionEntry,
+                                                                                      dictionaryService.isHint(manualRedactionEntry.getType(), dossierTemplateId)))
                 .toList();
     }
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/DocumentGraphMapper.java

@@ -6,6 +6,7 @@ import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
 import java.util.NoSuchElementException;
+import java.util.Set;
 
 import com.iqser.red.service.redaction.v1.server.model.document.DocumentTree;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Footer;
@@ -76,6 +77,7 @@ public class DocumentGraphMapper {
                 node.setLeafTextBlock(textBlock);
             }
             List<Integer> treeId = Arrays.stream(entryData.getTreeId()).boxed().toList();
+            entryData.getEngines().forEach(engine -> node.addEngine(engine));
             node.setTreeId(treeId);
 
             switch (entryData.getType()) {

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityCreationService.java

@@ -1,5 +1,6 @@
 package com.iqser.red.service.redaction.v1.server.service.document;
 
+import static com.iqser.red.service.redaction.v1.server.service.document.EntityCreationUtility.*;
 import static com.iqser.red.service.redaction.v1.server.utils.SeparatorUtils.boundaryIsSurroundedBySeparators;
 
 import java.util.Collection;
@@ -18,24 +19,21 @@ import org.kie.api.runtime.KieSession;
 
 import com.google.common.base.Functions;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine;
-import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
+import com.iqser.red.service.redaction.v1.server.model.NerEntities;
+import com.iqser.red.service.redaction.v1.server.model.dictionary.SearchImplementation;
 import com.iqser.red.service.redaction.v1.server.model.document.ConsecutiveBoundaryCollector;
 import com.iqser.red.service.redaction.v1.server.model.document.DocumentTree;
+import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.ManualChangeOverwrite;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
-import com.iqser.red.service.redaction.v1.server.model.document.entity.PositionOnPage;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.NodeType;
-import com.iqser.red.service.redaction.v1.server.model.document.nodes.Page;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Table;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
 import com.iqser.red.service.redaction.v1.server.utils.RectangleTransformations;
 import com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility;
-import com.iqser.red.service.redaction.v1.server.model.NerEntities;
-import com.iqser.red.service.redaction.v1.server.model.dictionary.SearchImplementation;
-import com.iqser.red.service.redaction.v1.server.utils.IdBuilder;
 import com.iqser.red.service.redaction.v1.server.utils.exception.NotFoundException;
 
 import lombok.RequiredArgsConstructor;
@@ -203,6 +201,7 @@ public class EntityCreationService {
         return betweenTextRanges(startTextRanges, stopTextRanges, type, entityType, node);
     }
 
+
     public Stream<TextEntity> shortestBetweenAnyStringIgnoreCase(List<String> starts, List<String> stops, String type, EntityType entityType, SemanticNode node, int limit) {
 
         checkIfBothStartAndEndAreEmpty(starts, stops);
@@ -233,9 +232,10 @@ public class EntityCreationService {
         return betweenTextRanges(startBoundaries, stopBoundaries, type, entityType, node);
     }
 
+
     public Stream<TextEntity> betweenTextRanges(List<TextRange> startBoundaries, List<TextRange> stopBoundaries, String type, EntityType entityType, SemanticNode node) {
 
-        return betweenTextRanges(startBoundaries, stopBoundaries, type, entityType, node,0);
+        return betweenTextRanges(startBoundaries, stopBoundaries, type, entityType, node, 0);
     }
 
 
@@ -282,22 +282,11 @@ public class EntityCreationService {
     }
 
 
-    private void checkIfBothStartAndEndAreEmpty(String start, String end) {
-        checkIfBothStartAndEndAreEmpty(List.of(start), List.of(end));
-    }
-
-
-    private <T> void checkIfBothStartAndEndAreEmpty(List<T> start, List<T> end) {
-        if ((start == null || start.isEmpty()) && (end == null || end.isEmpty())) {
-            throw new IllegalArgumentException("Start and end values are empty!");
-        }
-    }
-
-
     public Stream<TextEntity> bySearchImplementation(SearchImplementation searchImplementation, String type, EntityType entityType, SemanticNode node) {
 
         return searchImplementation.getBoundaries(node.getTextBlock(), node.getTextRange())
-                .stream().filter(boundary -> isValidEntityTextRange(node.getTextBlock(), boundary))
+                .stream()
+                .filter(boundary -> isValidEntityTextRange(node.getTextBlock(), boundary))
                 .map(bounds -> byTextRange(bounds, type, entityType, node))
                 .filter(Optional::isPresent)
                 .map(Optional::get);
@@ -309,7 +298,9 @@ public class EntityCreationService {
         TextBlock textBlock = node.getTextBlock();
         SearchImplementation searchImplementation = new SearchImplementation(strings, false);
         return searchImplementation.getBoundaries(textBlock, node.getTextRange())
-                .stream().map(boundary -> toLineAfterTextRange(textBlock, boundary)).filter(boundary -> isValidEntityTextRange(textBlock, boundary))
+                .stream()
+                .map(boundary -> toLineAfterTextRange(textBlock, boundary))
+                .filter(boundary -> isValidEntityTextRange(textBlock, boundary))
                 .map(boundary -> byTextRange(boundary, type, entityType, node))
                 .filter(Optional::isPresent)
                 .map(Optional::get);
@@ -321,7 +312,9 @@ public class EntityCreationService {
         TextBlock textBlock = node.getTextBlock();
         SearchImplementation searchImplementation = new SearchImplementation(strings, true);
         return searchImplementation.getBoundaries(textBlock, node.getTextRange())
-                .stream().map(boundary -> toLineAfterTextRange(textBlock, boundary)).filter(boundary -> isValidEntityTextRange(textBlock, boundary))
+                .stream()
+                .map(boundary -> toLineAfterTextRange(textBlock, boundary))
+                .filter(boundary -> isValidEntityTextRange(textBlock, boundary))
                 .map(boundary -> byTextRange(boundary, type, entityType, node))
                 .filter(Optional::isPresent)
                 .map(Optional::get);
@@ -332,7 +325,9 @@ public class EntityCreationService {
 
         TextBlock textBlock = node.getTextBlock();
         return RedactionSearchUtility.findTextRangesByString(string, textBlock)
-                .stream().map(boundary -> toLineAfterTextRange(textBlock, boundary)).filter(boundary -> isValidEntityTextRange(textBlock, boundary))
+                .stream()
+                .map(boundary -> toLineAfterTextRange(textBlock, boundary))
+                .filter(boundary -> isValidEntityTextRange(textBlock, boundary))
                 .map(boundary -> byTextRange(boundary, type, entityType, node))
                 .filter(Optional::isPresent)
                 .map(Optional::get);
@@ -343,7 +338,9 @@ public class EntityCreationService {
 
         TextBlock textBlock = node.getTextBlock();
         return RedactionSearchUtility.findTextRangesByStringIgnoreCase(string, textBlock)
-                .stream().map(boundary -> toLineAfterTextRange(textBlock, boundary)).filter(boundary -> isValidEntityTextRange(textBlock, boundary))
+                .stream()
+                .map(boundary -> toLineAfterTextRange(textBlock, boundary))
+                .filter(boundary -> isValidEntityTextRange(textBlock, boundary))
                 .map(boundary -> byTextRange(boundary, type, entityType, node))
                 .filter(Optional::isPresent)
                 .map(Optional::get);
@@ -352,7 +349,8 @@ public class EntityCreationService {
 
     public Stream<TextEntity> lineAfterStringAcrossColumns(String string, String type, EntityType entityType, Table tableNode) {
 
-        return tableNode.streamTableCells().flatMap(tableCell -> lineAfterBoundariesAcrossColumns(RedactionSearchUtility.findTextRangesByString(string, tableCell.getTextBlock()),
+        return tableNode.streamTableCells()
+                .flatMap(tableCell -> lineAfterBoundariesAcrossColumns(RedactionSearchUtility.findTextRangesByString(string, tableCell.getTextBlock()),
                         tableCell,
                         type,
                         entityType,
@@ -545,7 +543,7 @@ public class EntityCreationService {
     public Optional<TextEntity> byPrefixExpansionRegex(TextEntity entity, String regexPattern) {
 
         int expandedStart = RedactionSearchUtility.getExpandedStartByRegex(entity, regexPattern);
-        return byTextRange(new TextRange(expandedStart, entity.getTextRange().end()), entity.getType(), entity.getEntityType(), entity.getDeepestFullyContainingNode());
+        return byTextRange(new TextRange(expandedStart, entity.getTextRange().end()), entity.type(), entity.getEntityType(), entity.getDeepestFullyContainingNode());
     }
 
 
@@ -553,18 +551,10 @@ public class EntityCreationService {
 
         int expandedEnd = RedactionSearchUtility.getExpandedEndByRegex(entity, regexPattern);
         expandedEnd = truncateEndIfLineBreakIsBetween(entity.getTextRange().end(), expandedEnd, entity.getDeepestFullyContainingNode().getTextBlock());
-        return byTextRange(new TextRange(entity.getTextRange().start(), expandedEnd), entity.getType(), entity.getEntityType(), entity.getDeepestFullyContainingNode());
+        return byTextRange(new TextRange(entity.getTextRange().start(), expandedEnd), entity.type(), entity.getEntityType(), entity.getDeepestFullyContainingNode());
     }
 
 
-    private int truncateEndIfLineBreakIsBetween(int end, int expandedEnd, TextBlock textBlock) {
-
-        if (textBlock.getNextLinebreak(end) < expandedEnd) {
-            return end;
-        }
-        return expandedEnd;
-    }
-
     /**
      * Creates a redaction entity based on the given boundary, type, entity type, and semantic node.
      * If the document already contains an equal redaction entity, then the original Entity is returned.
@@ -581,6 +571,7 @@ public class EntityCreationService {
         return byTextRangeWithEngine(textRange, type, entityType, node, Set.of(Engine.RULE));
     }
 
+
     /**
      * Creates a redaction entity based on the given boundary, type, entity type, and semantic node.
      * If the document already contains an equal redaction entity, then the original Entity is returned.
@@ -599,9 +590,21 @@ public class EntityCreationService {
             throw new IllegalArgumentException(String.format("%s is not in the %s of the provided semantic node %s", textRange, node.getTextRange(), node));
         }
         TextRange trimmedTextRange = textRange.trim(node.getTextBlock());
-        TextEntity entity = TextEntity.initialEntityNode(trimmedTextRange, type, entityType);
+        TextEntity entity = TextEntity.initialEntityNode(trimmedTextRange, type, entityType, node);
         if (node.getEntities().contains(entity)) {
-            return node.getEntities().stream().filter(entity::equals).peek(e -> e.addEngines(engines)).findAny();
+            Optional<TextEntity> optionalTextEntity = node.getEntities().stream().filter(e -> e.equals(entity) && e.type().equals(type)).peek(e -> e.addEngines(engines)).findAny();
+            if (optionalTextEntity.isEmpty()) {
+                return optionalTextEntity; // Entity has been recategorized and should not be created at all.
+            }
+            TextEntity existingEntity = optionalTextEntity.get();
+            if (existingEntity.getTextRange().equals(textRange)) {
+                return optionalTextEntity; // exactly the same entity, return directly
+            }
+            if (!existingEntity.resized()) {
+                addDuplicateEntityToGraph(existingEntity, textRange, node);
+                return optionalTextEntity; // If Entity has not been resized, insert as duplicate at appropriate position
+            }
+            return Optional.empty(); // Entity has been resized, if there are duplicates they should be treated there
         }
         addEntityToGraph(entity, node);
         entity.addEngines(engines);
@@ -610,10 +613,23 @@ public class EntityCreationService {
     }
 
 
+    // Do not use anymore. This might not work correctly due to duplicate textranges not being taken into account here.
+    @Deprecated(forRemoval = true)
     public TextEntity forceByTextRange(TextRange textRange, String type, EntityType entityType, SemanticNode node) {
 
         TextRange trimmedTextRange = textRange.trim(node.getTextBlock());
-        TextEntity entity = TextEntity.initialEntityNode(trimmedTextRange, type, entityType);
+        TextEntity entity = TextEntity.initialEntityNode(trimmedTextRange, type, entityType, node);
+        addEntityToGraph(entity, node);
+        return entity;
+    }
+
+
+    // Do not use anymore. This might not work correctly due to duplicate textranges not being taken into account here.
+    @Deprecated(forRemoval = true)
+    public TextEntity forceByTextRange(TextRange textRange, String type, EntityType entityType, SemanticNode node, String id) {
+
+        TextRange trimmedTextRange = textRange.trim(node.getTextBlock());
+        TextEntity entity = TextEntity.initialEntityNode(trimmedTextRange, type, entityType, id);
         addEntityToGraph(entity, node);
         return entity;
     }
@@ -631,7 +647,7 @@ public class EntityCreationService {
             return entitiesToMerge.get(0);
         }
 
-        TextEntity mergedEntity = TextEntity.initialEntityNode(TextRange.merge(entitiesToMerge.stream().map(TextEntity::getTextRange).toList()), type, entityType);
+        TextEntity mergedEntity = TextEntity.initialEntityNode(TextRange.merge(entitiesToMerge.stream().map(TextEntity::getTextRange).toList()), type, entityType, node);
         mergedEntity.addEngines(entitiesToMerge.stream().flatMap(entityNode -> entityNode.getEngines().stream()).collect(Collectors.toSet()));
         entitiesToMerge.stream().map(TextEntity::getMatchedRuleList).flatMap(Collection::stream).forEach(matchedRule -> mergedEntity.getMatchedRuleList().add(matchedRule));
         entitiesToMerge.stream()
@@ -662,6 +678,7 @@ public class EntityCreationService {
         return newEntity;
     }
 
+
     public TextEntity copyEntityWithoutRules(TextEntity entity, String type, EntityType entityType, SemanticNode node) {
 
         TextEntity newEntity = byTextRangeWithEngine(entity.getTextRange(), type, entityType, node, entity.getEngines()).orElseThrow(() -> new NotFoundException(
@@ -683,7 +700,8 @@ public class EntityCreationService {
 
     public TextEntity byNerEntity(NerEntities.NerEntity nerEntity, EntityType entityType, SemanticNode semanticNode) {
 
-        return byTextRangeWithEngine(nerEntity.textRange(), nerEntity.type(), entityType, semanticNode, Set.of(Engine.NER)).orElseThrow(() -> new NotFoundException("No entity present!"));
+        return byTextRangeWithEngine(nerEntity.textRange(), nerEntity.type(), entityType, semanticNode, Set.of(Engine.NER)).orElseThrow(() -> new NotFoundException(
+                "No entity present!"));
     }
 
 
@@ -693,6 +711,18 @@ public class EntityCreationService {
     }
 
 
+    public Optional<TextEntity> optionalByNerEntity(NerEntities.NerEntity nerEntity, EntityType entityType, SemanticNode semanticNode) {
+
+        return byTextRangeWithEngine(nerEntity.textRange(), nerEntity.type(), entityType, semanticNode, Set.of(Engine.NER));
+    }
+
+
+    public Optional<TextEntity> optionalByNerEntity(NerEntities.NerEntity nerEntity, String type, EntityType entityType, SemanticNode semanticNode) {
+
+        return byTextRangeWithEngine(nerEntity.textRange(), type, entityType, semanticNode, Set.of(Engine.NER));
+    }
+
+
     public Stream<TextEntity> combineNerEntitiesToCbiAddressDefaults(NerEntities nerEntities, String type, EntityType entityType, SemanticNode semanticNode) {
 
         return NerEntitiesAdapter.combineNerEntitiesToCbiAddressDefaults(nerEntities)
@@ -702,21 +732,6 @@ public class EntityCreationService {
     }
 
 
-    public TextEntity byTableCellAsHighlight(TableCell tableCell, String type, EntityType entityType) {
-
-        TextEntity highlightEntity = TextEntity.initialEntityNode(new TextRange(tableCell.getTextRange().start(), tableCell.getTextRange().start()), type, entityType);
-
-        String positionId = IdBuilder.buildId(tableCell.getBBox().keySet(), tableCell.getBBox().values().stream().toList(), type, entityType.name());
-        highlightEntity.setPositionsOnPagePerPage(tableCell.getBBox()
-                .entrySet()
-                .stream()
-                .map(entry -> new PositionOnPage(positionId, entry.getKey(), List.of(entry.getValue())))
-                .toList());
-        addEntityToGraph(highlightEntity, tableCell);
-        return highlightEntity;
-    }
-
-
     public boolean isValidEntityTextRange(TextBlock textBlock, TextRange textRange) {
 
         return textRange.length() > 0 && boundaryIsSurroundedBySeparators(textBlock, textRange);
@@ -727,9 +742,17 @@ public class EntityCreationService {
 
         DocumentTree documentTree = node.getDocumentTree();
         try {
-            addEntityToGraph(entity, documentTree);
-            entity.addIntersectingNode(documentTree.getRoot().getNode());
-            documentTree.getRoot().getNode().getEntities().add(entity);
+            if (node.getEntities().contains(entity)) {
+                // If entity already exists and it has a different text range, we add the text range to the list of duplicated text ranges
+                node.getEntities().stream()//
+                        .filter(e -> e.equals(entity))//
+                        .filter(e -> !e.getTextRange().equals(entity.getTextRange()))//
+                        .findAny()//
+                        .ifPresent(entityToDuplicate -> addDuplicateEntityToGraph(entityToDuplicate, entity.getTextRange(), node));
+            } else {
+                entity.addIntersectingNode(documentTree.getRoot().getNode());
+                addEntityToGraph(entity, documentTree);
+            }
         } catch (NoSuchElementException e) {
             entity.setDeepestFullyContainingNode(documentTree.getRoot().getNode());
             entityEnrichmentService.enrichEntity(entity, entity.getDeepestFullyContainingNode().getTextBlock());
@@ -740,6 +763,33 @@ public class EntityCreationService {
     }
 
 
+    private void addDuplicateEntityToGraph(TextEntity entityToDuplicate, TextRange newTextRange, SemanticNode node) {
+
+        entityToDuplicate.addTextRange(newTextRange);
+
+        SemanticNode deepestSharedNode = entityToDuplicate.getIntersectingNodes()
+                .stream()
+                .sorted(Comparator.comparingInt(n -> -n.getTreeId().size()))
+                .filter(intersectingNode -> entityToDuplicate.getDuplicateTextRanges().stream().allMatch(tr -> intersectingNode.getTextRange().contains(tr)) && //
+                        intersectingNode.getTextRange().contains(entityToDuplicate.getTextRange()))
+                .findFirst()
+                .orElse(node.getDocumentTree().getRoot().getNode());
+
+        entityToDuplicate.setDeepestFullyContainingNode(deepestSharedNode);
+
+        Set<SemanticNode> additionalIntersectingNodes = findIntersectingSubNodes(deepestSharedNode, newTextRange);
+
+        additionalIntersectingNodes.forEach(additionalIntersectingNode -> {
+            if (entityToDuplicate.getIntersectingNodes().contains(additionalIntersectingNode)) {
+                return;
+            }
+            additionalIntersectingNode.getEntities().add(entityToDuplicate);
+            additionalIntersectingNode.getPages(newTextRange).forEach(page -> page.getEntities().add(entityToDuplicate));
+            entityToDuplicate.addIntersectingNode(additionalIntersectingNode);
+        });
+    }
+
+
     private void addEntityToGraph(TextEntity entity, DocumentTree documentTree) {
 
         SemanticNode containingNode = documentTree.childNodes(Collections.emptyList())
@@ -757,45 +807,4 @@ public class EntityCreationService {
     }
 
 
-    private static void addToPages(TextEntity entity) {
-
-        Set<Page> pages = entity.getDeepestFullyContainingNode().getPages(entity.getTextRange());
-        entity.getPages().addAll(pages);
-        pages.forEach(page -> page.getEntities().add(entity));
-    }
-
-
-    private static void addEntityToNodeEntitySets(TextEntity entity) {
-
-        entity.getIntersectingNodes().forEach(node -> node.getEntities().add(entity));
-    }
-
-
-    private static boolean allEntitiesIntersectAndHaveSameTypes(List<TextEntity> entitiesToMerge) {
-
-        if (entitiesToMerge.isEmpty()) {
-            return true;
-        }
-        TextEntity previousEntity = entitiesToMerge.get(0);
-        for (TextEntity textEntity : entitiesToMerge.subList(1, entitiesToMerge.size())) {
-            boolean typeMatches = textEntity.getType().equals(previousEntity.getType());
-            boolean entityTypeMatches = textEntity.getEntityType().equals(previousEntity.getEntityType());
-            boolean intersects = textEntity.intersects(previousEntity);
-            if (!typeMatches || !entityTypeMatches || !intersects) {
-                return false;
-            }
-        }
-        return true;
-    }
-
-
-    private static TextRange toLineAfterTextRange(TextBlock textBlock, TextRange textRange) {
-
-        if (textBlock.getTextRange().end() == textRange.end()) {
-            return new TextRange(textRange.end(), textRange.end());
-        }
-
-        return new TextRange(textRange.end(), textBlock.getNextLinebreak(textRange.end())).trim(textBlock);
-    }
-
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityCreationUtility.java

@@ -0,0 +1,91 @@
+package com.iqser.red.service.redaction.v1.server.service.document;
+
+import java.util.List;
+import java.util.Set;
+
+import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
+import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.Page;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
+import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
+
+public class EntityCreationUtility {
+
+
+    public static void checkIfBothStartAndEndAreEmpty(String start, String end) {
+
+        checkIfBothStartAndEndAreEmpty(List.of(start), List.of(end));
+    }
+
+
+    public static <T> void checkIfBothStartAndEndAreEmpty(List<T> start, List<T> end) {
+
+        if ((start == null || start.isEmpty()) && (end == null || end.isEmpty())) {
+            throw new IllegalArgumentException("Start and end values are empty!");
+        }
+    }
+
+
+    public static int truncateEndIfLineBreakIsBetween(int end, int expandedEnd, TextBlock textBlock) {
+
+        if (textBlock.getNextLinebreak(end) < expandedEnd) {
+            return end;
+        }
+        return expandedEnd;
+    }
+
+
+    public static Set<SemanticNode> findIntersectingSubNodes(SemanticNode initialIntersectingNode, TextRange textRange) {
+
+        IntersectingNodeVisitor visitor = new IntersectingNodeVisitor(textRange);
+
+        if (initialIntersectingNode.getTextRange().intersects(textRange)) {
+            initialIntersectingNode.accept(visitor);
+        }
+
+        return visitor.getIntersectingNodes();
+    }
+
+
+    public static void addToPages(TextEntity entity) {
+
+        Set<Page> pages = entity.getDeepestFullyContainingNode().getPages(entity.getTextRange());
+        entity.getPages().addAll(pages);
+        pages.forEach(page -> page.getEntities().add(entity));
+    }
+
+
+    public static void addEntityToNodeEntitySets(TextEntity entity) {
+
+        entity.getIntersectingNodes().forEach(node -> node.getEntities().add(entity));
+    }
+
+
+    public static boolean allEntitiesIntersectAndHaveSameTypes(List<TextEntity> entitiesToMerge) {
+
+        if (entitiesToMerge.isEmpty()) {
+            return true;
+        }
+        TextEntity previousEntity = entitiesToMerge.get(0);
+        for (TextEntity textEntity : entitiesToMerge.subList(1, entitiesToMerge.size())) {
+            boolean typeMatches = textEntity.type().equals(previousEntity.type());
+            boolean entityTypeMatches = textEntity.getEntityType().equals(previousEntity.getEntityType());
+            boolean intersects = textEntity.intersects(previousEntity);
+            if (!typeMatches || !entityTypeMatches || !intersects) {
+                return false;
+            }
+        }
+        return true;
+    }
+
+
+    public static TextRange toLineAfterTextRange(TextBlock textBlock, TextRange textRange) {
+
+        if (textBlock.getTextRange().end() == textRange.end()) {
+            return new TextRange(textRange.end(), textRange.end());
+        }
+
+        return new TextRange(textRange.end(), textBlock.getNextLinebreak(textRange.end())).trim(textBlock);
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityFindingUtility.java

@@ -5,6 +5,7 @@ import static java.util.stream.Collectors.groupingBy;
 
 import java.awt.geom.Rectangle2D;
 import java.util.Collection;
+import java.util.Collections;
 import java.util.Comparator;
 import java.util.List;
 import java.util.Locale;
@@ -17,7 +18,8 @@ import java.util.stream.Collectors;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
 
-import com.iqser.red.service.redaction.v1.server.model.ManualEntity;
+import com.iqser.red.service.redaction.v1.server.model.ClosestEntity;
+import com.iqser.red.service.redaction.v1.server.model.PrecursorEntity;
 import com.iqser.red.service.redaction.v1.server.model.RectangleWithPage;
 import com.iqser.red.service.redaction.v1.server.model.dictionary.SearchImplementation;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
@@ -45,41 +47,41 @@ public class EntityFindingUtility {
     }
 
 
-    public Optional<TextEntity> findClosestEntityAndReturnEmptyIfNotFound(ManualEntity manualEntity, Map<String, List<TextEntity>> entitiesWithSameValue, double matchThreshold) {
+    public Optional<TextEntity> findClosestEntityAndReturnEmptyIfNotFound(PrecursorEntity precursorEntity, Map<String, List<TextEntity>> entitiesWithSameValue, double matchThreshold) {
 
-        if (manualEntity.getValue() == null) {
+        if (precursorEntity.getValue() == null) {
             return Optional.empty();
         }
 
-        List<TextEntity> possibleEntities = entitiesWithSameValue.get(manualEntity.getValue().toLowerCase(Locale.ENGLISH));
+        List<TextEntity> possibleEntities = entitiesWithSameValue.get(precursorEntity.getValue().toLowerCase(Locale.ENGLISH));
 
         if (entityIdentifierValueNotFound(possibleEntities)) {
-            log.warn("Entity could not be created with manualEntity: {}, due to the value {} not being found anywhere.", manualEntity, manualEntity.getValue());
+            log.warn("Entity could not be created with precursorEntity: {}, due to the value {} not being found anywhere.", precursorEntity, precursorEntity.getValue());
             return Optional.empty();
         }
 
-        Optional<TextEntity> optionalClosestEntity = possibleEntities.stream()
-                .filter(entity -> pagesMatch(entity, manualEntity.getEntityPosition()))
-                .min(Comparator.comparingDouble(entity -> calculateMinDistance(manualEntity.getEntityPosition(), entity)));
+        Optional<ClosestEntity> optionalClosestEntity = possibleEntities.stream()
+                .filter(entity -> pagesMatch(entity, precursorEntity.getEntityPosition()))
+                .map(entity -> ClosestEntity.builder().distance(calculateMinDistance(precursorEntity.getEntityPosition(), entity)).textEntity(entity).build())
+                .min(Comparator.comparingDouble(ClosestEntity::getDistance));
 
         if (optionalClosestEntity.isEmpty()) {
-            log.warn("No Entity with value {} found on page {}", manualEntity.getValue(), manualEntity.getEntityPosition());
+            log.warn("No Entity with value {} found on page {}", precursorEntity.getValue(), precursorEntity.getEntityPosition());
             return Optional.empty();
         }
 
-        TextEntity closestEntity = optionalClosestEntity.get();
-        double distance = calculateMinDistance(manualEntity.getEntityPosition(), closestEntity);
-        if (distance > matchThreshold) {
+        ClosestEntity closestEntity = optionalClosestEntity.get();
+        if (closestEntity.getDistance() > matchThreshold) {
             log.warn("For entity {} on page {} with positions {} distance to closest found entity is {} and therefore higher than the threshold of {}",
-                    manualEntity.getValue(),
-                    manualEntity.getEntityPosition().get(0).pageNumber(),
-                    manualEntity.getEntityPosition().stream().map(RectangleWithPage::rectangle2D).toList(),
-                    distance,
+                    precursorEntity.getValue(),
+                    precursorEntity.getEntityPosition().get(0).pageNumber(),
+                    precursorEntity.getEntityPosition().stream().map(RectangleWithPage::rectangle2D).toList(),
+                    closestEntity.getDistance(),
                     matchThreshold);
             return Optional.empty();
         }
 
-        return Optional.of(closestEntity);
+        return Optional.of(closestEntity.getTextEntity());
     }
 
 
@@ -97,7 +99,7 @@ public class EntityFindingUtility {
     }
 
 
-    private double calculateMinDistance(List<RectangleWithPage> originalPositions, TextEntity entity) {
+    public static double calculateMinDistance(List<RectangleWithPage> originalPositions, TextEntity entity) {
 
         if (originalPositions.size() != countRectangles(entity)) {
             return Double.MAX_VALUE;
@@ -115,7 +117,7 @@ public class EntityFindingUtility {
     }
 
 
-    private double calculateMinDistancePerRectangle(TextEntity entity, int pageNumber, Rectangle2D originalRectangle) {
+    private static double calculateMinDistancePerRectangle(TextEntity entity, int pageNumber, Rectangle2D originalRectangle) {
 
         return entity.getPositionsOnPagePerPage()
                 .stream()
@@ -128,7 +130,7 @@ public class EntityFindingUtility {
     }
 
 
-    public double calculateDistance(Rectangle2D rectangle1, Rectangle2D rectangle2) {
+    public static double calculateDistance(Rectangle2D rectangle1, Rectangle2D rectangle2) {
 
         // mirrored coordinates safe comparison
         double minX1 = Math.min(rectangle1.getMinX(), rectangle1.getMaxX());
@@ -148,10 +150,10 @@ public class EntityFindingUtility {
     }
 
 
-    public Map<String, List<TextEntity>> findAllPossibleEntitiesAndGroupByValue(SemanticNode node, List<ManualEntity> manualEntities) {
+    public Map<String, List<TextEntity>> findAllPossibleEntitiesAndGroupByValue(SemanticNode node, List<PrecursorEntity> manualEntities) {
 
         Set<Integer> pageNumbers = manualEntities.stream().flatMap(entry -> entry.getEntityPosition().stream().map(RectangleWithPage::pageNumber)).collect(Collectors.toSet());
-        Set<String> entryValues = manualEntities.stream().map(ManualEntity::getValue).filter(Objects::nonNull).map(String::toLowerCase).collect(Collectors.toSet());
+        Set<String> entryValues = manualEntities.stream().map(PrecursorEntity::getValue).filter(Objects::nonNull).map(String::toLowerCase).collect(Collectors.toSet());
 
         if (!pageNumbers.stream().allMatch(node::onPage)) {
             throw new IllegalArgumentException(format("SemanticNode \"%s\" does not contain these pages %s, it has pages: %s",
@@ -163,7 +165,10 @@ public class EntityFindingUtility {
 
         return searchImplementation.getBoundaries(node.getTextBlock(), node.getTextRange())
                 .stream()
-                .map(boundary -> entityCreationService.forceByTextRange(boundary, "temp", EntityType.ENTITY, node))
+                .map(boundary -> entityCreationService.byTextRangeWithEngine(boundary, "temp", EntityType.ENTITY, node, Collections.emptySet()))
+                .filter(Optional::isPresent)
+                .map(Optional::get)
+                .distinct()
                 .collect(groupingBy(entity -> entity.getValue().toLowerCase(Locale.ROOT)));
     }
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityFromPrecursorCreationService.java

@@ -0,0 +1,161 @@
+package com.iqser.red.service.redaction.v1.server.service.document;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedactions;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.BaseAnnotation;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;
+import com.iqser.red.service.redaction.v1.server.model.PrecursorEntity;
+import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
+import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
+import com.iqser.red.service.redaction.v1.server.service.DictionaryService;
+
+import lombok.AccessLevel;
+import lombok.experimental.FieldDefaults;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+@Service
+@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
+public class EntityFromPrecursorCreationService {
+
+    static double MATCH_THRESHOLD = 10; // Is compared to the average sum of distances in pdf coordinates for each corner of the bounding box of the entities
+    EntityFindingUtility entityFindingUtility;
+    EntityCreationService entityCreationService;
+    DictionaryService dictionaryService;
+
+
+    @Autowired
+    public EntityFromPrecursorCreationService(EntityEnrichmentService entityEnrichmentService, DictionaryService dictionaryService, EntityFindingUtility entityFindingUtility) {
+
+        this.entityFindingUtility = entityFindingUtility;
+        entityCreationService = new EntityCreationService(entityEnrichmentService);
+        this.dictionaryService = dictionaryService;
+    }
+
+
+    public List<PrecursorEntity> createEntitiesIfFoundAndReturnNotFoundEntries(ManualRedactions manualRedactions, SemanticNode node, String dossierTemplateId) {
+
+        Set<IdRemoval> idRemovals = manualRedactions.getIdsToRemove();
+        List<PrecursorEntity> manualEntities = manualRedactions.getEntriesToAdd()
+                .stream()
+                .filter(BaseAnnotation::isLocal)
+                .filter(manualRedactionEntry -> idRemovals.stream()
+                        .filter(idRemoval -> idRemoval.getAnnotationId().equals(manualRedactionEntry.getAnnotationId()))
+                        .filter(idRemoval -> idRemoval.getRequestDate().isBefore(manualRedactionEntry.getRequestDate()))
+                        .findAny()//
+                        .isEmpty())
+                .map(manualRedactionEntry -> //
+                             PrecursorEntity.fromManualRedactionEntry(manualRedactionEntry, dictionaryService.isHint(manualRedactionEntry.getType(), dossierTemplateId)))
+                .peek(manualEntity -> {
+                    if (manualEntity.getEntityType().equals(EntityType.HINT)) {
+                        manualEntity.skip("MAN.5.1", "manual hint is skipped by default");
+                    } else {
+                        manualEntity.apply("MAN.5.0", "manual entries are applied by default", manualEntity.getLegalBasis());
+                    }
+                })
+                .toList();
+        return toTextEntity(manualEntities, node);
+    }
+
+
+    public List<PrecursorEntity> createEntitiesIfFoundAndReturnNotFoundEntries(ImportedRedactions importedRedactions, SemanticNode node) {
+
+        List<PrecursorEntity> importedEntities = importedRedactions.getImportedRedactions()
+                .stream()
+                .map(PrecursorEntity::fromImportedEntry)
+                .peek(importedEntity -> importedEntity.apply("IMP.0.0", importedEntity.getReason(), importedEntity.getLegalBasis()))
+                .toList();
+        //we could add image handling here as well and create not only text entities
+        return toTextEntity(importedEntities, node);
+    }
+
+
+    public List<PrecursorEntity> toTextEntity(List<PrecursorEntity> precursorEntities, SemanticNode node) {
+
+        var notFoundEntities = precursorEntities.stream()
+                .filter(PrecursorEntity::isRectangle)
+                .collect(Collectors.toList());
+
+        var findableEntities = precursorEntities.stream()
+                .filter(precursorEntity -> !precursorEntity.isRectangle())
+                .toList();
+
+        Map<String, List<TextEntity>> tempEntitiesByValue = entityFindingUtility.findAllPossibleEntitiesAndGroupByValue(node, findableEntities);
+
+        for (PrecursorEntity precursorEntity : findableEntities) {
+            Optional<TextEntity> optionalClosestEntity = entityFindingUtility.findClosestEntityAndReturnEmptyIfNotFound(precursorEntity, tempEntitiesByValue, MATCH_THRESHOLD);
+            if (optionalClosestEntity.isEmpty()) {
+                notFoundEntities.add(precursorEntity);
+                continue;
+            }
+            createCorrectEntity(precursorEntity, optionalClosestEntity.get());
+        }
+
+        tempEntitiesByValue.values()
+                .stream()
+                .flatMap(Collection::stream)
+                .forEach(TextEntity::removeFromGraph);
+
+        return notFoundEntities;
+    }
+
+
+    /**
+     * Deletes the temp Entity and creates a RedactionEntity with correct values, based on the given parameters.
+     *
+     * @param precursorEntity The entity identifier for the RedactionEntity.
+     * @param closestEntity   The closest Boundary to the RedactionEntity.
+     */
+    public static TextEntity createCorrectEntity(PrecursorEntity precursorEntity, TextEntity closestEntity) {
+
+        return createCorrectEntity(precursorEntity, closestEntity, false);
+    }
+
+
+    public static TextEntity createCorrectEntity(PrecursorEntity precursorEntity, TextEntity closestEntity, boolean generateId) {
+
+        TextEntity correctEntity;
+        if (generateId) {
+            correctEntity = TextEntity.initialEntityNode(closestEntity.getTextRange(),
+                                                         precursorEntity.type(),
+                                                         precursorEntity.getEntityType(),
+                                                         closestEntity.getDeepestFullyContainingNode());
+        } else {
+            correctEntity = TextEntity.initialEntityNode(closestEntity.getTextRange(), precursorEntity.type(), precursorEntity.getEntityType(), precursorEntity.getId());
+        }
+        correctEntity.setDeepestFullyContainingNode(closestEntity.getDeepestFullyContainingNode());
+        correctEntity.setIntersectingNodes(new ArrayList<>(closestEntity.getIntersectingNodes()));
+        correctEntity.setDuplicateTextRanges(new ArrayList<>(closestEntity.getDuplicateTextRanges()));
+        correctEntity.setPages(new HashSet<>(closestEntity.getPages()));
+
+        correctEntity.setValue(closestEntity.getValue());
+        correctEntity.setTextAfter(closestEntity.getTextAfter());
+        correctEntity.setTextBefore(closestEntity.getTextBefore());
+
+        correctEntity.getIntersectingNodes()
+                .forEach(n -> n.getEntities().add(correctEntity));
+        correctEntity.getPages()
+                .forEach(page -> page.getEntities().add(correctEntity));
+
+        correctEntity.addMatchedRules(precursorEntity.getMatchedRuleList());
+        correctEntity.setDictionaryEntry(precursorEntity.isDictionaryEntry());
+        correctEntity.setDossierDictionaryEntry(precursorEntity.isDossierDictionaryEntry());
+        correctEntity.getManualOverwrite().addChanges(precursorEntity.getManualOverwrite().getManualChangeLog());
+        correctEntity.addEngines(precursorEntity.getEngines());
+        return correctEntity;
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/ImportedRedactionEntryService.java

@@ -0,0 +1,60 @@
+package com.iqser.red.service.redaction.v1.server.service.document;
+
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.stream.Stream;
+
+import org.springframework.stereotype.Service;
+
+import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedactions;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.BaseAnnotation;
+import com.iqser.red.service.redaction.v1.server.model.PrecursorEntity;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
+
+import io.micrometer.observation.annotation.Observed;
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+@Service
+@RequiredArgsConstructor
+public class ImportedRedactionEntryService {
+
+    private final EntityFromPrecursorCreationService entityFromPrecursorCreationService;
+
+
+    @Observed(name = "ImportedRedactionEntryService", contextualName = "add-imported-redaction-entries")
+    public List<PrecursorEntity> addImportedEntriesAndReturnNotFoundEntries(AnalyzeRequest analyzeRequest, ImportedRedactions importedRedactions, Document document) {
+
+        List<PrecursorEntity> notFoundImportedEntries = Collections.emptyList();
+        if (!importedRedactions.getImportedRedactions().isEmpty()) {
+            notFoundImportedEntries = entityFromPrecursorCreationService.createEntitiesIfFoundAndReturnNotFoundEntries(importedRedactions, document);
+            log.info("Added Imported entries for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
+        }
+
+        if (notFoundImportedEntries.isEmpty()) {
+            return Collections.emptyList();
+        }
+        List<BaseAnnotation> manualChanges = allManualChangesExceptAdd(analyzeRequest.getManualRedactions());
+        for (PrecursorEntity notFoundManualRedactionEntry : notFoundImportedEntries) {
+            manualChanges.stream()
+                    .filter(change -> change.getAnnotationId().equals(notFoundManualRedactionEntry.getId()))
+                    .forEach(change -> notFoundManualRedactionEntry.getManualOverwrite().addChange(change));
+        }
+
+        return notFoundImportedEntries;
+    }
+
+
+    private List<BaseAnnotation> allManualChangesExceptAdd(ManualRedactions manualRedactions) {
+
+        return Stream.of(manualRedactions.getForceRedactions(),
+                manualRedactions.getResizeRedactions(),
+                manualRedactions.getRecategorizations(),
+                manualRedactions.getIdsToRemove(),
+                manualRedactions.getLegalBasisChanges()).flatMap(Collection::stream).map(baseAnnotation -> (BaseAnnotation) baseAnnotation).toList();
+    }
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/IntersectingNodeVisitor.java

@@ -0,0 +1,31 @@
+package com.iqser.red.service.redaction.v1.server.service.document;
+
+import java.util.HashSet;
+import java.util.Set;
+
+import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
+
+import lombok.Getter;
+
+public class IntersectingNodeVisitor implements NodeVisitor {
+
+    @Getter
+    private Set<SemanticNode> intersectingNodes;
+    private final TextRange textRange;
+
+    public IntersectingNodeVisitor(TextRange textRange) {
+
+        this.textRange = textRange;
+        this.intersectingNodes = new HashSet<>();
+    }
+
+    @Override
+    public void visit(SemanticNode node) {
+
+        if (node.getTextRange().intersects(textRange)) {
+            intersectingNodes.add(node);
+        }
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/ManualEntityCreationService.java

@@ -1,115 +0,0 @@
-package com.iqser.red.service.redaction.v1.server.service.document;
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Optional;
-import java.util.Set;
-
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.stereotype.Service;
-
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.BaseAnnotation;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
-import com.iqser.red.service.redaction.v1.server.model.ManualEntity;
-import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
-import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
-import com.iqser.red.service.redaction.v1.server.model.document.entity.PositionOnPage;
-import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
-import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
-import com.iqser.red.service.redaction.v1.server.service.DictionaryService;
-
-import lombok.AccessLevel;
-import lombok.experimental.FieldDefaults;
-import lombok.extern.slf4j.Slf4j;
-
-@Slf4j
-@Service
-@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
-public class ManualEntityCreationService {
-
-    static double MATCH_THRESHOLD = 10; // Is compared to the average sum of distances in pdf coordinates for each corner of the bounding box of the entities
-    EntityFindingUtility entityFindingUtility;
-    EntityCreationService entityCreationService;
-    DictionaryService dictionaryService;
-
-
-    @Autowired
-    public ManualEntityCreationService(EntityEnrichmentService entityEnrichmentService, DictionaryService dictionaryService, EntityFindingUtility entityFindingUtility) {
-
-        this.entityFindingUtility = entityFindingUtility;
-        entityCreationService = new EntityCreationService(entityEnrichmentService);
-        this.dictionaryService = dictionaryService;
-    }
-
-
-    public List<ManualEntity> createRedactionEntitiesIfFoundAndReturnNotFoundEntries(ManualRedactions manualRedactions,
-                                                                                     SemanticNode node,
-                                                                                     String dossierTemplateId) {
-
-        Set<IdRemoval> idRemovals = manualRedactions.getIdsToRemove();
-        List<ManualEntity> manualEntities = manualRedactions.getEntriesToAdd().stream()
-                .filter(manualRedactionEntry -> !(idRemovals.stream().map(BaseAnnotation::getAnnotationId).toList().contains(manualRedactionEntry.getAnnotationId()) &&
-                        manualRedactionEntry.getRequestDate().isBefore(idRemovals.stream().filter(idRemoval -> idRemoval.getAnnotationId().equals(manualRedactionEntry.getAnnotationId())).findFirst().get().getRequestDate())))
-                .filter(manualRedactionEntry -> !(manualRedactionEntry.isAddToDictionary() || manualRedactionEntry.isAddToDossierDictionary()))
-                .map(manualRedactionEntry -> ManualEntity.fromManualRedactionEntry(manualRedactionEntry,
-                        dictionaryService.isHint(manualRedactionEntry.getType(), dossierTemplateId))).peek(manualEntity -> {
-                    if (manualEntity.getEntityType().equals(EntityType.HINT)) {
-                        manualEntity.skip("MAN.5.1", "manual hint is skipped by default");
-                    } else {
-                        manualEntity.apply("MAN.5.0", "manual entries are applied by default", manualEntity.getLegalBasis());
-                    }
-                })
-                .toList();
-
-        return toTextEntity(manualEntities, node);
-    }
-
-
-    public List<ManualEntity> toTextEntity(List<ManualEntity> manualEntities, SemanticNode node) {
-
-        Map<String, List<TextEntity>> tempEntitiesByValue = entityFindingUtility.findAllPossibleEntitiesAndGroupByValue(node, manualEntities);
-
-        List<ManualEntity> notFoundManualEntities = new LinkedList<>();
-        for (ManualEntity manualEntity : manualEntities) {
-            Optional<TextEntity> optionalRedactionEntity = entityFindingUtility.findClosestEntityAndReturnEmptyIfNotFound(manualEntity, tempEntitiesByValue, MATCH_THRESHOLD);
-            if (optionalRedactionEntity.isEmpty()) {
-                notFoundManualEntities.add(manualEntity);
-                continue;
-            }
-            createCorrectEntity(manualEntity, node, optionalRedactionEntity.get().getTextRange());
-        }
-        tempEntitiesByValue.values().stream().flatMap(Collection::stream).forEach(TextEntity::removeFromGraph);
-        return notFoundManualEntities;
-    }
-
-
-    /**
-     * Deletes the temp Entity and creates a RedactionEntity with correct values, based on the given parameters.
-     *
-     * @param manualEntity     The entity identifier for the RedactionEntity.
-     * @param node             The SemanticNode associated with the RedactionEntity.
-     * @param closestTextRange The closest Boundary to the RedactionEntity.
-     */
-    private void createCorrectEntity(ManualEntity manualEntity, SemanticNode node, TextRange closestTextRange) {
-
-        TextEntity correctEntity = entityCreationService.forceByTextRange(closestTextRange, manualEntity.getType(), manualEntity.getEntityType(), node);
-
-        correctEntity.addMatchedRules(manualEntity.getMatchedRuleList());
-        correctEntity.setDictionaryEntry(manualEntity.isDictionaryEntry());
-        correctEntity.setDossierDictionaryEntry(manualEntity.isDossierDictionaryEntry());
-        correctEntity.getManualOverwrite().addChanges(manualEntity.getManualOverwrite().getManualChangeLog());
-
-        // AnnotationIds must match the IDs in the add requests, or comments break. Maybe think about migrating IDs on the fly!
-        List<PositionOnPage> redactionPositionsWithIdOfManualOnPage = new ArrayList<>(correctEntity.getPositionsOnPagePerPage().size());
-        for (PositionOnPage positionOnPage : correctEntity.getPositionsOnPagePerPage()) {
-            redactionPositionsWithIdOfManualOnPage.add(new PositionOnPage(manualEntity.getId(), positionOnPage.getPage(), positionOnPage.getRectanglePerLine()));
-        }
-        correctEntity.setPositionsOnPagePerPage(redactionPositionsWithIdOfManualOnPage);
-    }
-
-}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/ManualRedactionEntryService.java

@@ -11,7 +11,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequ
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.BaseAnnotation;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
-import com.iqser.red.service.redaction.v1.server.model.ManualEntity;
+import com.iqser.red.service.redaction.v1.server.model.PrecursorEntity;
 
 import io.micrometer.observation.annotation.Observed;
 import lombok.RequiredArgsConstructor;
@@ -22,23 +22,24 @@ import lombok.extern.slf4j.Slf4j;
 @RequiredArgsConstructor
 public class ManualRedactionEntryService {
 
-    private final ManualEntityCreationService manualEntityCreationService;
+    private final EntityFromPrecursorCreationService entityFromPrecursorCreationService;
 
 
     @Observed(name = "ManualRedactionEntryService", contextualName = "add-manual-redaction-entries")
-    public List<ManualEntity> addManualRedactionEntriesAndReturnNotFoundEntries(AnalyzeRequest analyzeRequest, Document document, String dossierTemplateId) {
+    public List<PrecursorEntity> addManualRedactionEntriesAndReturnNotFoundEntries(AnalyzeRequest analyzeRequest, Document document, String dossierTemplateId) {
 
-        List<ManualEntity> notFoundManualRedactionEntries = Collections.emptyList();
+        List<PrecursorEntity> notFoundManualRedactionEntries = Collections.emptyList();
         if (analyzeRequest.getManualRedactions() != null) {
-            notFoundManualRedactionEntries = manualEntityCreationService.createRedactionEntitiesIfFoundAndReturnNotFoundEntries(analyzeRequest.getManualRedactions(),
+            notFoundManualRedactionEntries = entityFromPrecursorCreationService.createEntitiesIfFoundAndReturnNotFoundEntries(analyzeRequest.getManualRedactions(),
                     document, dossierTemplateId);
             log.info("Added Manual redaction entries for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
         }
+
         if (notFoundManualRedactionEntries.isEmpty()) {
             return Collections.emptyList();
         }
         List<BaseAnnotation> manualChanges = allManualChangesExceptAdd(analyzeRequest.getManualRedactions());
-        for (ManualEntity notFoundManualRedactionEntry : notFoundManualRedactionEntries) {
+        for (PrecursorEntity notFoundManualRedactionEntry : notFoundManualRedactionEntries) {
             manualChanges.stream()
                     .filter(change -> change.getAnnotationId().equals(notFoundManualRedactionEntry.getId()))
                     .forEach(change -> notFoundManualRedactionEntry.getManualOverwrite().addChange(change));

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/NodeVisitor.java

@@ -0,0 +1,8 @@
+package com.iqser.red.service.redaction.v1.server.service.document;
+
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
+
+public interface NodeVisitor {
+
+    void visit(SemanticNode node);
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/SectionFinderService.java

@@ -13,6 +13,9 @@ import org.springframework.stereotype.Service;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedaction;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedactions;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;
@@ -40,7 +43,11 @@ import lombok.extern.slf4j.Slf4j;
 public class SectionFinderService {
 
     @Timed("redactmanager_findSectionsToReanalyse")
-    public Set<Integer> findSectionsToReanalyse(DictionaryIncrement dictionaryIncrement, EntityLog entityLog, Document document, AnalyzeRequest analyzeRequest) {
+    public Set<Integer> findSectionsToReanalyse(DictionaryIncrement dictionaryIncrement,
+                                                EntityLog entityLog,
+                                                Document document,
+                                                AnalyzeRequest analyzeRequest,
+                                                ImportedRedactions importedRedactions) {
 
         long start = System.currentTimeMillis();
         Set<String> relevantManuallyModifiedAnnotationIds = getRelevantManuallyModifiedAnnotationIds(analyzeRequest.getManualRedactions());
@@ -50,20 +57,26 @@ public class SectionFinderService {
                 if (entry.getContainingNodeId().isEmpty()) {
                     continue; // Empty list means either Entity has not been found or it is between main sections. Thus, this might lead to wrong reanalysis.
                 }
-                sectionsToReanalyse.add(entry.getContainingNodeId().get(0));
+                sectionsToReanalyse.add(entry.getContainingNodeId()
+                                                .get(0));
             }
         }
 
-        var dictionaryIncrementsSearch = new SearchImplementation(dictionaryIncrement.getValues().stream().map(DictionaryIncrementValue::getValue).collect(Collectors.toList()),
-                true);
+        var dictionaryIncrementsSearch = new SearchImplementation(dictionaryIncrement.getValues()
+                                                                          .stream()
+                                                                          .map(DictionaryIncrementValue::getValue)
+                                                                          .toList(), true);
 
-        document.streamChildren().forEach(mainNode -> {
-            if (dictionaryIncrementsSearch.atLeastOneMatches(mainNode.getTextBlock().getSearchText())) {
-                sectionsToReanalyse.add(mainNode.getTreeId().get(0));
-            }
-        });
+        document.streamChildren()
+                .forEach(mainNode -> {
+                    if (dictionaryIncrementsSearch.atLeastOneMatches(mainNode.getTextBlock().getSearchText())) {
+                        sectionsToReanalyse.add(mainNode.getTreeId()
+                                                        .get(0));
+                    }
+                });
 
         Set<Integer> relevantPagesForReanalysis = getRelevantPageNumbersForAddRedactions(analyzeRequest);
+        relevantPagesForReanalysis.addAll(getRelevantPageNumbersForImportedRedactions(importedRedactions, relevantManuallyModifiedAnnotationIds));
         if (!relevantPagesForReanalysis.isEmpty()) {
             sectionsToReanalyse.addAll(getSectionNumbersOnPages(document, relevantPagesForReanalysis));
         }
@@ -79,8 +92,11 @@ public class SectionFinderService {
         return document.getPages()
                 .stream()
                 .filter(page -> relevantPagesForReanalysis.contains(page.getNumber()))
-                .flatMap(page -> Stream.concat(page.getMainBody().stream().filter(node -> node.getType().equals(NodeType.SECTION)), Stream.of(page.getHeader(), page.getFooter())))
-                .map(node -> node.getTreeId().get(0))
+                .flatMap(page -> Stream.concat(page.getMainBody()
+                                                       .stream()
+                                                       .filter(node -> node.getType().equals(NodeType.SECTION)), Stream.of(page.getHeader(), page.getFooter())))
+                .map(node -> node.getTreeId()
+                        .get(0))
                 .toList();
     }
 
@@ -91,8 +107,7 @@ public class SectionFinderService {
             return Collections.emptySet();
         }
 
-        return analyzeRequest.getManualRedactions()
-                .getEntriesToAdd()
+        return analyzeRequest.getManualRedactions().getEntriesToAdd()
                 .stream()
                 .filter(addRedaction -> !(addRedaction.isAddToDictionary() || addRedaction.isAddToDossierDictionary()))
                 .map(ManualRedactionEntry::getPositions)
@@ -102,17 +117,44 @@ public class SectionFinderService {
     }
 
 
+    private static Set<Integer> getRelevantPageNumbersForImportedRedactions(ImportedRedactions importedRedactions, Set<String> idsWithUnprocessedManualChanges) {
+
+        if (importedRedactions == null || importedRedactions.getImportedRedactions() == null) {
+            return Collections.emptySet();
+        }
+
+        return importedRedactions.getImportedRedactions()
+                .stream()
+                .filter(importedRedaction -> idsWithUnprocessedManualChanges.contains(importedRedaction.getId()))
+                .map(ImportedRedaction::getPositions)
+                .flatMap(Collection::stream)
+                .map(Position::getPageNumber)
+                .collect(Collectors.toSet());
+    }
+
+
     private static Set<String> getRelevantManuallyModifiedAnnotationIds(ManualRedactions manualRedactions) {
 
         if (manualRedactions == null) {
             return new HashSet<>();
         }
 
-        return Stream.concat(manualRedactions.getResizeRedactions().stream().map(ManualResizeRedaction::getAnnotationId),
-                Stream.concat(manualRedactions.getLegalBasisChanges().stream().map(ManualLegalBasisChange::getAnnotationId),
-                        Stream.concat(manualRedactions.getRecategorizations().stream().map(ManualRecategorization::getAnnotationId),
-                                Stream.concat(manualRedactions.getIdsToRemove().stream().map(IdRemoval::getAnnotationId),
-                                        manualRedactions.getForceRedactions().stream().map(ManualForceRedaction::getAnnotationId))))).collect(Collectors.toSet());
+        return Stream.concat(manualRedactions.getResizeRedactions()
+                                     .stream()
+                                     .map(ManualResizeRedaction::getAnnotationId),
+                             Stream.concat(manualRedactions.getLegalBasisChanges()
+                                                   .stream()
+                                                   .map(ManualLegalBasisChange::getAnnotationId),
+                                           Stream.concat(manualRedactions.getRecategorizations()
+                                                                 .stream()
+                                                                 .map(ManualRecategorization::getAnnotationId),
+                                                         Stream.concat(manualRedactions.getIdsToRemove()
+                                                                               .stream()
+                                                                               .map(IdRemoval::getAnnotationId),
+                                                                       manualRedactions.getForceRedactions()
+                                                                               .stream()
+                                                                               .map(ManualForceRedaction::getAnnotationId)))))
+                .collect(Collectors.toSet());
     }
 
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/DroolsSyntaxValidationService.java

@@ -1,6 +1,9 @@
 package com.iqser.red.service.redaction.v1.server.service.drools;
 
+import java.util.ArrayList;
+import java.util.Comparator;
 import java.util.List;
+import java.util.Map;
 import java.util.Set;
 import java.util.regex.Pattern;
 import java.util.stream.Collectors;
@@ -12,24 +15,34 @@ import org.springframework.stereotype.Service;
 
 import com.google.common.collect.Sets;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType;
+import com.iqser.red.service.redaction.v1.model.DroolsSyntaxDeprecatedWarnings;
 import com.iqser.red.service.redaction.v1.model.DroolsSyntaxErrorMessage;
 import com.iqser.red.service.redaction.v1.model.DroolsSyntaxValidation;
 import com.iqser.red.service.redaction.v1.model.RuleValidationModel;
+import com.iqser.red.service.redaction.v1.server.DeprecatedElementsFinder;
+import com.iqser.red.service.redaction.v1.server.model.dictionary.SearchImplementation;
 import com.iqser.red.service.redaction.v1.server.model.drools.BasicQuery;
+import com.iqser.red.service.redaction.v1.server.model.drools.BasicRule;
+import com.iqser.red.service.redaction.v1.server.model.drools.RuleClass;
 import com.iqser.red.service.redaction.v1.server.model.drools.RuleFileBluePrint;
+import com.iqser.red.service.redaction.v1.server.model.drools.RuleUnit;
 import com.iqser.red.service.redaction.v1.server.storage.RuleManagementResources;
 
 import lombok.RequiredArgsConstructor;
 import lombok.SneakyThrows;
+import lombok.extern.slf4j.Slf4j;
 
 @Service
 @RequiredArgsConstructor
+@Slf4j
 public class DroolsSyntaxValidationService {
-    
+
     private static final Pattern allowedImportsPattern = Pattern.compile("^(?:import\\s+static\\s+)?(?:import\\s+)?(?:com\\.knecon\\.fforesight|com\\.iqser\\.red)\\..*;$");
 
     private final KieContainerCreationService kieContainerCreationService;
 
+    private final DeprecatedElementsFinder deprecatedElementsFinder;
+
 
     @SneakyThrows
     public DroolsSyntaxValidation testRules(RuleValidationModel rules) {
@@ -43,15 +56,74 @@ public class DroolsSyntaxValidationService {
         }
         DroolsSyntaxValidation droolsCompilerSyntaxValidation = buildDroolsCompilerSyntaxValidation(rules);
         droolsCompilerSyntaxValidation.getDroolsSyntaxErrorMessages().addAll(customDroolsSyntaxValidation.getDroolsSyntaxErrorMessages());
+        droolsCompilerSyntaxValidation.getDroolsSyntaxDeprecatedWarnings().addAll(customDroolsSyntaxValidation.getDroolsSyntaxDeprecatedWarnings());
         return droolsCompilerSyntaxValidation;
     }
 
+    private DroolsSyntaxDeprecatedWarnings getWarningsForDeprecatedImports(RuleFileBluePrint ruleFileBluePrint) {
+
+        if (!deprecatedElementsFinder.getDeprecatedClasses().isEmpty()) {
+            String imports = ruleFileBluePrint.getImports();
+            SearchImplementation classesSearchImplementation = deprecatedElementsFinder.getClassesSearchImplementation();
+            List<SearchImplementation.MatchPosition> matches = classesSearchImplementation.getMatches(imports);
+            if (!matches.isEmpty()) {
+                String sb = "Following imports are deprecated: \n" + matches.stream()
+                        .map(m -> imports.substring(m.startIndex(), m.endIndex()))
+                        .collect(Collectors.joining("\n"));
+                return DroolsSyntaxDeprecatedWarnings.builder().line(ruleFileBluePrint.getImportLine()).column(0).message(sb)
+                                            .build();
+            }
+        }
+        return null;
+    }
+
+    private List<DroolsSyntaxDeprecatedWarnings> getWarningsForDeprecatedRules(RuleFileBluePrint ruleFileBluePrint) {
+
+        List<DroolsSyntaxDeprecatedWarnings> warningMessages = new ArrayList<>();
+
+        // checks the rules for deprecateMethods
+        if (!deprecatedElementsFinder.getDeprecatedMethods().isEmpty()) {
+            SearchImplementation methodsSearchImplementation = deprecatedElementsFinder.getMethodsSearchImplementation();
+            Map<String, String> deprecatedMethodsSignatureMap = deprecatedElementsFinder.getDeprecatedMethodsSignaturesMap();
+
+            for (RuleClass ruleClass : ruleFileBluePrint.getRuleClasses()) {
+                for (RuleUnit ruleUnit : ruleClass.ruleUnits()) {
+                    for (BasicRule basicRule : ruleUnit.rules()) {
+                        List<SearchImplementation.MatchPosition> matches = methodsSearchImplementation.getMatches(basicRule.getCode());
+                        if (!matches.isEmpty()) {
+                            String warningMessage = matches.stream()
+                                    .map(m -> basicRule.getCode().substring(m.startIndex(), m.endIndex()))
+                                    .distinct()
+                                    .map(dm -> String.format("Method %s might be deprecated because of \n %s \n", dm, deprecatedMethodsSignatureMap.get(dm)))
+                                    .collect(Collectors.joining("\n"));
+                            warningMessages.add(DroolsSyntaxDeprecatedWarnings.builder().line(basicRule.getLine()).column(0).message(warningMessage)
+                                                        .build());
+                        }
+                    }
+                }
+            }
+
+        }
+
+        return warningMessages.stream()
+                .sorted(Comparator.comparingInt(DroolsSyntaxDeprecatedWarnings::getLine))
+                .collect(Collectors.toList());
+    }
+
 
     private DroolsSyntaxValidation buildCustomDroolsSyntaxValidation(String ruleString, RuleFileType ruleFileType) throws DroolsParserException {
 
         RuleFileBluePrint ruleFileBluePrint = RuleFileParser.buildBluePrintFromRulesString(ruleString);
+
         DroolsSyntaxValidation customSyntaxValidation = ruleFileBluePrint.getDroolsSyntaxValidation();
 
+        // find deprecated elements in the ruleFileBluePrint
+        DroolsSyntaxDeprecatedWarnings warningMessageForImports = getWarningsForDeprecatedImports(ruleFileBluePrint);
+        if (warningMessageForImports != null) {
+            customSyntaxValidation.getDroolsSyntaxDeprecatedWarnings().add(warningMessageForImports);
+        }
+        customSyntaxValidation.getDroolsSyntaxDeprecatedWarnings().addAll(getWarningsForDeprecatedRules(ruleFileBluePrint));
+
         RuleFileBluePrint baseRuleFileBluePrint = switch (ruleFileType) {
             case ENTITY -> RuleFileParser.buildBluePrintFromRulesString(RuleManagementResources.getBaseRuleFileString());
             case COMPONENT -> RuleFileParser.buildBluePrintFromRulesString(RuleManagementResources.getBaseComponentRuleFileString());
@@ -60,35 +132,36 @@ public class DroolsSyntaxValidationService {
         if (!importsAreValid(baseRuleFileBluePrint, ruleFileBluePrint)) {
             customSyntaxValidation.getDroolsSyntaxErrorMessages()
                     .add(DroolsSyntaxErrorMessage.builder()
-                            .line(ruleFileBluePrint.getImportLine())
-                            .column(0)
-                            .message(String.format("Changing the imports is not allowed! Must be: %n%s", baseRuleFileBluePrint.getImports()))
-                            .build());
+                                 .line(ruleFileBluePrint.getImportLine())
+                                 .column(0)
+                                 .message(String.format("Changing the imports is not allowed! Must be: %n%s", baseRuleFileBluePrint.getImports()))
+                                 .build());
         }
         if (!ruleFileBluePrint.getGlobals().equals(baseRuleFileBluePrint.getGlobals())) {
             customSyntaxValidation.getDroolsSyntaxErrorMessages()
                     .add(DroolsSyntaxErrorMessage.builder()
-                            .line(ruleFileBluePrint.getGlobalsLine())
-                            .column(0)
-                            .message(String.format("Changing the globals is not allowed! Must be: %n%s", baseRuleFileBluePrint.getGlobals()))
-                            .build());
+                                 .line(ruleFileBluePrint.getGlobalsLine())
+                                 .column(0)
+                                 .message(String.format("Changing the globals is not allowed! Must be: %n%s", baseRuleFileBluePrint.getGlobals()))
+                                 .build());
         }
-        baseRuleFileBluePrint.getQueries().forEach(basicQuery -> {
-            if (!validateQueryIsPresent(basicQuery, ruleFileBluePrint)) {
-                customSyntaxValidation.getDroolsSyntaxErrorMessages()
-                        .add(DroolsSyntaxErrorMessage.builder()
-                                .line(basicQuery.getLine())
-                                .column(0)
-                                .message(String.format("Changing or removing the query %s is not allowed! Must be: %n%s", basicQuery.getName(), basicQuery.getCode()))
-                                .build());
-            }
-        });
+        baseRuleFileBluePrint.getQueries()
+                .forEach(basicQuery -> {
+                    if (!validateQueryIsPresent(basicQuery, ruleFileBluePrint)) {
+                        customSyntaxValidation.getDroolsSyntaxErrorMessages()
+                                .add(DroolsSyntaxErrorMessage.builder()
+                                             .line(basicQuery.getLine())
+                                             .column(0)
+                                             .message(String.format("Changing or removing the query %s is not allowed! Must be: %n%s", basicQuery.getName(), basicQuery.getCode()))
+                                             .build());
+                    }
+                });
         if (ruleFileType.equals(RuleFileType.ENTITY)) {
             String requiredAgendaGroup = "LOCAL_DICTIONARY_ADDS";
             if (!validateAgendaGroupIsPresent(ruleFileBluePrint, requiredAgendaGroup)) {
-                customSyntaxValidation.getDroolsSyntaxErrorMessages().add(DroolsSyntaxErrorMessage.builder().line(0)
-                                .column(0).message(String.format("At least one rule with Agenda-Group '%s' required!", requiredAgendaGroup))
-                                .build());
+                customSyntaxValidation.getDroolsSyntaxErrorMessages()
+                        .add(DroolsSyntaxErrorMessage.builder().line(0).column(0).message(String.format("At least one rule with Agenda-Group '%s' required!", requiredAgendaGroup))
+                                     .build());
             }
         }
         return customSyntaxValidation;
@@ -97,7 +170,8 @@ public class DroolsSyntaxValidationService {
 
     private boolean validateAgendaGroupIsPresent(RuleFileBluePrint ruleFileBluePrint, String agendaGroupName) {
 
-        return ruleFileBluePrint.streamAllRules().anyMatch(basicRule -> basicRule.getAgendaGroup().equals(agendaGroupName));
+        return ruleFileBluePrint.streamAllRules()
+                .anyMatch(basicRule -> basicRule.getAgendaGroup().equals(agendaGroupName));
     }
 
 
@@ -109,13 +183,16 @@ public class DroolsSyntaxValidationService {
 
         Set<String> additionalImports = Sets.difference(imports, baseImports);
 
-        return additionalImports.stream().allMatch(additionalImport -> allowedImportsPattern.matcher(additionalImport).matches());
+        return additionalImports.stream()
+                .allMatch(additionalImport -> allowedImportsPattern.matcher(additionalImport).matches());
     }
 
 
     private static boolean validateQueryIsPresent(BasicQuery queryToCheckFor, RuleFileBluePrint ruleFileBluePrint) {
 
-        return ruleFileBluePrint.getQueries().stream().anyMatch(query -> query.getName().equals(queryToCheckFor.getName()) && query.getCode().equals(queryToCheckFor.getCode()));
+        return ruleFileBluePrint.getQueries()
+                .stream()
+                .anyMatch(query -> query.getName().equals(queryToCheckFor.getName()) && query.getCode().equals(queryToCheckFor.getCode()));
     }
 
 
@@ -124,9 +201,9 @@ public class DroolsSyntaxValidationService {
         var versionId = System.currentTimeMillis();
         var testRules = "test-rules";
         KieBuilder kieBuilder = kieContainerCreationService.registerNewKieContainerVersion(testRules,
-                versionId,
-                rules.getRulesString(),
-                RuleFileType.valueOf(rules.getRuleFileType()));
+                                                                                           versionId,
+                                                                                           rules.getRulesString(),
+                                                                                           RuleFileType.valueOf(rules.getRuleFileType()));
         return buildDroolsCompilerSyntaxValidation(kieBuilder);
     }
 
@@ -134,14 +211,18 @@ public class DroolsSyntaxValidationService {
     private DroolsSyntaxValidation buildDroolsCompilerSyntaxValidation(KieBuilder kieBuilder) {
 
         List<Message> errorMessages = kieBuilder.getResults().getMessages(Message.Level.ERROR);
-        List<DroolsSyntaxErrorMessage> droolsSyntaxErrorMessages = errorMessages.stream().map(this::buildDroolsSyntaxErrorMessage).collect(Collectors.toList());
-        return DroolsSyntaxValidation.builder().droolsSyntaxErrorMessages(droolsSyntaxErrorMessages).build();
+        List<DroolsSyntaxErrorMessage> droolsSyntaxErrorMessages = errorMessages.stream()
+                .map(this::buildDroolsSyntaxErrorMessage)
+                .collect(Collectors.toList());
+        return DroolsSyntaxValidation.builder().droolsSyntaxErrorMessages(droolsSyntaxErrorMessages)
+                .build();
     }
 
 
     private DroolsSyntaxErrorMessage buildDroolsSyntaxErrorMessage(Message message) {
 
-        return DroolsSyntaxErrorMessage.builder().line(message.getLine()).column(message.getColumn()).message(message.getText()).build();
+        return DroolsSyntaxErrorMessage.builder().line(message.getLine()).column(message.getColumn()).message(message.getText())
+                .build();
     }
 
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/EntityDroolsExecutionService.java

@@ -16,6 +16,7 @@ import org.springframework.stereotype.Service;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.BaseAnnotation;
 import com.iqser.red.service.redaction.v1.server.RedactionServiceSettings;
 import com.iqser.red.service.redaction.v1.server.model.NerEntities;
 import com.iqser.red.service.redaction.v1.server.model.dictionary.Dictionary;
@@ -42,7 +43,7 @@ public class EntityDroolsExecutionService {
 
     EntityEnrichmentService entityEnrichmentService;
     ObservationRegistry observationRegistry;
-
+    ManualChangesApplicationService manualChangesApplicationService;
     RedactionServiceSettings settings;
 
 
@@ -55,7 +56,14 @@ public class EntityDroolsExecutionService {
                                             ManualRedactions manualRedactions,
                                             NerEntities nerEntities) {
 
-        return executeRules(kieContainer, document, document.streamChildren().toList(), dictionary, fileAttributes, manualRedactions, nerEntities);
+        return executeRules(kieContainer,
+                            document,
+                            document.streamChildren()
+                                    .toList(),
+                            dictionary,
+                            fileAttributes,
+                            manualRedactions,
+                            nerEntities);
     }
 
 
@@ -73,7 +81,6 @@ public class EntityDroolsExecutionService {
 
         KieSession kieSession = kieContainer.newKieSession();
         EntityCreationService entityCreationService = new EntityCreationService(entityEnrichmentService, kieSession);
-        ManualChangesApplicationService manualChangesApplicationService = new ManualChangesApplicationService(entityCreationService);
 
         kieSession.setGlobal("document", document);
         kieSession.setGlobal("entityCreationService", entityCreationService);
@@ -81,19 +88,28 @@ public class EntityDroolsExecutionService {
         kieSession.setGlobal("dictionary", dictionary);
 
         kieSession.insert(document);
-        document.getEntities().forEach(kieSession::insert);
+
+        document.getEntities()
+                .forEach(kieSession::insert);
+
         sectionsToAnalyze.forEach(kieSession::insert);
-        sectionsToAnalyze.stream().flatMap(SemanticNode::streamAllSubNodes).forEach(kieSession::insert);
-        document.getPages().forEach(kieSession::insert);
-        fileAttributes.stream().filter(f -> f.getValue() != null).forEach(kieSession::insert);
+
+        sectionsToAnalyze.stream()
+                .flatMap(SemanticNode::streamAllSubNodes)
+                .forEach(kieSession::insert);
+
+        document.getPages()
+                .forEach(kieSession::insert);
+
+        fileAttributes.stream()
+                .filter(f -> f.getValue() != null)
+                .forEach(kieSession::insert);
 
         if (manualRedactions != null) {
-            manualRedactions.getResizeRedactions().stream().filter(manualResizeRedaction -> !manualResizeRedaction.getUpdateDictionary()).forEach(kieSession::insert);
-            manualRedactions.getRecategorizations().forEach(kieSession::insert);
-            manualRedactions.getEntriesToAdd().forEach(kieSession::insert);
-            manualRedactions.getForceRedactions().forEach(kieSession::insert);
-            manualRedactions.getIdsToRemove().forEach(kieSession::insert);
-            manualRedactions.getLegalBasisChanges().forEach(kieSession::insert);
+            manualRedactions.buildAll()
+                    .stream()
+                    .filter(BaseAnnotation::isLocal)
+                    .forEach(kieSession::insert);
         }
 
         kieSession.insert(nerEntities);
@@ -106,7 +122,8 @@ public class EntityDroolsExecutionService {
         });
 
         try {
-            completableFuture.orTimeout(settings.getDroolsExecutionTimeoutSecs(), TimeUnit.SECONDS).get();
+            completableFuture.orTimeout(settings.getDroolsExecutionTimeoutSecs(), TimeUnit.SECONDS)
+                    .get();
         } catch (ExecutionException e) {
             kieSession.dispose();
             if (e.getCause() instanceof TimeoutException) {

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/RedactionStorageService.java

@@ -3,6 +3,7 @@ package com.iqser.red.service.redaction.v1.server.storage;
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.InputStream;
+import java.util.List;
 import java.util.stream.Collectors;
 
 import org.springframework.cache.annotation.Cacheable;
@@ -11,6 +12,7 @@ import org.springframework.stereotype.Service;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentLog;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedactions;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedactionsPerPage;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog;
 import com.iqser.red.service.redaction.v1.server.client.model.NerEntitiesModel;
@@ -76,10 +78,26 @@ public class RedactionStorageService {
     @Timed("redactmanager_getImportedRedactions")
     public ImportedRedactions getImportedRedactions(String dossierId, String fileId) {
 
+        try {
+            ImportedRedactionsPerPage importedRedactionsPerPage = storageService.readJSONObject(TenantContext.getTenantId(),
+                    StorageIdUtils.getStorageId(dossierId, fileId, FileType.IMPORTED_REDACTIONS),
+                    ImportedRedactionsPerPage.class);
+            return new ImportedRedactions(importedRedactionsPerPage.getImportedRedactions().values().stream().flatMap(List::stream).collect(Collectors.toList()));
+        } catch (StorageObjectDoesNotExist e) {
+            log.debug("Imported redactions not available.");
+            return new ImportedRedactions();
+        }
+    }
+
+
+
+    @Timed("redactmanager_getImportedRedactions")
+    public ImportedRedactionsPerPage getImportedRedactionsPerPage(String dossierId, String fileId) {
+
         try {
             return storageService.readJSONObject(TenantContext.getTenantId(),
                     StorageIdUtils.getStorageId(dossierId, fileId, FileType.IMPORTED_REDACTIONS),
-                    ImportedRedactions.class);
+                    ImportedRedactionsPerPage.class);
         } catch (StorageObjectDoesNotExist e) {
             log.debug("Imported redactions not available.");
             return null;

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/utils/MigratedIdsCollector.java

@@ -1,5 +1,6 @@
 package com.iqser.red.service.redaction.v1.server.utils;
 
+import java.util.Collections;
 import java.util.LinkedList;
 import java.util.Set;
 import java.util.function.BiConsumer;
@@ -17,7 +18,7 @@ public class MigratedIdsCollector implements Collector<MigrationEntity, Migrated
     @Override
     public Supplier<MigratedIds> supplier() {
 
-        return () -> new MigratedIds(new LinkedList<>());
+        return () -> new MigratedIds(new LinkedList<>(), Collections.emptyList());
     }
 
 

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AbstractRedactionIntegrationTest.java

@@ -26,6 +26,7 @@ import org.springframework.core.io.ClassPathResource;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.iqser.red.service.dictionarymerge.commons.DictionaryEntry;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.configuration.Colors;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type;
@@ -474,6 +475,7 @@ public abstract class AbstractRedactionIntegrationTest {
                 .dossierId(TEST_DOSSIER_ID)
                 .fileId(TEST_FILE_ID)
                 .lastProcessed(OffsetDateTime.now())
+                .manualRedactions(ManualRedactions.builder().build())
                 .build();
 
         storageService.storeObject(TenantContext.getTenantId(), RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.ORIGIN), fileStream);

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/MigrationIntegrationTest.java

@@ -2,11 +2,15 @@ package com.iqser.red.service.redaction.v1.server;
 
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.Mockito.when;
 
+import java.awt.geom.Rectangle2D;
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.FileOutputStream;
+import java.io.IOException;
 import java.nio.file.Path;
+import java.util.Collections;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Locale;
@@ -15,7 +19,7 @@ import java.util.Objects;
 import java.util.Set;
 import java.util.stream.Collectors;
 
-import org.junit.jupiter.api.Disabled;
+import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.extension.ExtendWith;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -27,19 +31,26 @@ import org.springframework.test.context.junit.jupiter.SpringExtension;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.migration.MigratedIds;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
 import com.iqser.red.service.redaction.v1.server.annotate.AnnotateRequest;
 import com.iqser.red.service.redaction.v1.server.annotate.AnnotateResponse;
 import com.iqser.red.service.redaction.v1.server.document.graph.BuildDocumentIntegrationTest;
+import com.iqser.red.service.redaction.v1.server.migration.LegacyRedactionLogMergeService;
+import com.iqser.red.service.redaction.v1.server.migration.RedactionLogToEntityLogMigrationService;
 import com.iqser.red.service.redaction.v1.server.model.MigratedEntityLog;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.OsUtils;
-import com.iqser.red.service.redaction.v1.server.migration.RedactionLogToEntityLogMigrationService;
+import com.iqser.red.service.redaction.v1.server.service.DictionaryService;
+import com.iqser.red.service.redaction.v1.server.service.document.EntityFindingUtility;
+import com.knecon.fforesight.tenantcommons.TenantContext;
 
 import lombok.SneakyThrows;
 
@@ -52,15 +63,51 @@ public class MigrationIntegrationTest extends BuildDocumentIntegrationTest {
     @Autowired
     RedactionLogToEntityLogMigrationService redactionLogToEntityLogMigrationService;
 
+    @Autowired
+    LegacyRedactionLogMergeService legacyRedactionLogMergeService;
+
+    @Autowired
+    DictionaryService dictionaryService;
+
     @Autowired
     ObjectMapper mapper;
 
+    private final String TEST_DOSSIER_TEMPLATE_ID = "123";
+
+
+    @BeforeEach
+    public void stubClients() {
+
+        TenantContext.setTenantId("redaction");
+
+        loadDictionaryForTest();
+        loadTypeForTest();
+        when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L);
+        when(dictionaryClient.getAllTypesForDossierTemplate(TEST_DOSSIER_TEMPLATE_ID, true)).thenReturn(getTypeResponse());
+
+        when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L);
+        when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, true)).thenReturn(List.of(Type.builder()
+                                                                                                       .id(DOSSIER_REDACTIONS_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID)
+                                                                                                       .type(DOSSIER_REDACTIONS_INDICATOR)
+                                                                                                       .dossierTemplateId(TEST_DOSSIER_ID)
+                                                                                                       .hexColor("#ffe187")
+                                                                                                       .isHint(hintTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
+                                                                                                       .isCaseInsensitive(caseInSensitiveMap.get(DOSSIER_REDACTIONS_INDICATOR))
+                                                                                                       .isRecommendation(recommendationTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
+                                                                                                       .rank(rankTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
+                                                                                                       .build()));
+
+        mockDictionaryCalls(null);
+
+        when(dictionaryClient.getColors(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(colors);
+    }
+
 
     @Test
     @SneakyThrows
     public void testSave() {
 
-        MigratedIds ids = new MigratedIds(new LinkedList<>());
+        MigratedIds ids = new MigratedIds(new LinkedList<>(), null);
         ids.addMapping("123", "321");
         ids.addMapping("123", "321");
         ids.addMapping("123", "321");
@@ -77,33 +124,76 @@ public class MigrationIntegrationTest extends BuildDocumentIntegrationTest {
     @SneakyThrows
     public void testMigration() {
 
-        String filesPrefix = "files/migration/def8f960580f088b975ba806dfae1f87";
+        String filesPrefix = "files/migration/test";
 
         String fileName = filesPrefix + ".ORIGIN.pdf";
         String imageFileName = filesPrefix + ".IMAGE_INFO.json";
         String tableFileName = filesPrefix + ".TABLES.json";
+        String manualChangesFileName = filesPrefix + ".MANUAL_CHANGES.json";
+        String migratedIdsFileName = filesPrefix + ".MIGRATED_IDS.json";
 
         Document document = buildGraph(fileName, imageFileName, tableFileName);
         RedactionLog redactionLog;
+        RedactionLog mergedRedactionLog;
+
+        dictionaryService.updateDictionary(TEST_DOSSIER_TEMPLATE_ID, TEST_DOSSIER_ID);
+
         try (var in = new ClassPathResource(filesPrefix + ".REDACTION_LOG.json").getInputStream()) {
             redactionLog = mapper.readValue(in, RedactionLog.class);
         }
-        MigratedEntityLog migratedEntityLog = redactionLogToEntityLogMigrationService.migrate(redactionLog, document);
+        var manualChangesResource = new ClassPathResource(manualChangesFileName);
+        ManualRedactions manualRedactions;
+        if (manualChangesResource.exists()) {
+            try (var in = manualChangesResource.getInputStream()) {
+                manualRedactions = mapper.readValue(in, ManualRedactions.class);
+                if (manualRedactions.getEntriesToAdd() == null) {
+                    manualRedactions.setEntriesToAdd(Collections.emptySet());
+                }
+                if (manualRedactions.getForceRedactions() == null) {
+                    manualRedactions.setForceRedactions(Collections.emptySet());
+                }
+                if (manualRedactions.getIdsToRemove() == null) {
+                    manualRedactions.setIdsToRemove(Collections.emptySet());
+                }
+                if (manualRedactions.getLegalBasisChanges() == null) {
+                    manualRedactions.setLegalBasisChanges(Collections.emptySet());
+                }
+                if (manualRedactions.getRecategorizations() == null) {
+                    manualRedactions.setRecategorizations(Collections.emptySet());
+                }
+                if (manualRedactions.getResizeRedactions() == null) {
+                    manualRedactions.setResizeRedactions(Collections.emptySet());
+                }
+            }
+            MigratedIds migratedIds = getMigratedIds(migratedIdsFileName);
+            revertMigration(manualRedactions, migratedIds);
+            mergedRedactionLog = legacyRedactionLogMergeService.addManualAddEntriesAndRemoveSkippedImported(redactionLog, manualRedactions, TEST_DOSSIER_TEMPLATE_ID);
+        } else {
+            manualRedactions = new ManualRedactions();
+            mergedRedactionLog = redactionLog;
+        }
+
+        MigratedEntityLog migratedEntityLog = redactionLogToEntityLogMigrationService.migrate(mergedRedactionLog,
+                                                                                              document,
+                                                                                              TEST_DOSSIER_TEMPLATE_ID,
+                                                                                              manualRedactions,
+                                                                                              TEST_FILE_ID);
 
         redactionStorageService.storeObject(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.ENTITY_LOG, migratedEntityLog.getEntityLog());
-        assertEquals(redactionLog.getRedactionLogEntry().size(), migratedEntityLog.getEntityLog().getEntityLogEntry().size());
-        assertEquals(redactionLog.getRedactionLogEntry().stream().filter(entry -> !entry.getManualChanges().isEmpty()).count(), migratedEntityLog.getMigratedIds().getMappings().size());
+        assertEquals(mergedRedactionLog.getRedactionLogEntry().size(), migratedEntityLog.getEntityLog().getEntityLogEntry().size());
         EntityLog entityLog = migratedEntityLog.getEntityLog();
-        assertEquals(redactionLog.getAnalysisNumber(), entityLog.getAnalysisNumber());
-        assertEquals(redactionLog.getAnalysisVersion(), entityLog.getAnalysisVersion());
-        assertEquals(redactionLog.getDictionaryVersion(), entityLog.getDictionaryVersion());
-        assertEquals(redactionLog.getDossierDictionaryVersion(), entityLog.getDossierDictionaryVersion());
-        assertEquals(redactionLog.getLegalBasisVersion(), entityLog.getLegalBasisVersion());
-        assertEquals(redactionLog.getRulesVersion(), entityLog.getRulesVersion());
-        assertEquals(redactionLog.getLegalBasis().size(), entityLog.getLegalBasis().size());
+        assertEquals(mergedRedactionLog.getAnalysisNumber(), entityLog.getAnalysisNumber());
+        assertEquals(mergedRedactionLog.getAnalysisVersion(), entityLog.getAnalysisVersion());
+        assertEquals(mergedRedactionLog.getDictionaryVersion(), entityLog.getDictionaryVersion());
+        assertEquals(mergedRedactionLog.getDossierDictionaryVersion(), entityLog.getDossierDictionaryVersion());
+        assertEquals(mergedRedactionLog.getLegalBasisVersion(), entityLog.getLegalBasisVersion());
+        assertEquals(mergedRedactionLog.getRulesVersion(), entityLog.getRulesVersion());
+        assertEquals(mergedRedactionLog.getLegalBasis().size(), entityLog.getLegalBasis().size());
 
         Map<String, String> migratedIds = migratedEntityLog.getMigratedIds().buildOldToNewMapping();
-        migratedIds.forEach((oldId, newId) -> assertEntryIsEqual(oldId, newId, redactionLog, entityLog, migratedIds));
+//        assertEquals(legacyRedactionLogMergeService.getNumberOfAffectedAnnotations(manualRedactions), migratedIds.size());
+
+        migratedIds.forEach((oldId, newId) -> assertEntryIsEqual(oldId, newId, mergedRedactionLog, entityLog, migratedIds));
 
         AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder().dossierId(TEST_DOSSIER_ID).fileId(TEST_FILE_ID).build());
         File outputFile = Path.of(OsUtils.getTemporaryDirectory()).resolve(Path.of(fileName.replaceAll(".pdf", "_MIGRATED.pdf")).getFileName()).toFile();
@@ -114,56 +204,129 @@ public class MigrationIntegrationTest extends BuildDocumentIntegrationTest {
     }
 
 
+    private void revertMigration(ManualRedactions manualRedactions, MigratedIds migratedIds) {
+
+        var mapping = migratedIds.buildNewToOldMapping();
+        manualRedactions.getEntriesToAdd()
+                .stream()
+                .filter(e -> mapping.containsKey(e.getAnnotationId()))
+                .forEach(e -> e.setAnnotationId(mapping.get(e.getAnnotationId())));
+
+        manualRedactions.getRecategorizations()
+                .stream()
+                .filter(e -> mapping.containsKey(e.getAnnotationId()))
+                .forEach(e -> e.setAnnotationId(mapping.get(e.getAnnotationId())));
+
+        manualRedactions.getResizeRedactions()
+                .stream()
+                .filter(e -> mapping.containsKey(e.getAnnotationId()))
+                .forEach(e -> e.setAnnotationId(mapping.get(e.getAnnotationId())));
+
+        manualRedactions.getIdsToRemove()
+                .stream()
+                .filter(e -> mapping.containsKey(e.getAnnotationId()))
+                .forEach(e -> e.setAnnotationId(mapping.get(e.getAnnotationId())));
+
+        manualRedactions.getLegalBasisChanges()
+                .stream()
+                .filter(e -> mapping.containsKey(e.getAnnotationId()))
+                .forEach(e -> e.setAnnotationId(mapping.get(e.getAnnotationId())));
+
+        manualRedactions.getForceRedactions()
+                .stream()
+                .filter(e -> mapping.containsKey(e.getAnnotationId()))
+                .forEach(e -> e.setAnnotationId(mapping.get(e.getAnnotationId())));
+    }
+
+
+    private MigratedIds getMigratedIds(String migratedIdsFileName) throws IOException {
+
+        var migratedIdsResource = new ClassPathResource(migratedIdsFileName);
+        if (!migratedIdsResource.exists()) {
+            return new MigratedIds();
+        }
+        try (var in = migratedIdsResource.getInputStream()) {
+            return mapper.readValue(in, MigratedIds.class);
+        }
+    }
+
+
+    private static boolean hasManualChanges(RedactionLogEntry entry) {
+
+        return !entry.getManualChanges().isEmpty() || (entry.getComments() != null && !entry.getComments().isEmpty());
+    }
+
+
     private void assertEntryIsEqual(String oldId, String newId, RedactionLog redactionLog, EntityLog entityLog, Map<String, String> oldToNewMapping) {
 
-        RedactionLogEntry redactionLogEntry = redactionLog.getRedactionLogEntry().stream().filter(entry -> entry.getId().equals(oldId)).findAny().orElseThrow();
-        EntityLogEntry entityLogEntry = entityLog.getEntityLogEntry().stream().filter(entry -> entry.getId().equals(newId)).findAny().orElseThrow();
+        RedactionLogEntry redactionLogEntry = redactionLog.getRedactionLogEntry()
+                .stream()
+                .filter(entry -> entry.getId().equals(oldId))
+                .findAny()
+                .orElseThrow();
+        EntityLogEntry entityLogEntry = entityLog.getEntityLogEntry()
+                .stream()
+                .filter(entry -> entry.getId().equals(newId))
+                .findAny()
+                .orElseThrow();
 
         if (!redactionLogEntry.isImage()) {
             assertEquals(redactionLogEntry.getValue().toLowerCase(Locale.ENGLISH), entityLogEntry.getValue().toLowerCase(Locale.ENGLISH));
         }
+        if (entityLogEntry.getManualChanges()
+                .stream()
+                .noneMatch(mc -> mc.getManualRedactionType().equals(ManualRedactionType.RECATEGORIZE))) {
+            assertEquals(redactionLogEntry.getType(), entityLogEntry.getType());
+        }
         assertEquals(redactionLogEntry.getChanges().size(), entityLogEntry.getChanges().size());
-        assertEquals(redactionLogEntry.getManualChanges().size(), entityLogEntry.getManualChanges().size());
+        assertTrue(redactionLogEntry.getManualChanges().size() <= entityLogEntry.getManualChanges().size());
         assertEquals(redactionLogEntry.getPositions().size(), entityLogEntry.getPositions().size());
-        assertTrue(positionsAlmostEqual(redactionLogEntry.getPositions(), entityLogEntry.getPositions()));
-        assertEquals(redactionLogEntry.getColor(), entityLogEntry.getColor());
-        assertEqualsNullSafe(redactionLogEntry.getLegalBasis(), entityLogEntry.getLegalBasis());
-        assertEqualsNullSafe(redactionLogEntry.getReason(), entityLogEntry.getReason());
+        if (entityLogEntry.getManualChanges()
+                .stream()
+                .noneMatch(mc -> mc.getManualRedactionType().equals(ManualRedactionType.RESIZE) || mc.getManualRedactionType().equals(ManualRedactionType.RESIZE_IN_DICTIONARY))) {
+            assertTrue(positionsAlmostEqual(redactionLogEntry.getPositions(), entityLogEntry.getPositions()));
+        }
+        if (entityLogEntry.getManualChanges()
+                .stream()
+                .noneMatch(mc -> mc.getManualRedactionType().equals(ManualRedactionType.FORCE))) {
+            assertEqualsNullSafe(redactionLogEntry.getLegalBasis(), entityLogEntry.getLegalBasis());
+        }
         assertReferencesEqual(redactionLogEntry.getReference(), entityLogEntry.getReference(), oldToNewMapping);
         assertEquals(redactionLogEntry.isDictionaryEntry(), entityLogEntry.isDictionaryEntry());
         assertEquals(redactionLogEntry.isDossierDictionaryEntry(), entityLogEntry.isDossierDictionaryEntry());
         if (redactionLogEntry.getEngines() == null) {
             assertTrue(entityLogEntry.getEngines().isEmpty());
         } else {
-            assertEquals(redactionLogEntry.getEngines().stream().map(Enum::name).collect(Collectors.toSet()),
-                    entityLogEntry.getEngines().stream().map(Enum::name).collect(Collectors.toSet()));
+            assertEquals(redactionLogEntry.getEngines()
+                                 .stream()
+                                 .map(Enum::name)
+                                 .collect(Collectors.toSet()),
+                         entityLogEntry.getEngines()
+                                 .stream()
+                                 .map(Enum::name)
+                                 .collect(Collectors.toSet()));
         }
     }
 
 
     private boolean positionsAlmostEqual(List<Rectangle> positions1, List<Position> positions2) {
 
-        double tolerance = 3;
+        double tolerance = 10;
+        double averageDistance = 0;
         for (int i = 0; i < positions1.size(); i++) {
-            Rectangle p1 = positions1.get(0);
-            Position p2 = positions2.get(0);
-            if (p1.getPage() != p2.getPageNumber()) {
-                return false;
-            }
-            if (Math.abs(p1.getHeight() - p2.h()) > tolerance) {
-                return false;
-            }
-            if (Math.abs(p1.getWidth() - p2.w()) > tolerance) {
-                return false;
-            }
-            if (Math.abs(p1.getTopLeft().getX() - p2.x()) > tolerance) {
-                return false;
-            }
-            if (Math.abs(p1.getTopLeft().getY() - p2.y()) > tolerance) {
+            Rectangle r1 = positions1.get(i);
+            Rectangle2D p1 = new Rectangle2D.Double(r1.getTopLeft().getX(), r1.getTopLeft().getY(), r1.getWidth(), r1.getHeight());
+            Position p2 = positions2.get(i);
+            double distance = EntityFindingUtility.calculateDistance(p1, p2.toRectangle2D());
+            if (r1.getPage() != p2.getPageNumber()) {
                 return false;
             }
+            averageDistance += distance;
         }
-        return true;
+
+        averageDistance /= positions1.size();
+
+        return averageDistance <= tolerance;
     }
 
 
@@ -189,7 +352,9 @@ public class MigrationIntegrationTest extends BuildDocumentIntegrationTest {
             return;
         }
 
-        assertEquals(reference.stream().map(oldToNewMapping::get).collect(Collectors.toSet()), reference1);
+        assertEquals(reference.stream()
+                             .map(oldToNewMapping::get)
+                             .collect(Collectors.toSet()), reference1);
 
     }
 

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionAcceptanceTest.java

@@ -33,7 +33,6 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryType;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive;
@@ -208,7 +207,7 @@ public class RedactionAcceptanceTest extends AbstractRedactionIntegrationTest {
 
     private static IdRemoval buildIdRemoval(String id) {
 
-        return IdRemoval.builder().annotationId(id).requestDate(OffsetDateTime.now()).status(AnnotationStatus.APPROVED).fileId(TEST_FILE_ID).build();
+        return IdRemoval.builder().annotationId(id).requestDate(OffsetDateTime.now()).fileId(TEST_FILE_ID).build();
     }
 
 }

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java

@@ -50,7 +50,6 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileTyp
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryType;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;
@@ -120,15 +119,15 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
 
         when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L);
         when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, true)).thenReturn(List.of(Type.builder()
-                .id(DOSSIER_REDACTIONS_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID)
-                .type(DOSSIER_REDACTIONS_INDICATOR)
-                .dossierTemplateId(TEST_DOSSIER_ID)
-                .hexColor("#ffe187")
-                .isHint(hintTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
-                .isCaseInsensitive(caseInSensitiveMap.get(DOSSIER_REDACTIONS_INDICATOR))
-                .isRecommendation(recommendationTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
-                .rank(rankTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
-                .build()));
+                                                                                                       .id(DOSSIER_REDACTIONS_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID)
+                                                                                                       .type(DOSSIER_REDACTIONS_INDICATOR)
+                                                                                                       .dossierTemplateId(TEST_DOSSIER_ID)
+                                                                                                       .hexColor("#ffe187")
+                                                                                                       .isHint(hintTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
+                                                                                                       .isCaseInsensitive(caseInSensitiveMap.get(DOSSIER_REDACTIONS_INDICATOR))
+                                                                                                       .isRecommendation(recommendationTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
+                                                                                                       .rank(rankTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
+                                                                                                       .build()));
 
         mockDictionaryCalls(null);
 
@@ -170,9 +169,10 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
 
         var entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
 
-        entityLog.getEntityLogEntry().forEach(entry -> {
-            duplicates.computeIfAbsent(entry.getId(), v -> new ArrayList<>()).add(entry);
-        });
+        entityLog.getEntityLogEntry()
+                .forEach(entry -> {
+                    duplicates.computeIfAbsent(entry.getId(), v -> new ArrayList<>()).add(entry);
+                });
 
         duplicates.forEach((key, value) -> assertThat(value.size()).isEqualTo(1));
 
@@ -217,13 +217,12 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
 
         ManualRedactions manualRedactions = ManualRedactions.builder()
                 .resizeRedactions(Set.of(ManualResizeRedaction.builder()
-                        .annotationId("c6be5277f5ee60dc3d83527798b7fe02")
-                        .value("Dr. Alan")
-                        .positions(List.of(new Rectangle(236.8f, 182.90005f, 40.584f, 12.642f, 7)))
-                        .status(AnnotationStatus.APPROVED)
-                        .requestDate(OffsetDateTime.now())
-                        .updateDictionary(false)
-                        .build()))
+                                                 .annotationId("c6be5277f5ee60dc3d83527798b7fe02")
+                                                 .value("Dr. Alan")
+                                                 .positions(List.of(new Rectangle(236.8f, 182.90005f, 40.584f, 12.642f, 7)))
+                                                 .requestDate(OffsetDateTime.now())
+                                                 .updateDictionary(false)
+                                                 .build()))
                 .build();
 
         request.setManualRedactions(manualRedactions);
@@ -258,7 +257,10 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
 
         var redactionLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
 
-        var values = redactionLog.getEntityLogEntry().stream().map(EntityLogEntry::getValue).collect(Collectors.toList());
+        var values = redactionLog.getEntityLogEntry()
+                .stream()
+                .map(EntityLogEntry::getValue)
+                .collect(Collectors.toList());
 
         assertThat(values).containsExactlyInAnyOrder("Lastname M.", "Doe", "Doe J.", "M. Mustermann", "Mustermann M.", "F. Lastname");
     }
@@ -270,8 +272,8 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
         ClassPathResource importedRedactionClasspathResource = new ClassPathResource(
                 "files/ImportedRedactions/18 Chlorothalonil RAR 08 Volume 3CA B 6a Oct 2017.IMPORTED_REDACTIONS.json");
         storageService.storeObject(TenantContext.getTenantId(),
-                RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.IMPORTED_REDACTIONS),
-                importedRedactionClasspathResource.getInputStream());
+                                   RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.IMPORTED_REDACTIONS),
+                                   importedRedactionClasspathResource.getInputStream());
 
         AnalyzeRequest request = uploadFileToStorage("files/ImportedRedactions/18 Chlorothalonil RAR 08 Volume 3CA B 6a Oct 2017.pdf");
         System.out.println("Start Full integration test");
@@ -295,7 +297,7 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
     @Test
     public void titleExtraction() throws IOException {
 
-        AnalyzeRequest request = uploadFileToStorage("files/migration/def8f960580f088b975ba806dfae1f87.ORIGIN.pdf");
+        AnalyzeRequest request = uploadFileToStorage("files/Metolachlor/S-Metolachlor_RAR_01_Volume_1_2018-09-06.pdf");
         System.out.println("Start Full integration test");
         analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request);
         System.out.println("Finished structure analysis");
@@ -343,7 +345,6 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
 
         var toRemove = IdRemoval.builder()
                 .annotationId("c630599611e6e3db314518374bcf70f7")
-                .status(AnnotationStatus.APPROVED)
                 .user("test")
                 .removeFromDictionary(false)
                 .processedDate(OffsetDateTime.now())
@@ -356,10 +357,18 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
 
         var mergedEntityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
 
-        var cbiAddressBeforeHintRemoval = entityLog.getEntityLogEntry().stream().filter(re -> re.getType().equalsIgnoreCase("CBI_Address")).findAny().get();
+        var cbiAddressBeforeHintRemoval = entityLog.getEntityLogEntry()
+                .stream()
+                .filter(re -> re.getType().equalsIgnoreCase("CBI_Address"))
+                .findAny()
+                .get();
         assertThat(cbiAddressBeforeHintRemoval.getState().equals(EntryState.APPLIED)).isFalse();
 
-        var cbiAddressAfterHintRemoval = mergedEntityLog.getEntityLogEntry().stream().filter(re -> re.getType().equalsIgnoreCase("CBI_Address")).findAny().get();
+        var cbiAddressAfterHintRemoval = mergedEntityLog.getEntityLogEntry()
+                .stream()
+                .filter(re -> re.getType().equalsIgnoreCase("CBI_Address"))
+                .findAny()
+                .get();
         assertThat(cbiAddressAfterHintRemoval.getState().equals(EntryState.APPLIED)).isTrue();
     }
 
@@ -389,9 +398,10 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
 
                     var entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
 
-                    entityLog.getEntityLogEntry().forEach(entry -> {
-                        duplicates.computeIfAbsent(entry.getId(), v -> new ArrayList<>()).add(entry);
-                    });
+                    entityLog.getEntityLogEntry()
+                            .forEach(entry -> {
+                                duplicates.computeIfAbsent(entry.getId(), v -> new ArrayList<>()).add(entry);
+                            });
 
                     duplicates.forEach((id, redactionLogEntries) -> assertThat(redactionLogEntries.size()).isEqualTo(1));
 
@@ -424,11 +434,11 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
         AnalyzeRequest request = uploadFileToStorage(fileName);
 
         request.setFileAttributes(List.of(FileAttribute.builder()
-                .id("fileAttributeId")
-                .label("Vertebrate Study")
-                .placeholder("{fileattributes.vertebrateStudy}")
-                .value("true")
-                .build()));
+                                                  .id("fileAttributeId")
+                                                  .label("Vertebrate Study")
+                                                  .placeholder("{fileattributes.vertebrateStudy}")
+                                                  .value("true")
+                                                  .build()));
 
         analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request);
         AnalyzeResult result = analyzeService.analyze(request);
@@ -452,7 +462,10 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
                     correctFound++;
                     continue loop;
                 }
-                if (Objects.equals(entityLogEntry.getContainingNodeId().get(0), section.getTreeId().get(0))) {
+                if (Objects.equals(entityLogEntry.getContainingNodeId()
+                                           .get(0),
+                                   section.getTreeId()
+                                           .get(0))) {
                     String value = section.getTextBlock().subSequence(new TextRange(entityLogEntry.getStartOffset(), entityLogEntry.getEndOffset())).toString();
                     if (entityLogEntry.getValue().equalsIgnoreCase(value)) {
                         correctFound++;
@@ -484,12 +497,12 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
         ManualRedactions manualRedactions = new ManualRedactions();
 
         manualRedactions.setEntriesToAdd(Set.of(ManualRedactionEntry.builder()
-                .value("Redact")
-                .addToDictionary(true)
-                .addToDossierDictionary(true)
-                .positions(List.of(new Rectangle(new Point(95.96979999999999f, 515.7984f), 19.866899999999987f, 46.953f, 2)))
-                .type("dossier_redaction")
-                .build()));
+                                                        .value("Redact")
+                                                        .addToDictionary(true)
+                                                        .addToDossierDictionary(true)
+                                                        .positions(List.of(new Rectangle(new Point(95.96979999999999f, 515.7984f), 19.866899999999987f, 46.953f, 2)))
+                                                        .type("dossier_redaction")
+                                                        .build()));
 
         request.setManualRedactions(manualRedactions);
 
@@ -551,7 +564,11 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
 
         var entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
 
-        var changes = entityLog.getEntityLogEntry().stream().filter(entry -> entry.getValue() != null && entry.getValue().equals("report")).findFirst().get().getChanges();
+        var changes = entityLog.getEntityLogEntry()
+                .stream()
+                .filter(entry -> entry.getValue() != null && entry.getValue().equals("report"))
+                .findFirst()
+                .get().getChanges();
 
         assertThat(changes.size()).isEqualTo(2);
 
@@ -571,18 +588,18 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
 
         ClassPathResource responseJson = new ClassPathResource("files/crafted_document.NER_ENTITIES.json");
         storageService.storeObject(TenantContext.getTenantId(),
-                RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.NER_ENTITIES),
-                responseJson.getInputStream());
+                                   RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.NER_ENTITIES),
+                                   responseJson.getInputStream());
 
         long start = System.currentTimeMillis();
         AnalyzeRequest request = uploadFileToStorage(fileName);
 
         request.setFileAttributes(List.of(FileAttribute.builder()
-                .id("fileAttributeId")
-                .label("Vertebrate Study")
-                .placeholder("{fileattributes.vertebrateStudy}")
-                .value("true")
-                .build()));
+                                                  .id("fileAttributeId")
+                                                  .label("Vertebrate Study")
+                                                  .placeholder("{fileattributes.vertebrateStudy}")
+                                                  .value("true")
+                                                  .build()));
 
         analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request);
         AnalyzeResult result = analyzeService.analyze(request);
@@ -604,7 +621,11 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
                 .map(redactionLogEntry -> new TextRange(redactionLogEntry.getStartOffset(), redactionLogEntry.getEndOffset()))
                 .map(boundary -> documentGraph.getTextBlock().subSequence(boundary).toString())
                 .toList();
-        List<String> valuesInRedactionLog = entityLog.getEntityLogEntry().stream().filter(e -> !e.getEntryType().equals(EntryType.IMAGE)).map(EntityLogEntry::getValue).toList();
+        List<String> valuesInRedactionLog = entityLog.getEntityLogEntry()
+                .stream()
+                .filter(e -> !e.getEntryType().equals(EntryType.IMAGE))
+                .map(EntityLogEntry::getValue)
+                .toList();
 
         assertEquals(valuesInRedactionLog, valuesInDocument);
 
@@ -631,12 +652,11 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
         ManualRedactions manualRedactions = new ManualRedactions();
 
         manualRedactions.setRecategorizations(Set.of(ManualRecategorization.builder()
-                .annotationId("37eee3e9d589a5cc529bfec38c3ba479")
-                .fileId("fileId")
-                .status(AnnotationStatus.APPROVED)
-                .type("signature")
-                .requestDate(OffsetDateTime.now())
-                .build()));
+                                                             .annotationId("37eee3e9d589a5cc529bfec38c3ba479")
+                                                             .fileId("fileId")
+                                                             .type("signature")
+                                                             .requestDate(OffsetDateTime.now())
+                                                             .build()));
 
         request.setManualRedactions(manualRedactions);
 
@@ -687,43 +707,40 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
         ManualRedactions manualRedactions = new ManualRedactions();
         manualRedactions.getIdsToRemove()
                 .add(IdRemoval.builder()
-                        .annotationId("308dab9015bfafd911568cffe0a7f7de")
-                        .fileId(TEST_FILE_ID)
-                        .status(AnnotationStatus.APPROVED)
-                        .requestDate(OffsetDateTime.of(2022, 05, 23, 8, 30, 07, 475479, ZoneOffset.UTC))
-                        .processedDate(OffsetDateTime.of(2022, 05, 23, 8, 30, 07, 483651, ZoneOffset.UTC))
-                        .build());
+                             .annotationId("308dab9015bfafd911568cffe0a7f7de")
+                             .fileId(TEST_FILE_ID)
+                             .requestDate(OffsetDateTime.of(2022, 05, 23, 8, 30, 07, 475479, ZoneOffset.UTC))
+                             .processedDate(OffsetDateTime.of(2022, 05, 23, 8, 30, 07, 483651, ZoneOffset.UTC))
+                             .build());
 
         manualRedactions.getForceRedactions()
                 .add(ManualForceRedaction.builder()
-                        .annotationId("0b56ea1a87c83f351df177315af94f0d")
-                        .fileId(TEST_FILE_ID)
-                        .status(AnnotationStatus.APPROVED)
-                        .legalBasis("Something")
-                        .requestDate(OffsetDateTime.of(2022, 05, 23, 9, 30, 15, 4653, ZoneOffset.UTC))
-                        .processedDate(OffsetDateTime.of(2022, 05, 23, 9, 30, 15, 794, ZoneOffset.UTC))
-                        .build());
+                             .annotationId("0b56ea1a87c83f351df177315af94f0d")
+                             .fileId(TEST_FILE_ID)
+                             .legalBasis("Something")
+                             .requestDate(OffsetDateTime.of(2022, 05, 23, 9, 30, 15, 4653, ZoneOffset.UTC))
+                             .processedDate(OffsetDateTime.of(2022, 05, 23, 9, 30, 15, 794, ZoneOffset.UTC))
+                             .build());
 
         manualRedactions.getIdsToRemove()
                 .add(IdRemoval.builder()
-                        .annotationId("0b56ea1a87c83f351df177315af94f0d")
-                        .fileId(TEST_FILE_ID)
-                        .status(AnnotationStatus.APPROVED)
-                        .requestDate(OffsetDateTime.of(2022, 05, 23, 8, 30, 23, 961721, ZoneOffset.UTC))
-                        .processedDate(OffsetDateTime.of(2022, 05, 23, 8, 30, 23, 96528, ZoneOffset.UTC))
-                        .build());
+                             .annotationId("0b56ea1a87c83f351df177315af94f0d")
+                             .fileId(TEST_FILE_ID)
+                             .requestDate(OffsetDateTime.of(2022, 05, 23, 8, 30, 23, 961721, ZoneOffset.UTC))
+                             .processedDate(OffsetDateTime.of(2022, 05, 23, 8, 30, 23, 96528, ZoneOffset.UTC))
+                             .build());
 
         request.setManualRedactions(manualRedactions);
 
         AnalyzeResult result = analyzeService.analyze(request);
 
         AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder()
-                .manualRedactions(manualRedactions)
-                .colors(colors)
-                .types(types)
-                .dossierId(TEST_DOSSIER_ID)
-                .fileId(TEST_FILE_ID)
-                .build());
+                                                                               .manualRedactions(manualRedactions)
+                                                                               .colors(colors)
+                                                                               .types(types)
+                                                                               .dossierId(TEST_DOSSIER_ID)
+                                                                               .fileId(TEST_FILE_ID)
+                                                                               .build());
 
         try (FileOutputStream fileOutputStream = new FileOutputStream(OsUtils.getTemporaryDirectory() + "/Annotated.pdf")) {
             fileOutputStream.write(annotateResponse.getDocument());
@@ -921,7 +938,6 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
         ManualResizeRedaction manualResizeRedaction = ManualResizeRedaction.builder()
                 .annotationId("ca2b437e2480a4b5966cb8386020d454")
                 .fileId(TEST_FILE_ID)
-                .status(AnnotationStatus.APPROVED)
                 .requestDate(OffsetDateTime.of(2022, 05, 23, 8, 30, 07, 475479, ZoneOffset.UTC))
                 .processedDate(OffsetDateTime.of(2022, 05, 23, 8, 30, 07, 483651, ZoneOffset.UTC))
                 .value("Bera P")
@@ -940,12 +956,12 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
         var entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
 
         AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder()
-                .manualRedactions(manualRedactions)
-                .colors(colors)
-                .types(types)
-                .dossierId(TEST_DOSSIER_ID)
-                .fileId(TEST_FILE_ID)
-                .build());
+                                                                               .manualRedactions(manualRedactions)
+                                                                               .colors(colors)
+                                                                               .types(types)
+                                                                               .dossierId(TEST_DOSSIER_ID)
+                                                                               .fileId(TEST_FILE_ID)
+                                                                               .build());
 
         try (FileOutputStream fileOutputStream = new FileOutputStream(OsUtils.getTemporaryDirectory() + "/Annotated.pdf")) {
             fileOutputStream.write(annotateResponse.getDocument());
@@ -968,15 +984,16 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
 
         var redactionLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
 
-        redactionLog.getEntityLogEntry().forEach(entry -> {
-            if (!entry.getEntryType().equals(EntryType.HINT)) {
-                if (entry.getType().equals("CBI_author")) {
-                    assertThat(entry.getReason()).isEqualTo("Not redacted because it's row does not belong to a vertebrate study");
-                } else if (entry.getType().equals("CBI_address")) {
-                    assertThat(entry.getReason()).isEqualTo("No vertebrate found");
-                }
-            }
-        });
+        redactionLog.getEntityLogEntry()
+                .forEach(entry -> {
+                    if (!entry.getEntryType().equals(EntryType.HINT)) {
+                        if (entry.getType().equals("CBI_author")) {
+                            assertThat(entry.getReason()).isEqualTo("Not redacted because it's row does not belong to a vertebrate study");
+                        } else if (entry.getType().equals("CBI_address")) {
+                            assertThat(entry.getReason()).isEqualTo("No vertebrate found");
+                        }
+                    }
+                });
     }
 
 
@@ -1013,32 +1030,29 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
         String manualAddId = UUID.randomUUID().toString();
 
         manualRedactions.setIdsToRemove(Set.of(IdRemoval.builder()
-                .annotationId("5b940b2cb401ed9f5be6fc24f6e77bcf")
-                .fileId("fileId")
-                .status(AnnotationStatus.DECLINED)
-                .processedDate(OffsetDateTime.now())
-                .requestDate(OffsetDateTime.now())
-                .build()));
+                                                       .annotationId("5b940b2cb401ed9f5be6fc24f6e77bcf")
+                                                       .fileId("fileId")
+                                                       .processedDate(OffsetDateTime.now())
+                                                       .requestDate(OffsetDateTime.now())
+                                                       .build()));
         manualRedactions.setForceRedactions(Set.of(ManualForceRedaction.builder()
-                .annotationId("675eba69b0c2917de55462c817adaa05")
-                .fileId("fileId")
-                .legalBasis("Something")
-                .status(AnnotationStatus.APPROVED)
-                .requestDate(OffsetDateTime.now())
-                .processedDate(OffsetDateTime.now())
-                .build()));
+                                                           .annotationId("675eba69b0c2917de55462c817adaa05")
+                                                           .fileId("fileId")
+                                                           .legalBasis("Something")
+                                                           .requestDate(OffsetDateTime.now())
+                                                           .processedDate(OffsetDateTime.now())
+                                                           .build()));
 
         ManualRedactionEntry manualRedactionEntry = new ManualRedactionEntry();
         manualRedactionEntry.setAnnotationId(manualAddId);
         manualRedactionEntry.setFileId("fileId");
-        manualRedactionEntry.setStatus(AnnotationStatus.APPROVED);
         manualRedactionEntry.setType("CBI_author");
         manualRedactionEntry.setValue("O'Loughlin C.K.");
         manualRedactionEntry.setReason("Manual Redaction");
         manualRedactionEntry.setProcessedDate(OffsetDateTime.now());
         manualRedactionEntry.setRequestDate(OffsetDateTime.now());
         manualRedactionEntry.setPositions(List.of(Rectangle.builder().topLeftX(375.61096f).topLeftY(241.282f).width(7.648041f).height(43.72262f).page(1).build(),
-                Rectangle.builder().topLeftX(384.83517f).topLeftY(241.282f).width(7.648041f).height(17.043358f).page(1).build()));
+                                                  Rectangle.builder().topLeftX(384.83517f).topLeftY(241.282f).width(7.648041f).height(17.043358f).page(1).build()));
 
 //        manualRedactions.getEntriesToAdd().add(manualRedactionEntry);
 
@@ -1049,42 +1063,63 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
 
         manualRedactions.getEntriesToAdd().add(manualRedactionEntry);
         manualRedactions.setIdsToRemove(Set.of(IdRemoval.builder()
-                .annotationId("5b940b2cb401ed9f5be6fc24f6e77bcf")
-                .fileId("fileId")
-                .status(AnnotationStatus.APPROVED)
-                .requestDate(OffsetDateTime.now())
-                .processedDate(OffsetDateTime.now())
-                .build()));
+                                                       .annotationId("5b940b2cb401ed9f5be6fc24f6e77bcf")
+                                                       .fileId("fileId")
+                                                       .requestDate(OffsetDateTime.now())
+                                                       .processedDate(OffsetDateTime.now())
+                                                       .build()));
         manualRedactions.setLegalBasisChanges((Set.of(ManualLegalBasisChange.builder()
-                .annotationId("675eba69b0c2917de55462c817adaa05")
-                .fileId("fileId")
-                .legalBasis("Manual Legal Basis Change")
-                .status(AnnotationStatus.APPROVED)
-                .processedDate(OffsetDateTime.now())
-                .requestDate(OffsetDateTime.now())
-                .build())));
+                                                              .annotationId("675eba69b0c2917de55462c817adaa05")
+                                                              .fileId("fileId")
+                                                              .legalBasis("Manual Legal Basis Change")
+                                                              .processedDate(OffsetDateTime.now())
+                                                              .requestDate(OffsetDateTime.now())
+                                                              .build())));
         manualRedactions.setResizeRedactions(Set.of(ManualResizeRedaction.builder()
-                .annotationId("fc287b74be2421156ab2895c7474ccdd")
-                .fileId("fileId")
-                .processedDate(OffsetDateTime.now())
-                .requestDate(OffsetDateTime.now())
-                .value("Syngenta Crop Protection AG, Basel, Switzerland RCC Ltd., Itingen, Switzerland")
-                .positions(List.of(Rectangle.builder().topLeftX(289.44595f).topLeftY(327.567f).width(7.648041f).height(82.51475f).page(1).build(),
-                        Rectangle.builder().topLeftX(298.67056f).topLeftY(327.567f).width(7.648041f).height(75.32377f).page(1).build(),
-                        Rectangle.builder().topLeftX(307.89517f).topLeftY(327.567f).width(7.648041f).height(61.670967f).page(1).build(),
-                        Rectangle.builder().topLeftX(316.99985f).topLeftY(327.567f).width(7.648041f).height(38.104286f).page(1).build()))
-                .status(AnnotationStatus.APPROVED)
-                .updateDictionary(false)
-                .build()));
+                                                            .annotationId("fc287b74be2421156ab2895c7474ccdd")
+                                                            .fileId("fileId")
+                                                            .processedDate(OffsetDateTime.now())
+                                                            .requestDate(OffsetDateTime.now())
+                                                            .value("Syngenta Crop Protection AG, Basel, Switzerland RCC Ltd., Itingen, Switzerland")
+                                                            .positions(List.of(Rectangle.builder()
+                                                                                       .topLeftX(289.44595f)
+                                                                                       .topLeftY(327.567f)
+                                                                                       .width(7.648041f)
+                                                                                       .height(82.51475f)
+                                                                                       .page(1)
+                                                                                       .build(),
+                                                                               Rectangle.builder()
+                                                                                       .topLeftX(298.67056f)
+                                                                                       .topLeftY(327.567f)
+                                                                                       .width(7.648041f)
+                                                                                       .height(75.32377f)
+                                                                                       .page(1)
+                                                                                       .build(),
+                                                                               Rectangle.builder()
+                                                                                       .topLeftX(307.89517f)
+                                                                                       .topLeftY(327.567f)
+                                                                                       .width(7.648041f)
+                                                                                       .height(61.670967f)
+                                                                                       .page(1)
+                                                                                       .build(),
+                                                                               Rectangle.builder()
+                                                                                       .topLeftX(316.99985f)
+                                                                                       .topLeftY(327.567f)
+                                                                                       .width(7.648041f)
+                                                                                       .height(38.104286f)
+                                                                                       .page(1)
+                                                                                       .build()))
+                                                            .updateDictionary(false)
+                                                            .build()));
 
         analyzeService.reanalyze(request);
 
         AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder()
-                .dossierId(TEST_DOSSIER_ID)
-                .fileId(TEST_FILE_ID)
-                .dossierTemplateId(TEST_DOSSIER_TEMPLATE_ID)
-                .manualRedactions(manualRedactions)
-                .build());
+                                                                               .dossierId(TEST_DOSSIER_ID)
+                                                                               .fileId(TEST_FILE_ID)
+                                                                               .dossierTemplateId(TEST_DOSSIER_TEMPLATE_ID)
+                                                                               .manualRedactions(manualRedactions)
+                                                                               .build());
 
         try (FileOutputStream fileOutputStream = new FileOutputStream(OsUtils.getTemporaryDirectory() + "/Annotated.pdf")) {
             fileOutputStream.write(annotateResponse.getDocument());
@@ -1124,8 +1159,8 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
 
         AnalyzeRequest request = uploadFileToStorage("files/ImportedRedactions/RotateTestFile_without_highlights.pdf");
         storageService.storeObject(TenantContext.getTenantId(),
-                RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.IMPORTED_REDACTIONS),
-                importedRedactions.getInputStream());
+                                   RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.IMPORTED_REDACTIONS),
+                                   importedRedactions.getInputStream());
 
         analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request);
         AnalyzeResult result = analyzeService.analyze(request);
@@ -1138,17 +1173,18 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
             fileOutputStream.write(annotateResponse.getDocument());
         }
 
-        entityLog.getEntityLogEntry().forEach(entry -> {
-            if (entry.getValue() == null) {
-                return;
-            }
-            if (entry.getValue().equals("David")) {
-                assertThat(entry.getImportedRedactionIntersections()).hasSize(1);
-            }
-            if (entry.getValue().equals("annotation")) {
-                assertThat(entry.getImportedRedactionIntersections()).isEmpty();
-            }
-        });
+        entityLog.getEntityLogEntry()
+                .forEach(entry -> {
+                    if (entry.getValue() == null) {
+                        return;
+                    }
+                    if (entry.getValue().equals("David")) {
+                        assertThat(entry.getImportedRedactionIntersections()).hasSize(1);
+                    }
+                    if (entry.getValue().equals("annotation")) {
+                        assertThat(entry.getImportedRedactionIntersections()).isEmpty();
+                    }
+                });
     }
 
 
@@ -1177,7 +1213,10 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
         }
 
         var entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
-        var values = entityLog.getEntityLogEntry().stream().map(EntityLogEntry::getValue).collect(Collectors.toList());
+        var values = entityLog.getEntityLogEntry()
+                .stream()
+                .map(EntityLogEntry::getValue)
+                .collect(Collectors.toList());
 
         assertThat(values).contains("Mrs. Robinson");
         assertThat(values).contains("Mr. Bojangles");
@@ -1185,6 +1224,95 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
     }
 
 
+    @Test
+    public void signaturesAreRedactionAfterReanalyse() throws IOException {
+
+        AnalyzeRequest request = uploadFileToStorage("files/new/SYNGENTA_EFSA_sanitisation_GFL_v1 (1).pdf");
+        ClassPathResource imageServiceResponseFileResource = new ClassPathResource("files/new/SYNGENTA_EFSA_sanitisation_GFL_v1 (1).IMAGE_INFO.json");
+
+        storageService.storeObject(TenantContext.getTenantId(),
+                                   RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.IMAGE_INFO),
+                                   imageServiceResponseFileResource.getInputStream());
+
+        System.out.println("Start Full integration test");
+        analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request);
+        System.out.println("Finished structure analysis");
+        analyzeService.analyze(request);
+        System.out.println("Finished analysis");
+
+        request.setManualRedactions(ManualRedactions.builder()
+                                            .legalBasisChanges(Set.of(ManualLegalBasisChange.builder()
+                                                                              .annotationId("3029651d0842a625f2d23f8375c23600")
+                                                                              .section("[19, 2]: Paragraph: Contact point: LexCo Contact:")
+                                                                              .value("0049 331 441 551 14")
+                                                                              .requestDate(OffsetDateTime.now())
+                                                                              .fileId(TEST_FILE_ID)
+                                                                              .legalBasis("Article 39(e)(2) of Regulation (EC) No 178/2002")
+                                                                              .build()))
+                                            .build());
+
+        analyzeService.reanalyze(request);
+        System.out.println("Finished reanalysis");
+
+        var entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
+        entityLog.getEntityLogEntry()
+                .stream()
+                .filter(entityLogEntry -> entityLogEntry.getType().equals("signature"))
+                .forEach(entityLogEntry -> {
+                    assertThat(entityLogEntry.getState() == EntryState.APPLIED).isTrue();
+                });
+    }
+
+
+    @Test
+    public void entityIsAppliedAfterRecateorize() throws IOException {
+
+        AnalyzeRequest request = uploadFileToStorage("files/new/SYNGENTA_EFSA_sanitisation_GFL_v1 (1).pdf");
+        ClassPathResource imageServiceResponseFileResource = new ClassPathResource("files/new/SYNGENTA_EFSA_sanitisation_GFL_v1 (1).IMAGE_INFO.json");
+
+        storageService.storeObject(TenantContext.getTenantId(),
+                                   RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.IMAGE_INFO),
+                                   imageServiceResponseFileResource.getInputStream());
+
+        System.out.println("Start Full integration test");
+        analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request);
+        System.out.println("Finished structure analysis");
+        analyzeService.analyze(request);
+        System.out.println("Finished analysis");
+
+        request.setManualRedactions(ManualRedactions.builder()
+                                            .legalBasisChanges(Set.of(ManualLegalBasisChange.builder()
+                                                                              .annotationId("3029651d0842a625f2d23f8375c23600")
+                                                                              .section("[19, 2]: Paragraph: Contact point: LexCo Contact:")
+                                                                              .value("0049 331 441 551 14")
+                                                                              .requestDate(OffsetDateTime.now())
+                                                                              .fileId(TEST_FILE_ID)
+                                                                              .legalBasis("Article 39(e)(2) of Regulation (EC) No 178/2002")
+                                                                              .build()))
+                                            .recategorizations(Set.of(ManualRecategorization.builder()
+                                                                              .annotationId("3029651d0842a625f2d23f8375c23600")
+                                                                              .type("CBI_author")
+                                                                              .legalBasis("")
+                                                                              .requestDate(OffsetDateTime.now())
+                                                                              .fileId(TEST_FILE_ID)
+                                                                              .build()))
+                                            .build());
+
+        analyzeService.reanalyze(request);
+        System.out.println("Finished reanalysis");
+
+        var entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
+        var changedAnnotation = entityLog.getEntityLogEntry()
+                .stream()
+                .filter(entityLogEntry -> entityLogEntry.getId().equals("3029651d0842a625f2d23f8375c23600"))
+                .findFirst()
+                .get();
+
+        assertThat(changedAnnotation.getState() == EntryState.APPLIED).isTrue();
+
+    }
+
+
     @Test
     @SneakyThrows
     public void testRemovingAndAddingRedactionOnTheSameValue() {
@@ -1194,8 +1322,12 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
         String manualAddId = UUID.randomUUID().toString();
         String manualAddId2 = UUID.randomUUID().toString();
         List<Rectangle> positions = List.of(Rectangle.builder().topLeftX(305.35f).topLeftY(332.5033f).width(71.40744f).height(13.645125f).page(1).build());
-        ManualRedactionEntry manualRedactionEntry = getManualRedactionEntry(manualAddId, positions, "the manufacturing or production process, including the method and innovative aspects thereof, as well as other technical and industrial specifications inherent to that process or method, except for information which is relevant to the assessment of safety");
-        ManualRedactionEntry manualRedactionEntry2 = getManualRedactionEntry(manualAddId2, positions, "commercial information revealing sourcing, market shares or business strategy of the applicant");
+        ManualRedactionEntry manualRedactionEntry = getManualRedactionEntry(manualAddId,
+                                                                            positions,
+                                                                            "the manufacturing or production process, including the method and innovative aspects thereof, as well as other technical and industrial specifications inherent to that process or method, except for information which is relevant to the assessment of safety");
+        ManualRedactionEntry manualRedactionEntry2 = getManualRedactionEntry(manualAddId2,
+                                                                             positions,
+                                                                             "commercial information revealing sourcing, market shares or business strategy of the applicant");
 
         IdRemoval idRemoval = getIdRemoval(manualAddId);
         IdRemoval idRemoval2 = getIdRemoval(manualAddId2);
@@ -1207,59 +1339,150 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
 
         var entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
 
-        assertTrue(entityLog.getEntityLogEntry().stream().anyMatch(entityLogEntry -> entityLogEntry.getId().equals(manualAddId)));
-        assertEquals(entityLog.getEntityLogEntry().stream().filter(entityLogEntry -> entityLogEntry.getId().equals(manualAddId)).findFirst().get().getState(), EntryState.APPLIED);
+        assertTrue(entityLog.getEntityLogEntry()
+                           .stream()
+                           .anyMatch(entityLogEntry -> entityLogEntry.getId().equals(manualAddId)));
+        assertEquals(entityLog.getEntityLogEntry()
+                             .stream()
+                             .filter(entityLogEntry -> entityLogEntry.getId().equals(manualAddId))
+                             .findFirst()
+                             .get().getState(), EntryState.APPLIED);
 
         request.setManualRedactions(ManualRedactions.builder().entriesToAdd(Set.of(manualRedactionEntry)).idsToRemove(Set.of(idRemoval)).build());
         analyzeService.reanalyze(request);
 
         entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
 
-        assertTrue(entityLog.getEntityLogEntry().stream().anyMatch(entityLogEntry -> entityLogEntry.getId().equals(manualAddId)));
-        assertEquals(entityLog.getEntityLogEntry().stream().filter(entityLogEntry -> entityLogEntry.getId().equals(manualAddId)).findFirst().get().getState(), EntryState.REMOVED);
+        assertTrue(entityLog.getEntityLogEntry()
+                           .stream()
+                           .anyMatch(entityLogEntry -> entityLogEntry.getId().equals(manualAddId)));
+        assertEquals(entityLog.getEntityLogEntry()
+                             .stream()
+                             .filter(entityLogEntry -> entityLogEntry.getId().equals(manualAddId))
+                             .findFirst()
+                             .get().getState(), EntryState.REMOVED);
 
         request.setManualRedactions(ManualRedactions.builder().entriesToAdd(Set.of(manualRedactionEntry, manualRedactionEntry2)).idsToRemove(Set.of(idRemoval)).build());
         analyzeService.reanalyze(request);
 
         entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
 
-        assertTrue(entityLog.getEntityLogEntry().stream().anyMatch(entityLogEntry -> entityLogEntry.getId().equals(manualAddId)));
-        assertEquals(entityLog.getEntityLogEntry().stream().filter(entityLogEntry -> entityLogEntry.getId().equals(manualAddId)).findFirst().get().getState(), EntryState.REMOVED);
-        assertTrue(entityLog.getEntityLogEntry().stream().anyMatch(entityLogEntry -> entityLogEntry.getId().equals(manualAddId2)));
-        assertEquals(entityLog.getEntityLogEntry().stream().filter(entityLogEntry -> entityLogEntry.getId().equals(manualAddId2)).findFirst().get().getState(), EntryState.APPLIED);
+        assertTrue(entityLog.getEntityLogEntry()
+                           .stream()
+                           .anyMatch(entityLogEntry -> entityLogEntry.getId().equals(manualAddId)));
+        assertEquals(entityLog.getEntityLogEntry()
+                             .stream()
+                             .filter(entityLogEntry -> entityLogEntry.getId().equals(manualAddId))
+                             .findFirst()
+                             .get().getState(), EntryState.REMOVED);
+        assertTrue(entityLog.getEntityLogEntry()
+                           .stream()
+                           .anyMatch(entityLogEntry -> entityLogEntry.getId().equals(manualAddId2)));
+        assertEquals(entityLog.getEntityLogEntry()
+                             .stream()
+                             .filter(entityLogEntry -> entityLogEntry.getId().equals(manualAddId2))
+                             .findFirst()
+                             .get().getState(), EntryState.APPLIED);
 
-        request.setManualRedactions(ManualRedactions.builder().entriesToAdd(Set.of(manualRedactionEntry, manualRedactionEntry2)).idsToRemove(Set.of(idRemoval, idRemoval2)).build());
+        request.setManualRedactions(ManualRedactions.builder()
+                                            .entriesToAdd(Set.of(manualRedactionEntry, manualRedactionEntry2))
+                                            .idsToRemove(Set.of(idRemoval, idRemoval2))
+                                            .build());
         analyzeService.reanalyze(request);
 
         entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
 
-        assertTrue(entityLog.getEntityLogEntry().stream().anyMatch(entityLogEntry -> entityLogEntry.getId().equals(manualAddId)));
-        assertEquals(entityLog.getEntityLogEntry().stream().filter(entityLogEntry -> entityLogEntry.getId().equals(manualAddId)).findFirst().get().getState(), EntryState.REMOVED);
-        assertTrue(entityLog.getEntityLogEntry().stream().anyMatch(entityLogEntry -> entityLogEntry.getId().equals(manualAddId2)));
-        assertEquals(entityLog.getEntityLogEntry().stream().filter(entityLogEntry -> entityLogEntry.getId().equals(manualAddId2)).findFirst().get().getState(), EntryState.REMOVED);
+        assertTrue(entityLog.getEntityLogEntry()
+                           .stream()
+                           .anyMatch(entityLogEntry -> entityLogEntry.getId().equals(manualAddId)));
+        assertEquals(entityLog.getEntityLogEntry()
+                             .stream()
+                             .filter(entityLogEntry -> entityLogEntry.getId().equals(manualAddId))
+                             .findFirst()
+                             .get().getState(), EntryState.REMOVED);
+        assertTrue(entityLog.getEntityLogEntry()
+                           .stream()
+                           .anyMatch(entityLogEntry -> entityLogEntry.getId().equals(manualAddId2)));
+        assertEquals(entityLog.getEntityLogEntry()
+                             .stream()
+                             .filter(entityLogEntry -> entityLogEntry.getId().equals(manualAddId2))
+                             .findFirst()
+                             .get().getState(), EntryState.REMOVED);
 
         manualRedactionEntry.setRequestDate(OffsetDateTime.now());
-        request.setManualRedactions(ManualRedactions.builder().entriesToAdd(Set.of(manualRedactionEntry, manualRedactionEntry2)).idsToRemove(Set.of(idRemoval, idRemoval2)).build());
+        request.setManualRedactions(ManualRedactions.builder().entriesToAdd(Set.of(manualRedactionEntry, manualRedactionEntry2)).idsToRemove(Set.of(idRemoval2)).build());
         analyzeService.reanalyze(request);
 
         entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
 
-        assertTrue(entityLog.getEntityLogEntry().stream().anyMatch(entityLogEntry -> entityLogEntry.getId().equals(manualAddId)));
-        assertEquals(entityLog.getEntityLogEntry().stream().filter(entityLogEntry -> entityLogEntry.getId().equals(manualAddId)).findFirst().get().getState(), EntryState.APPLIED);
-        assertTrue(entityLog.getEntityLogEntry().stream().anyMatch(entityLogEntry -> entityLogEntry.getId().equals(manualAddId2)));
-        assertEquals(entityLog.getEntityLogEntry().stream().filter(entityLogEntry -> entityLogEntry.getId().equals(manualAddId2)).findFirst().get().getState(), EntryState.REMOVED);
+        assertTrue(entityLog.getEntityLogEntry()
+                           .stream()
+                           .anyMatch(entityLogEntry -> entityLogEntry.getId().equals(manualAddId)));
+        assertEquals(entityLog.getEntityLogEntry()
+                             .stream()
+                             .filter(entityLogEntry -> entityLogEntry.getId().equals(manualAddId))
+                             .findFirst()
+                             .get().getState(), EntryState.APPLIED);
+        assertTrue(entityLog.getEntityLogEntry()
+                           .stream()
+                           .anyMatch(entityLogEntry -> entityLogEntry.getId().equals(manualAddId2)));
+        assertEquals(entityLog.getEntityLogEntry()
+                             .stream()
+                             .filter(entityLogEntry -> entityLogEntry.getId().equals(manualAddId2))
+                             .findFirst()
+                             .get().getState(), EntryState.REMOVED);
+    }
+
+
+    @Test
+    @SneakyThrows
+    public void testResizeWithUpdateDictionaryTrue() {
+
+        String pdfFile = "files/new/crafted document.pdf";
+
+        AnalyzeRequest request = uploadFileToStorage(pdfFile);
+        analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request);
+        analyzeService.analyze(request);
+
+        var entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
+        var david = entityLog.getEntityLogEntry()
+                .stream()
+                .filter(e -> e.getValue().equals("David"))
+                .findFirst()
+                .get();
+
+        request.setManualRedactions(ManualRedactions.builder()
+                                            .resizeRedactions(Set.of(ManualResizeRedaction.builder()
+                                                                             .updateDictionary(true)
+                                                                             .annotationId(david.getId())
+                                                                             .requestDate(OffsetDateTime.now())
+                                                                             .value("David Ksenia")
+                                                                             .positions(List.of(Rectangle.builder()
+                                                                                                        .topLeftX(56.8f)
+                                                                                                        .topLeftY(293.564f)
+                                                                                                        .width(65.592f)
+                                                                                                        .height(15.408f)
+                                                                                                        .page(1)
+                                                                                                        .build()))
+                                                                             .addToAllDossiers(false)
+                                                                             .build()))
+                                            .build());
+        analyzeService.reanalyze(request);
+        entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
+        var resizedEntity = entityLog.getEntityLogEntry()
+                .stream()
+                .filter(e -> e.getId().equals(david.getId()))
+                .findFirst()
+                .get();
+        assertEquals(resizedEntity.getState(), EntryState.APPLIED);
+        assertEquals(resizedEntity.getValue(), "David");
+        assertEquals(0, resizedEntity.getManualChanges().size());
     }
 
 
     private IdRemoval getIdRemoval(String id) {
 
-        return IdRemoval.builder()
-                .annotationId(id)
-                .removeFromAllDossiers(false)
-                .removeFromDictionary(false)
-                .status(AnnotationStatus.APPROVED)
-                .requestDate(OffsetDateTime.now())
-                .build();
+        return IdRemoval.builder().annotationId(id).removeFromAllDossiers(false).removeFromDictionary(false).requestDate(OffsetDateTime.now()).build();
     }
 
 

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/annotate/AnnotationService.java

@@ -7,6 +7,7 @@ import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
@@ -23,10 +24,12 @@ import org.apache.pdfbox.pdmodel.interactive.annotation.PDAnnotationHighlight;
 import org.springframework.stereotype.Service;
 
 import com.google.common.primitives.Floats;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Change;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryType;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualChange;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
 import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
@@ -60,8 +63,8 @@ public class AnnotationService {
     public AnnotateResponse annotate(AnnotateRequest annotateRequest) {
 
         var storedObjectFile = redactionStorageService.getStoredObjectFile(RedactionStorageService.StorageIdUtils.getStorageId(annotateRequest.getDossierId(),
-                annotateRequest.getFileId(),
-                FileType.VIEWER_DOCUMENT));
+                                                                                                                               annotateRequest.getFileId(),
+                                                                                                                               FileType.VIEWER_DOCUMENT));
 
         var entityLog = redactionStorageService.getEntityLog(annotateRequest.getDossierId(), annotateRequest.getFileId());
 
@@ -76,7 +79,8 @@ public class AnnotationService {
 
             try (ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream()) {
                 pdDocument.save(byteArrayOutputStream);
-                return AnnotateResponse.builder().document(byteArrayOutputStream.toByteArray()).build();
+                return AnnotateResponse.builder().document(byteArrayOutputStream.toByteArray())
+                        .build();
             }
         }
     }
@@ -134,7 +138,10 @@ public class AnnotationService {
 
         List<PDAnnotation> annotations = new ArrayList<>();
 
-        List<Position> rectangles = redactionLogEntry.getPositions().stream().filter(pos -> pos.getPageNumber() == page).collect(Collectors.toList());
+        List<Position> rectangles = redactionLogEntry.getPositions()
+                .stream()
+                .filter(pos -> pos.getPageNumber() == page)
+                .collect(Collectors.toList());
 
         if (rectangles.isEmpty()) {
             return annotations;
@@ -145,12 +152,33 @@ public class AnnotationService {
         PDRectangle pdRectangle = toPDRectangleBBox(rectangles);
         annotation.setRectangle(pdRectangle);
         annotation.setQuadPoints(Floats.toArray(toQuadPoints(rectangles)));
-        if (!(redactionLogEntry.getEntryType().equals(EntryType.HINT) || redactionLogEntry.getState().equals(EntryState.IGNORED))) {
-            annotation.setContents(redactionLogEntry.getValue() + " " + createAnnotationContent(redactionLogEntry));
-        }
-        annotation.setTitlePopup(redactionLogEntry.getId());
+        annotation.setContents(redactionLogEntry.getValue() + " " + createAnnotationContent(redactionLogEntry));
+        annotation.setTitlePopup(redactionLogEntry.getEntryType().name() + ":" + redactionLogEntry.getState().name() + ":" + redactionLogEntry.getId());
         annotation.setAnnotationName(redactionLogEntry.getId());
-        annotation.setColor(new PDColor(redactionLogEntry.getColor(), PDDeviceRGB.INSTANCE));
+        float[] color;
+        if (redactionLogEntry.getEntryType().equals(EntryType.RECOMMENDATION)) {
+            color = new float[]{0, 0.8f, 0};
+        } else if ((redactionLogEntry.getEntryType().equals(EntryType.ENTITY) || redactionLogEntry.getEntryType().equals(EntryType.IMAGE)) //
+                   && redactionLogEntry.getState().equals(EntryState.APPLIED)) {
+            color = new float[]{0.5764706f, 0.59607846f, 0.627451f};
+        } else if ((redactionLogEntry.getEntryType().equals(EntryType.ENTITY) || redactionLogEntry.getEntryType().equals(EntryType.IMAGE)) //
+                   && (redactionLogEntry.getState().equals(EntryState.SKIPPED) || redactionLogEntry.getState().equals(EntryState.IGNORED))) {
+            color = new float[]{0.76862746f, 0.59607846f, 0.98039216f};
+        } else if (redactionLogEntry.getEntryType().equals(EntryType.HINT) && redactionLogEntry.getState().equals(EntryState.SKIPPED) && redactionLogEntry.getType()
+                .equals("published_information")) {
+            color = new float[]{0.52156866f, 0.92156863f, 1.0f};
+        } else if (redactionLogEntry.getEntryType().equals(EntryType.HINT) && redactionLogEntry.getState().equals(EntryState.SKIPPED)) {
+            color = new float[]{0.98039216f, 0.59607846f, 0.96862745f};
+        } else if (redactionLogEntry.getEntryType().equals(EntryType.HINT) && redactionLogEntry.getState().equals(EntryState.IGNORED)) {
+            color = new float[]{0.76862746f, 0.59607846f, 0.98039216f};
+        } else if (redactionLogEntry.getState().equals(EntryState.APPLIED)) {
+            color = Optional.ofNullable(redactionLogEntry.getColor())
+                    .orElse(new float[]{0.5764706f, 0.59607846f, 0.627451f});
+        } else {
+            color = Optional.ofNullable(redactionLogEntry.getColor())
+                    .orElse(new float[]{0.76862746f, 0.59607846f, 0.98039216f});
+        }
+        annotation.setColor(new PDColor(color, PDDeviceRGB.INSTANCE));
         annotation.setNoRotate(false);
         annotations.add(annotation);
 
@@ -173,7 +201,10 @@ public class AnnotationService {
 
     public static List<Double> toQuadPoints(List<Position> rectangles) {
 
-        return rectangles.stream().map(Position::toRectangle2D).flatMap(AnnotationService::toQuadPoints).toList();
+        return rectangles.stream()
+                .map(Position::toRectangle2D)
+                .flatMap(AnnotationService::toQuadPoints)
+                .toList();
     }
 
 
@@ -196,7 +227,21 @@ public class AnnotationService {
 
     private String createAnnotationContent(EntityLogEntry redactionLogEntry) {
 
-        return redactionLogEntry.getType() + " \nRule " + redactionLogEntry.getMatchedRule() + " matched\n\n" + redactionLogEntry.getReason() + "\n\nLegal basis:" + redactionLogEntry.getLegalBasis() + "\n\nIn section: \"" + redactionLogEntry.getSection() + "\"";
+        return redactionLogEntry.getType() +//
+               " \nRule " + redactionLogEntry.getMatchedRule() +//
+               " matched\n\n" + redactionLogEntry.getReason() +//
+               "\n\nLegal basis:" + redactionLogEntry.getLegalBasis() +//
+               "\n\nIn section: \"" + redactionLogEntry.getSection() + "\"" +//
+               "\n\nChanges: " + redactionLogEntry.getChanges()
+                       .stream()
+                       .map(Change::getType)
+                       .map(Enum::name)
+                       .collect(Collectors.joining("\n")) +//
+               "\n\nManualChanges: " + redactionLogEntry.getManualChanges()
+                       .stream()
+                       .map(ManualChange::getManualRedactionType)
+                       .map(Enum::name)
+                       .collect(Collectors.joining("\n"));
     }
 
 }

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/entity/TextEntityTest.java

@@ -5,16 +5,18 @@ import static org.junit.jupiter.api.Assertions.assertThrows;
 
 import org.junit.jupiter.api.Test;
 
-import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
+import com.iqser.red.service.redaction.v1.server.model.PrecursorEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
-import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
 
 public class TextEntityTest {
 
     @Test
     public void testMatchedRule() {
 
-        TextEntity entity = TextEntity.initialEntityNode(new TextRange(1, 100), "PII", EntityType.ENTITY);
+        PrecursorEntity entity = PrecursorEntity.builder()
+                .type("PII")
+                .entityType(EntityType.ENTITY)
+                .build();
         entity.skip("CBI.1.0", "");
         entity.skip("CBI.2.0", "");
         entity.skip("CBI.3.0", "");
@@ -29,7 +31,10 @@ public class TextEntityTest {
     @Test
     public void testMatchedRuleWithNonsense() {
 
-        TextEntity entity = TextEntity.initialEntityNode(new TextRange(1, 100), "PII", EntityType.ENTITY);
+        PrecursorEntity entity = PrecursorEntity.builder()
+                .type("PII")
+                .entityType(EntityType.ENTITY)
+                .build();
         assertThrows(IllegalArgumentException.class, () -> {
             entity.skip("", "");
         });

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentIEntityInsertionIntegrationTest.java

@@ -49,6 +49,21 @@ public class DocumentIEntityInsertionIntegrationTest extends BuildDocumentIntegr
         entityCreationService = new EntityCreationService(entityEnrichmentService, kieSession);
     }
 
+    @Test
+    public void assertEntitiesAreDuplicatedWithTheirTableCell() {
+
+        Document document = buildGraph("files/Minimal Examples/Meto1_Page22.pdf");
+        List<TextEntity> entities = entityCreationService.byString("Surface Water", "test", EntityType.ENTITY, document).toList();
+        assertEquals(3, entities.size());
+        assertEquals(1, entities.stream().distinct().count());
+        assertEquals(2, entities.get(0).getDuplicateTextRanges().size());
+
+        var node = entities.get(0).getDeepestFullyContainingNode();
+
+        assertTrue(node.getTextRange().contains(entities.get(0).getTextRange()));
+        assertTrue(entities.get(0).getDuplicateTextRanges().stream().allMatch(tr -> node.getTextRange().contains(tr)));
+    }
+
 
     @Test
     public void assertCollectAllEntitiesWorks() {
@@ -82,7 +97,7 @@ public class DocumentIEntityInsertionIntegrationTest extends BuildDocumentIntegr
         assert start != -1;
 
         TextRange textRange = new TextRange(start, start + searchTerm.length());
-        TextEntity textEntity = TextEntity.initialEntityNode(textRange, "123", EntityType.ENTITY);
+        TextEntity textEntity = TextEntity.initialEntityNode(textRange, "123", EntityType.ENTITY, document);
         entityCreationService.addEntityToGraph(textEntity, document);
         return textEntity;
     }
@@ -241,7 +256,7 @@ public class DocumentIEntityInsertionIntegrationTest extends BuildDocumentIntegr
         assert start != -1;
 
         TextRange textRange = new TextRange(start, start + searchTerm.length());
-        TextEntity textEntity = TextEntity.initialEntityNode(textRange, "123", EntityType.ENTITY);
+        TextEntity textEntity = TextEntity.initialEntityNode(textRange, "123", EntityType.ENTITY, document);
         entityCreationService.addEntityToGraph(textEntity, document);
 
         assertEquals("2.6.1 Summary of ", textEntity.getTextBefore());
@@ -299,7 +314,7 @@ public class DocumentIEntityInsertionIntegrationTest extends BuildDocumentIntegr
         assert start != -1;
 
         TextRange textRange = new TextRange(start, start + searchTerm.length());
-        TextEntity textEntity = TextEntity.initialEntityNode(textRange, "123", EntityType.ENTITY);
+        TextEntity textEntity = TextEntity.initialEntityNode(textRange, "123", EntityType.ENTITY, document);
         entityCreationService.addEntityToGraph(textEntity, document);
         Page pageNode = document.getPages().stream().filter(page -> page.getNumber() == pageNumber).findFirst().orElseThrow();
 

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentPerformanceIntegrationTest.java

@@ -1,7 +1,7 @@
 package com.iqser.red.service.redaction.v1.server.document.graph;
 
 import static com.iqser.red.service.redaction.v1.server.utils.SeparatorUtils.boundaryIsSurroundedBySeparators;
-import static org.mockito.Mockito.doThrow;
+import static org.junit.jupiter.api.Assertions.assertTrue;
 import static org.mockito.Mockito.when;
 
 import java.awt.Color;
@@ -34,6 +34,10 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileTyp
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type;
+import com.iqser.red.service.redaction.v1.server.model.NerEntities;
+import com.iqser.red.service.redaction.v1.server.model.dictionary.Dictionary;
+import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryModel;
+import com.iqser.red.service.redaction.v1.server.model.dictionary.SearchImplementation;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
@@ -41,16 +45,11 @@ import com.iqser.red.service.redaction.v1.server.model.document.nodes.Page;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Section;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
+import com.iqser.red.service.redaction.v1.server.service.DictionaryService;
 import com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService;
 import com.iqser.red.service.redaction.v1.server.service.document.EntityEnrichmentService;
-import com.iqser.red.service.redaction.v1.server.utils.ExceptionProvider;
-import com.iqser.red.service.redaction.v1.server.utils.PdfVisualisationUtility;
-import com.iqser.red.service.redaction.v1.server.model.NerEntities;
-import com.iqser.red.service.redaction.v1.server.model.dictionary.Dictionary;
-import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryModel;
-import com.iqser.red.service.redaction.v1.server.model.dictionary.SearchImplementation;
-import com.iqser.red.service.redaction.v1.server.service.DictionaryService;
 import com.iqser.red.service.redaction.v1.server.service.drools.EntityDroolsExecutionService;
+import com.iqser.red.service.redaction.v1.server.utils.PdfVisualisationUtility;
 import com.knecon.fforesight.tenantcommons.TenantContext;
 
 import lombok.SneakyThrows;
@@ -245,8 +244,12 @@ public class DocumentPerformanceIntegrationTest extends BuildDocumentIntegration
         System.out.printf("%d Searches took %s s, average %.2f ms\n", numberOfRuns, ((float) totalSearchTime / 1000), totalSearchTime / numberOfRuns);
         System.out.printf("%d Insertions took %s s, average %.2f ms\n", numberOfRuns, ((float) totalInsertTime / 1000), totalInsertTime / numberOfRuns);
         System.out.printf("Found %d entities and saved %d\n", foundEntities.size(), document.getEntities().size());
-        assert document.getEntities().size() == foundEntities.size();
-
+        for (TextEntity entity : document.getEntities()) {
+            var foundEntity = foundEntities.stream().filter(f -> f.getId().equals(entity.getId())).findFirst().get();
+            assertTrue(foundEntity.getTextRange().equals(entity.getTextRange()) || foundEntity.getDuplicateTextRanges().contains(entity.getTextRange()));
+        }
+        assert document.getEntities().stream().mapToInt(e -> e.getDuplicateTextRanges().size() + 1).sum() == foundEntities.size();
+        assert foundEntities.stream().map(TextEntity::getId).distinct().count() == document.getEntities().size();
         drawAllEntities(filename, document);
     }
 
@@ -300,7 +303,7 @@ public class DocumentPerformanceIntegrationTest extends BuildDocumentIntegration
         searchImplementation.getBoundaries(textBlock, textBlock.getTextRange())
                 .stream()
                 .filter(boundary -> boundaryIsSurroundedBySeparators(textBlock, boundary))
-                .map(bounds -> TextEntity.initialEntityNode(bounds, type, entityType))
+                .map(bounds -> TextEntity.initialEntityNode(bounds, type, entityType, document))
                 .forEach(foundEntities::add);
     }
 

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/TableTest.java

@@ -0,0 +1,162 @@
+package com.iqser.red.service.redaction.v1.server.document.graph;
+
+import static com.iqser.red.service.redaction.v1.server.utils.EntityVisualizationUtility.ENTITY_LAYER;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+import java.awt.Color;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.nio.file.Path;
+import java.util.List;
+import java.util.Set;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+
+import org.drools.io.ClassPathResource;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.springframework.beans.factory.annotation.Autowired;
+
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
+import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
+import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.NodeType;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.Table;
+import com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService;
+import com.iqser.red.service.redaction.v1.server.service.document.EntityEnrichmentService;
+import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
+import com.iqser.red.service.redaction.v1.server.utils.EntityVisualizationUtility;
+import com.knecon.fforesight.service.viewerdoc.model.Visualizations;
+import com.knecon.fforesight.service.viewerdoc.service.ViewerDocumentService;
+import com.knecon.fforesight.tenantcommons.TenantContext;
+
+import lombok.SneakyThrows;
+
+public class TableTest extends BuildDocumentIntegrationTest {
+
+    private static final boolean DRAW_FILE = false;
+
+    @Autowired
+    private EntityEnrichmentService entityEnrichmentService;
+
+    private EntityCreationService entityCreationService;
+
+    private static final String TYPE_1 = "type1";
+    private static final String TYPE_2 = "type2";
+    private static final String TYPE_3 = "type3";
+    private static final String TYPE_4 = "type4";
+
+    private Table table;
+
+    private Set<TextEntity> entities;
+
+
+    @SneakyThrows
+    @BeforeEach
+    public void createTable() {
+
+        entityCreationService = new EntityCreationService(entityEnrichmentService);
+
+        String fileName = "files/Minimal Examples/BasicTable.pdf";
+
+        Document document = buildGraph(fileName);
+
+        table = (Table) document.streamAllSubNodesOfType(NodeType.TABLE)
+                .findAny()
+                .orElseThrow();
+
+        entities = List.of(//
+                           entityCreationService.byString("Cell11", TYPE_1, EntityType.ENTITY, document),
+                           entityCreationService.byString("Cell21", TYPE_1, EntityType.ENTITY, document),
+                           entityCreationService.byString("Cell31", TYPE_1, EntityType.ENTITY, document),
+                           entityCreationService.byString("Cell41", TYPE_1, EntityType.ENTITY, document),
+                           entityCreationService.byString("Cell51", TYPE_1, EntityType.ENTITY, document),
+
+                           entityCreationService.byString("Cell12", TYPE_2, EntityType.ENTITY, document),
+                           entityCreationService.byString("Cell32", TYPE_2, EntityType.ENTITY, document),
+                           entityCreationService.byString("Cell42", TYPE_2, EntityType.ENTITY, document),
+
+                           entityCreationService.byString("Cell23", TYPE_3, EntityType.ENTITY, document),
+                           entityCreationService.byString("Cell53", TYPE_3, EntityType.ENTITY, document),
+
+                           entityCreationService.byString("Cell14", TYPE_4, EntityType.ENTITY, document),
+                           entityCreationService.byString("Cell34", TYPE_4, EntityType.ENTITY, document))
+                .stream()
+                .flatMap(Function.identity())
+                .collect(Collectors.toSet());
+
+        if (DRAW_FILE) {
+            File file = new File("/tmp/" + Path.of(fileName).getFileName().toString());
+            storageService.downloadTo(TenantContext.getTenantId(),
+                                      RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.VIEWER_DOCUMENT),
+                                      file);
+            ViewerDocumentService viewerDocumentService = new ViewerDocumentService(null);
+
+            var visualizationsOnPage = EntityVisualizationUtility.createVisualizationsOnPage(document.getEntities(), Color.MAGENTA);
+
+            viewerDocumentService.addVisualizationsOnPage(file,
+                                                          file,
+                                                          Visualizations.builder()
+                                                                  .layer(ENTITY_LAYER)
+                                                                  .visualizationsOnPages(visualizationsOnPage)
+                                                                  .layerVisibilityDefaultValue(true)
+                                                                  .build());
+        }
+
+    }
+
+
+    @Test
+    public void testStreamEntitiesWhereRowContainsEntitiesOfType() {
+
+        int type_2_count = table.getEntitiesOfType(TYPE_2).size();
+
+        assertEquals(type_2_count,
+                     table.streamEntitiesWhereRowContainsEntitiesOfType(List.of(TYPE_1))
+                             .filter(textEntity -> textEntity.type().equals(TYPE_2))
+                             .count());
+
+        assertEquals(type_2_count,
+                     table.streamEntitiesWhereRowContainsEntitiesOfType(List.of(TYPE_1, TYPE_4))
+                             .filter(textEntity -> textEntity.type().equals(TYPE_2))
+                             .count());
+
+        assertEquals(2,
+                     table.streamEntitiesWhereRowContainsEntitiesOfEachType(List.of(TYPE_1, TYPE_4))
+                             .filter(textEntity -> textEntity.type().equals(TYPE_2))
+                             .count());
+
+        assertEquals(0,
+                     table.streamEntitiesWhereRowContainsEntitiesOfEachType(List.of(TYPE_1, TYPE_3))
+                             .filter(textEntity -> textEntity.type().equals(TYPE_2))
+                             .count());
+
+        assertEquals(0,
+                     table.streamEntitiesWhereRowContainsEntitiesOfEachType(List.of(TYPE_1, TYPE_3, TYPE_4))
+                             .filter(textEntity -> textEntity.type().equals(TYPE_2))
+                             .count());
+
+        assertEquals(type_2_count,
+                     table.streamEntitiesWhereRowContainsEntitiesOfEachType(List.of())
+                             .filter(textEntity -> textEntity.type().equals(TYPE_2))
+                             .count());
+
+        assertEquals(3,
+                     table.streamTextEntitiesInRow(1)
+                             .count());
+
+        assertEquals(2,
+                     table.streamTextEntitiesInRow(4)
+                             .count());
+
+        assertEquals(5,
+                     table.streamTextEntitiesInCol(1)
+                             .count());
+
+        assertEquals(3,
+                     table.streamTextEntitiesInRow(3)
+                             .count());
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/drools/files/management/services/DroolsSyntaxValidationServiceTest.java

@@ -1,5 +1,6 @@
 package com.iqser.red.service.redaction.v1.server.drools.files.management.services;
 
+import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertFalse;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 
@@ -18,6 +19,7 @@ import org.springframework.core.io.ClassPathResource;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType;
 import com.iqser.red.service.redaction.v1.model.DroolsSyntaxValidation;
 import com.iqser.red.service.redaction.v1.model.RuleValidationModel;
+import com.iqser.red.service.redaction.v1.server.DeprecatedElementsFinder;
 import com.iqser.red.service.redaction.v1.server.client.RulesClient;
 import com.iqser.red.service.redaction.v1.server.model.drools.RuleFileBluePrint;
 import com.iqser.red.service.redaction.v1.server.service.document.EntityEnrichmentService;
@@ -47,13 +49,15 @@ class DroolsSyntaxValidationServiceTest {
     @SneakyThrows
     void testRules() {
 
-        DroolsSyntaxValidationService droolsSyntaxValidationService = new DroolsSyntaxValidationService(new KieContainerCreationService(rulesClient));
+        DroolsSyntaxValidationService droolsSyntaxValidationService = new DroolsSyntaxValidationService(new KieContainerCreationService(rulesClient),
+                                                                                                        new DeprecatedElementsFinder());
         var rulesFile = new ClassPathResource("drools/rules.drl");
 
         String rulesString = new String(rulesFile.getInputStream().readAllBytes());
 
         DroolsSyntaxValidation droolsSyntaxValidation = droolsSyntaxValidationService.testRules(new RuleValidationModel(RuleFileType.ENTITY.name(), rulesString));
-        droolsSyntaxValidation.getDroolsSyntaxErrorMessages().forEach(System.out::println);
+        droolsSyntaxValidation.getDroolsSyntaxErrorMessages()
+                .forEach(System.out::println);
         assertTrue(droolsSyntaxValidation.isCompiled());
     }
 
@@ -62,14 +66,16 @@ class DroolsSyntaxValidationServiceTest {
     @SneakyThrows
     void testRulesWithRemovedImportsAndChangedName() {
 
-        DroolsSyntaxValidationService droolsSyntaxValidationService = new DroolsSyntaxValidationService(new KieContainerCreationService(rulesClient));
+        DroolsSyntaxValidationService droolsSyntaxValidationService = new DroolsSyntaxValidationService(new KieContainerCreationService(rulesClient),
+                                                                                                        new DeprecatedElementsFinder());
         var rulesFile = new ClassPathResource("drools/rules.drl");
 
         String rulesString = new String(rulesFile.getInputStream().readAllBytes());
         rulesString = rulesString.replaceAll("import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell;", "");
         rulesString = rulesString.replaceAll("rule \"LDS.0.0: Run local dictionary search\"", "rule \"LDS.0.0: run local dictionary search\"");
         DroolsSyntaxValidation droolsSyntaxValidation = droolsSyntaxValidationService.testRules(new RuleValidationModel(RuleFileType.ENTITY.name(), rulesString));
-        droolsSyntaxValidation.getDroolsSyntaxErrorMessages().forEach(System.out::println);
+        droolsSyntaxValidation.getDroolsSyntaxErrorMessages()
+                .forEach(System.out::println);
         assertTrue(droolsSyntaxValidation.isCompiled());
     }
 
@@ -77,102 +83,32 @@ class DroolsSyntaxValidationServiceTest {
     @SneakyThrows
     void testRulesWithAddedImports() {
 
-        DroolsSyntaxValidationService droolsSyntaxValidationService = new DroolsSyntaxValidationService(new KieContainerCreationService(rulesClient));
+        DroolsSyntaxValidationService droolsSyntaxValidationService = new DroolsSyntaxValidationService(new KieContainerCreationService(rulesClient),
+                                                                                                        new DeprecatedElementsFinder());
         var rulesFile = new ClassPathResource("drools/rules.drl");
 
         String rulesString = new String(rulesFile.getInputStream().readAllBytes());
-        rulesString = rulesString.replaceAll("import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell;", "import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell;\nimport com.iqser.red.service.redaction.v1.server.service.document.EntityComparators;");
+        rulesString = rulesString.replaceAll("import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell;",
+                                             "import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell;\nimport com.iqser.red.service.redaction.v1.server.service.document.EntityComparators;");
         DroolsSyntaxValidation droolsSyntaxValidation = droolsSyntaxValidationService.testRules(new RuleValidationModel(RuleFileType.ENTITY.name(), rulesString));
-        droolsSyntaxValidation.getDroolsSyntaxErrorMessages().forEach(System.out::println);
+        droolsSyntaxValidation.getDroolsSyntaxErrorMessages()
+                .forEach(System.out::println);
         assertTrue(droolsSyntaxValidation.isCompiled());
     }
 
-    @Test
-    @SneakyThrows
-    void testRulesWithUnallowedAddedImports() {
-
-        DroolsSyntaxValidationService droolsSyntaxValidationService = new DroolsSyntaxValidationService(new KieContainerCreationService(rulesClient));
-        var rulesFile = new ClassPathResource("drools/rules.drl");
-
-        String rulesString = new String(rulesFile.getInputStream().readAllBytes());
-        rulesString = rulesString.replaceAll("import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell;", "import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell;\nimport com.google.common.collect.Sets;");
-        DroolsSyntaxValidation droolsSyntaxValidation = droolsSyntaxValidationService.testRules(new RuleValidationModel(RuleFileType.ENTITY.name(), rulesString));
-        droolsSyntaxValidation.getDroolsSyntaxErrorMessages().forEach(System.out::println);
-        assertFalse(droolsSyntaxValidation.isCompiled());
-    }
-
-
-    @Test
-    @SneakyThrows
-    void testAllRules() {
-
-        DroolsSyntaxValidationService droolsSyntaxValidationService = new DroolsSyntaxValidationService(new KieContainerCreationService(rulesClient));
-        var rulesFile = new ClassPathResource("drools/all_redact_manager_rules.drl");
-
-        String rulesString = new String(rulesFile.getInputStream().readAllBytes());
-
-        DroolsSyntaxValidation droolsSyntaxValidation = droolsSyntaxValidationService.testRules(new RuleValidationModel(RuleFileType.ENTITY.name(), rulesString));
-        droolsSyntaxValidation.getDroolsSyntaxErrorMessages().forEach(System.out::println);
-        assertTrue(droolsSyntaxValidation.isCompiled());
-    }
-
-
-    @Test
-    @SneakyThrows
-    void testAcceptanceRules() {
-
-        DroolsSyntaxValidationService droolsSyntaxValidationService = new DroolsSyntaxValidationService(new KieContainerCreationService(rulesClient));
-        var rulesFile = new ClassPathResource("drools/acceptance_rules.drl");
-
-        String rulesString = new String(rulesFile.getInputStream().readAllBytes());
-
-        DroolsSyntaxValidation droolsSyntaxValidation = droolsSyntaxValidationService.testRules(new RuleValidationModel(RuleFileType.ENTITY.name(), rulesString));
-        droolsSyntaxValidation.getDroolsSyntaxErrorMessages().forEach(System.out::println);
-        assertTrue(droolsSyntaxValidation.isCompiled());
-    }
-
-
-    @Test
-    @SneakyThrows
-    void testManualRedactionRules() {
-
-        DroolsSyntaxValidationService droolsSyntaxValidationService = new DroolsSyntaxValidationService(new KieContainerCreationService(rulesClient));
-        var rulesFile = new ClassPathResource("drools/manual_redaction_rules.drl");
-
-        String rulesString = new String(rulesFile.getInputStream().readAllBytes());
-
-        DroolsSyntaxValidation droolsSyntaxValidation = droolsSyntaxValidationService.testRules(new RuleValidationModel(RuleFileType.ENTITY.name(), rulesString));
-        droolsSyntaxValidation.getDroolsSyntaxErrorMessages().forEach(System.out::println);
-        assertTrue(droolsSyntaxValidation.isCompiled());
-    }
-
-
-    @Test
-    @SneakyThrows
-    void testRulesV2() {
-
-        DroolsSyntaxValidationService droolsSyntaxValidationService = new DroolsSyntaxValidationService(new KieContainerCreationService(rulesClient));
-        var rulesFile = new ClassPathResource("drools/rules_v2.drl");
-
-        String rulesString = new String(rulesFile.getInputStream().readAllBytes());
-
-        DroolsSyntaxValidation droolsSyntaxValidation = droolsSyntaxValidationService.testRules(new RuleValidationModel(RuleFileType.ENTITY.name(), rulesString));
-        droolsSyntaxValidation.getDroolsSyntaxErrorMessages().forEach(System.out::println);
-        assertTrue(droolsSyntaxValidation.isCompiled());
-    }
-
-
     @Test
     @SneakyThrows
     void testDocumineRules() {
 
-        DroolsSyntaxValidationService droolsSyntaxValidationService = new DroolsSyntaxValidationService(new KieContainerCreationService(rulesClient));
+        DroolsSyntaxValidationService droolsSyntaxValidationService = new DroolsSyntaxValidationService(new KieContainerCreationService(rulesClient),
+                                                                                                        new DeprecatedElementsFinder());
         var rulesFile = new ClassPathResource("drools/documine_flora.drl");
 
         String rulesString = new String(rulesFile.getInputStream().readAllBytes());
 
         DroolsSyntaxValidation droolsSyntaxValidation = droolsSyntaxValidationService.testRules(new RuleValidationModel(RuleFileType.ENTITY.name(), rulesString));
-        droolsSyntaxValidation.getDroolsSyntaxErrorMessages().forEach(System.out::println);
+        droolsSyntaxValidation.getDroolsSyntaxErrorMessages()
+                .forEach(System.out::println);
         assertTrue(droolsSyntaxValidation.isCompiled());
     }
 
@@ -181,7 +117,8 @@ class DroolsSyntaxValidationServiceTest {
     @SneakyThrows
     void testCorruptedRules() {
 
-        DroolsSyntaxValidationService droolsSyntaxValidationService = new DroolsSyntaxValidationService(new KieContainerCreationService(rulesClient));
+        DroolsSyntaxValidationService droolsSyntaxValidationService = new DroolsSyntaxValidationService(new KieContainerCreationService(rulesClient),
+                                                                                                        new DeprecatedElementsFinder());
         var rulesFile = new ClassPathResource("drools/rules.drl");
 
         String rulesString = new String(rulesFile.getInputStream().readAllBytes());
@@ -196,12 +133,13 @@ class DroolsSyntaxValidationServiceTest {
     @SneakyThrows
     void testCorruptedImports() {
 
-        DroolsSyntaxValidationService droolsSyntaxValidationService = new DroolsSyntaxValidationService(new KieContainerCreationService(rulesClient));
+        DroolsSyntaxValidationService droolsSyntaxValidationService = new DroolsSyntaxValidationService(new KieContainerCreationService(rulesClient),
+                                                                                                        new DeprecatedElementsFinder());
         var rulesFile = new ClassPathResource("drools/rules.drl");
 
         String rulesString = new String(rulesFile.getInputStream().readAllBytes());
         String corruptedRules = rulesString.replaceAll("import com.iqser.red.service.redaction.v1.server.model.document.nodes.ImageType;",
-                "import com.iqser.red.service.redaction.v1.server.model.document.nodes.ImageType;\nimport com.iqser.red.service.redaction.v1.server.model.document.nodes.SomethingElse;");
+                                                       "import com.iqser.red.service.redaction.v1.server.model.document.nodes.ImageType;\nimport com.iqser.red.service.redaction.v1.server.model.document.nodes.SomethingElse;");
 
         DroolsSyntaxValidation droolsSyntaxValidation = droolsSyntaxValidationService.testRules(new RuleValidationModel(RuleFileType.ENTITY.name(), corruptedRules));
         assertFalse(droolsSyntaxValidation.isCompiled());
@@ -212,14 +150,16 @@ class DroolsSyntaxValidationServiceTest {
     @SneakyThrows
     void testRemoveQuery() {
 
-        DroolsSyntaxValidationService droolsSyntaxValidationService = new DroolsSyntaxValidationService(new KieContainerCreationService(rulesClient));
+        DroolsSyntaxValidationService droolsSyntaxValidationService = new DroolsSyntaxValidationService(new KieContainerCreationService(rulesClient),
+                                                                                                        new DeprecatedElementsFinder());
         var rulesFile = new ClassPathResource("drools/rules.drl");
 
         String rulesString = new String(rulesFile.getInputStream().readAllBytes());
         String corruptedRules = rulesString.replaceAll("query \"getFileAttributes\"\n.*\n *end", "");
 
         DroolsSyntaxValidation droolsSyntaxValidation = droolsSyntaxValidationService.testRules(new RuleValidationModel(RuleFileType.ENTITY.name(), corruptedRules));
-        droolsSyntaxValidation.getDroolsSyntaxErrorMessages().forEach(System.out::println);
+        droolsSyntaxValidation.getDroolsSyntaxErrorMessages()
+                .forEach(System.out::println);
         assertFalse(droolsSyntaxValidation.isCompiled());
     }
 
@@ -228,13 +168,15 @@ class DroolsSyntaxValidationServiceTest {
     @SneakyThrows
     void testComponentDrools() {
 
-        DroolsSyntaxValidationService droolsSyntaxValidationService = new DroolsSyntaxValidationService(new KieContainerCreationService(rulesClient));
+        DroolsSyntaxValidationService droolsSyntaxValidationService = new DroolsSyntaxValidationService(new KieContainerCreationService(rulesClient),
+                                                                                                        new DeprecatedElementsFinder());
         var rulesFile = new ClassPathResource("drools/documine_flora_components.drl");
 
         String rulesString = new String(rulesFile.getInputStream().readAllBytes());
 
         DroolsSyntaxValidation droolsSyntaxValidation = droolsSyntaxValidationService.testRules(new RuleValidationModel(RuleFileType.COMPONENT.name(), rulesString));
-        droolsSyntaxValidation.getDroolsSyntaxErrorMessages().forEach(System.out::println);
+        droolsSyntaxValidation.getDroolsSyntaxErrorMessages()
+                .forEach(System.out::println);
         assertTrue(droolsSyntaxValidation.isCompiled());
     }
 
@@ -243,13 +185,15 @@ class DroolsSyntaxValidationServiceTest {
     @SneakyThrows
     void testComponentDroolsAsEntityDrools() {
 
-        DroolsSyntaxValidationService droolsSyntaxValidationService = new DroolsSyntaxValidationService(new KieContainerCreationService(rulesClient));
+        DroolsSyntaxValidationService droolsSyntaxValidationService = new DroolsSyntaxValidationService(new KieContainerCreationService(rulesClient),
+                                                                                                        new DeprecatedElementsFinder());
         var rulesFile = new ClassPathResource("drools/documine_flora_components.drl");
 
         String rulesString = new String(rulesFile.getInputStream().readAllBytes());
 
         DroolsSyntaxValidation droolsSyntaxValidation = droolsSyntaxValidationService.testRules(new RuleValidationModel(RuleFileType.ENTITY.name(), rulesString));
-        droolsSyntaxValidation.getDroolsSyntaxErrorMessages().forEach(System.out::println);
+        droolsSyntaxValidation.getDroolsSyntaxErrorMessages()
+                .forEach(System.out::println);
         assertFalse(droolsSyntaxValidation.isCompiled());
     }
 
@@ -259,13 +203,15 @@ class DroolsSyntaxValidationServiceTest {
     @Disabled
     void attemptImportsFixToAllRuleFiles() {
 
-        DroolsSyntaxValidationService droolsSyntaxValidationService = new DroolsSyntaxValidationService(new KieContainerCreationService(rulesClient));
+        DroolsSyntaxValidationService droolsSyntaxValidationService = new DroolsSyntaxValidationService(new KieContainerCreationService(rulesClient),
+                                                                                                        new DeprecatedElementsFinder());
 
-        List<String> ruleFiles = List.of("drools/rules.drl", "drools/all_redact_manager_rules.drl",
-                "drools/documine_flora.drl",
-                "drools/manual_redaction_rules.drl",
-                "drools/acceptance_rules.drl",
-                "drools/rules_v2.drl");
+        List<String> ruleFiles = List.of("drools/rules.drl",
+                                         "drools/all_redact_manager_rules.drl",
+                                         "drools/documine_flora.drl",
+                                         "drools/manual_redaction_rules.drl",
+                                         "drools/acceptance_rules.drl",
+                                         "drools/rules_v2.drl");
 
         for (String ruleFile : ruleFiles) {
             var rulesFile = new ClassPathResource(ruleFile);
@@ -288,4 +234,109 @@ class DroolsSyntaxValidationServiceTest {
 
     }
 
+
+    @Test
+    @SneakyThrows
+    void testRulesWithUnallowedAddedImports() {
+
+        DroolsSyntaxValidationService droolsSyntaxValidationService = new DroolsSyntaxValidationService(new KieContainerCreationService(rulesClient),
+                                                                                                        new DeprecatedElementsFinder());
+        var rulesFile = new ClassPathResource("drools/rules.drl");
+
+        String rulesString = new String(rulesFile.getInputStream().readAllBytes());
+        rulesString = rulesString.replaceAll("import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell;",
+                                             "import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell;\nimport com.google.common.collect.Sets;");
+        DroolsSyntaxValidation droolsSyntaxValidation = droolsSyntaxValidationService.testRules(new RuleValidationModel(RuleFileType.ENTITY.name(), rulesString));
+        droolsSyntaxValidation.getDroolsSyntaxErrorMessages()
+                .forEach(System.out::println);
+        assertFalse(droolsSyntaxValidation.isCompiled());
+    }
+
+
+    @Test
+    @SneakyThrows
+    void testAllRules() {
+
+        DroolsSyntaxValidationService droolsSyntaxValidationService = new DroolsSyntaxValidationService(new KieContainerCreationService(rulesClient),
+                                                                                                        new DeprecatedElementsFinder());
+        var rulesFile = new ClassPathResource("drools/all_redact_manager_rules.drl");
+
+        String rulesString = new String(rulesFile.getInputStream().readAllBytes());
+
+        DroolsSyntaxValidation droolsSyntaxValidation = droolsSyntaxValidationService.testRules(new RuleValidationModel(RuleFileType.ENTITY.name(), rulesString));
+        droolsSyntaxValidation.getDroolsSyntaxErrorMessages()
+                .forEach(System.out::println);
+        assertTrue(droolsSyntaxValidation.isCompiled());
+    }
+
+
+    @Test
+    @SneakyThrows
+    void testAcceptanceRules() {
+
+        DroolsSyntaxValidationService droolsSyntaxValidationService = new DroolsSyntaxValidationService(new KieContainerCreationService(rulesClient),
+                                                                                                        new DeprecatedElementsFinder());
+        var rulesFile = new ClassPathResource("drools/acceptance_rules.drl");
+
+        String rulesString = new String(rulesFile.getInputStream().readAllBytes());
+
+        DroolsSyntaxValidation droolsSyntaxValidation = droolsSyntaxValidationService.testRules(new RuleValidationModel(RuleFileType.ENTITY.name(), rulesString));
+        droolsSyntaxValidation.getDroolsSyntaxErrorMessages()
+                .forEach(System.out::println);
+        assertTrue(droolsSyntaxValidation.isCompiled());
+    }
+
+
+    @Test
+    @SneakyThrows
+    void testManualRedactionRules() {
+
+        DroolsSyntaxValidationService droolsSyntaxValidationService = new DroolsSyntaxValidationService(new KieContainerCreationService(rulesClient),
+                                                                                                        new DeprecatedElementsFinder());
+        var rulesFile = new ClassPathResource("drools/manual_redaction_rules.drl");
+
+        String rulesString = new String(rulesFile.getInputStream().readAllBytes());
+
+        DroolsSyntaxValidation droolsSyntaxValidation = droolsSyntaxValidationService.testRules(new RuleValidationModel(RuleFileType.ENTITY.name(), rulesString));
+        droolsSyntaxValidation.getDroolsSyntaxErrorMessages()
+                .forEach(System.out::println);
+        assertTrue(droolsSyntaxValidation.isCompiled());
+    }
+
+
+    @Test
+    @SneakyThrows
+    void testRulesV2() {
+
+        DroolsSyntaxValidationService droolsSyntaxValidationService = new DroolsSyntaxValidationService(new KieContainerCreationService(rulesClient),
+                                                                                                        new DeprecatedElementsFinder());
+        var rulesFile = new ClassPathResource("drools/rules_v2.drl");
+
+        String rulesString = new String(rulesFile.getInputStream().readAllBytes());
+
+        DroolsSyntaxValidation droolsSyntaxValidation = droolsSyntaxValidationService.testRules(new RuleValidationModel(RuleFileType.ENTITY.name(), rulesString));
+        droolsSyntaxValidation.getDroolsSyntaxErrorMessages()
+                .forEach(System.out::println);
+        assertTrue(droolsSyntaxValidation.isCompiled());
+    }
+
+
+    @Test
+    @SneakyThrows
+    void testRulesWithDeprecatedFunctions() {
+
+        DroolsSyntaxValidationService droolsSyntaxValidationService = new DroolsSyntaxValidationService(new KieContainerCreationService(rulesClient),
+                                                                                                        new DeprecatedElementsFinder());
+        var rulesFile = new ClassPathResource("drools/rules.drl");
+
+        String rulesString = new String(rulesFile.getInputStream().readAllBytes());
+        rulesString = rulesString.replaceAll(".resize\\(", ".resizeEntityAndReinsert(");
+
+        DroolsSyntaxValidation droolsSyntaxValidation = droolsSyntaxValidationService.testRules(new RuleValidationModel(RuleFileType.ENTITY.name(), rulesString));
+        droolsSyntaxValidation.getDroolsSyntaxErrorMessages()
+                .forEach(System.out::println);
+        assertTrue(droolsSyntaxValidation.isCompiled());
+        assertEquals(droolsSyntaxValidation.getDroolsSyntaxDeprecatedWarnings().size(), 2);
+    }
+
 }

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesEnd2EndTest.java

@@ -4,11 +4,13 @@ import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertFalse;
 import static org.junit.jupiter.api.Assertions.assertNotEquals;
 import static org.mockito.Mockito.when;
+import static org.wildfly.common.Assert.assertTrue;
 
 import java.awt.geom.Rectangle2D;
 import java.io.File;
 import java.io.FileOutputStream;
 import java.io.IOException;
+import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.time.OffsetDateTime;
 import java.util.Comparator;
@@ -36,11 +38,11 @@ import com.iqser.red.commons.jackson.ObjectMapperFactory;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeResult;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;
@@ -57,7 +59,6 @@ import com.iqser.red.service.redaction.v1.server.Application;
 import com.iqser.red.service.redaction.v1.server.annotate.AnnotateRequest;
 import com.iqser.red.service.redaction.v1.server.annotate.AnnotateResponse;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
-import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.OsUtils;
 import com.iqser.red.service.redaction.v1.server.service.document.DocumentGraphMapper;
@@ -122,6 +123,7 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest {
         loadNerForTest();
         when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L);
         when(dictionaryClient.getAllTypesForDossierTemplate(TEST_DOSSIER_TEMPLATE_ID, false)).thenReturn(getTypeResponse());
+        when(dictionaryClient.getAllTypesForDossierTemplate(TEST_DOSSIER_TEMPLATE_ID, true)).thenReturn(getTypeResponse());
 
         when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L);
         when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, false)).thenReturn(List.of(Type.builder()
@@ -148,7 +150,7 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest {
         String filePath = "files/new/crafted document.pdf";
         AnalyzeRequest request = uploadFileToStorage(filePath);
         analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request);
-        AnalyzeResult result = analyzeService.analyze(request);
+        analyzeService.analyze(request);
 
         String testEntityValue1 = "Desiree";
         String testEntityValue2 = "Melanie";
@@ -158,7 +160,7 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest {
 
         Document document = DocumentGraphMapper.toDocumentGraph(redactionStorageService.getDocumentData(TEST_DOSSIER_ID, TEST_FILE_ID));
         String expandedEntityKeyword = "Lorem ipsum dolor sit amet, consectetur adipiscing elit Desiree et al sed do eiusmod tempor incididunt ut labore et dolore magna aliqua Melanie et al. Reference No 12345 Lorem ipsum.";
-        TextEntity expandedEntity = entityCreationService.byString(expandedEntityKeyword, "PII", EntityType.ENTITY, document).findFirst().get();
+        entityCreationService.byString(expandedEntityKeyword, "PII", EntityType.ENTITY, document).findFirst().get();
 
         String idToResize = redactionLog.getEntityLogEntry()
                 .stream()
@@ -166,25 +168,18 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest {
                 .max(Comparator.comparingInt(EntityLogEntry::getStartOffset))
                 .get()
                 .getId();
-        List<Rectangle> resizedPositions = expandedEntity.getPositionsOnPagePerPage()
-                .get(0)
-                .getRectanglePerLine()
-                .stream()
-                .map(rectangle2D -> toAnnotationRectangle(rectangle2D, 3))
-                .toList();
-        ManualResizeRedaction manualResizeRedaction = ManualResizeRedaction.builder()
-                .annotationId(idToResize)
-                .value(expandedEntityKeyword)
-                .positions(resizedPositions)
-                .requestDate(OffsetDateTime.now())
-                .status(AnnotationStatus.APPROVED)
-                .updateDictionary(false)
-                .build();
-        manualResizeRedaction.setUpdateDictionary(false);
         ManualRedactions manualRedactions = new ManualRedactions();
-        manualRedactions.getResizeRedactions().add(manualResizeRedaction);
+        manualRedactions.getResizeRedactions().add(ManualResizeRedaction.builder()
+                        .annotationId(idToResize)
+                        .value(expandedEntityKeyword)
+                        .positions(List.of(Rectangle.builder().topLeftX(56.8f).topLeftY(454.664f).height(15.408f).width(493.62f).page(3).build(),
+                                Rectangle.builder().topLeftX(56.8f).topLeftY(440.864f).height(15.408f).width(396f).page(3).build()))
+                        .addToAllDossiers(false)
+                        .updateDictionary(false)
+                        .requestDate(OffsetDateTime.now())
+                .build());
         request.setManualRedactions(manualRedactions);
-        AnalyzeResult reanalyzeResult = analyzeService.reanalyze(request);
+        analyzeService.reanalyze(request);
 
         redactionLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
         String annotatedFileName = Paths.get(filePath).getFileName().toString().replace(".pdf", "_annotated2.pdf");
@@ -222,18 +217,17 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest {
 
         String manualAddId = UUID.randomUUID().toString();
 
-        manualRedactions.setIdsToRemove(Set.of(IdRemoval.builder().annotationId("5b940b2cb401ed9f5be6fc24f6e77bcf").fileId("fileId").status(AnnotationStatus.DECLINED).build()));
+        manualRedactions.setIdsToRemove(Set.of(IdRemoval.builder().annotationId("5b940b2cb401ed9f5be6fc24f6e77bcf").fileId("fileId").build()));
         manualRedactions.setForceRedactions(Set.of(ManualForceRedaction.builder()
                 .annotationId("675eba69b0c2917de55462c817adaa05")
                 .fileId("fileId")
                 .legalBasis("Something")
-                .status(AnnotationStatus.APPROVED)
                 .build()));
 
         ManualRedactionEntry manualRedactionEntry = new ManualRedactionEntry();
         manualRedactionEntry.setAnnotationId(manualAddId);
         manualRedactionEntry.setFileId("fileId");
-        manualRedactionEntry.setStatus(AnnotationStatus.REQUESTED);
+
         manualRedactionEntry.setType("name");
         manualRedactionEntry.setValue("O'Loughlin C.K.");
         manualRedactionEntry.setReason("Manual Redaction");
@@ -246,12 +240,11 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest {
         AnalyzeResult result = analyzeService.analyze(request);
 
         manualRedactions.getEntriesToAdd().add(manualRedactionEntry);
-        manualRedactions.setIdsToRemove(Set.of(IdRemoval.builder().annotationId("5b940b2cb401ed9f5be6fc24f6e77bcf").fileId("fileId").status(AnnotationStatus.APPROVED).build()));
+        manualRedactions.setIdsToRemove(Set.of(IdRemoval.builder().annotationId("5b940b2cb401ed9f5be6fc24f6e77bcf").fileId("fileId").build()));
         manualRedactions.setLegalBasisChanges((Set.of(ManualLegalBasisChange.builder()
                 .annotationId("675eba69b0c2917de55462c817adaa05")
                 .fileId("fileId")
                 .legalBasis("Manual Legal Basis Change")
-                .status(AnnotationStatus.APPROVED)
                 .requestDate(OffsetDateTime.now())
                 .build())));
 
@@ -271,6 +264,45 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest {
     }
 
 
+    @Test
+    public void testManualRedactionIsDuplicatedWithTableCells() throws IOException {
+
+        System.out.println("testManualRedaction");
+        long start = System.currentTimeMillis();
+        String pdfFile = "files/Minimal Examples/Meto1_Page22.pdf";
+
+        ManualRedactions manualRedactions = new ManualRedactions();
+
+        String manualAddId = UUID.randomUUID().toString();
+
+        ManualRedactionEntry manualRedactionEntry = new ManualRedactionEntry();
+        manualRedactionEntry.setAnnotationId(manualAddId);
+        manualRedactionEntry.setFileId("fileId");
+        manualRedactionEntry.setType("name");
+        manualRedactionEntry.setValue("Surface Water");
+        manualRedactionEntry.setReason("Dictionary Request");
+        manualRedactionEntry.setPositions(List.of(Rectangle.builder().topLeftX(76.584f).topLeftY(506.56238f).width(57.4329f).height(9.065156f).page(1).build()));
+        manualRedactions.setEntriesToAdd(Set.of(manualRedactionEntry));
+        AnalyzeRequest request = uploadFileToStorage(pdfFile);
+        request.setManualRedactions(manualRedactions);
+        analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request);
+        AnalyzeResult result = analyzeService.analyze(request);
+
+        var redactionLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
+        AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder().dossierId(TEST_DOSSIER_ID).fileId(TEST_FILE_ID).build());
+
+        try (FileOutputStream fileOutputStream = new FileOutputStream(OsUtils.getTemporaryDirectory() + "/" + Path.of(pdfFile).getFileName() + "MANUAL_REDACTION_TEST.pdf")) {
+            fileOutputStream.write(annotateResponse.getDocument());
+        }
+        long end = System.currentTimeMillis();
+        var optionalEntry = redactionLog.getEntityLogEntry().stream().filter(entityLogEntry -> entityLogEntry.getId().equals(manualAddId)).findAny();
+        assertTrue(optionalEntry.isPresent());
+        assertEquals(2, optionalEntry.get().getContainingNodeId().size()); // 2 is the depth of the table instead of the table cell
+        System.out.println("duration: " + (end - start));
+        System.out.println("numberOfPages: " + result.getNumberOfPages());
+    }
+
+
     @Test
     public void testReCategorizeToVertebrateChangesCbiAuthor() {
 
@@ -286,6 +318,7 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest {
                 .filter(entry -> entry.getValue().equals("Oxford University Press"))
                 .findFirst()
                 .get();
+        assertFalse(oxfordUniversityPress.getEngines().contains(Engine.MANUAL));
 
         var asyaLyon = redactionLog.getEntityLogEntry()
                 .stream()
@@ -300,7 +333,6 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest {
 
         ManualRecategorization recategorization = ManualRecategorization.builder()
                 .requestDate(OffsetDateTime.now())
-                .status(AnnotationStatus.APPROVED)
                 .type("vertebrate")
                 .annotationId(oxfordUniversityPress.getId())
                 .fileId(TEST_FILE_ID)
@@ -333,6 +365,7 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest {
 
         assertEquals(asyaLyon2.getState(), EntryState.APPLIED);
         assertEquals(1, oxfordUniversityPressRecategorized.getManualChanges().size());
+        assertTrue(oxfordUniversityPressRecategorized.getEngines().contains(Engine.MANUAL));
     }
 
 
@@ -350,19 +383,18 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest {
         manualRedactions.setEntriesToAdd(Set.of(ManualRedactionEntry.builder()
                 .annotationId(annotationId)
                 .requestDate(OffsetDateTime.now())
+                .type("manual")
                 .value("Expand to Hint Clarissa’s Donut ← not added to Dict, should be not annotated Simpson's Tower ← added to Authors-Dict, should be annotated")
                 .positions(List.of(//
                         new Rectangle(new Point(56.8f, 496.27f), 61.25f, 12.83f, 2), //
                         new Rectangle(new Point(56.8f, 482.26f), 303.804f, 15.408f, 2), //
                         new Rectangle(new Point(56.8f, 468.464f), 314.496f, 15.408f, 2))) //
-                .status(AnnotationStatus.APPROVED)
                 .build()));
         ManualResizeRedaction manualResizeRedaction = ManualResizeRedaction.builder()
                 .annotationId(annotationId)
                 .requestDate(OffsetDateTime.now())
                 .value("Expand to Hint")
                 .positions(List.of(new Rectangle(new Point(56.8f, 496.27f), 61.25f, 12.83f, 2)))
-                .status(AnnotationStatus.APPROVED)
                 .updateDictionary(false)
                 .build();
         manualRedactions.setResizeRedactions(Set.of(manualResizeRedaction));
@@ -371,10 +403,58 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest {
         analyzeService.reanalyze(request);
 
         EntityLog entityLog = redactionStorageService.getEntityLog(request.getDossierId(), request.getFileId());
-        EntityLogEntry entityLogEntry = entityLog.getEntityLogEntry().stream().filter(entry -> entry.getId().equals(annotationId)).findFirst().orElseThrow();
+        EntityLogEntry entityLogEntry = entityLog.getEntityLogEntry()
+                .stream()
+                .filter(entry -> entry.getId().equals(annotationId))
+                .findFirst()
+                .orElseThrow();
         assertEquals("Expand to Hint", entityLogEntry.getValue());
         assertEquals(1, entityLogEntry.getPositions().size());
-        assertEquals(ManualRedactionType.RESIZE, entityLogEntry.getManualChanges().get(entityLogEntry.getManualChanges().size() - 1).getManualRedactionType());
+        assertEquals(ManualRedactionType.RESIZE,
+                     entityLogEntry.getManualChanges()
+                             .get(entityLogEntry.getManualChanges().size() - 1).getManualRedactionType());
+        assertTrue(entityLogEntry.getEngines().contains(Engine.MANUAL));
+    }
+
+
+    @Test
+    @SneakyThrows
+    public void testAddEngineManualToResizeDictionaryEntry() {
+
+        String filePath = "files/new/crafted document.pdf";
+        AnalyzeRequest request = uploadFileToStorage(filePath);
+        analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request);
+        AnalyzeResult result = analyzeService.analyze(request);
+        ManualRedactions manualRedactions = new ManualRedactions();
+
+        EntityLog entityLog = redactionStorageService.getEntityLog(request.getDossierId(), request.getFileId());
+        var dictionaryEntry = entityLog.getEntityLogEntry()
+                .stream()
+                .filter(entry -> entry.isDictionaryEntry() || entry.isDossierDictionaryEntry())
+                .findFirst()
+                .get();
+        ManualResizeRedaction manualResizeRedaction = ManualResizeRedaction.builder()
+                .annotationId(dictionaryEntry.getId())
+                .requestDate(OffsetDateTime.now())
+                .value("Image")
+                .positions(List.of(new Rectangle(new Point(56.8f, 496.27f), 61.25f, 12.83f, 1)))
+                .updateDictionary(true)
+                .build();
+        manualRedactions.setResizeRedactions(Set.of(manualResizeRedaction));
+        request.setManualRedactions(manualRedactions);
+
+        analyzeService.reanalyze(request);
+
+        entityLog = redactionStorageService.getEntityLog(request.getDossierId(), request.getFileId());
+        EntityLogEntry entityLogEntry = entityLog.getEntityLogEntry()
+                .stream()
+                .filter(entry -> entry.getId().equals(dictionaryEntry.getId()))
+                .findFirst()
+                .orElseThrow();
+        assertEquals(ManualRedactionType.RESIZE_IN_DICTIONARY,
+                     entityLogEntry.getManualChanges()
+                             .get(entityLogEntry.getManualChanges().size() - 1).getManualRedactionType());
+        assertTrue(entityLogEntry.getEngines().contains(Engine.MANUAL));
     }
 
 }

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesIntegrationTest.java

@@ -7,13 +7,14 @@ import static org.wildfly.common.Assert.assertFalse;
 import java.awt.geom.Rectangle2D;
 import java.time.OffsetDateTime;
 import java.util.Collection;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
 import java.util.stream.Collectors;
 
 import org.junit.jupiter.api.Test;
 
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus;
+import com.google.common.collect.Sets;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction;
@@ -43,7 +44,6 @@ public class ManualChangesIntegrationTest extends RulesIntegrationTest {
                 .annotationId(initialId)
                 .value(biggerEntity.getValue())
                 .positions(toAnnotationRectangles(biggerEntity.getPositionsOnPagePerPage().get(0)))
-                .status(AnnotationStatus.APPROVED)
                 .requestDate(OffsetDateTime.now())
                 .updateDictionary(false)
                 .build();
@@ -52,7 +52,7 @@ public class ManualChangesIntegrationTest extends RulesIntegrationTest {
 
         assertEquals(biggerEntity.getTextRange(), entity.getTextRange());
         assertEquals(biggerEntity.getDeepestFullyContainingNode(), entity.getDeepestFullyContainingNode());
-        assertEquals(biggerEntity.getIntersectingNodes(), entity.getIntersectingNodes());
+        assertTrue(Sets.difference(new HashSet<>(biggerEntity.getIntersectingNodes()), new HashSet<>(entity.getIntersectingNodes())).isEmpty());
         assertEquals(biggerEntity.getPages(), entity.getPages());
         assertEquals(biggerEntity.getValue(), entity.getValue());
         assertEquals(initialId, entity.getPositionsOnPagePerPage().get(0).getId());
@@ -70,12 +70,7 @@ public class ManualChangesIntegrationTest extends RulesIntegrationTest {
         TextEntity entity = entities.stream().filter(e -> e.getPages().stream().anyMatch(p -> p.getNumber() == 1)).findFirst().get();
 
         String initialId = entity.getPositionsOnPagePerPage().get(0).getId();
-        ManualForceRedaction manualForceRedaction = ManualForceRedaction.builder()
-                .annotationId(initialId)
-                .status(AnnotationStatus.APPROVED)
-                .legalBasis("Something")
-                .requestDate(OffsetDateTime.now())
-                .build();
+        ManualForceRedaction manualForceRedaction = ManualForceRedaction.builder().annotationId(initialId).legalBasis("Something").requestDate(OffsetDateTime.now()).build();
 
         doAnalysis(document, List.of(manualForceRedaction));
 
@@ -100,7 +95,7 @@ public class ManualChangesIntegrationTest extends RulesIntegrationTest {
         TextEntity entity = entities.stream().filter(e -> e.getPages().stream().anyMatch(p -> p.getNumber() == 1)).findFirst().get();
 
         String initialId = entity.getPositionsOnPagePerPage().get(0).getId();
-        IdRemoval idRemoval = IdRemoval.builder().annotationId(initialId).status(AnnotationStatus.APPROVED).requestDate(OffsetDateTime.now()).build();
+        IdRemoval idRemoval = IdRemoval.builder().annotationId(initialId).requestDate(OffsetDateTime.now()).build();
 
         doAnalysis(document, List.of(idRemoval));
 
@@ -119,13 +114,8 @@ public class ManualChangesIntegrationTest extends RulesIntegrationTest {
         TextEntity entity = entities.stream().filter(e -> e.getPages().stream().anyMatch(p -> p.getNumber() == 1)).findFirst().get();
 
         String initialId = entity.getPositionsOnPagePerPage().get(0).getId();
-        IdRemoval idRemoval = IdRemoval.builder().annotationId(initialId).status(AnnotationStatus.APPROVED).requestDate(OffsetDateTime.now()).build();
-        ManualForceRedaction manualForceRedaction = ManualForceRedaction.builder()
-                .annotationId(initialId)
-                .status(AnnotationStatus.APPROVED)
-                .legalBasis("Something")
-                .requestDate(OffsetDateTime.now())
-                .build();
+        IdRemoval idRemoval = IdRemoval.builder().annotationId(initialId).requestDate(OffsetDateTime.now()).build();
+        ManualForceRedaction manualForceRedaction = ManualForceRedaction.builder().annotationId(initialId).legalBasis("Something").requestDate(OffsetDateTime.now()).build();
 
         doAnalysis(document, List.of(manualForceRedaction));
 
@@ -139,29 +129,6 @@ public class ManualChangesIntegrationTest extends RulesIntegrationTest {
     }
 
 
-    @Test
-    public void manualIDRemovalNotApprovedTest() {
-
-        Document document = buildGraph("files/new/crafted document");
-        Set<TextEntity> entities = entityCreationService.byString("David Ksenia", "CBI_author", EntityType.ENTITY, document).collect(Collectors.toUnmodifiableSet());
-
-        TextEntity entity = entities.stream().filter(e -> e.getPages().stream().anyMatch(p -> p.getNumber() == 1)).findFirst().get();
-
-        String initialId = entity.getPositionsOnPagePerPage().get(0).getId();
-        IdRemoval idRemoval = IdRemoval.builder().annotationId(initialId).status(AnnotationStatus.REQUESTED).build();
-
-        doAnalysis(document, List.of(idRemoval));
-
-        assertEquals(Paragraph.class, entity.getDeepestFullyContainingNode().getClass());
-        assertFalse(entity.getIntersectingNodes().isEmpty());
-        assertEquals(1, entity.getPages().size());
-        assertEquals("David Ksenia", entity.getValue());
-        assertEquals(initialId, entity.getPositionsOnPagePerPage().get(0).getId());
-        assertFalse(entity.ignored());
-        assertFalse(entity.removed());
-    }
-
-
     private void assertRectanglesAlmostEqual(Collection<Rectangle2D> rects1, Collection<Rectangle2D> rects2) {
 
         if (rects1.stream().allMatch(rect1 -> rects2.stream().anyMatch(rect2 -> rectanglesAlmostEqual(rect1, rect2)))) {

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesUnitTest.java

@@ -11,11 +11,10 @@ import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 import org.springframework.beans.factory.annotation.Autowired;
 
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
 import com.iqser.red.service.redaction.v1.server.document.graph.BuildDocumentIntegrationTest;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
@@ -56,7 +55,7 @@ public class ManualChangesUnitTest extends BuildDocumentIntegrationTest {
         String annotationId = entity.getPositionsOnPagePerPage().get(0).getId();
 
         // remove first
-        IdRemoval removal = IdRemoval.builder().requestDate(start).fileId(TEST_FILE_ID).annotationId(annotationId).status(AnnotationStatus.APPROVED).build();
+        IdRemoval removal = IdRemoval.builder().requestDate(start).fileId(TEST_FILE_ID).annotationId(annotationId).build();
         entity.getManualOverwrite().addChange(removal);
         assertTrue(entity.ignored());
         assertFalse(entity.applied());
@@ -68,7 +67,6 @@ public class ManualChangesUnitTest extends BuildDocumentIntegrationTest {
                 .fileId(TEST_FILE_ID)
                 .annotationId(annotationId)
                 .legalBasis("coolio")
-                .status(AnnotationStatus.APPROVED)
                 .build();
         entity.getManualOverwrite().addChange(forceRedaction);
         assertTrue(entity.applied());
@@ -78,7 +76,7 @@ public class ManualChangesUnitTest extends BuildDocumentIntegrationTest {
         assertEquals("coolio", entity.getManualOverwrite().getLegalBasis().orElse(entity.getMatchedRule().getLegalBasis()));
 
         // remove again
-        IdRemoval removal2 = IdRemoval.builder().requestDate(start.plusSeconds(3)).fileId(TEST_FILE_ID).annotationId(annotationId).status(AnnotationStatus.APPROVED).build();
+        IdRemoval removal2 = IdRemoval.builder().requestDate(start.plusSeconds(3)).fileId(TEST_FILE_ID).annotationId(annotationId).build();
         entity.getManualOverwrite().addChange(removal2);
         assertTrue(entity.ignored());
         assertFalse(entity.applied());
@@ -90,7 +88,6 @@ public class ManualChangesUnitTest extends BuildDocumentIntegrationTest {
                 .fileId(TEST_FILE_ID)
                 .annotationId(annotationId)
                 .legalBasis("coolio")
-                .status(AnnotationStatus.APPROVED)
                 .build();
         entity.getManualOverwrite().addChange(forceRedaction2);
         assertTrue(entity.ignored());
@@ -106,7 +103,6 @@ public class ManualChangesUnitTest extends BuildDocumentIntegrationTest {
                 .annotationId(annotationId)
                 .requestDate(start.plusSeconds(4))
                 .section(section)
-                .status(AnnotationStatus.APPROVED)
                 .user("peter")
                 .value(value)
                 .build();
@@ -123,14 +119,15 @@ public class ManualChangesUnitTest extends BuildDocumentIntegrationTest {
                 .type("type")
                 .requestDate(start.plusSeconds(5))
                 .annotationId(annotationId)
-                .status(AnnotationStatus.APPROVED)
+                .legalBasis("")
                 .build();
         entity.getManualOverwrite().addChange(imageRecategorizationRequest);
         assertTrue(entity.getManualOverwrite().getRecategorized().isPresent());
         assertTrue(entity.getManualOverwrite().getRecategorized().get());
-        assertEquals("type", entity.getManualOverwrite().getType().orElse(entity.getType()));
+        assertEquals("type", entity.getManualOverwrite().getType().orElse(entity.type()));
     }
 
+
     @Test
     public void testRemoveHintForceHint() {
 
@@ -149,7 +146,7 @@ public class ManualChangesUnitTest extends BuildDocumentIntegrationTest {
         String annotationId = entity.getPositionsOnPagePerPage().get(0).getId();
 
         // remove first
-        IdRemoval removal = IdRemoval.builder().requestDate(start).fileId(TEST_FILE_ID).annotationId(annotationId).status(AnnotationStatus.APPROVED).build();
+        IdRemoval removal = IdRemoval.builder().requestDate(start).fileId(TEST_FILE_ID).annotationId(annotationId).build();
         entity.getManualOverwrite().addChange(removal);
         assertTrue(entity.ignored());
         assertFalse(entity.applied());
@@ -161,7 +158,6 @@ public class ManualChangesUnitTest extends BuildDocumentIntegrationTest {
                 .fileId(TEST_FILE_ID)
                 .annotationId(annotationId)
                 .legalBasis("coolio")
-                .status(AnnotationStatus.APPROVED)
                 .build();
         entity.getManualOverwrite().addChange(forceRedaction);
         assertFalse(entity.applied());

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/PrecursorEntityTest.java

@@ -25,7 +25,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Point;
 import com.iqser.red.service.redaction.v1.server.document.graph.BuildDocumentIntegrationTest;
-import com.iqser.red.service.redaction.v1.server.model.ManualEntity;
+import com.iqser.red.service.redaction.v1.server.model.PrecursorEntity;
 import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryVersion;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.IEntity;
@@ -35,17 +35,17 @@ import com.iqser.red.service.redaction.v1.server.service.DictionaryService;
 import com.iqser.red.service.redaction.v1.server.service.EntityLogCreatorService;
 import com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService;
 import com.iqser.red.service.redaction.v1.server.service.document.EntityEnrichmentService;
-import com.iqser.red.service.redaction.v1.server.service.document.ManualEntityCreationService;
+import com.iqser.red.service.redaction.v1.server.service.document.EntityFromPrecursorCreationService;
 
 import lombok.SneakyThrows;
 
-public class ManualEntityTest extends BuildDocumentIntegrationTest {
+public class PrecursorEntityTest extends BuildDocumentIntegrationTest {
 
     @Autowired
     private EntityEnrichmentService entityEnrichmentService;
 
     @Autowired
-    private ManualEntityCreationService manualEntityCreationService;
+    private EntityFromPrecursorCreationService entityFromPrecursorCreationService;
 
     @Autowired
     private EntityLogCreatorService entityLogCreatorService;
@@ -122,7 +122,7 @@ public class ManualEntityTest extends BuildDocumentIntegrationTest {
 
         assertTrue(document.getEntities().isEmpty());
 
-        List<ManualEntity> notFoundManualEntities = manualEntityCreationService.createRedactionEntitiesIfFoundAndReturnNotFoundEntries(ManualRedactions.builder().entriesToAdd(Set.of(manualRedactionEntry)).build(),
+        List<PrecursorEntity> notFoundManualEntities = entityFromPrecursorCreationService.createEntitiesIfFoundAndReturnNotFoundEntries(ManualRedactions.builder().entriesToAdd(Set.of(manualRedactionEntry)).build(),
                 document,
                 TEST_DOSSIER_TEMPLATE_ID);
         assertEquals(1, notFoundManualEntities.size());
@@ -162,7 +162,7 @@ public class ManualEntityTest extends BuildDocumentIntegrationTest {
                 .reason("reason")
                 .legalBasis("n-a")
                 .section(tempEntity.getDeepestFullyContainingNode().toString())
-                .rectangle(true)
+                .rectangle(false)
                 .positions(positions)
                 .requestDate(OffsetDateTime.now())
                 .textAfter("")
@@ -172,7 +172,7 @@ public class ManualEntityTest extends BuildDocumentIntegrationTest {
         tempEntity.removeFromGraph();
         assertTrue(document.getEntities().isEmpty());
 
-        List<ManualEntity> notFoundManualEntities = manualEntityCreationService.createRedactionEntitiesIfFoundAndReturnNotFoundEntries(ManualRedactions.builder().entriesToAdd(Set.of(manualRedactionEntry)).build(),
+        List<PrecursorEntity> notFoundManualEntities = entityFromPrecursorCreationService.createEntitiesIfFoundAndReturnNotFoundEntries(ManualRedactions.builder().entriesToAdd(Set.of(manualRedactionEntry)).build(),
                 document,
                 TEST_DOSSIER_TEMPLATE_ID);
         assertTrue(notFoundManualEntities.isEmpty());

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/adapter/NerEntitiesAdapterTest.java

@@ -167,14 +167,14 @@ class NerEntitiesAdapterTest extends BuildDocumentIntegrationTest {
 
     private List<Rectangle2D> getPositionsFromEntityOfType(String type, List<TextEntity> entities) {
 
-        return getPositionsFromEntities(entities.stream().filter(e -> e.getType().equals(type)));
+        return getPositionsFromEntities(entities.stream().filter(e -> e.type().equals(type)));
 
     }
 
 
     private List<Rectangle2D> getPositionsFromEntityNotOfType(List<String> types, List<TextEntity> entities) {
 
-        return getPositionsFromEntities(entities.stream().filter(e -> types.stream().noneMatch(type -> e.getType().equals(type))));
+        return getPositionsFromEntities(entities.stream().filter(e -> types.stream().noneMatch(type -> e.type().equals(type))));
 
     }
 

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/rules/RulesIntegrationTest.java

@@ -28,6 +28,8 @@ public class RulesIntegrationTest extends BuildDocumentIntegrationTest {
 
     @Autowired
     protected EntityEnrichmentService entityEnrichmentService;
+    @Autowired
+    protected ManualChangesApplicationService manualChangesApplicationService;
     protected EntityCreationService entityCreationService;
     protected KieSession kieSession;
 
@@ -72,7 +74,6 @@ public class RulesIntegrationTest extends BuildDocumentIntegrationTest {
 
         kieSession = kieContainer.newKieSession();
         entityCreationService = new EntityCreationService(entityEnrichmentService, kieSession);
-        ManualChangesApplicationService manualChangesApplicationService = new ManualChangesApplicationService(entityCreationService);
         kieSession.setGlobal("manualChangesApplicationService", manualChangesApplicationService);
         kieSession.setGlobal("entityCreationService", entityCreationService);
     }

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/service/document/SemanticNodeComparatorsTest.java

@@ -16,8 +16,8 @@ class SemanticNodeComparatorsTest {
     @Test
     public void testFirstSemanticNode() {
 
-        var node = new Section(List.of(0, 1), null, null, null, null);
-        var otherNode = new Section(List.of(0, 2), null, null, null, null);
+        var node = new Section(List.of(0, 1), null, null, null, null,null);
+        var otherNode = new Section(List.of(0, 2), null, null, null, null,null);
         List<SemanticNode> list = new ArrayList<>();
         list.add(otherNode);
         list.add(node);
@@ -29,8 +29,8 @@ class SemanticNodeComparatorsTest {
     @Test
     public void testFirstSemanticNode2() {
 
-        var node = new Section(Collections.emptyList(), null, null, null, null);
-        var otherNode = new Section(List.of(0, 2), null, null, null, null);
+        var node = new Section(Collections.emptyList(), null, null, null, null, null);
+        var otherNode = new Section(List.of(0, 2), null, null, null, null, null);
         List<SemanticNode> list = new ArrayList<>();
         list.add(otherNode);
         list.add(node);
@@ -42,8 +42,8 @@ class SemanticNodeComparatorsTest {
     @Test
     public void testFirstSemanticNode3() {
 
-        var node = new Section(List.of(1, 5, 8), null, null, null, null);
-        var otherNode = new Section(List.of(0, 2), null, null, null, null);
+        var node = new Section(List.of(1, 5, 8), null, null, null, null, null);
+        var otherNode = new Section(List.of(0, 2), null, null, null, null, null);
         List<SemanticNode> list = new ArrayList<>();
         list.add(otherNode);
         list.add(node);
@@ -55,8 +55,8 @@ class SemanticNodeComparatorsTest {
     @Test
     public void testFirstSemanticNode4() {
 
-        var node = new Section(List.of(1, 5, 8), null, null, null, null);
-        var otherNode = new Section(List.of(1, 5, 9), null, null, null, null);
+        var node = new Section(List.of(1, 5, 8), null, null, null, null,null);
+        var otherNode = new Section(List.of(1, 5, 9), null, null, null, null,null);
         List<SemanticNode> list = new ArrayList<>();
         list.add(otherNode);
         list.add(node);

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/service/document/UnprocessedChangesServiceTest.java

@@ -174,7 +174,7 @@ public class UnprocessedChangesServiceTest extends AbstractRedactionIntegrationT
 
         analyzeService.reanalyze(request);
 
-        List<Rectangle> positions = List.of(Rectangle.builder().topLeftX(286.1072f).topLeftY(266.18945f).width(98.7528f).height(10.048125f).build());
+        List<Rectangle> positions = List.of(Rectangle.builder().topLeftX(286.1072f).topLeftY(266.18945f).width(98.7528f).height(10.048125f).page(1).build());
         ManualResizeRedaction manualResizeRedaction = prepareManualSizeRedaction(aoelId, positions, "was above the AOEL");
 
         request.setManualRedactions(ManualRedactions.builder().resizeRedactions(Set.of(manualResizeRedaction)).build());
@@ -221,7 +221,7 @@ public class UnprocessedChangesServiceTest extends AbstractRedactionIntegrationT
         analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request);
         analyzeService.analyze(request);
 
-        List<Rectangle> positions = List.of(Rectangle.builder().topLeftX(369.024f).topLeftY(240.8695f).width(29.32224f).height(10.048125f).build());
+        List<Rectangle> positions = List.of(Rectangle.builder().topLeftX(369.024f).topLeftY(240.8695f).width(29.32224f).height(10.048125f).page(1).build());
         List<Rectangle> positions2 = List.of(Rectangle.builder().topLeftX(129.86f).topLeftY(505.7295f).width(80.144233125f).height(10.048125f).page(1).build());
         List<Rectangle> positions3 = List.of(Rectangle.builder().topLeftX(70.944f).topLeftY(291.5095f).width(107.01071999999994f).height(10.048125f).page(1).build());
         ManualResizeRedaction manualResizeRedaction = prepareManualSizeRedaction(aoelId, positions, "AOEL");
@@ -285,7 +285,7 @@ public class UnprocessedChangesServiceTest extends AbstractRedactionIntegrationT
         analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request);
         analyzeService.analyze(request);
 
-        List<Rectangle> positions = List.of(Rectangle.builder().topLeftX(369.024f).topLeftY(240.8695f).width(29.32224f).height(10.048125f).build());
+        List<Rectangle> positions = List.of(Rectangle.builder().topLeftX(369.024f).topLeftY(240.8695f).width(29.32224f).height(10.048125f).page(1).build());
         ManualResizeRedaction manualResizeRedaction = prepareManualSizeRedaction(aoelId, positions, "AOEL");
 
         request.setManualRedactions(ManualRedactions.builder().resizeRedactions(Set.of(manualResizeRedaction)).build());
@@ -323,7 +323,7 @@ public class UnprocessedChangesServiceTest extends AbstractRedactionIntegrationT
         analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request);
         analyzeService.analyze(request);
 
-        List<Rectangle> positions = List.of(Rectangle.builder().topLeftX(149.94624f).topLeftY(417.1695f).width(37.23792f).height(10.048125f).build());
+        List<Rectangle> positions = List.of(Rectangle.builder().topLeftX(149.94624f).topLeftY(417.1695f).width(37.23792f).height(10.048125f).page(1).build());
         ManualResizeRedaction manualResizeRedaction = prepareManualSizeRedaction(aoelId, positions, "AAOEL");
 
         request.setManualRedactions(ManualRedactions.builder().resizeRedactions(Set.of(manualResizeRedaction)).build());
@@ -361,7 +361,7 @@ public class UnprocessedChangesServiceTest extends AbstractRedactionIntegrationT
         analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request);
         analyzeService.analyze(request);
 
-        List<Rectangle> positions = List.of(Rectangle.builder().topLeftX(293.024f).topLeftY(240.8695f).width(29.32224f).height(10.048125f).build());
+        List<Rectangle> positions = List.of(Rectangle.builder().topLeftX(293.024f).topLeftY(240.8695f).width(29.32224f).height(10.048125f).page(1).build());
         ManualResizeRedaction manualResizeRedaction = prepareManualSizeRedaction(aoelId, positions, "Does Not Exist");
 
         request.setManualRedactions(ManualRedactions.builder().resizeRedactions(Set.of(manualResizeRedaction)).build());
@@ -392,7 +392,6 @@ public class UnprocessedChangesServiceTest extends AbstractRedactionIntegrationT
         ManualRedactionEntry manualRedactionEntry = new ManualRedactionEntry();
         manualRedactionEntry.setAnnotationId(id);
         manualRedactionEntry.setFileId("fileId");
-        manualRedactionEntry.setStatus(AnnotationStatus.APPROVED);
         manualRedactionEntry.setType("CBI_author");
         manualRedactionEntry.setValue(value);
         manualRedactionEntry.setReason("Manual Redaction");

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/utils/EntityVisualizationUtility.java

@@ -0,0 +1,61 @@
+package com.iqser.red.service.redaction.v1.server.utils;
+
+import java.awt.Color;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import org.apache.pdfbox.cos.COSName;
+
+import com.iqser.red.service.redaction.v1.server.model.document.entity.PositionOnPage;
+import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.Page;
+import com.knecon.fforesight.service.viewerdoc.ContentStreams;
+import com.knecon.fforesight.service.viewerdoc.model.ColoredRectangle;
+import com.knecon.fforesight.service.viewerdoc.model.VisualizationsOnPage;
+
+import lombok.experimental.UtilityClass;
+
+@UtilityClass
+public class EntityVisualizationUtility {
+
+    public static final ContentStreams.Identifier ENTITY_LAYER = new ContentStreams.Identifier("Entities", COSName.getPDFName("KNECON_ENTITIES"), true);
+
+
+    public Map<Integer, VisualizationsOnPage> createVisualizationsOnPage(Collection<TextEntity> entity, Color color) {
+
+        Map<Integer, VisualizationsOnPage> visualizations = new HashMap<>();
+        Set<Page> pages = entity.stream()
+                .map(TextEntity::getPages)
+                .flatMap(Collection::stream)
+                .collect(Collectors.toSet());
+
+        pages.forEach(page -> visualizations.put(page.getNumber() - 1, buildVisualizationsOnPage(color, page)));
+
+        return visualizations;
+    }
+
+
+    private static VisualizationsOnPage buildVisualizationsOnPage(Color color, Page page) {
+
+        return VisualizationsOnPage.builder().coloredRectangles(getEntityRectangles(color, page)).build();
+    }
+
+
+    private static List<ColoredRectangle> getEntityRectangles(Color color, Page page) {
+
+        return page.getEntities()
+                .stream()
+                .map(TextEntity::getPositionsOnPagePerPage)
+                .flatMap(Collection::stream)
+                .filter(p -> p.getPage().equals(page))
+                .map(PositionOnPage::getRectanglePerLine)
+                .flatMap(Collection::stream)
+                .map(r -> new ColoredRectangle(r, color, 1))
+                .toList();
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/utils/LayoutParsingRequestProvider.java

@@ -29,11 +29,14 @@ public class LayoutParsingRequestProvider {
                 .originFileStorageId(originFileStorageId)
                 .tablesFileStorageId(Optional.of(tablesFileStorageId))
                 .imagesFileStorageId(Optional.of(imagesFileStorageId))
+                .visualLayoutParsingFileId(Optional.empty())
                 .structureFileStorageId(structureFileStorageId)
                 .textBlockFileStorageId(textBlockFileStorageId)
                 .positionBlockFileStorageId(positionBlockFileStorageId)
                 .pageFileStorageId(pageFileStorageId)
-                .simplifiedTextStorageId(simplifiedTextStorageId).viewerDocumentStorageId(viewerDocumentStorageId)
+                .simplifiedTextStorageId(simplifiedTextStorageId)
+                .viewerDocumentStorageId(viewerDocumentStorageId)
+                .visualLayoutParsingFileId(Optional.empty())
                 .build();
     }
 

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/acceptance_rules.drl

@@ -53,7 +53,8 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus;
+
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
 
 global Document document
 global EntityCreationService entityCreationService
@@ -85,7 +86,7 @@ rule "SYN.1.0: Recommend CTL/BL laboratory that start with BL or CTL"
 rule "CBI.0.0: Redact CBI Authors (non vertebrate Study)"
     when
         not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
-        $entity: TextEntity(type == "CBI_author", dictionaryEntry)
+        $entity: TextEntity(type() == "CBI_author", dictionaryEntry)
     then
         $entity.redact("CBI.0.0", "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
     end
@@ -93,7 +94,7 @@ rule "CBI.0.0: Redact CBI Authors (non vertebrate Study)"
 rule "CBI.0.1: Redact CBI Authors (vertebrate Study)"
     when
         FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
-        $entity: TextEntity(type == "CBI_author", dictionaryEntry)
+        $entity: TextEntity(type() == "CBI_author", dictionaryEntry)
     then
         $entity.redact("CBI.0.1", "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
     end
@@ -103,7 +104,7 @@ rule "CBI.0.1: Redact CBI Authors (vertebrate Study)"
 rule "CBI.1.0: Do not redact CBI Address (non vertebrate Study)"
     when
         not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
-        $entity: TextEntity(type == "CBI_address", dictionaryEntry)
+        $entity: TextEntity(type() == "CBI_address", dictionaryEntry)
     then
         $entity.skip("CBI.1.0", "Address found for Non Vertebrate Study");
     end
@@ -111,7 +112,7 @@ rule "CBI.1.0: Do not redact CBI Address (non vertebrate Study)"
 rule "CBI.1.1: Redact CBI Address (vertebrate Study)"
     when
         FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
-        $entity: TextEntity(type == "CBI_address", dictionaryEntry)
+        $entity: TextEntity(type() == "CBI_address", dictionaryEntry)
     then
         $entity.redact("CBI.1.1", "Address found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
     end
@@ -120,7 +121,7 @@ rule "CBI.1.1: Redact CBI Address (vertebrate Study)"
 // Rule unit: CBI.2
 rule "CBI.2.0: Do not redact genitive CBI Author"
     when
-        $entity: TextEntity(type == "CBI_author", anyMatch(textAfter, "['’’'ʼˈ´`‘′ʻ’']s"))
+        $entity: TextEntity(type() == "CBI_author", anyMatch(textAfter, "['’’'ʼˈ´`‘′ʻ’']s"))
     then
         entityCreationService.byTextRange($entity.getTextRange(), "CBI_author", EntityType.FALSE_POSITIVE, document)
         .ifPresent(falsePositive -> falsePositive.skip("CBI.2.0", "Genitive Author found"));
@@ -149,7 +150,7 @@ rule "CBI.7.1: Do not redact Names and Addresses if published information found
         $table: Table(hasEntitiesOfType("published_information"), hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address"))
         $cellsWithPublishedInformation: TableCell() from $table.streamTableCellsWhichContainType("published_information").toList()
         $tableCell: TableCell(row == $cellsWithPublishedInformation.row) from $table.streamTableCells().toList()
-        $authorOrAddress: TextEntity(type == "CBI_author" || type == "CBI_address", active()) from $tableCell.getEntities()
+        $authorOrAddress: TextEntity(type() == "CBI_author" || type() == "CBI_address", active()) from $tableCell.getEntities()
     then
         $authorOrAddress.skipWithReferences("CBI.7.1", "Published Information found in row", $table.getEntitiesOfTypeInSameRow("published_information", $authorOrAddress));
     end
@@ -302,7 +303,7 @@ rule "CBI.20.1: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJEC
 rule "PII.0.0: Redact all PII (non vertebrate study)"
     when
         not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
-        $pii: TextEntity(type == "PII", dictionaryEntry)
+        $pii: TextEntity(type() == "PII", dictionaryEntry)
     then
         $pii.redact("PII.0.0", "Personal Information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
     end
@@ -310,7 +311,7 @@ rule "PII.0.0: Redact all PII (non vertebrate study)"
 rule "PII.0.1: Redact all PII (vertebrate study)"
     when
         FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
-        $pii: TextEntity(type == "PII", dictionaryEntry)
+        $pii: TextEntity(type() == "PII", dictionaryEntry)
     then
         $pii.redact("PII.0.1", "Personal Information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
     end
@@ -374,6 +375,214 @@ rule "PII.2.1: Redact Phone and Fax by RegEx (vertebrate study)"
     end
 
 
+// Rule unit: PII.3
+rule "PII.3.0: Redact telephone numbers by RegEx (Non vertebrate study)"
+    when
+        not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
+        $section: Section(matchesRegex("[+]\\d{1,}"))
+    then
+        entityCreationService.byRegex("((([+]\\d{1,3} (\\d{7,12})\\b)|([+]\\d{1,3}(\\d{3,12})\\b|[+]\\d{1,3}([ -]\\(?\\d{1,6}\\)?){2,4})|[+]\\d{1,3} ?((\\d{2,6}\\)?)([ -]\\d{2,6}){1,4}))(-\\d{1,3})?\\b)", "PII", EntityType.ENTITY, 1, $section)
+        .forEach(entity -> entity.redact("PII.3.0", "Telephone number found by regex", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
+    end
+
+rule "PII.3.1: Redact telephone numbers by RegEx (vertebrate study)"
+    when
+        FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
+        $section: Section(matchesRegex("[+]\\d{1,}"))
+    then
+        entityCreationService.byRegex("((([+]\\d{1,3} (\\d{7,12})\\b)|([+]\\d{1,3}(\\d{3,12})\\b|[+]\\d{1,3}([ -]\\(?\\d{1,6}\\)?){2,4})|[+]\\d{1,3} ?((\\d{2,6}\\)?)([ -]\\d{2,6}){1,4}))(-\\d{1,3})?\\b)", "PII", EntityType.ENTITY, 1, $section)
+        .forEach(entity -> entity.redact("PII.3.1", "Telephone number found by regex", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
+    end
+
+
+// Rule unit: PII.4
+rule "PII.4.0: Redact line after contact information keywords (non vertebrate study)"
+    when
+        not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
+        $contactKeyword: String() from List.of("Contact point:",
+                                         "Contact:",
+                                         "Alternative contact:",
+                                         "European contact:",
+                                         "No:",
+                                         "Contact:",
+                                         "Tel.:",
+                                         "Tel:",
+                                         "Telephone number:",
+                                         "Telephone No:",
+                                         "Telephone:",
+                                         "Phone No.",
+                                         "Phone:",
+                                         "Fax number:",
+                                         "Fax:",
+                                         "E-mail:",
+                                         "Email:",
+                                         "e-mail:",
+                                         "E-mail address:")
+        $section: Section(containsString($contactKeyword))
+    then
+        entityCreationService.lineAfterString($contactKeyword, "PII", EntityType.ENTITY, $section)
+            .forEach(contactEntity -> contactEntity.redact("PII.4.0", "Found after \"" + $contactKeyword + "\" contact keyword", "Reg (EC) No 1107/2009 Art. 63 (2e)"));
+    end
+
+rule "PII.4.1: Redact line after contact information keywords (vertebrate study)"
+    when
+        FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
+        $contactKeyword: String() from List.of("Contact point:",
+                                         "Contact:",
+                                         "Alternative contact:",
+                                         "European contact:",
+                                         "No:",
+                                         "Contact:",
+                                         "Tel.:",
+                                         "Tel:",
+                                         "Telephone number:",
+                                         "Telephone No:",
+                                         "Telephone:",
+                                         "Phone No.",
+                                         "Phone:",
+                                         "Fax number:",
+                                         "Fax:",
+                                         "E-mail:",
+                                         "Email:",
+                                         "e-mail:",
+                                         "E-mail address:")
+        $section: Section(containsString($contactKeyword))
+    then
+        entityCreationService.lineAfterString($contactKeyword, "PII", EntityType.ENTITY, $section)
+            .forEach(contactEntity -> contactEntity.redact("PII.4.1", "Found after \"" + $contactKeyword + "\" contact keyword", "Reg (EC) No 1107/2009 Art. 63 (2e)"));
+    end
+
+
+// Rule unit: PII.5
+rule "PII.5.0: Redact line after contact information keywords reduced (non vertebrate study)"
+    when
+        not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
+        $contactKeyword: String() from List.of("Contact point:",
+                                         "Contact:",
+                                         "Alternative contact:",
+                                         "European contact:")
+        $section: Section(containsString($contactKeyword))
+    then
+        entityCreationService.lineAfterString($contactKeyword, "PII", EntityType.ENTITY, $section)
+            .forEach(contactEntity -> contactEntity.redact("PII.5.0", "Found after \"" + $contactKeyword + "\" contact keyword", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
+    end
+
+rule "PII.5.1: Redact line after contact information keywords reduced (Vertebrate study)"
+    when
+        FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
+        $contactKeyword: String() from List.of("Contact point:",
+                                               "Contact:",
+                                               "Alternative contact:",
+                                               "European contact:")
+        $section: Section(containsString($contactKeyword))
+    then
+        entityCreationService.lineAfterString($contactKeyword, "PII", EntityType.ENTITY, $section)
+            .forEach(contactEntity -> contactEntity.redact("PII.5.1", "Found after \"" + $contactKeyword + "\" contact keyword", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
+    end
+
+
+// Rule unit: PII.6
+rule "PII.6.0: Redact line between contact keywords (non vertebrate study)"
+    when
+        not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
+        $section: Section((containsString("No:") && containsString("Fax")) || (containsString("Contact:") && containsString("Tel")))
+    then
+        Stream.concat(
+            entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section),
+            entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section)
+        )
+            .forEach(contactEntity -> contactEntity.redact("PII.6.0", "Found between contact keywords", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
+    end
+
+rule "PII.6.1: Redact line between contact keywords (vertebrate study)"
+    when
+        FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
+        $section: Section((containsString("No:") && containsString("Fax")) || (containsString("Contact:") && containsString("Tel")))
+    then
+        Stream.concat(
+            entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section),
+            entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section)
+        )
+            .forEach(contactEntity -> contactEntity.redact("PII.6.1", "Found between contact keywords", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
+    end
+
+
+// Rule unit: PII.7
+rule "PII.7.0: Redact contact information if applicant is found (non vertebrate study)"
+    when
+        not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
+        $section: Section(getHeadline().containsString("applicant") ||
+                          getHeadline().containsString("Primary contact") ||
+                          getHeadline().containsString("Alternative contact") ||
+                          containsString("Applicant") ||
+                          containsString("Telephone number:"))
+    then
+        Stream.concat(entityCreationService.lineAfterStrings(List.of("Contact point:", "Contact:", "Alternative contact:", "European contact:", "No:", "Contact:", "Tel.:", "Tel:", "Telephone number:",
+                            "Telephone No:", "Telephone:", "Phone No.", "Phone:", "Fax number:", "Fax:", "E-mail:", "Email:", "e-mail:", "E-mail address:"), "PII", EntityType.ENTITY, $section),
+             Stream.concat(
+                 entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section),
+                 entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section)
+             ))
+            .forEach(entity -> entity.redact("PII.7.0", "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
+    end
+
+rule "PII.7.1: Redact contact information if applicant is found (vertebrate study)"
+    when
+        FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
+        $section: Section(getHeadline().containsString("applicant") ||
+                          getHeadline().containsString("Primary contact") ||
+                          getHeadline().containsString("Alternative contact") ||
+                          containsString("Applicant") ||
+                          containsString("Telephone number:"))
+    then
+        Stream.concat(entityCreationService.lineAfterStrings(List.of("Contact point:", "Contact:", "Alternative contact:", "European contact:", "No:", "Contact:", "Tel.:", "Tel:", "Telephone number:",
+                            "Telephone No:", "Telephone:", "Phone No.", "Phone:", "Fax number:", "Fax:", "E-mail:", "Email:", "e-mail:", "E-mail address:"), "PII", EntityType.ENTITY, $section),
+             Stream.concat(
+                 entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section),
+                 entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section)
+             ))
+            .forEach(entity -> entity.redact("PII.7.1", "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
+    end
+
+
+// Rule unit: PII.8
+rule "PII.8.0: Redact contact information if producer is found (non vertebrate study)"
+    when
+        not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
+        $section: Section(containsStringIgnoreCase("producer of the plant protection") ||
+                          containsStringIgnoreCase("producer of the active substance") ||
+                          containsStringIgnoreCase("manufacturer of the active substance") ||
+                          containsStringIgnoreCase("manufacturer:") ||
+                          containsStringIgnoreCase("Producer or producers of the active substance"))
+    then
+        Stream.concat(entityCreationService.lineAfterStrings(List.of("Contact point:", "Contact:", "Alternative contact:", "European contact:", "No:", "Contact:", "Tel.:", "Tel:", "Telephone number:",
+                            "Telephone No:", "Telephone:", "Phone No.", "Phone:", "Fax number:", "Fax:", "E-mail:", "Email:", "e-mail:", "E-mail address:"), "PII", EntityType.ENTITY, $section),
+             Stream.concat(
+                 entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section),
+                 entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section)
+             ))
+            .forEach(entity -> entity.redact("PII.8.0", "Producer was found", "Reg (EC) No 1107/2009 Art. 63 (2e)"));
+    end
+
+rule "PII.8.1: Redact contact information if producer is found (vertebrate study)"
+    when
+        FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
+        $section: Section(containsStringIgnoreCase("producer of the plant protection") ||
+                          containsStringIgnoreCase("producer of the active substance") ||
+                          containsStringIgnoreCase("manufacturer of the active substance") ||
+                          containsStringIgnoreCase("manufacturer:") ||
+                          containsStringIgnoreCase("Producer or producers of the active substance"))
+    then
+        Stream.concat(entityCreationService.lineAfterStrings(List.of("Contact point:", "Contact:", "Alternative contact:", "European contact:", "No:", "Contact:", "Tel.:", "Tel:", "Telephone number:",
+                            "Telephone No:", "Telephone:", "Phone No.", "Phone:", "Fax number:", "Fax:", "E-mail:", "Email:", "e-mail:", "E-mail address:"), "PII", EntityType.ENTITY, $section),
+             Stream.concat(
+                 entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section),
+                 entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section)
+             ))
+            .forEach(entity -> entity.redact("PII.8.1", "Producer was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
+    end
+
+
 // Rule unit: PII.9
 rule "PII.9.0: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\" (non vertebrate study)"
     when
@@ -394,6 +603,46 @@ rule "PII.9.1: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\" (ver
     end
 
 
+// Rule unit: PII.10
+rule "PII.10.0: Redact study director abbreviation"
+    when
+        $section: Section(containsString("KATH") || containsString("BECH") || containsString("KML"))
+    then
+        entityCreationService.byRegexIgnoreCase("((KATH)|(BECH)|(KML)) ?(\\d{4})","PII", EntityType.ENTITY, 1, $section)
+        .forEach(entity -> entity.redact("PII.10.0", "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
+     end
+
+
+// Rule unit: PII.11
+rule "PII.11.0: Redact On behalf of Sequani Ltd.:"
+    when
+        $section: Section(!hasTables(), containsString("On behalf of Sequani Ltd.: Name Title"))
+    then
+        entityCreationService.betweenStrings("On behalf of Sequani Ltd.: Name Title", "On behalf of", "PII", EntityType.ENTITY, $section)
+         .forEach(authorEntity -> authorEntity.redact("PII.11.0", "On behalf of Sequani Ltd.: Name Title was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
+    end
+
+
+// Rule unit: PII.12
+rule "PII.12.0: Expand PII entities with salutation prefix"
+    when
+        not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
+        $entityToExpand: TextEntity(type() == "PII", anyMatch(textBefore, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*"))
+    then
+        entityCreationService.byPrefixExpansionRegex($entityToExpand, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*")
+        .ifPresent(expandedEntity -> expandedEntity.apply("PII.12.0", "Expanded PII with salutation prefix", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
+    end
+
+rule "PII.12.1: Expand PII entities with salutation prefix"
+    when
+        FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
+        $entityToExpand: TextEntity(type() == "PII", anyMatch(textBefore, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*"))
+    then
+        entityCreationService.byPrefixExpansionRegex($entityToExpand, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*")
+        .ifPresent(expandedEntity -> expandedEntity.apply("PII.12.1", "Expanded PII with salutation prefix", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
+    end
+
+
 //------------------------------------ Other rules ------------------------------------
 
 // Rule unit: ETC.0
@@ -443,13 +692,22 @@ rule "ETC.3.1: Redact logos (vertebrate study)"
 
 
 // Rule unit: ETC.5
-rule "ETC.5.0: Ignore dossier_redaction entries if confidentiality is not 'confidential'"
+rule "ETC.5.0: Skip dossier_redaction entries if confidentiality is 'confidential'"
+    when
+        FileAttribute(label == "Confidentiality", value == "confidential")
+        $dossierRedaction: TextEntity(type() == "dossier_redaction")
+    then
+        $dossierRedaction.skip("ETC.5.0", "Ignore dossier_redaction when confidential");
+        $dossierRedaction.getIntersectingNodes().forEach(node -> update(node));
+    end
+
+rule "ETC.5.1: Remove dossier_redaction entries if confidentiality is not 'confidential'"
     when
         not FileAttribute(label == "Confidentiality", value == "confidential")
-        $dossierRedaction: TextEntity(type == "dossier_redaction")
+        $dossierRedaction: TextEntity(type() == "dossier_redaction")
     then
-        $dossierRedaction.ignore("ETC.5.0", "Ignore dossier redactions, when not confidential");
-        $dossierRedaction.getIntersectingNodes().forEach(node -> update(node));
+        $dossierRedaction.remove("ETC.5.1", "Remove dossier_redaction when not confidential");
+        retract($dossierRedaction);
     end
 
 
@@ -462,7 +720,7 @@ rule "AI.0.0: Add all NER Entities of type CBI_author"
         nerEntities: NerEntities(hasEntitiesOfType("CBI_author"))
     then
         nerEntities.streamEntitiesOfType("CBI_author")
-            .forEach(nerEntity -> entityCreationService.byNerEntity(nerEntity, EntityType.RECOMMENDATION, document));
+            .forEach(nerEntity -> entityCreationService.optionalByNerEntity(nerEntity, EntityType.RECOMMENDATION, document));
     end
 
 
@@ -482,11 +740,11 @@ rule "AI.1.0: Combine and add NER Entities as CBI_address"
 rule "MAN.0.0: Apply manual resize redaction"
     salience 128
     when
-        $resizeRedaction: ManualResizeRedaction($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate)
+        $resizeRedaction: ManualResizeRedaction($id: annotationId, $requestDate: requestDate)
         not ManualResizeRedaction(annotationId == $id, requestDate.isBefore($requestDate))
         $entityToBeResized: TextEntity(matchesAnnotationId($id))
     then
-        manualChangesApplicationService.resizeEntityAndReinsert($entityToBeResized, $resizeRedaction);
+        manualChangesApplicationService.resize($entityToBeResized, $resizeRedaction);
         retract($resizeRedaction);
         update($entityToBeResized);
         $entityToBeResized.getIntersectingNodes().forEach(node -> update(node));
@@ -495,7 +753,7 @@ rule "MAN.0.0: Apply manual resize redaction"
 rule "MAN.0.1: Apply manual resize redaction"
     salience 128
     when
-        $resizeRedaction: ManualResizeRedaction($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate)
+        $resizeRedaction: ManualResizeRedaction($id: annotationId, $requestDate: requestDate)
         not ManualResizeRedaction(annotationId == $id, requestDate.isBefore($requestDate))
         $imageToBeResized: Image(id == $id)
     then
@@ -510,7 +768,7 @@ rule "MAN.0.1: Apply manual resize redaction"
 rule "MAN.1.0: Apply id removals that are valid and not in forced redactions to Entity"
     salience 128
     when
-        $idRemoval: IdRemoval($id: annotationId, !removeFromDictionary, !removeFromAllDossiers, status == AnnotationStatus.APPROVED)
+        $idRemoval: IdRemoval($id: annotationId, !removeFromDictionary, !removeFromAllDossiers)
         $entityToBeRemoved: TextEntity(matchesAnnotationId($id))
     then
         $entityToBeRemoved.getManualOverwrite().addChange($idRemoval);
@@ -522,7 +780,7 @@ rule "MAN.1.0: Apply id removals that are valid and not in forced redactions to
 rule "MAN.1.1: Apply id removals that are valid and not in forced redactions to Image"
     salience 128
     when
-        $idRemoval: IdRemoval($id: annotationId, status == AnnotationStatus.APPROVED)
+        $idRemoval: IdRemoval($id: annotationId)
         $imageEntityToBeRemoved: Image($id == id)
     then
         $imageEntityToBeRemoved.getManualOverwrite().addChange($idRemoval);
@@ -536,7 +794,7 @@ rule "MAN.1.1: Apply id removals that are valid and not in forced redactions to
 rule "MAN.2.0: Apply force redaction"
     salience 128
     when
-        $force: ManualForceRedaction($id: annotationId, status == AnnotationStatus.APPROVED)
+        $force: ManualForceRedaction($id: annotationId)
         $entityToForce: TextEntity(matchesAnnotationId($id))
     then
         $entityToForce.getManualOverwrite().addChange($force);
@@ -548,7 +806,7 @@ rule "MAN.2.0: Apply force redaction"
 rule "MAN.2.1: Apply force redaction to images"
     salience 128
     when
-        $force: ManualForceRedaction($id: annotationId, status == AnnotationStatus.APPROVED)
+        $force: ManualForceRedaction($id: annotationId)
         $imageToForce: Image(id == $id)
     then
         $imageToForce.getManualOverwrite().addChange($force);
@@ -562,23 +820,22 @@ rule "MAN.2.1: Apply force redaction to images"
 rule "MAN.3.0: Apply entity recategorization"
     salience 128
     when
-        $recategorization: ManualRecategorization($id: annotationId, $type: type, status == AnnotationStatus.APPROVED, $requestDate: requestDate)
+        $recategorization: ManualRecategorization($id: annotationId, $type: type, $requestDate: requestDate)
         not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))
-        $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type != $type)
+        $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() != $type)
     then
         $entityToBeRecategorized.getIntersectingNodes().forEach(node -> update(node));
-        manualChangesApplicationService.recategorize($entityToBeRecategorized, $recategorization);
+        $entityToBeRecategorized.getManualOverwrite().addChange($recategorization);
+        update($entityToBeRecategorized);
         retract($recategorization);
-        // Entity is copied and inserted, so the old entity needs to be retracted to avoid duplication.
-        retract($entityToBeRecategorized);
     end
 
 rule "MAN.3.1: Apply entity recategorization of same type"
     salience 128
     when
-        $recategorization: ManualRecategorization($id: annotationId, $type: type, status == AnnotationStatus.APPROVED, $requestDate: requestDate)
+        $recategorization: ManualRecategorization($id: annotationId, $type: type, $requestDate: requestDate)
         not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))
-        $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type == $type)
+        $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() == $type)
     then
         $entityToBeRecategorized.getManualOverwrite().addChange($recategorization);
         retract($recategorization);
@@ -587,7 +844,7 @@ rule "MAN.3.1: Apply entity recategorization of same type"
 rule "MAN.3.2: Apply image recategorization"
     salience 128
     when
-        $recategorization: ManualRecategorization($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate)
+        $recategorization: ManualRecategorization($id: annotationId, $requestDate: requestDate)
         not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))
         $imageToBeRecategorized: Image($id == id)
     then
@@ -597,24 +854,36 @@ rule "MAN.3.2: Apply image recategorization"
         retract($recategorization);
     end
 
+rule "MAN.3.3: Apply recategorization entities by default"
+    salience 128
+    when
+        $entity: TextEntity(getManualOverwrite().getRecategorized().orElse(false), !dictionary.isHint(type()))
+    then
+        $entity.apply("MAN.3.3", "Recategorized entities are applied by default.", $entity.legalBasis());
+    end
+
 
 // Rule unit: MAN.4
 rule "MAN.4.0: Apply legal basis change"
     salience 128
     when
-        $legalbasisChange: ManualLegalBasisChange($id: annotationId, status == AnnotationStatus.APPROVED)
+        $legalBasisChange: ManualLegalBasisChange($id: annotationId)
         $imageToBeRecategorized: Image($id == id)
     then
-        $imageToBeRecategorized.getManualOverwrite().addChange($legalbasisChange);
+        $imageToBeRecategorized.getManualOverwrite().addChange($legalBasisChange);
+        update($imageToBeRecategorized)
+        retract($legalBasisChange)
     end
 
 rule "MAN.4.1: Apply legal basis change"
     salience 128
     when
-        $legalBasisChange: ManualLegalBasisChange($id: annotationId, status == AnnotationStatus.APPROVED)
+        $legalBasisChange: ManualLegalBasisChange($id: annotationId)
         $entityToBeChanged: TextEntity(matchesAnnotationId($id))
     then
         $entityToBeChanged.getManualOverwrite().addChange($legalBasisChange);
+        update($entityToBeChanged)
+        retract($legalBasisChange)
     end
 
 
@@ -624,8 +893,8 @@ rule "MAN.4.1: Apply legal basis change"
 rule "X.0.0: Remove Entity contained by Entity of same type"
     salience 65
     when
-        $larger: TextEntity($type: type, $entityType: entityType, active())
-        $contained: TextEntity(containedBy($larger), type == $type, entityType == $entityType, this != $larger, !hasManualChanges(), active())
+        $larger: TextEntity($type: type(), $entityType: entityType, active())
+        $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges())
     then
         $contained.remove("X.0.0", "remove Entity contained by Entity of same type");
         retract($contained);
@@ -636,8 +905,8 @@ rule "X.0.0: Remove Entity contained by Entity of same type"
 rule "X.1.0: Merge intersecting Entities of same type"
     salience 64
     when
-        $first: TextEntity($type: type, $entityType: entityType, !resized(), active())
-        $second: TextEntity(intersects($first), type == $type, entityType == $entityType, this != $first, !hasManualChanges(), active())
+        $first: TextEntity($type: type(), $entityType: entityType, !resized(), active())
+        $second: TextEntity(intersects($first), type() == $type, entityType == $entityType, this != $first, !hasManualChanges(), active())
     then
         TextEntity mergedEntity = entityCreationService.mergeEntitiesOfSameType(List.of($first, $second), $type, $entityType, document);
         $first.remove("X.1.0", "merge intersecting Entities of same type");
@@ -652,8 +921,8 @@ rule "X.1.0: Merge intersecting Entities of same type"
 rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
     salience 64
     when
-        $falsePositive: TextEntity($type: type, entityType == EntityType.FALSE_POSITIVE, active())
-        $entity: TextEntity(containedBy($falsePositive), type == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges(), active())
+        $falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active())
+        $entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges())
     then
         $entity.getIntersectingNodes().forEach(node -> update(node));
         $entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE");
@@ -665,8 +934,8 @@ rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
 rule "X.3.0: Remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"
     salience 64
     when
-        $falseRecommendation: TextEntity($type: type, entityType == EntityType.FALSE_RECOMMENDATION, active())
-        $recommendation: TextEntity(containedBy($falseRecommendation), type == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
+        $falseRecommendation: TextEntity($type: type(), entityType == EntityType.FALSE_RECOMMENDATION, active())
+        $recommendation: TextEntity(containedBy($falseRecommendation), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges())
     then
         $recommendation.remove("X.3.0", "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION");
         retract($recommendation);
@@ -674,51 +943,105 @@ rule "X.3.0: Remove Entity of type RECOMMENDATION when contained by FALSE_RECOMM
 
 
 // Rule unit: X.4
-rule "X.4.0: Remove Entity of type RECOMMENDATION when intersected by ENTITY with same type"
+rule "X.4.0: Remove Entity of type RECOMMENDATION when text range equals ENTITY with same type"
     salience 256
     when
-        $entity: TextEntity($type: type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
-        $recommendation: TextEntity(intersects($entity), type == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
+        $entity: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
+        $recommendation: TextEntity(getTextRange().equals($entity.getTextRange()), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges())
     then
         $entity.addEngines($recommendation.getEngines());
-        $recommendation.remove("X.4.0", "remove Entity of type RECOMMENDATION when intersected by ENTITY with same type");
+        $recommendation.remove("X.4.0", "remove Entity of type RECOMMENDATION when text range equals ENTITY with same type");
         retract($recommendation);
     end
 
 
 // Rule unit: X.5
-rule "X.5.0: Remove Entity of type RECOMMENDATION when contained by ENTITY"
+rule "X.5.0: Remove Entity of type RECOMMENDATION when intersected by ENTITY"
     salience 256
     when
         $entity: TextEntity((entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
-        $recommendation: TextEntity(containedBy($entity), entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
+        $recommendation: TextEntity(intersects($entity), entityType == EntityType.RECOMMENDATION, !hasManualChanges())
     then
-        $recommendation.remove("X.5.0", "remove Entity of type RECOMMENDATION when contained by ENTITY");
+        $recommendation.remove("X.5.0", "remove Entity of type RECOMMENDATION when intersected by ENTITY");
+        retract($recommendation);
+    end
+
+rule "X.5.1: Remove Entity of type RECOMMENDATION when contained by RECOMMENDATION"
+    salience 256
+    when
+        $entity: TextEntity($type: type(), entityType == EntityType.RECOMMENDATION, active())
+        $recommendation: TextEntity(containedBy($entity), type() != $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges())
+    then
+        $recommendation.remove("X.5.1", "remove Entity of type RECOMMENDATION when contained by RECOMMENDATION");
         retract($recommendation);
     end
 
 
 // Rule unit: X.6
-rule "X.6.0: Remove Entity of lower rank, when contained by by entity of type ENTITY"
+rule "X.6.0: Remove Entity of lower rank, when contained by entity of type ENTITY or HINT"
     salience 32
     when
-        $higherRank: TextEntity($type: type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
-        $lowerRank: TextEntity(containedBy($higherRank), type != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active())
+        $higherRank: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
+        $lowerRank: TextEntity(containedBy($higherRank), type() != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges())
     then
         $lowerRank.getIntersectingNodes().forEach(node -> update(node));
-        $lowerRank.remove("X.6.0", "remove Entity of lower rank, when contained by entity of type ENTITY");
+        $lowerRank.remove("X.6.0", "remove Entity of lower rank, when contained by entity of type ENTITY or HINT");
         retract($lowerRank);
     end
 
-rule "X.6.1: remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity"
+rule "X.6.1: remove Entity, when contained in another entity of type ENTITY or HINT with larger text range"
     salience 32
     when
-        $higherRank: TextEntity($type: type, $value: value, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
-        $lowerRank: TextEntity(intersects($higherRank), type != $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active(), $lowerRank.getValue().length() > $value.length())
+         $outer: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
+         $inner: TextEntity(containedBy($outer), type() != $type, $outer.getTextRange().length > getTextRange().length(), !hasManualChanges())
     then
-        $higherRank.getIntersectingNodes().forEach(node -> update(node));
-        $higherRank.remove("X.6.1", "remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity");
-        retract($higherRank);
+        $inner.getIntersectingNodes().forEach(node -> update(node));
+        $inner.remove("X.6.1", "remove Entity, when contained in another entity of type ENTITY or HINT with larger text range");
+        retract($inner);
+    end
+
+
+// Rule unit: X.8
+rule "X.8.0: Remove Entity when text range and type equals to imported Entity"
+    salience 257
+    when
+        $entity: TextEntity($type: type(), engines contains Engine.IMPORTED, active())
+        $other: TextEntity(getTextRange().equals($entity.getTextRange()), this != $entity, type() == $type, engines not contains Engine.IMPORTED)
+    then
+        $other.remove("X.8.0", "remove Entity when text range and type equals to imported Entity");
+        $entity.addEngines($other.getEngines());
+        retract($other);
+    end
+
+rule "X.8.1: Remove Entity when intersected by imported Entity"
+    salience 256
+    when
+        $entity: TextEntity(engines contains Engine.IMPORTED, active())
+        $other: TextEntity(intersects($entity), this != $entity, engines not contains Engine.IMPORTED)
+    then
+        $other.remove("X.8.1", "remove Entity when intersected by imported Entity");
+        retract($other);
+    end
+
+
+// Rule unit: X.9
+rule "X.9.0: Merge mostly contained signatures"
+    when
+        $aiSignature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.AI)
+        $signature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.ALGORITHM, mostlyContains($aiSignature, 0.8))
+    then
+        $aiSignature.remove("X.9.0", "removed because already contained by alogrithm signature");
+        $signature.addEngine(LayoutEngine.AI);
+    end
+
+
+// Rule unit: X.10
+rule "X.10.0: remove false positives of ai"
+    when
+        $anyImage: Image(engines contains LayoutEngine.ALGORITHM)
+        $aiSignature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.AI, !mostlyContainedBy($anyImage, 0.8))
+    then
+        $aiSignature.remove("X.10.0", "Removed because false positive");
     end
 
 

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/all_redact_manager_rules.drl

@@ -53,7 +53,8 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus;
+
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
 
 global Document document
 global EntityCreationService entityCreationService
@@ -98,7 +99,7 @@ rule "SYN.1.0: Recommend CTL/BL laboratory that start with BL or CTL"
 rule "CBI.0.0: Redact CBI Authors (non vertebrate Study)"
     when
         not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
-        $entity: TextEntity(type == "CBI_author", dictionaryEntry)
+        $entity: TextEntity(type() == "CBI_author", dictionaryEntry)
     then
         $entity.redact("CBI.0.0", "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
     end
@@ -106,7 +107,7 @@ rule "CBI.0.0: Redact CBI Authors (non vertebrate Study)"
 rule "CBI.0.1: Redact CBI Authors (vertebrate Study)"
     when
         FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
-        $entity: TextEntity(type == "CBI_author", dictionaryEntry)
+        $entity: TextEntity(type() == "CBI_author", dictionaryEntry)
     then
         $entity.redact("CBI.0.1", "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
     end
@@ -116,7 +117,7 @@ rule "CBI.0.1: Redact CBI Authors (vertebrate Study)"
 rule "CBI.1.0: Do not redact CBI Address (non vertebrate Study)"
     when
         not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
-        $entity: TextEntity(type == "CBI_address", dictionaryEntry)
+        $entity: TextEntity(type() == "CBI_address", dictionaryEntry)
     then
         $entity.skip("CBI.1.0", "Address found for Non Vertebrate Study");
     end
@@ -124,7 +125,7 @@ rule "CBI.1.0: Do not redact CBI Address (non vertebrate Study)"
 rule "CBI.1.1: Redact CBI Address (vertebrate Study)"
     when
         FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
-        $entity: TextEntity(type == "CBI_address", dictionaryEntry)
+        $entity: TextEntity(type() == "CBI_address", dictionaryEntry)
     then
         $entity.redact("CBI.1.1", "Address found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
     end
@@ -133,7 +134,7 @@ rule "CBI.1.1: Redact CBI Address (vertebrate Study)"
 // Rule unit: CBI.2
 rule "CBI.2.0: Do not redact genitive CBI Author"
     when
-        $entity: TextEntity(type == "CBI_author", anyMatch(textAfter, "['’’'ʼˈ´`‘′ʻ’']s"))
+        $entity: TextEntity(type() == "CBI_author", anyMatch(textAfter, "['’’'ʼˈ´`‘′ʻ’']s"))
     then
         entityCreationService.byTextRange($entity.getTextRange(), "CBI_author", EntityType.FALSE_POSITIVE, document)
         .ifPresent(falsePositive -> falsePositive.skip("CBI.2.0", "Genitive Author found"));
@@ -158,18 +159,17 @@ rule "CBI.3.0: Redacted because Section contains a vertebrate"
 
 rule "CBI.3.1: Redacted because table row contains a vertebrate"
     when
-        $table: Table(hasEntitiesOfType("vertebrate"), (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
+        $table: Table(hasEntitiesOfType("vertebrate"), hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address"))
+        $cellsWithvertebrate: TableCell() from $table.streamTableCellsWhichContainType("vertebrate").toList()
+        $tableCell: TableCell(row == $cellsWithvertebrate.row) from $table.streamTableCells().toList()
+        $authorOrAddress: TextEntity(type() == "CBI_author" || type() == "CBI_address", active()) from $tableCell.getEntities()
     then
-        $table.streamEntitiesWhereRowContainsEntitiesOfType(List.of("vertebrate"))
-            .filter(entity -> entity.getType().equals("CBI_author") || entity.getType().equals("CBI_address"))
-            .forEach(entity -> {
-                entity.applyWithReferences(
+        $authorOrAddress.applyWithReferences(
                         "CBI.3.1",
                         "Vertebrate found",
                         "Reg (EC) No 1107/2009 Art. 63 (2g)",
-                        $table.getEntitiesOfTypeInSameRow("vertebrate", entity)
+                        $table.getEntitiesOfTypeInSameRow("vertebrate", $authorOrAddress)
                 );
-            });
     end
 
 rule "CBI.3.2: Do not redact because Section does not contain a vertebrate"
@@ -208,23 +208,21 @@ rule "CBI.4.0: Do not redact Names and Addresses if no_redaction_indicator is fo
             });
     end
 
-rule "CBI.4.1: Do not redact Names and Addresses if no_redaction_indicator is found in table row"
+rule "CBI.4.1: Redacted because table row contains a vertebrate"
     when
-        $table: Table(hasEntitiesOfType("no_redaction_indicator"),
-                      hasEntitiesOfType("vertebrate"),
-                      (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
+        $table: Table(hasEntitiesOfType("no_redaction_indicator"), hasEntitiesOfType("vertebrate"), hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address"))
+        TableCell($row: row) from $table.streamTableCellsWhichContainType("vertebrate").toList()
+        TableCell(row == $row) from $table.streamTableCellsWhichContainType("no_redaction_indicator").toList()
+        $tableCell: TableCell(row == $row) from $table.streamTableCells().toList()
+        $authorOrAddress: TextEntity(type() == "CBI_author" || type() == "CBI_address", active()) from $tableCell.getEntities()
     then
-        $table.streamEntitiesWhereRowContainsEntitiesOfType(List.of("vertebrate", "no-redaction_indicator"))
-            .filter(entity -> entity.getType().equals("CBI_author") || entity.getType().equals("CBI_address"))
-            .forEach(entity -> {
-                entity.skipWithReferences(
+        $authorOrAddress.skipWithReferences(
                         "CBI.4.1",
                         "Vertebrate but a no redaction indicator found",
                         Stream.concat(
-                            $table.getEntitiesOfTypeInSameRow("vertebrate", entity).stream(),
-                            $table.getEntitiesOfTypeInSameRow("no_redaction_indicator", entity).stream()).toList()
+                            $table.getEntitiesOfTypeInSameRow("vertebrate", $authorOrAddress).stream(),
+                            $table.getEntitiesOfTypeInSameRow("no_redaction_indicator", $authorOrAddress).stream()).toList()
                 );
-            });
     end
 
 
@@ -251,22 +249,20 @@ rule "CBI.5.0: Redact Names and Addresses if no_redaction_indicator but also red
 
 rule "CBI.5.1: Redact Names and Addresses if no_redaction_indicator but also redaction_indicator is found in table row"
     when
-        $table: Table(hasEntitiesOfType("no_redaction_indicator"),
-                      hasEntitiesOfType("redaction_indicator"),
-                      (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
+        $table: Table(hasEntitiesOfType("no_redaction_indicator"), hasEntitiesOfType("redaction_indicator"), hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address"))
+        TableCell($row: row) from $table.streamTableCellsWhichContainType("redaction_indicator").toList()
+        TableCell(row == $row) from $table.streamTableCellsWhichContainType("no_redaction_indicator").toList()
+        $tableCell: TableCell(row == $row) from $table.streamTableCells().toList()
+        $entity: TextEntity(type() == "CBI_author" || type() == "CBI_address", active()) from $tableCell.getEntities()
     then
-        $table.streamEntitiesWhereRowContainsEntitiesOfType(List.of("redaction_indicator", "no_redaction_indicator"))
-            .filter(entity -> entity.getType().equals("CBI_author") || entity.getType().equals("CBI_address"))
-            .forEach(entity -> {
-                entity.applyWithReferences(
+        $entity.applyWithReferences(
                         "CBI.5.1",
                         "no_redaction_indicator but also redaction_indicator found",
                         "Reg (EC) No 1107/2009 Art. 63 (2g)",
                         Stream.concat(
-                            $table.getEntitiesOfTypeInSameRow("vertebrate", entity).stream(),
-                            $table.getEntitiesOfTypeInSameRow("no_redaction_indicator", entity).stream()).toList()
+                            $table.getEntitiesOfTypeInSameRow("redaction_indicator", $entity).stream(),
+                            $table.getEntitiesOfTypeInSameRow("no_redaction_indicator", $entity).stream()).toList()
                 );
-            });
     end
 
 
@@ -332,7 +328,7 @@ rule "CBI.7.1: Do not redact Names and Addresses if published information found
         $table: Table(hasEntitiesOfType("published_information"), hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address"))
         $cellsWithPublishedInformation: TableCell() from $table.streamTableCellsWhichContainType("published_information").toList()
         $tableCell: TableCell(row == $cellsWithPublishedInformation.row) from $table.streamTableCells().toList()
-        $authorOrAddress: TextEntity(type == "CBI_author" || type == "CBI_address", active()) from $tableCell.getEntities()
+        $authorOrAddress: TextEntity(type() == "CBI_author" || type() == "CBI_address", active()) from $tableCell.getEntities()
     then
         $authorOrAddress.skipWithReferences("CBI.7.1", "Published Information found in row", $table.getEntitiesOfTypeInSameRow("published_information", $authorOrAddress));
     end
@@ -356,18 +352,17 @@ rule "CBI.8.0: Redacted because Section contains must_redact entity"
 
 rule "CBI.8.1: Redacted because table row contains must_redact entity"
     when
-        $table: Table(hasEntitiesOfType("must_redact"), (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
+        $table: Table(hasEntitiesOfType("must_redact"), hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address"))
+        $cellsWithMustRedact: TableCell() from $table.streamTableCellsWhichContainType("must_redact").toList()
+        $tableCell: TableCell(row == $cellsWithMustRedact.row) from $table.streamTableCells().toList()
+        $authorOrAddress: TextEntity(type() == "CBI_author" || type() == "CBI_address", active()) from $tableCell.getEntities()
     then
-        $table.streamEntitiesWhereRowContainsEntitiesOfType(List.of("must_redact"))
-            .filter(entity -> entity.getType().equals("CBI_author") || entity.getType().equals("CBI_address"))
-            .forEach(entity -> {
-                entity.applyWithReferences(
+        $authorOrAddress.applyWithReferences(
                         "CBI.8.1",
-                        "must_redact entity found",
+                        "Must_redact found",
                         "Reg (EC) No 1107/2009 Art. 63 (2g)",
-                        $table.getEntitiesOfTypeInSameRow("must_redact", entity)
+                        $table.getEntitiesOfTypeInSameRow("must_redact", $authorOrAddress)
                 );
-            });
     end
 
 
@@ -449,7 +444,6 @@ rule "CBI.12.0: Redact and recommend TableCell with header 'Author' or 'Author(s
         TableCell(!header, containsAnyString("Yes", "Y"), $rowWithYes: row) from $table.streamCol($vertebrateCol).toList()
         $authorCell: TableCell(row == $rowWithYes) from $table.streamCol($authorCol).toList()
     then
-
         entityCreationService.bySemanticNode($authorCell, "CBI_author", EntityType.ENTITY)
           .ifPresent(authorEntity -> {
               authorEntity.redact("CBI.12.0", "Redacted because it's row belongs to a vertebrate study", "Article 39(e)(3) of Regulation (EC) No 178/2002");
@@ -494,7 +488,7 @@ rule "CBI.12.2: Skip TableCell with header 'Author' or 'Author(s)' and header 'V
 rule "CBI.13.0: Ignore CBI Address recommendations"
     when
         not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
-        $entity: TextEntity(type == "CBI_address", entityType == EntityType.RECOMMENDATION)
+        $entity: TextEntity(type() == "CBI_address", entityType == EntityType.RECOMMENDATION)
     then
         $entity.ignore("CBI.13.0", "Ignore CBI Address Recommendations");
         retract($entity)
@@ -504,7 +498,7 @@ rule "CBI.13.0: Ignore CBI Address recommendations"
 // Rule unit: CBI.14
 rule "CBI.14.0: Redact CBI_sponsor entities if preceded by \"batches produced at\""
     when
-        $sponsorEntity: TextEntity(type == "CBI_sponsor", textBefore.contains("batches produced at"))
+        $sponsorEntity: TextEntity(type() == "CBI_sponsor", textBefore.contains("batches produced at"))
     then
         $sponsorEntity.redact("CBI.14.0", "Redacted because it represents a sponsor company", "Reg (EC) No 1107/2009 Art. 63 (2g)");
     end
@@ -607,7 +601,7 @@ rule "CBI.17.1: Add recommendation for Addresses in Test Organism sections, with
 rule "CBI.18.0: Expand CBI_author entities with firstname initials"
     no-loop true
     when
-        $entityToExpand: TextEntity(type == "CBI_author",
+        $entityToExpand: TextEntity(type() == "CBI_author",
           value.matches("[^\\s]+"),
           textAfter.startsWith(" "),
           anyMatch(textAfter, "(,? [A-Z]\\.?( ?[A-Z]\\.?)?( ?[A-Z]\\.?)?\\b\\.?)")
@@ -625,7 +619,7 @@ rule "CBI.18.0: Expand CBI_author entities with firstname initials"
 // Rule unit: CBI.19
 rule "CBI.19.0: Expand CBI_author entities with salutation prefix"
     when
-        $entityToExpand: TextEntity(type == "CBI_author", anyMatch(textBefore, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*"))
+        $entityToExpand: TextEntity(type() == "CBI_author", anyMatch(textBefore, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*"))
     then
         entityCreationService.byPrefixExpansionRegex($entityToExpand, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*")
          .ifPresent(expandedEntity -> {
@@ -669,7 +663,7 @@ rule "CBI.21.0: Redact short Authors section (non vertebrate study)"
     when
         not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
         $section: Section(containsAnyStringIgnoreCase("author(s)", "author", "authors"), length() < 50, getTreeId().get(0) <= 20) //TODO: evaluate the reason of this rule
-        not TextEntity(type == "CBI_author", engines contains Engine.NER) from $section.getEntities()
+        not TextEntity(type() == "CBI_author", engines contains Engine.NER) from $section.getEntities()
     then
         entityCreationService.byRegexIgnoreCase("(?<=author\\(?s\\)?\\s\\n?)([\\p{Lu}\\p{L} ]{5,15}(,|\\n)?){1,3}", "CBI_author", EntityType.ENTITY, $section)
             .forEach(entity -> {
@@ -681,7 +675,7 @@ rule "CBI.21.1: Redact short Authors section (vertebrate study)"
     when
         FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
         $section: Section(containsAnyStringIgnoreCase("author(s)", "author", "authors"), length() < 50, getTreeId().get(0) <= 20) //TODO: evaluate the reason of this rule
-        not TextEntity(type == "CBI_author", engines contains Engine.NER) from $section.getEntities()
+        not TextEntity(type() == "CBI_author", engines contains Engine.NER) from $section.getEntities()
     then
         entityCreationService.byRegexIgnoreCase("(?<=author\\(?s\\)?\\s\\n?)([\\p{Lu}\\p{L} ]{5,15}(,|\\n)?){1,3}", "CBI_author", EntityType.ENTITY, $section)
             .forEach(entity -> {
@@ -708,7 +702,7 @@ rule "CBI.22.0: Redact Addresses in Reference Tables for vertebrate studies in n
 rule "PII.0.0: Redact all PII (non vertebrate study)"
     when
         not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
-        $pii: TextEntity(type == "PII", dictionaryEntry)
+        $pii: TextEntity(type() == "PII", dictionaryEntry)
     then
         $pii.redact("PII.0.0", "Personal Information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
     end
@@ -716,7 +710,7 @@ rule "PII.0.0: Redact all PII (non vertebrate study)"
 rule "PII.0.1: Redact all PII (vertebrate study)"
     when
         FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
-        $pii: TextEntity(type == "PII", dictionaryEntry)
+        $pii: TextEntity(type() == "PII", dictionaryEntry)
     then
         $pii.redact("PII.0.1", "Personal Information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
     end
@@ -745,7 +739,7 @@ rule "PII.1.2: Redact typoed Emails with indicator"
     when
         $section: Section(containsString("@") || containsStringIgnoreCase("mail"))
     then
-        entityCreationService.byRegexIgnoreCase("mail[:\\.\\s]{1,2}([\\w\\/\\-\\{\\(\\. ]{3,20}(@|a|f)\\s?[\\w\\/\\-\\{\\(\\. ]{3,20}(\\. \\w{2,4}\\b|\\.\\B|\\.\\w{1,4}\\b))", "PII", EntityType.ENTITY, $section)
+        entityCreationService.byRegexIgnoreCase("mail[:\\.\\s]{1,2}([\\w\\/\\-\\{\\(\\. ]{3,20}(@|a|f)\\s?[\\w\\/\\-\\{\\(\\. ]{3,20}(\\. \\w{2,4}\\b|\\.\\B|\\.\\w{1,4}\\b))", "PII", EntityType.ENTITY, 1, $section)
             .forEach(emailEntity -> emailEntity.redact("PII.1.2", "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
     end
 
@@ -1048,7 +1042,7 @@ rule "PII.11.0: Redact On behalf of Sequani Ltd.:"
 rule "PII.12.0: Expand PII entities with salutation prefix"
     when
         not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
-        $entityToExpand: TextEntity(type == "PII", anyMatch(textBefore, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*"))
+        $entityToExpand: TextEntity(type() == "PII", anyMatch(textBefore, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*"))
     then
         entityCreationService.byPrefixExpansionRegex($entityToExpand, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*")
         .ifPresent(expandedEntity -> expandedEntity.apply("PII.12.0", "Expanded PII with salutation prefix", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
@@ -1135,34 +1129,43 @@ rule "ETC.3.1: Redact logos (vertebrate study)"
 // Rule unit: ETC.4
 rule "ETC.4.0: Redact dossier dictionary entries"
     when
-        $dossierRedaction: TextEntity(type == "dossier_redaction")
+        $dossierRedaction: TextEntity(type() == "dossier_redaction")
     then
         $dossierRedaction.redact("ETC.4.0", "Specification of impurity found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
     end
 
 rule "ETC.4.1: Redact dossier dictionary entries"
     when
-        $dossierRedaction: TextEntity(type == "dossier_redaction")
+        $dossierRedaction: TextEntity(type() == "dossier_redaction")
     then
         $dossierRedaction.redact("ETC.4.1", "Dossier Redaction found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
     end
 
 rule "ETC.4.2: Redact dossier dictionary entries"
     when
-        $dossierRedaction: TextEntity(type == "dossier_redaction")
+        $dossierRedaction: TextEntity(type() == "dossier_redaction")
     then
         $dossierRedaction.redact("ETC.4.2", "Dossier redaction found", "Article 63(2)(a) of Regulation (EC) No 1107/2009 (making reference to Article 39 of Regulation EC No  178/2002)");
     end
 
 
 // Rule unit: ETC.5
-rule "ETC.5.0: Ignore dossier_redaction entries if confidentiality is not 'confidential'"
+rule "ETC.5.0: Skip dossier_redaction entries if confidentiality is 'confidential'"
+    when
+        FileAttribute(label == "Confidentiality", value == "confidential")
+        $dossierRedaction: TextEntity(type() == "dossier_redaction")
+    then
+        $dossierRedaction.skip("ETC.5.0", "Ignore dossier_redaction when confidential");
+        $dossierRedaction.getIntersectingNodes().forEach(node -> update(node));
+    end
+
+rule "ETC.5.1: Remove dossier_redaction entries if confidentiality is not 'confidential'"
     when
         not FileAttribute(label == "Confidentiality", value == "confidential")
-        $dossierRedaction: TextEntity(type == "dossier_redaction")
+        $dossierRedaction: TextEntity(type() == "dossier_redaction")
     then
-        $dossierRedaction.ignore("ETC.5.0", "Ignore dossier redactions, when not confidential");
-        $dossierRedaction.getIntersectingNodes().forEach(node -> update(node));
+        $dossierRedaction.remove("ETC.5.1", "Remove dossier_redaction when not confidential");
+        retract($dossierRedaction);
     end
 
 
@@ -1213,7 +1216,7 @@ rule "ETC.8.1: Redact formulas (non vertebrate study)"
 rule "ETC.9.0: Redact skipped impurities"
     when
         FileAttribute(label == "Redact Skipped Impurities", value soundslike "Yes" || value.toLowerCase() == "y")
-        $skippedImpurities: TextEntity(type == "skipped_impurities")
+        $skippedImpurities: TextEntity(type() == "skipped_impurities")
     then
         $skippedImpurities.redact("ETC.9.0", "Occasional Impurity found", "Article 63(2)(b) of Regulation (EC) No 1107/2009");
     end
@@ -1221,7 +1224,7 @@ rule "ETC.9.0: Redact skipped impurities"
 rule "ETC.9.1: Redact impurities"
     when
         FileAttribute(label == "Redact Impurities", value soundslike "Yes" || value.toLowerCase() == "y")
-        $skippedImpurities: TextEntity(type == "impurities")
+        $skippedImpurities: TextEntity(type() == "impurities")
     then
         $skippedImpurities.redact("ETC.9.1", "Impurity found", "Article 63(2)(b) of Regulation (EC) No 1107/2009");
     end
@@ -1230,7 +1233,7 @@ rule "ETC.9.1: Redact impurities"
 // Rule unit: ETC.10
 rule "ETC.10.0: Redact Product Composition Information"
     when
-        $compositionInformation: TextEntity(type == "product_composition")
+        $compositionInformation: TextEntity(type() == "product_composition")
     then
         $compositionInformation.redact("ETC.10.0", "Product Composition Information found", "Article 63(2)(d) of Regulation (EC) No 1107/2009");
     end
@@ -1257,7 +1260,7 @@ rule "AI.0.0: Add all NER Entities of type CBI_author"
         nerEntities: NerEntities(hasEntitiesOfType("CBI_author"))
     then
         nerEntities.streamEntitiesOfType("CBI_author")
-            .forEach(nerEntity -> entityCreationService.byNerEntity(nerEntity, EntityType.RECOMMENDATION, document));
+            .forEach(nerEntity -> entityCreationService.optionalByNerEntity(nerEntity, EntityType.RECOMMENDATION, document));
     end
 
 
@@ -1279,7 +1282,7 @@ rule "AI.2.0: Add all NER Entities of any type except CBI_author"
     then
         nerEntities.getNerEntityList().stream()
             .filter(nerEntity -> !nerEntity.type().equals("CBI_author"))
-            .forEach(nerEntity -> entityCreationService.byNerEntity(nerEntity, nerEntity.type().toLowerCase(), EntityType.RECOMMENDATION, document));
+            .forEach(nerEntity -> entityCreationService.optionalByNerEntity(nerEntity, nerEntity.type().toLowerCase(), EntityType.RECOMMENDATION, document));
     end
 
 
@@ -1290,7 +1293,7 @@ rule "AI.3.0: Recommend authors from AI as PII"
         nerEntities: NerEntities(hasEntitiesOfType("CBI_author"))
     then
         nerEntities.streamEntitiesOfType("CBI_author")
-            .forEach(nerEntity -> entityCreationService.byNerEntity(nerEntity, "PII", EntityType.RECOMMENDATION, document));
+            .forEach(nerEntity -> entityCreationService.optionalByNerEntity(nerEntity, "PII", EntityType.RECOMMENDATION, document));
     end
 
 
@@ -1300,11 +1303,11 @@ rule "AI.3.0: Recommend authors from AI as PII"
 rule "MAN.0.0: Apply manual resize redaction"
     salience 128
     when
-        $resizeRedaction: ManualResizeRedaction($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate)
+        $resizeRedaction: ManualResizeRedaction($id: annotationId, $requestDate: requestDate)
         not ManualResizeRedaction(annotationId == $id, requestDate.isBefore($requestDate))
         $entityToBeResized: TextEntity(matchesAnnotationId($id))
     then
-        manualChangesApplicationService.resizeEntityAndReinsert($entityToBeResized, $resizeRedaction);
+        manualChangesApplicationService.resize($entityToBeResized, $resizeRedaction);
         retract($resizeRedaction);
         update($entityToBeResized);
         $entityToBeResized.getIntersectingNodes().forEach(node -> update(node));
@@ -1313,7 +1316,7 @@ rule "MAN.0.0: Apply manual resize redaction"
 rule "MAN.0.1: Apply manual resize redaction"
     salience 128
     when
-        $resizeRedaction: ManualResizeRedaction($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate)
+        $resizeRedaction: ManualResizeRedaction($id: annotationId, $requestDate: requestDate)
         not ManualResizeRedaction(annotationId == $id, requestDate.isBefore($requestDate))
         $imageToBeResized: Image(id == $id)
     then
@@ -1328,7 +1331,7 @@ rule "MAN.0.1: Apply manual resize redaction"
 rule "MAN.1.0: Apply id removals that are valid and not in forced redactions to Entity"
     salience 128
     when
-        $idRemoval: IdRemoval($id: annotationId, !removeFromDictionary, !removeFromAllDossiers, status == AnnotationStatus.APPROVED)
+        $idRemoval: IdRemoval($id: annotationId, !removeFromDictionary, !removeFromAllDossiers)
         $entityToBeRemoved: TextEntity(matchesAnnotationId($id))
     then
         $entityToBeRemoved.getManualOverwrite().addChange($idRemoval);
@@ -1340,7 +1343,7 @@ rule "MAN.1.0: Apply id removals that are valid and not in forced redactions to
 rule "MAN.1.1: Apply id removals that are valid and not in forced redactions to Image"
     salience 128
     when
-        $idRemoval: IdRemoval($id: annotationId, status == AnnotationStatus.APPROVED)
+        $idRemoval: IdRemoval($id: annotationId)
         $imageEntityToBeRemoved: Image($id == id)
     then
         $imageEntityToBeRemoved.getManualOverwrite().addChange($idRemoval);
@@ -1354,7 +1357,7 @@ rule "MAN.1.1: Apply id removals that are valid and not in forced redactions to
 rule "MAN.2.0: Apply force redaction"
     salience 128
     when
-        $force: ManualForceRedaction($id: annotationId, status == AnnotationStatus.APPROVED)
+        $force: ManualForceRedaction($id: annotationId)
         $entityToForce: TextEntity(matchesAnnotationId($id))
     then
         $entityToForce.getManualOverwrite().addChange($force);
@@ -1366,7 +1369,7 @@ rule "MAN.2.0: Apply force redaction"
 rule "MAN.2.1: Apply force redaction to images"
     salience 128
     when
-        $force: ManualForceRedaction($id: annotationId, status == AnnotationStatus.APPROVED)
+        $force: ManualForceRedaction($id: annotationId)
         $imageToForce: Image(id == $id)
     then
         $imageToForce.getManualOverwrite().addChange($force);
@@ -1380,23 +1383,22 @@ rule "MAN.2.1: Apply force redaction to images"
 rule "MAN.3.0: Apply entity recategorization"
     salience 128
     when
-        $recategorization: ManualRecategorization($id: annotationId, $type: type, status == AnnotationStatus.APPROVED, $requestDate: requestDate)
+        $recategorization: ManualRecategorization($id: annotationId, $type: type, $requestDate: requestDate)
         not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))
-        $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type != $type)
+        $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() != $type)
     then
         $entityToBeRecategorized.getIntersectingNodes().forEach(node -> update(node));
-        manualChangesApplicationService.recategorize($entityToBeRecategorized, $recategorization);
+        $entityToBeRecategorized.getManualOverwrite().addChange($recategorization);
+        update($entityToBeRecategorized);
         retract($recategorization);
-        // Entity is copied and inserted, so the old entity needs to be retracted to avoid duplication.
-        retract($entityToBeRecategorized);
     end
 
 rule "MAN.3.1: Apply entity recategorization of same type"
     salience 128
     when
-        $recategorization: ManualRecategorization($id: annotationId, $type: type, status == AnnotationStatus.APPROVED, $requestDate: requestDate)
+        $recategorization: ManualRecategorization($id: annotationId, $type: type, $requestDate: requestDate)
         not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))
-        $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type == $type)
+        $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() == $type)
     then
         $entityToBeRecategorized.getManualOverwrite().addChange($recategorization);
         retract($recategorization);
@@ -1405,7 +1407,7 @@ rule "MAN.3.1: Apply entity recategorization of same type"
 rule "MAN.3.2: Apply image recategorization"
     salience 128
     when
-        $recategorization: ManualRecategorization($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate)
+        $recategorization: ManualRecategorization($id: annotationId, $requestDate: requestDate)
         not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))
         $imageToBeRecategorized: Image($id == id)
     then
@@ -1415,24 +1417,36 @@ rule "MAN.3.2: Apply image recategorization"
         retract($recategorization);
     end
 
+rule "MAN.3.3: Apply recategorization entities by default"
+    salience 128
+    when
+        $entity: TextEntity(getManualOverwrite().getRecategorized().orElse(false), !dictionary.isHint(type()))
+    then
+        $entity.apply("MAN.3.3", "Recategorized entities are applied by default.", $entity.legalBasis());
+    end
+
 
 // Rule unit: MAN.4
 rule "MAN.4.0: Apply legal basis change"
     salience 128
     when
-        $legalbasisChange: ManualLegalBasisChange($id: annotationId, status == AnnotationStatus.APPROVED)
+        $legalBasisChange: ManualLegalBasisChange($id: annotationId)
         $imageToBeRecategorized: Image($id == id)
     then
-        $imageToBeRecategorized.getManualOverwrite().addChange($legalbasisChange);
+        $imageToBeRecategorized.getManualOverwrite().addChange($legalBasisChange);
+        update($imageToBeRecategorized)
+        retract($legalBasisChange)
     end
 
 rule "MAN.4.1: Apply legal basis change"
     salience 128
     when
-        $legalBasisChange: ManualLegalBasisChange($id: annotationId, status == AnnotationStatus.APPROVED)
+        $legalBasisChange: ManualLegalBasisChange($id: annotationId)
         $entityToBeChanged: TextEntity(matchesAnnotationId($id))
     then
         $entityToBeChanged.getManualOverwrite().addChange($legalBasisChange);
+        update($entityToBeChanged)
+        retract($legalBasisChange)
     end
 
 
@@ -1442,8 +1456,8 @@ rule "MAN.4.1: Apply legal basis change"
 rule "X.0.0: Remove Entity contained by Entity of same type"
     salience 65
     when
-        $larger: TextEntity($type: type, $entityType: entityType, active())
-        $contained: TextEntity(containedBy($larger), type == $type, entityType == $entityType, this != $larger, !hasManualChanges(), active())
+        $larger: TextEntity($type: type(), $entityType: entityType, active())
+        $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges())
     then
         $contained.remove("X.0.0", "remove Entity contained by Entity of same type");
         retract($contained);
@@ -1454,8 +1468,8 @@ rule "X.0.0: Remove Entity contained by Entity of same type"
 rule "X.1.0: Merge intersecting Entities of same type"
     salience 64
     when
-        $first: TextEntity($type: type, $entityType: entityType, !resized(), active())
-        $second: TextEntity(intersects($first), type == $type, entityType == $entityType, this != $first, !hasManualChanges(), active())
+        $first: TextEntity($type: type(), $entityType: entityType, !resized(), active())
+        $second: TextEntity(intersects($first), type() == $type, entityType == $entityType, this != $first, !hasManualChanges(), active())
     then
         TextEntity mergedEntity = entityCreationService.mergeEntitiesOfSameType(List.of($first, $second), $type, $entityType, document);
         $first.remove("X.1.0", "merge intersecting Entities of same type");
@@ -1470,8 +1484,8 @@ rule "X.1.0: Merge intersecting Entities of same type"
 rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
     salience 64
     when
-        $falsePositive: TextEntity($type: type, entityType == EntityType.FALSE_POSITIVE, active())
-        $entity: TextEntity(containedBy($falsePositive), type == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges(), active())
+        $falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active())
+        $entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges())
     then
         $entity.getIntersectingNodes().forEach(node -> update(node));
         $entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE");
@@ -1483,8 +1497,8 @@ rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
 rule "X.3.0: Remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"
     salience 64
     when
-        $falseRecommendation: TextEntity($type: type, entityType == EntityType.FALSE_RECOMMENDATION, active())
-        $recommendation: TextEntity(containedBy($falseRecommendation), type == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
+        $falseRecommendation: TextEntity($type: type(), entityType == EntityType.FALSE_RECOMMENDATION, active())
+        $recommendation: TextEntity(containedBy($falseRecommendation), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges())
     then
         $recommendation.remove("X.3.0", "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION");
         retract($recommendation);
@@ -1492,51 +1506,105 @@ rule "X.3.0: Remove Entity of type RECOMMENDATION when contained by FALSE_RECOMM
 
 
 // Rule unit: X.4
-rule "X.4.0: Remove Entity of type RECOMMENDATION when intersected by ENTITY with same type"
+rule "X.4.0: Remove Entity of type RECOMMENDATION when text range equals ENTITY with same type"
     salience 256
     when
-        $entity: TextEntity($type: type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
-        $recommendation: TextEntity(intersects($entity), type == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
+        $entity: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
+        $recommendation: TextEntity(getTextRange().equals($entity.getTextRange()), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges())
     then
         $entity.addEngines($recommendation.getEngines());
-        $recommendation.remove("X.4.0", "remove Entity of type RECOMMENDATION when intersected by ENTITY with same type");
+        $recommendation.remove("X.4.0", "remove Entity of type RECOMMENDATION when text range equals ENTITY with same type");
         retract($recommendation);
     end
 
 
 // Rule unit: X.5
-rule "X.5.0: Remove Entity of type RECOMMENDATION when contained by ENTITY"
+rule "X.5.0: Remove Entity of type RECOMMENDATION when intersected by ENTITY"
     salience 256
     when
         $entity: TextEntity((entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
-        $recommendation: TextEntity(containedBy($entity), entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
+        $recommendation: TextEntity(intersects($entity), entityType == EntityType.RECOMMENDATION, !hasManualChanges())
     then
-        $recommendation.remove("X.5.0", "remove Entity of type RECOMMENDATION when contained by ENTITY");
+        $recommendation.remove("X.5.0", "remove Entity of type RECOMMENDATION when intersected by ENTITY");
+        retract($recommendation);
+    end
+
+rule "X.5.1: Remove Entity of type RECOMMENDATION when contained by RECOMMENDATION"
+    salience 256
+    when
+        $entity: TextEntity($type: type(), entityType == EntityType.RECOMMENDATION, active())
+        $recommendation: TextEntity(containedBy($entity), type() != $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges())
+    then
+        $recommendation.remove("X.5.1", "remove Entity of type RECOMMENDATION when contained by RECOMMENDATION");
         retract($recommendation);
     end
 
 
 // Rule unit: X.6
-rule "X.6.0: Remove Entity of lower rank, when contained by by entity of type ENTITY"
+rule "X.6.0: Remove Entity of lower rank, when contained by entity of type ENTITY or HINT"
     salience 32
     when
-        $higherRank: TextEntity($type: type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
-        $lowerRank: TextEntity(containedBy($higherRank), type != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active())
+        $higherRank: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
+        $lowerRank: TextEntity(containedBy($higherRank), type() != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges())
     then
         $lowerRank.getIntersectingNodes().forEach(node -> update(node));
-        $lowerRank.remove("X.6.0", "remove Entity of lower rank, when contained by entity of type ENTITY");
+        $lowerRank.remove("X.6.0", "remove Entity of lower rank, when contained by entity of type ENTITY or HINT");
         retract($lowerRank);
     end
 
-rule "X.6.1: remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity"
+rule "X.6.1: remove Entity, when contained in another entity of type ENTITY or HINT with larger text range"
     salience 32
     when
-        $higherRank: TextEntity($type: type, $value: value, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
-        $lowerRank: TextEntity(intersects($higherRank), type != $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active(), $lowerRank.getValue().length() > $value.length())
+         $outer: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
+         $inner: TextEntity(containedBy($outer), type() != $type, $outer.getTextRange().length > getTextRange().length(), !hasManualChanges())
     then
-        $higherRank.getIntersectingNodes().forEach(node -> update(node));
-        $higherRank.remove("X.6.1", "remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity");
-        retract($higherRank);
+        $inner.getIntersectingNodes().forEach(node -> update(node));
+        $inner.remove("X.6.1", "remove Entity, when contained in another entity of type ENTITY or HINT with larger text range");
+        retract($inner);
+    end
+
+
+// Rule unit: X.8
+rule "X.8.0: Remove Entity when text range and type equals to imported Entity"
+    salience 257
+    when
+        $entity: TextEntity($type: type(), engines contains Engine.IMPORTED, active())
+        $other: TextEntity(getTextRange().equals($entity.getTextRange()), this != $entity, type() == $type, engines not contains Engine.IMPORTED)
+    then
+        $other.remove("X.8.0", "remove Entity when text range and type equals to imported Entity");
+        $entity.addEngines($other.getEngines());
+        retract($other);
+    end
+
+rule "X.8.1: Remove Entity when intersected by imported Entity"
+    salience 256
+    when
+        $entity: TextEntity(engines contains Engine.IMPORTED, active())
+        $other: TextEntity(intersects($entity), this != $entity, engines not contains Engine.IMPORTED)
+    then
+        $other.remove("X.8.1", "remove Entity when intersected by imported Entity");
+        retract($other);
+    end
+
+
+// Rule unit: X.9
+rule "X.9.0: Merge mostly contained signatures"
+    when
+        $aiSignature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.AI)
+        $signature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.ALGORITHM, mostlyContains($aiSignature, 0.8))
+    then
+        $aiSignature.remove("X.9.0", "removed because already contained by alogrithm signature");
+        $signature.addEngine(LayoutEngine.AI);
+    end
+
+
+// Rule unit: X.10
+rule "X.10.0: remove false positives of ai"
+    when
+        $anyImage: Image(engines contains LayoutEngine.ALGORITHM)
+        $aiSignature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.AI, !mostlyContainedBy($anyImage, 0.8))
+    then
+        $aiSignature.remove("X.10.0", "Removed because false positive");
     end
 
 

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora.drl

@@ -53,7 +53,8 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus;
+
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
 
 global Document document
 global EntityCreationService entityCreationService
@@ -467,7 +468,9 @@ rule "DOC.7.1: Performing Laboratory (Country)"
      then
          nerEntities.streamEntitiesOfType("COUNTRY")
                      .filter(nerEntity -> $section.getTextRange().contains(nerEntity.textRange()))
-                     .map(nerEntity -> entityCreationService.byNerEntity(nerEntity, "laboratory_country", EntityType.ENTITY, $section))
+                     .map(nerEntity -> entityCreationService.optionalByNerEntity(nerEntity, "laboratory_country", EntityType.ENTITY, $section))
+                     .flatMap(Optional::stream)
+                     .collect(Collectors.toList())
                      .forEach(entity -> {
                           entity.apply("DOC.7.1", "Performing Laboratory found", "n-a");
                      });
@@ -476,7 +479,7 @@ rule "DOC.7.1: Performing Laboratory (Country)"
 rule "DOC.7.2: Performing Laboratory (Country & Name) from dict"
     when
         $section: Section(containsString("PERFORMING LABORATORY:") || (containsString("PERFORMING") && containsString("LABORATORY:")))
-        $countryOrNameFromDictionary: TextEntity(type == "laboratory_country" || type == "laboratory_name", $type: type, isDictionaryEntry()) from $section.getEntities()
+        $countryOrNameFromDictionary: TextEntity(type() == "laboratory_country" || type() == "laboratory_name", $type: type, isDictionaryEntry()) from $section.getEntities()
     then
         $countryOrNameFromDictionary.apply("DOC.7.2", "Performing " + $type + " dictionary entry found.");
     end
@@ -1158,11 +1161,11 @@ rule "DOC.35.0: Doses (mg/kg bodyweight)"
 rule "MAN.0.0: Apply manual resize redaction"
     salience 128
     when
-        $resizeRedaction: ManualResizeRedaction($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate)
+        $resizeRedaction: ManualResizeRedaction($id: annotationId, $requestDate: requestDate)
         not ManualResizeRedaction(annotationId == $id, requestDate.isBefore($requestDate))
         $entityToBeResized: TextEntity(matchesAnnotationId($id))
     then
-        manualChangesApplicationService.resizeEntityAndReinsert($entityToBeResized, $resizeRedaction);
+        manualChangesApplicationService.resize($entityToBeResized, $resizeRedaction);
         retract($resizeRedaction);
         update($entityToBeResized);
         $entityToBeResized.getIntersectingNodes().forEach(node -> update(node));
@@ -1171,7 +1174,7 @@ rule "MAN.0.0: Apply manual resize redaction"
 rule "MAN.0.1: Apply manual resize redaction"
     salience 128
     when
-        $resizeRedaction: ManualResizeRedaction($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate)
+        $resizeRedaction: ManualResizeRedaction($id: annotationId, $requestDate: requestDate)
         not ManualResizeRedaction(annotationId == $id, requestDate.isBefore($requestDate))
         $imageToBeResized: Image(id == $id)
     then
@@ -1186,7 +1189,7 @@ rule "MAN.0.1: Apply manual resize redaction"
 rule "MAN.1.0: Apply id removals that are valid and not in forced redactions to Entity"
     salience 128
     when
-        $idRemoval: IdRemoval($id: annotationId, !removeFromDictionary, !removeFromAllDossiers, status == AnnotationStatus.APPROVED)
+        $idRemoval: IdRemoval($id: annotationId, !removeFromDictionary, !removeFromAllDossiers)
         $entityToBeRemoved: TextEntity(matchesAnnotationId($id))
     then
         $entityToBeRemoved.getManualOverwrite().addChange($idRemoval);
@@ -1198,7 +1201,7 @@ rule "MAN.1.0: Apply id removals that are valid and not in forced redactions to
 rule "MAN.1.1: Apply id removals that are valid and not in forced redactions to Image"
     salience 128
     when
-        $idRemoval: IdRemoval($id: annotationId, status == AnnotationStatus.APPROVED)
+        $idRemoval: IdRemoval($id: annotationId)
         $imageEntityToBeRemoved: Image($id == id)
     then
         $imageEntityToBeRemoved.getManualOverwrite().addChange($idRemoval);
@@ -1212,7 +1215,7 @@ rule "MAN.1.1: Apply id removals that are valid and not in forced redactions to
 rule "MAN.2.0: Apply force redaction"
     salience 128
     when
-        $force: ManualForceRedaction($id: annotationId, status == AnnotationStatus.APPROVED)
+        $force: ManualForceRedaction($id: annotationId)
         $entityToForce: TextEntity(matchesAnnotationId($id))
     then
         $entityToForce.getManualOverwrite().addChange($force);
@@ -1224,7 +1227,7 @@ rule "MAN.2.0: Apply force redaction"
 rule "MAN.2.1: Apply force redaction to images"
     salience 128
     when
-        $force: ManualForceRedaction($id: annotationId, status == AnnotationStatus.APPROVED)
+        $force: ManualForceRedaction($id: annotationId)
         $imageToForce: Image(id == $id)
     then
         $imageToForce.getManualOverwrite().addChange($force);
@@ -1238,23 +1241,22 @@ rule "MAN.2.1: Apply force redaction to images"
 rule "MAN.3.0: Apply entity recategorization"
     salience 128
     when
-        $recategorization: ManualRecategorization($id: annotationId, $type: type, status == AnnotationStatus.APPROVED, $requestDate: requestDate)
+        $recategorization: ManualRecategorization($id: annotationId, $type: type, $requestDate: requestDate)
         not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))
-        $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type != $type)
+        $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() != $type)
     then
         $entityToBeRecategorized.getIntersectingNodes().forEach(node -> update(node));
-        manualChangesApplicationService.recategorize($entityToBeRecategorized, $recategorization);
+        $entityToBeRecategorized.getManualOverwrite().addChange($recategorization);
+        update($entityToBeRecategorized);
         retract($recategorization);
-        // Entity is copied and inserted, so the old entity needs to be retracted to avoid duplication.
-        retract($entityToBeRecategorized);
     end
 
 rule "MAN.3.1: Apply entity recategorization of same type"
     salience 128
     when
-        $recategorization: ManualRecategorization($id: annotationId, $type: type, status == AnnotationStatus.APPROVED, $requestDate: requestDate)
+        $recategorization: ManualRecategorization($id: annotationId, $type: type, $requestDate: requestDate)
         not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))
-        $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type == $type)
+        $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() == $type)
     then
         $entityToBeRecategorized.getManualOverwrite().addChange($recategorization);
         retract($recategorization);
@@ -1263,7 +1265,7 @@ rule "MAN.3.1: Apply entity recategorization of same type"
 rule "MAN.3.2: Apply image recategorization"
     salience 128
     when
-        $recategorization: ManualRecategorization($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate)
+        $recategorization: ManualRecategorization($id: annotationId, $requestDate: requestDate)
         not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))
         $imageToBeRecategorized: Image($id == id)
     then
@@ -1276,9 +1278,9 @@ rule "MAN.3.2: Apply image recategorization"
 rule "MAN.3.3: Apply recategorization entities by default"
     salience 128
     when
-        $entity: IEntity(getManualOverwrite().getRecategorized().orElse(false))
+        $entity: TextEntity(getManualOverwrite().getRecategorized().orElse(false), !dictionary.isHint(type()))
     then
-        $entity.apply("MAN.3.3", "Recategorized entities are applied by default.");
+        $entity.apply("MAN.3.3", "Recategorized entities are applied by default.", $entity.legalBasis());
     end
 
 
@@ -1286,19 +1288,23 @@ rule "MAN.3.3: Apply recategorization entities by default"
 rule "MAN.4.0: Apply legal basis change"
     salience 128
     when
-        $legalbasisChange: ManualLegalBasisChange($id: annotationId, status == AnnotationStatus.APPROVED)
+        $legalBasisChange: ManualLegalBasisChange($id: annotationId)
         $imageToBeRecategorized: Image($id == id)
     then
-        $imageToBeRecategorized.getManualOverwrite().addChange($legalbasisChange);
+        $imageToBeRecategorized.getManualOverwrite().addChange($legalBasisChange);
+        update($imageToBeRecategorized)
+        retract($legalBasisChange)
     end
 
 rule "MAN.4.1: Apply legal basis change"
     salience 128
     when
-        $legalBasisChange: ManualLegalBasisChange($id: annotationId, status == AnnotationStatus.APPROVED)
+        $legalBasisChange: ManualLegalBasisChange($id: annotationId)
         $entityToBeChanged: TextEntity(matchesAnnotationId($id))
     then
         $entityToBeChanged.getManualOverwrite().addChange($legalBasisChange);
+        update($entityToBeChanged)
+        retract($legalBasisChange)
     end
 
 
@@ -1308,8 +1314,8 @@ rule "MAN.4.1: Apply legal basis change"
 rule "X.0.0: Remove Entity contained by Entity of same type"
     salience 65
     when
-        $larger: TextEntity($type: type, $entityType: entityType, active())
-        $contained: TextEntity(containedBy($larger), type == $type, entityType == $entityType, this != $larger, !hasManualChanges(), active())
+        $larger: TextEntity($type: type(), $entityType: entityType, active())
+        $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges())
     then
         $contained.remove("X.0.0", "remove Entity contained by Entity of same type");
         retract($contained);
@@ -1320,8 +1326,8 @@ rule "X.0.0: Remove Entity contained by Entity of same type"
 rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
     salience 64
     when
-        $falsePositive: TextEntity($type: type, entityType == EntityType.FALSE_POSITIVE, active())
-        $entity: TextEntity(containedBy($falsePositive), type == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges(), active())
+        $falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active())
+        $entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges())
     then
         $entity.getIntersectingNodes().forEach(node -> update(node));
         $entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE");
@@ -1333,8 +1339,8 @@ rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
 rule "X.3.0: Remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"
     salience 64
     when
-        $falseRecommendation: TextEntity($type: type, entityType == EntityType.FALSE_RECOMMENDATION, active())
-        $recommendation: TextEntity(containedBy($falseRecommendation), type == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
+        $falseRecommendation: TextEntity($type: type(), entityType == EntityType.FALSE_RECOMMENDATION, active())
+        $recommendation: TextEntity(containedBy($falseRecommendation), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges())
     then
         $recommendation.remove("X.3.0", "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION");
         retract($recommendation);
@@ -1342,26 +1348,36 @@ rule "X.3.0: Remove Entity of type RECOMMENDATION when contained by FALSE_RECOMM
 
 
 // Rule unit: X.4
-rule "X.4.0: Remove Entity of type RECOMMENDATION when intersected by ENTITY with same type"
+rule "X.4.0: Remove Entity of type RECOMMENDATION when text range equals ENTITY with same type"
     salience 256
     when
-        $entity: TextEntity($type: type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
-        $recommendation: TextEntity(intersects($entity), type == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
+        $entity: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
+        $recommendation: TextEntity(getTextRange().equals($entity.getTextRange()), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges())
     then
         $entity.addEngines($recommendation.getEngines());
-        $recommendation.remove("X.4.0", "remove Entity of type RECOMMENDATION when intersected by ENTITY with same type");
+        $recommendation.remove("X.4.0", "remove Entity of type RECOMMENDATION when text range equals ENTITY with same type");
         retract($recommendation);
     end
 
 
 // Rule unit: X.5
-rule "X.5.0: Remove Entity of type RECOMMENDATION when contained by ENTITY"
+rule "X.5.0: Remove Entity of type RECOMMENDATION when intersected by ENTITY"
     salience 256
     when
         $entity: TextEntity((entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
-        $recommendation: TextEntity(containedBy($entity), entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
+        $recommendation: TextEntity(intersects($entity), entityType == EntityType.RECOMMENDATION, !hasManualChanges())
     then
-        $recommendation.remove("X.5.0", "remove Entity of type RECOMMENDATION when contained by ENTITY");
+        $recommendation.remove("X.5.0", "remove Entity of type RECOMMENDATION when intersected by ENTITY");
+        retract($recommendation);
+    end
+
+rule "X.5.1: Remove Entity of type RECOMMENDATION when contained by RECOMMENDATION"
+    salience 256
+    when
+        $entity: TextEntity($type: type(), entityType == EntityType.RECOMMENDATION, active())
+        $recommendation: TextEntity(containedBy($entity), type() != $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges())
+    then
+        $recommendation.remove("X.5.1", "remove Entity of type RECOMMENDATION when contained by RECOMMENDATION");
         retract($recommendation);
     end
 
@@ -1377,6 +1393,50 @@ rule "X.7.0: Remove all images"
     end
 
 
+// Rule unit: X.8
+rule "X.8.0: Remove Entity when text range and type equals to imported Entity"
+    salience 257
+    when
+        $entity: TextEntity($type: type(), engines contains Engine.IMPORTED, active())
+        $other: TextEntity(getTextRange().equals($entity.getTextRange()), this != $entity, type() == $type, engines not contains Engine.IMPORTED)
+    then
+        $other.remove("X.8.0", "remove Entity when text range and type equals to imported Entity");
+        $entity.addEngines($other.getEngines());
+        retract($other);
+    end
+
+rule "X.8.1: Remove Entity when intersected by imported Entity"
+    salience 256
+    when
+        $entity: TextEntity(engines contains Engine.IMPORTED, active())
+        $other: TextEntity(intersects($entity), this != $entity, engines not contains Engine.IMPORTED)
+    then
+        $other.remove("X.8.1", "remove Entity when intersected by imported Entity");
+        retract($other);
+    end
+
+
+// Rule unit: X.9
+rule "X.9.0: Merge mostly contained signatures"
+    when
+        $aiSignature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.AI)
+        $signature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.ALGORITHM, mostlyContains($aiSignature, 0.8))
+    then
+        $aiSignature.remove("X.9.0", "removed because already contained by alogrithm signature");
+        $signature.addEngine(LayoutEngine.AI);
+    end
+
+
+// Rule unit: X.10
+rule "X.10.0: remove false positives of ai"
+    when
+        $anyImage: Image(engines contains LayoutEngine.ALGORITHM)
+        $aiSignature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.AI, !mostlyContainedBy($anyImage, 0.8))
+    then
+        $aiSignature.remove("X.10.0", "Removed because false positive");
+    end
+
+
 //------------------------------------ File attributes rules ------------------------------------
 
 // Rule unit: FA.1

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/manual_redaction_rules.drl

@@ -53,7 +53,8 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus;
+
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
 
 global Document document
 global EntityCreationService entityCreationService
@@ -72,11 +73,11 @@ query "getFileAttributes"
 rule "MAN.0.0: Apply manual resize redaction"
     salience 128
     when
-        $resizeRedaction: ManualResizeRedaction($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate)
+        $resizeRedaction: ManualResizeRedaction($id: annotationId, $requestDate: requestDate)
         not ManualResizeRedaction(annotationId == $id, requestDate.isBefore($requestDate))
         $entityToBeResized: TextEntity(matchesAnnotationId($id))
     then
-        manualChangesApplicationService.resizeEntityAndReinsert($entityToBeResized, $resizeRedaction);
+        manualChangesApplicationService.resize($entityToBeResized, $resizeRedaction);
         retract($resizeRedaction);
         update($entityToBeResized);
         $entityToBeResized.getIntersectingNodes().forEach(node -> update(node));
@@ -85,7 +86,7 @@ rule "MAN.0.0: Apply manual resize redaction"
 rule "MAN.0.1: Apply manual resize redaction"
     salience 128
     when
-        $resizeRedaction: ManualResizeRedaction($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate)
+        $resizeRedaction: ManualResizeRedaction($id: annotationId, $requestDate: requestDate)
         not ManualResizeRedaction(annotationId == $id, requestDate.isBefore($requestDate))
         $imageToBeResized: Image(id == $id)
     then
@@ -100,7 +101,7 @@ rule "MAN.0.1: Apply manual resize redaction"
 rule "MAN.1.0: Apply id removals that are valid and not in forced redactions to Entity"
     salience 128
     when
-        $idRemoval: IdRemoval($id: annotationId, !removeFromDictionary, !removeFromAllDossiers, status == AnnotationStatus.APPROVED)
+        $idRemoval: IdRemoval($id: annotationId, !removeFromDictionary, !removeFromAllDossiers)
         $entityToBeRemoved: TextEntity(matchesAnnotationId($id))
     then
         $entityToBeRemoved.getManualOverwrite().addChange($idRemoval);
@@ -112,7 +113,7 @@ rule "MAN.1.0: Apply id removals that are valid and not in forced redactions to
 rule "MAN.1.1: Apply id removals that are valid and not in forced redactions to Image"
     salience 128
     when
-        $idRemoval: IdRemoval($id: annotationId, status == AnnotationStatus.APPROVED)
+        $idRemoval: IdRemoval($id: annotationId)
         $imageEntityToBeRemoved: Image($id == id)
     then
         $imageEntityToBeRemoved.getManualOverwrite().addChange($idRemoval);
@@ -126,7 +127,7 @@ rule "MAN.1.1: Apply id removals that are valid and not in forced redactions to
 rule "MAN.2.0: Apply force redaction"
     salience 128
     when
-        $force: ManualForceRedaction($id: annotationId, status == AnnotationStatus.APPROVED)
+        $force: ManualForceRedaction($id: annotationId)
         $entityToForce: TextEntity(matchesAnnotationId($id))
     then
         $entityToForce.getManualOverwrite().addChange($force);
@@ -138,7 +139,7 @@ rule "MAN.2.0: Apply force redaction"
 rule "MAN.2.1: Apply force redaction to images"
     salience 128
     when
-        $force: ManualForceRedaction($id: annotationId, status == AnnotationStatus.APPROVED)
+        $force: ManualForceRedaction($id: annotationId)
         $imageToForce: Image(id == $id)
     then
         $imageToForce.getManualOverwrite().addChange($force);
@@ -152,23 +153,22 @@ rule "MAN.2.1: Apply force redaction to images"
 rule "MAN.3.0: Apply entity recategorization"
     salience 128
     when
-        $recategorization: ManualRecategorization($id: annotationId, $type: type, status == AnnotationStatus.APPROVED, $requestDate: requestDate)
+        $recategorization: ManualRecategorization($id: annotationId, $type: type, $requestDate: requestDate)
         not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))
-        $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type != $type)
+        $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() != $type)
     then
         $entityToBeRecategorized.getIntersectingNodes().forEach(node -> update(node));
-        manualChangesApplicationService.recategorize($entityToBeRecategorized, $recategorization);
+        $entityToBeRecategorized.getManualOverwrite().addChange($recategorization);
+        update($entityToBeRecategorized);
         retract($recategorization);
-        // Entity is copied and inserted, so the old entity needs to be retracted to avoid duplication.
-        retract($entityToBeRecategorized);
     end
 
 rule "MAN.3.1: Apply entity recategorization of same type"
     salience 128
     when
-        $recategorization: ManualRecategorization($id: annotationId, $type: type, status == AnnotationStatus.APPROVED, $requestDate: requestDate)
+        $recategorization: ManualRecategorization($id: annotationId, $type: type, $requestDate: requestDate)
         not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))
-        $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type == $type)
+        $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() == $type)
     then
         $entityToBeRecategorized.getManualOverwrite().addChange($recategorization);
         retract($recategorization);
@@ -177,7 +177,7 @@ rule "MAN.3.1: Apply entity recategorization of same type"
 rule "MAN.3.2: Apply image recategorization"
     salience 128
     when
-        $recategorization: ManualRecategorization($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate)
+        $recategorization: ManualRecategorization($id: annotationId, $requestDate: requestDate)
         not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))
         $imageToBeRecategorized: Image($id == id)
     then
@@ -187,24 +187,94 @@ rule "MAN.3.2: Apply image recategorization"
         retract($recategorization);
     end
 
+rule "MAN.3.3: Apply recategorization entities by default"
+    salience 128
+    when
+        $entity: TextEntity(getManualOverwrite().getRecategorized().orElse(false), !dictionary.isHint(type()))
+    then
+        $entity.apply("MAN.3.3", "Recategorized entities are applied by default.", $entity.legalBasis());
+    end
+
 
 // Rule unit: MAN.4
 rule "MAN.4.0: Apply legal basis change"
     salience 128
     when
-        $legalbasisChange: ManualLegalBasisChange($id: annotationId, status == AnnotationStatus.APPROVED)
+        $legalBasisChange: ManualLegalBasisChange($id: annotationId)
         $imageToBeRecategorized: Image($id == id)
     then
-        $imageToBeRecategorized.getManualOverwrite().addChange($legalbasisChange);
+        $imageToBeRecategorized.getManualOverwrite().addChange($legalBasisChange);
+        update($imageToBeRecategorized)
+        retract($legalBasisChange)
     end
 
 rule "MAN.4.1: Apply legal basis change"
     salience 128
     when
-        $legalBasisChange: ManualLegalBasisChange($id: annotationId, status == AnnotationStatus.APPROVED)
+        $legalBasisChange: ManualLegalBasisChange($id: annotationId)
         $entityToBeChanged: TextEntity(matchesAnnotationId($id))
     then
         $entityToBeChanged.getManualOverwrite().addChange($legalBasisChange);
+        update($entityToBeChanged)
+        retract($legalBasisChange)
+    end
+
+
+//------------------------------------ Entity merging rules ------------------------------------
+
+// Rule unit: X.5
+rule "X.5.1: Remove Entity of type RECOMMENDATION when contained by RECOMMENDATION"
+    salience 256
+    when
+        $entity: TextEntity($type: type(), entityType == EntityType.RECOMMENDATION, active())
+        $recommendation: TextEntity(containedBy($entity), type() != $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges())
+    then
+        $recommendation.remove("X.5.1", "remove Entity of type RECOMMENDATION when contained by RECOMMENDATION");
+        retract($recommendation);
+    end
+
+
+// Rule unit: X.8
+rule "X.8.0: Remove Entity when text range and type equals to imported Entity"
+    salience 257
+    when
+        $entity: TextEntity($type: type(), engines contains Engine.IMPORTED, active())
+        $other: TextEntity(getTextRange().equals($entity.getTextRange()), this != $entity, type() == $type, engines not contains Engine.IMPORTED)
+    then
+        $other.remove("X.8.0", "remove Entity when text range and type equals to imported Entity");
+        $entity.addEngines($other.getEngines());
+        retract($other);
+    end
+
+rule "X.8.1: Remove Entity when intersected by imported Entity"
+    salience 256
+    when
+        $entity: TextEntity(engines contains Engine.IMPORTED, active())
+        $other: TextEntity(intersects($entity), this != $entity, engines not contains Engine.IMPORTED)
+    then
+        $other.remove("X.8.1", "remove Entity when intersected by imported Entity");
+        retract($other);
+    end
+
+
+// Rule unit: X.9
+rule "X.9.0: Merge mostly contained signatures"
+    when
+        $aiSignature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.AI)
+        $signature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.ALGORITHM, mostlyContains($aiSignature, 0.8))
+    then
+        $aiSignature.remove("X.9.0", "removed because already contained by alogrithm signature");
+        $signature.addEngine(LayoutEngine.AI);
+    end
+
+
+// Rule unit: X.10
+rule "X.10.0: remove false positives of ai"
+    when
+        $anyImage: Image(engines contains LayoutEngine.ALGORITHM)
+        $aiSignature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.AI, !mostlyContainedBy($anyImage, 0.8))
+    then
+        $aiSignature.remove("X.10.0", "Removed because false positive");
     end
 
 

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl

@@ -53,7 +53,8 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus;
+
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
 
 global Document document
 global EntityCreationService entityCreationService
@@ -101,18 +102,17 @@ rule "CBI.3.0: Redacted because Section contains a vertebrate"
 
 rule "CBI.3.1: Redacted because table row contains a vertebrate"
     when
-        $table: Table(hasEntitiesOfType("vertebrate"), (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
+        $table: Table(hasEntitiesOfType("vertebrate"), hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address"))
+        $cellsWithvertebrate: TableCell() from $table.streamTableCellsWhichContainType("vertebrate").toList()
+        $tableCell: TableCell(row == $cellsWithvertebrate.row) from $table.streamTableCells().toList()
+        $authorOrAddress: TextEntity(type() == "CBI_author" || type() == "CBI_address", active()) from $tableCell.getEntities()
     then
-        $table.streamEntitiesWhereRowContainsEntitiesOfType(List.of("vertebrate"))
-            .filter(entity -> entity.getType().equals("CBI_author") || entity.getType().equals("CBI_address"))
-            .forEach(entity -> {
-                entity.applyWithReferences(
+        $authorOrAddress.applyWithReferences(
                         "CBI.3.1",
                         "Vertebrate found",
                         "Reg (EC) No 1107/2009 Art. 63 (2g)",
-                        $table.getEntitiesOfTypeInSameRow("vertebrate", entity)
+                        $table.getEntitiesOfTypeInSameRow("vertebrate", $authorOrAddress)
                 );
-            });
     end
 
 rule "CBI.3.2: Do not redact because Section does not contain a vertebrate"
@@ -151,23 +151,21 @@ rule "CBI.4.0: Do not redact Names and Addresses if no_redaction_indicator is fo
             });
     end
 
-rule "CBI.4.1: Do not redact Names and Addresses if no_redaction_indicator is found in table row"
+rule "CBI.4.1: Redacted because table row contains a vertebrate"
     when
-        $table: Table(hasEntitiesOfType("no_redaction_indicator"),
-                      hasEntitiesOfType("vertebrate"),
-                      (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
+        $table: Table(hasEntitiesOfType("no_redaction_indicator"), hasEntitiesOfType("vertebrate"), hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address"))
+        TableCell($row: row) from $table.streamTableCellsWhichContainType("vertebrate").toList()
+        TableCell(row == $row) from $table.streamTableCellsWhichContainType("no_redaction_indicator").toList()
+        $tableCell: TableCell(row == $row) from $table.streamTableCells().toList()
+        $authorOrAddress: TextEntity(type() == "CBI_author" || type() == "CBI_address", active()) from $tableCell.getEntities()
     then
-        $table.streamEntitiesWhereRowContainsEntitiesOfType(List.of("vertebrate", "no-redaction_indicator"))
-            .filter(entity -> entity.getType().equals("CBI_author") || entity.getType().equals("CBI_address"))
-            .forEach(entity -> {
-                entity.skipWithReferences(
+        $authorOrAddress.skipWithReferences(
                         "CBI.4.1",
                         "Vertebrate but a no redaction indicator found",
                         Stream.concat(
-                            $table.getEntitiesOfTypeInSameRow("vertebrate", entity).stream(),
-                            $table.getEntitiesOfTypeInSameRow("no_redaction_indicator", entity).stream()).toList()
+                            $table.getEntitiesOfTypeInSameRow("vertebrate", $authorOrAddress).stream(),
+                            $table.getEntitiesOfTypeInSameRow("no_redaction_indicator", $authorOrAddress).stream()).toList()
                 );
-            });
     end
 
 
@@ -194,22 +192,20 @@ rule "CBI.5.0: Redact Names and Addresses if no_redaction_indicator but also red
 
 rule "CBI.5.1: Redact Names and Addresses if no_redaction_indicator but also redaction_indicator is found in table row"
     when
-        $table: Table(hasEntitiesOfType("no_redaction_indicator"),
-                      hasEntitiesOfType("redaction_indicator"),
-                      (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
+        $table: Table(hasEntitiesOfType("no_redaction_indicator"), hasEntitiesOfType("redaction_indicator"), hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address"))
+        TableCell($row: row) from $table.streamTableCellsWhichContainType("redaction_indicator").toList()
+        TableCell(row == $row) from $table.streamTableCellsWhichContainType("no_redaction_indicator").toList()
+        $tableCell: TableCell(row == $row) from $table.streamTableCells().toList()
+        $entity: TextEntity(type() == "CBI_author" || type() == "CBI_address", active()) from $tableCell.getEntities()
     then
-        $table.streamEntitiesWhereRowContainsEntitiesOfType(List.of("redaction_indicator", "no_redaction_indicator"))
-            .filter(entity -> entity.getType().equals("CBI_author") || entity.getType().equals("CBI_address"))
-            .forEach(entity -> {
-                entity.applyWithReferences(
+        $entity.applyWithReferences(
                         "CBI.5.1",
                         "no_redaction_indicator but also redaction_indicator found",
                         "Reg (EC) No 1107/2009 Art. 63 (2g)",
                         Stream.concat(
-                            $table.getEntitiesOfTypeInSameRow("vertebrate", entity).stream(),
-                            $table.getEntitiesOfTypeInSameRow("no_redaction_indicator", entity).stream()).toList()
+                            $table.getEntitiesOfTypeInSameRow("redaction_indicator", $entity).stream(),
+                            $table.getEntitiesOfTypeInSameRow("no_redaction_indicator", $entity).stream()).toList()
                 );
-            });
     end
 
 
@@ -231,18 +227,17 @@ rule "CBI.8.0: Redacted because Section contains must_redact entity"
 
 rule "CBI.8.1: Redacted because table row contains must_redact entity"
     when
-        $table: Table(hasEntitiesOfType("must_redact"), (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
+        $table: Table(hasEntitiesOfType("must_redact"), hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address"))
+        $cellsWithMustRedact: TableCell() from $table.streamTableCellsWhichContainType("must_redact").toList()
+        $tableCell: TableCell(row == $cellsWithMustRedact.row) from $table.streamTableCells().toList()
+        $authorOrAddress: TextEntity(type() == "CBI_author" || type() == "CBI_address", active()) from $tableCell.getEntities()
     then
-        $table.streamEntitiesWhereRowContainsEntitiesOfType(List.of("must_redact"))
-            .filter(entity -> entity.getType().equals("CBI_author") || entity.getType().equals("CBI_address"))
-            .forEach(entity -> {
-                entity.applyWithReferences(
+        $authorOrAddress.applyWithReferences(
                         "CBI.8.1",
-                        "must_redact entity found",
+                        "Must_redact found",
                         "Reg (EC) No 1107/2009 Art. 63 (2g)",
-                        $table.getEntitiesOfTypeInSameRow("must_redact", entity)
+                        $table.getEntitiesOfTypeInSameRow("must_redact", $authorOrAddress)
                 );
-            });
     end
 
 
@@ -296,7 +291,6 @@ rule "CBI.12.0: Redact and recommend TableCell with header 'Author' or 'Author(s
         TableCell(!header, containsAnyString("Yes", "Y"), $rowWithYes: row) from $table.streamCol($vertebrateCol).toList()
         $authorCell: TableCell(row == $rowWithYes) from $table.streamCol($authorCol).toList()
     then
-
         entityCreationService.bySemanticNode($authorCell, "CBI_author", EntityType.ENTITY)
           .ifPresent(authorEntity -> {
               authorEntity.redact("CBI.12.0", "Redacted because it's row belongs to a vertebrate study", "Article 39(e)(3) of Regulation (EC) No 178/2002");
@@ -340,7 +334,7 @@ rule "CBI.12.2: Skip TableCell with header 'Author' or 'Author(s)' and header 'V
 // Rule unit: CBI.14
 rule "CBI.14.0: Redact CBI_sponsor entities if preceded by \"batches produced at\""
     when
-        $sponsorEntity: TextEntity(type == "CBI_sponsor", textBefore.contains("batches produced at"))
+        $sponsorEntity: TextEntity(type() == "CBI_sponsor", textBefore.contains("batches produced at"))
     then
         $sponsorEntity.redact("CBI.14.0", "Redacted because it represents a sponsor company", "Reg (EC) No 1107/2009 Art. 63 (2g)");
     end
@@ -443,7 +437,7 @@ rule "CBI.17.1: Add recommendation for Addresses in Test Organism sections, with
 rule "CBI.18.0: Expand CBI_author entities with firstname initials"
     no-loop true
     when
-        $entityToExpand: TextEntity(type == "CBI_author",
+        $entityToExpand: TextEntity(type() == "CBI_author",
           value.matches("[^\\s]+"),
           textAfter.startsWith(" "),
           anyMatch(textAfter, "(,? [A-Z]\\.?( ?[A-Z]\\.?)?( ?[A-Z]\\.?)?\\b\\.?)")
@@ -461,7 +455,7 @@ rule "CBI.18.0: Expand CBI_author entities with firstname initials"
 // Rule unit: CBI.19
 rule "CBI.19.0: Expand CBI_author entities with salutation prefix"
     when
-        $entityToExpand: TextEntity(type == "CBI_author", anyMatch(textBefore, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*"))
+        $entityToExpand: TextEntity(type() == "CBI_author", anyMatch(textBefore, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*"))
     then
         entityCreationService.byPrefixExpansionRegex($entityToExpand, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*")
          .ifPresent(expandedEntity -> {
@@ -506,7 +500,7 @@ rule "CBI.20.1: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJEC
 rule "PII.0.0: Redact all PII (non vertebrate study)"
     when
         not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
-        $pii: TextEntity(type == "PII", dictionaryEntry)
+        $pii: TextEntity(type() == "PII", dictionaryEntry)
     then
         $pii.redact("PII.0.0", "Personal Information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
     end
@@ -514,7 +508,7 @@ rule "PII.0.0: Redact all PII (non vertebrate study)"
 rule "PII.0.1: Redact all PII (vertebrate study)"
     when
         FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
-        $pii: TextEntity(type == "PII", dictionaryEntry)
+        $pii: TextEntity(type() == "PII", dictionaryEntry)
     then
         $pii.redact("PII.0.1", "Personal Information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
     end
@@ -734,7 +728,7 @@ rule "PII.11.0: Redact On behalf of Sequani Ltd.:"
 rule "PII.12.0: Expand PII entities with salutation prefix"
     when
         not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
-        $entityToExpand: TextEntity(type == "PII", anyMatch(textBefore, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*"))
+        $entityToExpand: TextEntity(type() == "PII", anyMatch(textBefore, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*"))
     then
         entityCreationService.byPrefixExpansionRegex($entityToExpand, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*")
         .ifPresent(expandedEntity -> expandedEntity.apply("PII.12.0", "Expanded PII with salutation prefix", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
@@ -743,7 +737,7 @@ rule "PII.12.0: Expand PII entities with salutation prefix"
 rule "PII.12.1: Expand PII entities with salutation prefix"
     when
         FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
-        $entityToExpand: TextEntity(type == "PII", anyMatch(textBefore, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*"))
+        $entityToExpand: TextEntity(type() == "PII", anyMatch(textBefore, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*"))
     then
         entityCreationService.byPrefixExpansionRegex($entityToExpand, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*")
         .ifPresent(expandedEntity -> expandedEntity.apply("PII.12.1", "Expanded PII with salutation prefix", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
@@ -801,20 +795,29 @@ rule "ETC.3.1: Redact logos (vertebrate study)"
 // Rule unit: ETC.4
 rule "ETC.4.0: Redact dossier dictionary entries"
     when
-        $dossierRedaction: TextEntity(type == "dossier_redaction")
+        $dossierRedaction: TextEntity(type() == "dossier_redaction")
     then
         $dossierRedaction.redact("ETC.4.0", "Specification of impurity found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
     end
 
 
 // Rule unit: ETC.5
-rule "ETC.5.0: Ignore dossier_redaction entries if confidentiality is not 'confidential'"
+rule "ETC.5.0: Skip dossier_redaction entries if confidentiality is 'confidential'"
+    when
+        FileAttribute(label == "Confidentiality", value == "confidential")
+        $dossierRedaction: TextEntity(type() == "dossier_redaction")
+    then
+        $dossierRedaction.skip("ETC.5.0", "Ignore dossier_redaction when confidential");
+        $dossierRedaction.getIntersectingNodes().forEach(node -> update(node));
+    end
+
+rule "ETC.5.1: Remove dossier_redaction entries if confidentiality is not 'confidential'"
     when
         not FileAttribute(label == "Confidentiality", value == "confidential")
-        $dossierRedaction: TextEntity(type == "dossier_redaction")
+        $dossierRedaction: TextEntity(type() == "dossier_redaction")
     then
-        $dossierRedaction.ignore("ETC.5.0", "Ignore dossier redactions, when not confidential");
-        $dossierRedaction.getIntersectingNodes().forEach(node -> update(node));
+        $dossierRedaction.remove("ETC.5.1", "Remove dossier_redaction when not confidential");
+        retract($dossierRedaction);
     end
 
 
@@ -870,7 +873,7 @@ rule "AI.0.0: Add all NER Entities of type CBI_author"
         nerEntities: NerEntities(hasEntitiesOfType("CBI_author"))
     then
         nerEntities.streamEntitiesOfType("CBI_author")
-            .forEach(nerEntity -> entityCreationService.byNerEntity(nerEntity, EntityType.RECOMMENDATION, document));
+            .forEach(nerEntity -> entityCreationService.optionalByNerEntity(nerEntity, EntityType.RECOMMENDATION, document));
     end
 
 
@@ -890,11 +893,11 @@ rule "AI.1.0: Combine and add NER Entities as CBI_address"
 rule "MAN.0.0: Apply manual resize redaction"
     salience 128
     when
-        $resizeRedaction: ManualResizeRedaction($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate)
+        $resizeRedaction: ManualResizeRedaction($id: annotationId, $requestDate: requestDate)
         not ManualResizeRedaction(annotationId == $id, requestDate.isBefore($requestDate))
         $entityToBeResized: TextEntity(matchesAnnotationId($id))
     then
-        manualChangesApplicationService.resizeEntityAndReinsert($entityToBeResized, $resizeRedaction);
+        manualChangesApplicationService.resize($entityToBeResized, $resizeRedaction);
         retract($resizeRedaction);
         update($entityToBeResized);
         $entityToBeResized.getIntersectingNodes().forEach(node -> update(node));
@@ -903,7 +906,7 @@ rule "MAN.0.0: Apply manual resize redaction"
 rule "MAN.0.1: Apply manual resize redaction"
     salience 128
     when
-        $resizeRedaction: ManualResizeRedaction($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate)
+        $resizeRedaction: ManualResizeRedaction($id: annotationId, $requestDate: requestDate)
         not ManualResizeRedaction(annotationId == $id, requestDate.isBefore($requestDate))
         $imageToBeResized: Image(id == $id)
     then
@@ -918,7 +921,7 @@ rule "MAN.0.1: Apply manual resize redaction"
 rule "MAN.1.0: Apply id removals that are valid and not in forced redactions to Entity"
     salience 128
     when
-        $idRemoval: IdRemoval($id: annotationId, !removeFromDictionary, !removeFromAllDossiers, status == AnnotationStatus.APPROVED)
+        $idRemoval: IdRemoval($id: annotationId, !removeFromDictionary, !removeFromAllDossiers)
         $entityToBeRemoved: TextEntity(matchesAnnotationId($id))
     then
         $entityToBeRemoved.getManualOverwrite().addChange($idRemoval);
@@ -930,7 +933,7 @@ rule "MAN.1.0: Apply id removals that are valid and not in forced redactions to
 rule "MAN.1.1: Apply id removals that are valid and not in forced redactions to Image"
     salience 128
     when
-        $idRemoval: IdRemoval($id: annotationId, status == AnnotationStatus.APPROVED)
+        $idRemoval: IdRemoval($id: annotationId)
         $imageEntityToBeRemoved: Image($id == id)
     then
         $imageEntityToBeRemoved.getManualOverwrite().addChange($idRemoval);
@@ -944,7 +947,7 @@ rule "MAN.1.1: Apply id removals that are valid and not in forced redactions to
 rule "MAN.2.0: Apply force redaction"
     salience 128
     when
-        $force: ManualForceRedaction($id: annotationId, status == AnnotationStatus.APPROVED)
+        $force: ManualForceRedaction($id: annotationId)
         $entityToForce: TextEntity(matchesAnnotationId($id))
     then
         $entityToForce.getManualOverwrite().addChange($force);
@@ -956,7 +959,7 @@ rule "MAN.2.0: Apply force redaction"
 rule "MAN.2.1: Apply force redaction to images"
     salience 128
     when
-        $force: ManualForceRedaction($id: annotationId, status == AnnotationStatus.APPROVED)
+        $force: ManualForceRedaction($id: annotationId)
         $imageToForce: Image(id == $id)
     then
         $imageToForce.getManualOverwrite().addChange($force);
@@ -970,23 +973,22 @@ rule "MAN.2.1: Apply force redaction to images"
 rule "MAN.3.0: Apply entity recategorization"
     salience 128
     when
-        $recategorization: ManualRecategorization($id: annotationId, $type: type, status == AnnotationStatus.APPROVED, $requestDate: requestDate)
+        $recategorization: ManualRecategorization($id: annotationId, $type: type, $requestDate: requestDate)
         not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))
-        $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type != $type)
+        $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() != $type)
     then
         $entityToBeRecategorized.getIntersectingNodes().forEach(node -> update(node));
-        manualChangesApplicationService.recategorize($entityToBeRecategorized, $recategorization);
+        $entityToBeRecategorized.getManualOverwrite().addChange($recategorization);
+        update($entityToBeRecategorized);
         retract($recategorization);
-        // Entity is copied and inserted, so the old entity needs to be retracted to avoid duplication.
-        retract($entityToBeRecategorized);
     end
 
 rule "MAN.3.1: Apply entity recategorization of same type"
     salience 128
     when
-        $recategorization: ManualRecategorization($id: annotationId, $type: type, status == AnnotationStatus.APPROVED, $requestDate: requestDate)
+        $recategorization: ManualRecategorization($id: annotationId, $type: type, $requestDate: requestDate)
         not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))
-        $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type == $type)
+        $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() == $type)
     then
         $entityToBeRecategorized.getManualOverwrite().addChange($recategorization);
         retract($recategorization);
@@ -995,7 +997,7 @@ rule "MAN.3.1: Apply entity recategorization of same type"
 rule "MAN.3.2: Apply image recategorization"
     salience 128
     when
-        $recategorization: ManualRecategorization($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate)
+        $recategorization: ManualRecategorization($id: annotationId, $requestDate: requestDate)
         not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))
         $imageToBeRecategorized: Image($id == id)
     then
@@ -1005,24 +1007,36 @@ rule "MAN.3.2: Apply image recategorization"
         retract($recategorization);
     end
 
+rule "MAN.3.3: Apply recategorization entities by default"
+    salience 128
+    when
+        $entity: TextEntity(getManualOverwrite().getRecategorized().orElse(false), !dictionary.isHint(type()))
+    then
+        $entity.apply("MAN.3.3", "Recategorized entities are applied by default.", $entity.legalBasis());
+    end
+
 
 // Rule unit: MAN.4
 rule "MAN.4.0: Apply legal basis change"
     salience 128
     when
-        $legalbasisChange: ManualLegalBasisChange($id: annotationId, status == AnnotationStatus.APPROVED)
+        $legalBasisChange: ManualLegalBasisChange($id: annotationId)
         $imageToBeRecategorized: Image($id == id)
     then
-        $imageToBeRecategorized.getManualOverwrite().addChange($legalbasisChange);
+        $imageToBeRecategorized.getManualOverwrite().addChange($legalBasisChange);
+        update($imageToBeRecategorized)
+        retract($legalBasisChange)
     end
 
 rule "MAN.4.1: Apply legal basis change"
     salience 128
     when
-        $legalBasisChange: ManualLegalBasisChange($id: annotationId, status == AnnotationStatus.APPROVED)
+        $legalBasisChange: ManualLegalBasisChange($id: annotationId)
         $entityToBeChanged: TextEntity(matchesAnnotationId($id))
     then
         $entityToBeChanged.getManualOverwrite().addChange($legalBasisChange);
+        update($entityToBeChanged)
+        retract($legalBasisChange)
     end
 
 
@@ -1032,8 +1046,8 @@ rule "MAN.4.1: Apply legal basis change"
 rule "X.0.0: Remove Entity contained by Entity of same type"
     salience 65
     when
-        $larger: TextEntity($type: type, $entityType: entityType, active())
-        $contained: TextEntity(containedBy($larger), type == $type, entityType == $entityType, this != $larger, !hasManualChanges(), active())
+        $larger: TextEntity($type: type(), $entityType: entityType, active())
+        $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges())
     then
         $contained.remove("X.0.0", "remove Entity contained by Entity of same type");
         retract($contained);
@@ -1044,8 +1058,8 @@ rule "X.0.0: Remove Entity contained by Entity of same type"
 rule "X.1.0: Merge intersecting Entities of same type"
     salience 64
     when
-        $first: TextEntity($type: type, $entityType: entityType, !resized(), active())
-        $second: TextEntity(intersects($first), type == $type, entityType == $entityType, this != $first, !hasManualChanges(), active())
+        $first: TextEntity($type: type(), $entityType: entityType, !resized(), active())
+        $second: TextEntity(intersects($first), type() == $type, entityType == $entityType, this != $first, !hasManualChanges(), active())
     then
         TextEntity mergedEntity = entityCreationService.mergeEntitiesOfSameType(List.of($first, $second), $type, $entityType, document);
         $first.remove("X.1.0", "merge intersecting Entities of same type");
@@ -1060,8 +1074,8 @@ rule "X.1.0: Merge intersecting Entities of same type"
 rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
     salience 64
     when
-        $falsePositive: TextEntity($type: type, entityType == EntityType.FALSE_POSITIVE, active())
-        $entity: TextEntity(containedBy($falsePositive), type == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges(), active())
+        $falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active())
+        $entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges())
     then
         $entity.getIntersectingNodes().forEach(node -> update(node));
         $entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE");
@@ -1073,8 +1087,8 @@ rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
 rule "X.3.0: Remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"
     salience 64
     when
-        $falseRecommendation: TextEntity($type: type, entityType == EntityType.FALSE_RECOMMENDATION, active())
-        $recommendation: TextEntity(containedBy($falseRecommendation), type == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
+        $falseRecommendation: TextEntity($type: type(), entityType == EntityType.FALSE_RECOMMENDATION, active())
+        $recommendation: TextEntity(containedBy($falseRecommendation), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges())
     then
         $recommendation.remove("X.3.0", "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION");
         retract($recommendation);
@@ -1082,51 +1096,105 @@ rule "X.3.0: Remove Entity of type RECOMMENDATION when contained by FALSE_RECOMM
 
 
 // Rule unit: X.4
-rule "X.4.0: Remove Entity of type RECOMMENDATION when intersected by ENTITY with same type"
+rule "X.4.0: Remove Entity of type RECOMMENDATION when text range equals ENTITY with same type"
     salience 256
     when
-        $entity: TextEntity($type: type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
-        $recommendation: TextEntity(intersects($entity), type == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
+        $entity: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
+        $recommendation: TextEntity(getTextRange().equals($entity.getTextRange()), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges())
     then
         $entity.addEngines($recommendation.getEngines());
-        $recommendation.remove("X.4.0", "remove Entity of type RECOMMENDATION when intersected by ENTITY with same type");
+        $recommendation.remove("X.4.0", "remove Entity of type RECOMMENDATION when text range equals ENTITY with same type");
         retract($recommendation);
     end
 
 
 // Rule unit: X.5
-rule "X.5.0: Remove Entity of type RECOMMENDATION when contained by ENTITY"
+rule "X.5.0: Remove Entity of type RECOMMENDATION when intersected by ENTITY"
     salience 256
     when
         $entity: TextEntity((entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
-        $recommendation: TextEntity(containedBy($entity), entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
+        $recommendation: TextEntity(intersects($entity), entityType == EntityType.RECOMMENDATION, !hasManualChanges())
     then
-        $recommendation.remove("X.5.0", "remove Entity of type RECOMMENDATION when contained by ENTITY");
+        $recommendation.remove("X.5.0", "remove Entity of type RECOMMENDATION when intersected by ENTITY");
+        retract($recommendation);
+    end
+
+rule "X.5.1: Remove Entity of type RECOMMENDATION when contained by RECOMMENDATION"
+    salience 256
+    when
+        $entity: TextEntity($type: type(), entityType == EntityType.RECOMMENDATION, active())
+        $recommendation: TextEntity(containedBy($entity), type() != $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges())
+    then
+        $recommendation.remove("X.5.1", "remove Entity of type RECOMMENDATION when contained by RECOMMENDATION");
         retract($recommendation);
     end
 
 
 // Rule unit: X.6
-rule "X.6.0: Remove Entity of lower rank, when contained by by entity of type ENTITY"
+rule "X.6.0: Remove Entity of lower rank, when contained by entity of type ENTITY or HINT"
     salience 32
     when
-        $higherRank: TextEntity($type: type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
-        $lowerRank: TextEntity(containedBy($higherRank), type != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active())
+        $higherRank: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
+        $lowerRank: TextEntity(containedBy($higherRank), type() != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges())
     then
         $lowerRank.getIntersectingNodes().forEach(node -> update(node));
-        $lowerRank.remove("X.6.0", "remove Entity of lower rank, when contained by entity of type ENTITY");
+        $lowerRank.remove("X.6.0", "remove Entity of lower rank, when contained by entity of type ENTITY or HINT");
         retract($lowerRank);
     end
 
-rule "X.6.1: remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity"
+rule "X.6.1: remove Entity, when contained in another entity of type ENTITY or HINT with larger text range"
     salience 32
     when
-        $higherRank: TextEntity($type: type, $value: value, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
-        $lowerRank: TextEntity(intersects($higherRank), type != $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active(), $lowerRank.getValue().length() > $value.length())
+         $outer: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
+         $inner: TextEntity(containedBy($outer), type() != $type, $outer.getTextRange().length > getTextRange().length(), !hasManualChanges())
     then
-        $higherRank.getIntersectingNodes().forEach(node -> update(node));
-        $higherRank.remove("X.6.1", "remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity");
-        retract($higherRank);
+        $inner.getIntersectingNodes().forEach(node -> update(node));
+        $inner.remove("X.6.1", "remove Entity, when contained in another entity of type ENTITY or HINT with larger text range");
+        retract($inner);
+    end
+
+
+// Rule unit: X.8
+rule "X.8.0: Remove Entity when text range and type equals to imported Entity"
+    salience 257
+    when
+        $entity: TextEntity($type: type(), engines contains Engine.IMPORTED, active())
+        $other: TextEntity(getTextRange().equals($entity.getTextRange()), this != $entity, type() == $type, engines not contains Engine.IMPORTED)
+    then
+        $other.remove("X.8.0", "remove Entity when text range and type equals to imported Entity");
+        $entity.addEngines($other.getEngines());
+        retract($other);
+    end
+
+rule "X.8.1: Remove Entity when intersected by imported Entity"
+    salience 256
+    when
+        $entity: TextEntity(engines contains Engine.IMPORTED, active())
+        $other: TextEntity(intersects($entity), this != $entity, engines not contains Engine.IMPORTED)
+    then
+        $other.remove("X.8.1", "remove Entity when intersected by imported Entity");
+        retract($other);
+    end
+
+
+// Rule unit: X.9
+rule "X.9.0: Merge mostly contained signatures"
+    when
+        $aiSignature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.AI)
+        $signature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.ALGORITHM, mostlyContains($aiSignature, 0.8))
+    then
+        $aiSignature.remove("X.9.0", "removed because already contained by alogrithm signature");
+        $signature.addEngine(LayoutEngine.AI);
+    end
+
+
+// Rule unit: X.10
+rule "X.10.0: remove false positives of ai"
+    when
+        $anyImage: Image(engines contains LayoutEngine.ALGORITHM)
+        $aiSignature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.AI, !mostlyContainedBy($anyImage, 0.8))
+    then
+        $aiSignature.remove("X.10.0", "Removed because false positive");
     end
 
 

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_v2.drl

@@ -53,7 +53,8 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus;
+
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
 
 global Document document
 global EntityCreationService entityCreationService
@@ -72,7 +73,7 @@ query "getFileAttributes"
 rule "CBI.0.0: Redact CBI Authors (non vertebrate Study)"
     when
         not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
-        $entity: TextEntity(type == "CBI_author", dictionaryEntry)
+        $entity: TextEntity(type() == "CBI_author", dictionaryEntry)
     then
         $entity.redact("CBI.0.0", "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
     end
@@ -84,7 +85,7 @@ rule "CBI.0.0: Redact CBI Authors (non vertebrate Study)"
 rule "PII.0.0: Redact all PII (non vertebrate study)"
     when
         not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
-        $pii: TextEntity(type == "PII", dictionaryEntry)
+        $pii: TextEntity(type() == "PII", dictionaryEntry)
     then
         $pii.redact("PII.0.0", "Personal Information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
     end
@@ -99,7 +100,7 @@ rule "AI.0.0: Add all NER Entities of type CBI_author"
         nerEntities: NerEntities(hasEntitiesOfType("CBI_author"))
     then
         nerEntities.streamEntitiesOfType("CBI_author")
-            .forEach(nerEntity -> entityCreationService.byNerEntity(nerEntity, EntityType.RECOMMENDATION, document));
+            .forEach(nerEntity -> entityCreationService.optionalByNerEntity(nerEntity, EntityType.RECOMMENDATION, document));
     end
 
 
@@ -109,11 +110,11 @@ rule "AI.0.0: Add all NER Entities of type CBI_author"
 rule "MAN.0.0: Apply manual resize redaction"
     salience 128
     when
-        $resizeRedaction: ManualResizeRedaction($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate)
+        $resizeRedaction: ManualResizeRedaction($id: annotationId, $requestDate: requestDate)
         not ManualResizeRedaction(annotationId == $id, requestDate.isBefore($requestDate))
         $entityToBeResized: TextEntity(matchesAnnotationId($id))
     then
-        manualChangesApplicationService.resizeEntityAndReinsert($entityToBeResized, $resizeRedaction);
+        manualChangesApplicationService.resize($entityToBeResized, $resizeRedaction);
         retract($resizeRedaction);
         update($entityToBeResized);
         $entityToBeResized.getIntersectingNodes().forEach(node -> update(node));
@@ -122,7 +123,7 @@ rule "MAN.0.0: Apply manual resize redaction"
 rule "MAN.0.1: Apply manual resize redaction"
     salience 128
     when
-        $resizeRedaction: ManualResizeRedaction($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate)
+        $resizeRedaction: ManualResizeRedaction($id: annotationId, $requestDate: requestDate)
         not ManualResizeRedaction(annotationId == $id, requestDate.isBefore($requestDate))
         $imageToBeResized: Image(id == $id)
     then
@@ -137,7 +138,7 @@ rule "MAN.0.1: Apply manual resize redaction"
 rule "MAN.1.0: Apply id removals that are valid and not in forced redactions to Entity"
     salience 128
     when
-        $idRemoval: IdRemoval($id: annotationId, !removeFromDictionary, !removeFromAllDossiers, status == AnnotationStatus.APPROVED)
+        $idRemoval: IdRemoval($id: annotationId, !removeFromDictionary, !removeFromAllDossiers)
         $entityToBeRemoved: TextEntity(matchesAnnotationId($id))
     then
         $entityToBeRemoved.getManualOverwrite().addChange($idRemoval);
@@ -149,7 +150,7 @@ rule "MAN.1.0: Apply id removals that are valid and not in forced redactions to
 rule "MAN.1.1: Apply id removals that are valid and not in forced redactions to Image"
     salience 128
     when
-        $idRemoval: IdRemoval($id: annotationId, status == AnnotationStatus.APPROVED)
+        $idRemoval: IdRemoval($id: annotationId)
         $imageEntityToBeRemoved: Image($id == id)
     then
         $imageEntityToBeRemoved.getManualOverwrite().addChange($idRemoval);
@@ -163,7 +164,7 @@ rule "MAN.1.1: Apply id removals that are valid and not in forced redactions to
 rule "MAN.2.0: Apply force redaction"
     salience 128
     when
-        $force: ManualForceRedaction($id: annotationId, status == AnnotationStatus.APPROVED)
+        $force: ManualForceRedaction($id: annotationId)
         $entityToForce: TextEntity(matchesAnnotationId($id))
     then
         $entityToForce.getManualOverwrite().addChange($force);
@@ -175,7 +176,7 @@ rule "MAN.2.0: Apply force redaction"
 rule "MAN.2.1: Apply force redaction to images"
     salience 128
     when
-        $force: ManualForceRedaction($id: annotationId, status == AnnotationStatus.APPROVED)
+        $force: ManualForceRedaction($id: annotationId)
         $imageToForce: Image(id == $id)
     then
         $imageToForce.getManualOverwrite().addChange($force);
@@ -189,46 +190,57 @@ rule "MAN.2.1: Apply force redaction to images"
 rule "MAN.3.0: Apply entity recategorization"
     salience 128
     when
-        $recategorization: ManualRecategorization($id: annotationId, $type: type, status == AnnotationStatus.APPROVED, $requestDate: requestDate)
+        $recategorization: ManualRecategorization($id: annotationId, $type: type, $requestDate: requestDate)
         not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))
-        $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type != $type)
+        $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() != $type)
     then
         $entityToBeRecategorized.getIntersectingNodes().forEach(node -> update(node));
-        manualChangesApplicationService.recategorize($entityToBeRecategorized, $recategorization);
+        $entityToBeRecategorized.getManualOverwrite().addChange($recategorization);
+        update($entityToBeRecategorized);
         retract($recategorization);
-        // Entity is copied and inserted, so the old entity needs to be retracted to avoid duplication.
-        retract($entityToBeRecategorized);
     end
 
 rule "MAN.3.1: Apply entity recategorization of same type"
     salience 128
     when
-        $recategorization: ManualRecategorization($id: annotationId, $type: type, status == AnnotationStatus.APPROVED, $requestDate: requestDate)
+        $recategorization: ManualRecategorization($id: annotationId, $type: type, $requestDate: requestDate)
         not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))
-        $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type == $type)
+        $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() == $type)
     then
         $entityToBeRecategorized.getManualOverwrite().addChange($recategorization);
         retract($recategorization);
     end
 
+rule "MAN.3.3: Apply recategorization entities by default"
+    salience 128
+    when
+        $entity: TextEntity(getManualOverwrite().getRecategorized().orElse(false), !dictionary.isHint(type()))
+    then
+        $entity.apply("MAN.3.3", "Recategorized entities are applied by default.", $entity.legalBasis());
+    end
+
 
 // Rule unit: MAN.4
 rule "MAN.4.0: Apply legal basis change"
     salience 128
     when
-        $legalbasisChange: ManualLegalBasisChange($id: annotationId, status == AnnotationStatus.APPROVED)
+        $legalBasisChange: ManualLegalBasisChange($id: annotationId)
         $imageToBeRecategorized: Image($id == id)
     then
-        $imageToBeRecategorized.getManualOverwrite().addChange($legalbasisChange);
+        $imageToBeRecategorized.getManualOverwrite().addChange($legalBasisChange);
+        update($imageToBeRecategorized)
+        retract($legalBasisChange)
     end
 
 rule "MAN.4.1: Apply legal basis change"
     salience 128
     when
-        $legalBasisChange: ManualLegalBasisChange($id: annotationId, status == AnnotationStatus.APPROVED)
+        $legalBasisChange: ManualLegalBasisChange($id: annotationId)
         $entityToBeChanged: TextEntity(matchesAnnotationId($id))
     then
         $entityToBeChanged.getManualOverwrite().addChange($legalBasisChange);
+        update($entityToBeChanged)
+        retract($legalBasisChange)
     end
 
 
@@ -238,8 +250,8 @@ rule "MAN.4.1: Apply legal basis change"
 rule "X.0.0: Remove Entity contained by Entity of same type"
     salience 65
     when
-        $larger: TextEntity($type: type, $entityType: entityType, active())
-        $contained: TextEntity(containedBy($larger), type == $type, entityType == $entityType, this != $larger, !hasManualChanges(), active())
+        $larger: TextEntity($type: type(), $entityType: entityType, active())
+        $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges())
     then
         $contained.remove("X.0.0", "remove Entity contained by Entity of same type");
         retract($contained);
@@ -250,8 +262,8 @@ rule "X.0.0: Remove Entity contained by Entity of same type"
 rule "X.1.0: Merge intersecting Entities of same type"
     salience 64
     when
-        $first: TextEntity($type: type, $entityType: entityType, !resized(), active())
-        $second: TextEntity(intersects($first), type == $type, entityType == $entityType, this != $first, !hasManualChanges(), active())
+        $first: TextEntity($type: type(), $entityType: entityType, !resized(), active())
+        $second: TextEntity(intersects($first), type() == $type, entityType == $entityType, this != $first, !hasManualChanges(), active())
     then
         TextEntity mergedEntity = entityCreationService.mergeEntitiesOfSameType(List.of($first, $second), $type, $entityType, document);
         $first.remove("X.1.0", "merge intersecting Entities of same type");
@@ -266,8 +278,8 @@ rule "X.1.0: Merge intersecting Entities of same type"
 rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
     salience 64
     when
-        $falsePositive: TextEntity($type: type, entityType == EntityType.FALSE_POSITIVE, active())
-        $entity: TextEntity(containedBy($falsePositive), type == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges(), active())
+        $falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active())
+        $entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges())
     then
         $entity.getIntersectingNodes().forEach(node -> update(node));
         $entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE");
@@ -279,8 +291,8 @@ rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
 rule "X.3.0: Remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"
     salience 64
     when
-        $falseRecommendation: TextEntity($type: type, entityType == EntityType.FALSE_RECOMMENDATION, active())
-        $recommendation: TextEntity(containedBy($falseRecommendation), type == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
+        $falseRecommendation: TextEntity($type: type(), entityType == EntityType.FALSE_RECOMMENDATION, active())
+        $recommendation: TextEntity(containedBy($falseRecommendation), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges())
     then
         $recommendation.remove("X.3.0", "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION");
         retract($recommendation);
@@ -288,51 +300,105 @@ rule "X.3.0: Remove Entity of type RECOMMENDATION when contained by FALSE_RECOMM
 
 
 // Rule unit: X.4
-rule "X.4.0: Remove Entity of type RECOMMENDATION when intersected by ENTITY with same type"
+rule "X.4.0: Remove Entity of type RECOMMENDATION when text range equals ENTITY with same type"
     salience 256
     when
-        $entity: TextEntity($type: type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
-        $recommendation: TextEntity(intersects($entity), type == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
+        $entity: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
+        $recommendation: TextEntity(getTextRange().equals($entity.getTextRange()), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges())
     then
         $entity.addEngines($recommendation.getEngines());
-        $recommendation.remove("X.4.0", "remove Entity of type RECOMMENDATION when intersected by ENTITY with same type");
+        $recommendation.remove("X.4.0", "remove Entity of type RECOMMENDATION when text range equals ENTITY with same type");
         retract($recommendation);
     end
 
 
 // Rule unit: X.5
-rule "X.5.0: Remove Entity of type RECOMMENDATION when contained by ENTITY"
+rule "X.5.0: Remove Entity of type RECOMMENDATION when intersected by ENTITY"
     salience 256
     when
         $entity: TextEntity((entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
-        $recommendation: TextEntity(containedBy($entity), entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
+        $recommendation: TextEntity(intersects($entity), entityType == EntityType.RECOMMENDATION, !hasManualChanges())
     then
-        $recommendation.remove("X.5.0", "remove Entity of type RECOMMENDATION when contained by ENTITY");
+        $recommendation.remove("X.5.0", "remove Entity of type RECOMMENDATION when intersected by ENTITY");
+        retract($recommendation);
+    end
+
+rule "X.5.1: Remove Entity of type RECOMMENDATION when contained by RECOMMENDATION"
+    salience 256
+    when
+        $entity: TextEntity($type: type(), entityType == EntityType.RECOMMENDATION, active())
+        $recommendation: TextEntity(containedBy($entity), type() != $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges())
+    then
+        $recommendation.remove("X.5.1", "remove Entity of type RECOMMENDATION when contained by RECOMMENDATION");
         retract($recommendation);
     end
 
 
 // Rule unit: X.6
-rule "X.6.0: Remove Entity of lower rank, when contained by by entity of type ENTITY"
+rule "X.6.0: Remove Entity of lower rank, when contained by entity of type ENTITY or HINT"
     salience 32
     when
-        $higherRank: TextEntity($type: type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
-        $lowerRank: TextEntity(containedBy($higherRank), type != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active())
+        $higherRank: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
+        $lowerRank: TextEntity(containedBy($higherRank), type() != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges())
     then
         $lowerRank.getIntersectingNodes().forEach(node -> update(node));
-        $lowerRank.remove("X.6.0", "remove Entity of lower rank, when contained by entity of type ENTITY");
+        $lowerRank.remove("X.6.0", "remove Entity of lower rank, when contained by entity of type ENTITY or HINT");
         retract($lowerRank);
     end
 
-rule "X.6.1: remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity"
+rule "X.6.1: remove Entity, when contained in another entity of type ENTITY or HINT with larger text range"
     salience 32
     when
-        $higherRank: TextEntity($type: type, $value: value, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
-        $lowerRank: TextEntity(intersects($higherRank), type != $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active(), $lowerRank.getValue().length() > $value.length())
+         $outer: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
+         $inner: TextEntity(containedBy($outer), type() != $type, $outer.getTextRange().length > getTextRange().length(), !hasManualChanges())
     then
-        $higherRank.getIntersectingNodes().forEach(node -> update(node));
-        $higherRank.remove("X.6.1", "remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity");
-        retract($higherRank);
+        $inner.getIntersectingNodes().forEach(node -> update(node));
+        $inner.remove("X.6.1", "remove Entity, when contained in another entity of type ENTITY or HINT with larger text range");
+        retract($inner);
+    end
+
+
+// Rule unit: X.8
+rule "X.8.0: Remove Entity when text range and type equals to imported Entity"
+    salience 257
+    when
+        $entity: TextEntity($type: type(), engines contains Engine.IMPORTED, active())
+        $other: TextEntity(getTextRange().equals($entity.getTextRange()), this != $entity, type() == $type, engines not contains Engine.IMPORTED)
+    then
+        $other.remove("X.8.0", "remove Entity when text range and type equals to imported Entity");
+        $entity.addEngines($other.getEngines());
+        retract($other);
+    end
+
+rule "X.8.1: Remove Entity when intersected by imported Entity"
+    salience 256
+    when
+        $entity: TextEntity(engines contains Engine.IMPORTED, active())
+        $other: TextEntity(intersects($entity), this != $entity, engines not contains Engine.IMPORTED)
+    then
+        $other.remove("X.8.1", "remove Entity when intersected by imported Entity");
+        retract($other);
+    end
+
+
+// Rule unit: X.9
+rule "X.9.0: Merge mostly contained signatures"
+    when
+        $aiSignature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.AI)
+        $signature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.ALGORITHM, mostlyContains($aiSignature, 0.8))
+    then
+        $aiSignature.remove("X.9.0", "removed because already contained by alogrithm signature");
+        $signature.addEngine(LayoutEngine.AI);
+    end
+
+
+// Rule unit: X.10
+rule "X.10.0: remove false positives of ai"
+    when
+        $anyImage: Image(engines contains LayoutEngine.ALGORITHM)
+        $aiSignature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.AI, !mostlyContainedBy($anyImage, 0.8))
+    then
+        $aiSignature.remove("X.10.0", "Removed because false positive");
     end
 
 

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/table_demo.drl

@@ -53,7 +53,8 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus;
+
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
 
 global Document document
 global EntityCreationService entityCreationService
@@ -222,11 +223,11 @@ rule "TAB.7.0: Indicator (Species)"
 rule "MAN.0.0: Apply manual resize redaction"
     salience 128
     when
-        $resizeRedaction: ManualResizeRedaction($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate)
+        $resizeRedaction: ManualResizeRedaction($id: annotationId, $requestDate: requestDate)
         not ManualResizeRedaction(annotationId == $id, requestDate.isBefore($requestDate))
         $entityToBeResized: TextEntity(matchesAnnotationId($id))
     then
-        manualChangesApplicationService.resizeEntityAndReinsert($entityToBeResized, $resizeRedaction);
+        manualChangesApplicationService.resize($entityToBeResized, $resizeRedaction);
         retract($resizeRedaction);
         update($entityToBeResized);
         $entityToBeResized.getIntersectingNodes().forEach(node -> update(node));
@@ -235,7 +236,7 @@ rule "MAN.0.0: Apply manual resize redaction"
 rule "MAN.0.1: Apply manual resize redaction"
     salience 128
     when
-        $resizeRedaction: ManualResizeRedaction($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate)
+        $resizeRedaction: ManualResizeRedaction($id: annotationId, $requestDate: requestDate)
         not ManualResizeRedaction(annotationId == $id, requestDate.isBefore($requestDate))
         $imageToBeResized: Image(id == $id)
     then
@@ -250,7 +251,7 @@ rule "MAN.0.1: Apply manual resize redaction"
 rule "MAN.1.0: Apply id removals that are valid and not in forced redactions to Entity"
     salience 128
     when
-        $idRemoval: IdRemoval($id: annotationId, !removeFromDictionary, !removeFromAllDossiers, status == AnnotationStatus.APPROVED)
+        $idRemoval: IdRemoval($id: annotationId, !removeFromDictionary, !removeFromAllDossiers)
         $entityToBeRemoved: TextEntity(matchesAnnotationId($id))
     then
         $entityToBeRemoved.getManualOverwrite().addChange($idRemoval);
@@ -262,7 +263,7 @@ rule "MAN.1.0: Apply id removals that are valid and not in forced redactions to
 rule "MAN.1.1: Apply id removals that are valid and not in forced redactions to Image"
     salience 128
     when
-        $idRemoval: IdRemoval($id: annotationId, status == AnnotationStatus.APPROVED)
+        $idRemoval: IdRemoval($id: annotationId)
         $imageEntityToBeRemoved: Image($id == id)
     then
         $imageEntityToBeRemoved.getManualOverwrite().addChange($idRemoval);
@@ -276,7 +277,7 @@ rule "MAN.1.1: Apply id removals that are valid and not in forced redactions to
 rule "MAN.2.0: Apply force redaction"
     salience 128
     when
-        $force: ManualForceRedaction($id: annotationId, status == AnnotationStatus.APPROVED)
+        $force: ManualForceRedaction($id: annotationId)
         $entityToForce: TextEntity(matchesAnnotationId($id))
     then
         $entityToForce.getManualOverwrite().addChange($force);
@@ -288,7 +289,7 @@ rule "MAN.2.0: Apply force redaction"
 rule "MAN.2.1: Apply force redaction to images"
     salience 128
     when
-        $force: ManualForceRedaction($id: annotationId, status == AnnotationStatus.APPROVED)
+        $force: ManualForceRedaction($id: annotationId)
         $imageToForce: Image(id == $id)
     then
         $imageToForce.getManualOverwrite().addChange($force);
@@ -302,23 +303,22 @@ rule "MAN.2.1: Apply force redaction to images"
 rule "MAN.3.0: Apply entity recategorization"
     salience 128
     when
-        $recategorization: ManualRecategorization($id: annotationId, $type: type, status == AnnotationStatus.APPROVED, $requestDate: requestDate)
+        $recategorization: ManualRecategorization($id: annotationId, $type: type, $requestDate: requestDate)
         not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))
-        $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type != $type)
+        $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() != $type)
     then
         $entityToBeRecategorized.getIntersectingNodes().forEach(node -> update(node));
-        manualChangesApplicationService.recategorize($entityToBeRecategorized, $recategorization);
+        $entityToBeRecategorized.getManualOverwrite().addChange($recategorization);
+        update($entityToBeRecategorized);
         retract($recategorization);
-        // Entity is copied and inserted, so the old entity needs to be retracted to avoid duplication.
-        retract($entityToBeRecategorized);
     end
 
 rule "MAN.3.1: Apply entity recategorization of same type"
     salience 128
     when
-        $recategorization: ManualRecategorization($id: annotationId, $type: type, status == AnnotationStatus.APPROVED, $requestDate: requestDate)
+        $recategorization: ManualRecategorization($id: annotationId, $type: type, $requestDate: requestDate)
         not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))
-        $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type == $type)
+        $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() == $type)
     then
         $entityToBeRecategorized.getManualOverwrite().addChange($recategorization);
         retract($recategorization);
@@ -327,7 +327,7 @@ rule "MAN.3.1: Apply entity recategorization of same type"
 rule "MAN.3.2: Apply image recategorization"
     salience 128
     when
-        $recategorization: ManualRecategorization($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate)
+        $recategorization: ManualRecategorization($id: annotationId, $requestDate: requestDate)
         not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))
         $imageToBeRecategorized: Image($id == id)
     then
@@ -337,24 +337,36 @@ rule "MAN.3.2: Apply image recategorization"
         retract($recategorization);
     end
 
+rule "MAN.3.3: Apply recategorization entities by default"
+    salience 128
+    when
+        $entity: TextEntity(getManualOverwrite().getRecategorized().orElse(false), !dictionary.isHint(type()))
+    then
+        $entity.apply("MAN.3.3", "Recategorized entities are applied by default.", $entity.legalBasis());
+    end
+
 
 // Rule unit: MAN.4
 rule "MAN.4.0: Apply legal basis change"
     salience 128
     when
-        $legalbasisChange: ManualLegalBasisChange($id: annotationId, status == AnnotationStatus.APPROVED)
+        $legalBasisChange: ManualLegalBasisChange($id: annotationId)
         $imageToBeRecategorized: Image($id == id)
     then
-        $imageToBeRecategorized.getManualOverwrite().addChange($legalbasisChange);
+        $imageToBeRecategorized.getManualOverwrite().addChange($legalBasisChange);
+        update($imageToBeRecategorized)
+        retract($legalBasisChange)
     end
 
 rule "MAN.4.1: Apply legal basis change"
     salience 128
     when
-        $legalBasisChange: ManualLegalBasisChange($id: annotationId, status == AnnotationStatus.APPROVED)
+        $legalBasisChange: ManualLegalBasisChange($id: annotationId)
         $entityToBeChanged: TextEntity(matchesAnnotationId($id))
     then
         $entityToBeChanged.getManualOverwrite().addChange($legalBasisChange);
+        update($entityToBeChanged)
+        retract($legalBasisChange)
     end
 
 
@@ -364,8 +376,8 @@ rule "MAN.4.1: Apply legal basis change"
 rule "X.0.0: Remove Entity contained by Entity of same type"
     salience 65
     when
-        $larger: TextEntity($type: type, $entityType: entityType, active())
-        $contained: TextEntity(containedBy($larger), type == $type, entityType == $entityType, this != $larger, !hasManualChanges(), active())
+        $larger: TextEntity($type: type(), $entityType: entityType, active())
+        $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges())
     then
         $contained.remove("X.0.0", "remove Entity contained by Entity of same type");
         retract($contained);
@@ -376,8 +388,8 @@ rule "X.0.0: Remove Entity contained by Entity of same type"
 rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
     salience 64
     when
-        $falsePositive: TextEntity($type: type, entityType == EntityType.FALSE_POSITIVE, active())
-        $entity: TextEntity(containedBy($falsePositive), type == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges(), active())
+        $falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active())
+        $entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges())
     then
         $entity.getIntersectingNodes().forEach(node -> update(node));
         $entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE");
@@ -389,8 +401,8 @@ rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
 rule "X.3.0: Remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"
     salience 64
     when
-        $falseRecommendation: TextEntity($type: type, entityType == EntityType.FALSE_RECOMMENDATION, active())
-        $recommendation: TextEntity(containedBy($falseRecommendation), type == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
+        $falseRecommendation: TextEntity($type: type(), entityType == EntityType.FALSE_RECOMMENDATION, active())
+        $recommendation: TextEntity(containedBy($falseRecommendation), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges())
     then
         $recommendation.remove("X.3.0", "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION");
         retract($recommendation);
@@ -398,51 +410,61 @@ rule "X.3.0: Remove Entity of type RECOMMENDATION when contained by FALSE_RECOMM
 
 
 // Rule unit: X.4
-rule "X.4.0: Remove Entity of type RECOMMENDATION when intersected by ENTITY with same type"
+rule "X.4.0: Remove Entity of type RECOMMENDATION when text range equals ENTITY with same type"
     salience 256
     when
-        $entity: TextEntity($type: type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
-        $recommendation: TextEntity(intersects($entity), type == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
+        $entity: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
+        $recommendation: TextEntity(getTextRange().equals($entity.getTextRange()), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges())
     then
         $entity.addEngines($recommendation.getEngines());
-        $recommendation.remove("X.4.0", "remove Entity of type RECOMMENDATION when intersected by ENTITY with same type");
+        $recommendation.remove("X.4.0", "remove Entity of type RECOMMENDATION when text range equals ENTITY with same type");
         retract($recommendation);
     end
 
 
 // Rule unit: X.5
-rule "X.5.0: Remove Entity of type RECOMMENDATION when contained by ENTITY"
+rule "X.5.0: Remove Entity of type RECOMMENDATION when intersected by ENTITY"
     salience 256
     when
         $entity: TextEntity((entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
-        $recommendation: TextEntity(containedBy($entity), entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
+        $recommendation: TextEntity(intersects($entity), entityType == EntityType.RECOMMENDATION, !hasManualChanges())
     then
-        $recommendation.remove("X.5.0", "remove Entity of type RECOMMENDATION when contained by ENTITY");
+        $recommendation.remove("X.5.0", "remove Entity of type RECOMMENDATION when intersected by ENTITY");
+        retract($recommendation);
+    end
+
+rule "X.5.1: Remove Entity of type RECOMMENDATION when contained by RECOMMENDATION"
+    salience 256
+    when
+        $entity: TextEntity($type: type(), entityType == EntityType.RECOMMENDATION, active())
+        $recommendation: TextEntity(containedBy($entity), type() != $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges())
+    then
+        $recommendation.remove("X.5.1", "remove Entity of type RECOMMENDATION when contained by RECOMMENDATION");
         retract($recommendation);
     end
 
 
 // Rule unit: X.6
-rule "X.6.0: Remove Entity of lower rank, when contained by by entity of type ENTITY"
+rule "X.6.0: Remove Entity of lower rank, when contained by entity of type ENTITY or HINT"
     salience 32
     when
-        $higherRank: TextEntity($type: type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
-        $lowerRank: TextEntity(containedBy($higherRank), type != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active())
+        $higherRank: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
+        $lowerRank: TextEntity(containedBy($higherRank), type() != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges())
     then
         $lowerRank.getIntersectingNodes().forEach(node -> update(node));
-        $lowerRank.remove("X.6.0", "remove Entity of lower rank, when contained by entity of type ENTITY");
+        $lowerRank.remove("X.6.0", "remove Entity of lower rank, when contained by entity of type ENTITY or HINT");
         retract($lowerRank);
     end
 
-rule "X.6.1: remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity"
+rule "X.6.1: remove Entity, when contained in another entity of type ENTITY or HINT with larger text range"
     salience 32
     when
-        $higherRank: TextEntity($type: type, $value: value, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
-        $lowerRank: TextEntity(intersects($higherRank), type != $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active(), $lowerRank.getValue().length() > $value.length())
+         $outer: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
+         $inner: TextEntity(containedBy($outer), type() != $type, $outer.getTextRange().length > getTextRange().length(), !hasManualChanges())
     then
-        $higherRank.getIntersectingNodes().forEach(node -> update(node));
-        $higherRank.remove("X.6.1", "remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity");
-        retract($higherRank);
+        $inner.getIntersectingNodes().forEach(node -> update(node));
+        $inner.remove("X.6.1", "remove Entity, when contained in another entity of type ENTITY or HINT with larger text range");
+        retract($inner);
     end
 
 
@@ -457,6 +479,50 @@ rule "X.7.0: Remove all images"
     end
 
 
+// Rule unit: X.8
+rule "X.8.0: Remove Entity when text range and type equals to imported Entity"
+    salience 257
+    when
+        $entity: TextEntity($type: type(), engines contains Engine.IMPORTED, active())
+        $other: TextEntity(getTextRange().equals($entity.getTextRange()), this != $entity, type() == $type, engines not contains Engine.IMPORTED)
+    then
+        $other.remove("X.8.0", "remove Entity when text range and type equals to imported Entity");
+        $entity.addEngines($other.getEngines());
+        retract($other);
+    end
+
+rule "X.8.1: Remove Entity when intersected by imported Entity"
+    salience 256
+    when
+        $entity: TextEntity(engines contains Engine.IMPORTED, active())
+        $other: TextEntity(intersects($entity), this != $entity, engines not contains Engine.IMPORTED)
+    then
+        $other.remove("X.8.1", "remove Entity when intersected by imported Entity");
+        retract($other);
+    end
+
+
+// Rule unit: X.9
+rule "X.9.0: Merge mostly contained signatures"
+    when
+        $aiSignature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.AI)
+        $signature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.ALGORITHM, mostlyContains($aiSignature, 0.8))
+    then
+        $aiSignature.remove("X.9.0", "removed because already contained by alogrithm signature");
+        $signature.addEngine(LayoutEngine.AI);
+    end
+
+
+// Rule unit: X.10
+rule "X.10.0: remove false positives of ai"
+    when
+        $anyImage: Image(engines contains LayoutEngine.ALGORITHM)
+        $aiSignature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.AI, !mostlyContainedBy($anyImage, 0.8))
+    then
+        $aiSignature.remove("X.10.0", "Removed because false positive");
+    end
+
+
 //------------------------------------ File attributes rules ------------------------------------
 
 // Rule unit: FA.1

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/test_rules.drl

@@ -53,7 +53,8 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus;
+
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
 
 global Document document
 global EntityCreationService entityCreationService
@@ -122,11 +123,11 @@ rule "TAB.6.0: Targeted cell extraction (Experimental Stop date)"
 rule "MAN.0.0: Apply manual resize redaction"
     salience 128
     when
-        $resizeRedaction: ManualResizeRedaction($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate)
+        $resizeRedaction: ManualResizeRedaction($id: annotationId, $requestDate: requestDate)
         not ManualResizeRedaction(annotationId == $id, requestDate.isBefore($requestDate))
         $entityToBeResized: TextEntity(matchesAnnotationId($id))
     then
-        manualChangesApplicationService.resizeEntityAndReinsert($entityToBeResized, $resizeRedaction);
+        manualChangesApplicationService.resize($entityToBeResized, $resizeRedaction);
         retract($resizeRedaction);
         update($entityToBeResized);
         $entityToBeResized.getIntersectingNodes().forEach(node -> update(node));
@@ -135,7 +136,7 @@ rule "MAN.0.0: Apply manual resize redaction"
 rule "MAN.0.1: Apply manual resize redaction"
     salience 128
     when
-        $resizeRedaction: ManualResizeRedaction($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate)
+        $resizeRedaction: ManualResizeRedaction($id: annotationId, $requestDate: requestDate)
         not ManualResizeRedaction(annotationId == $id, requestDate.isBefore($requestDate))
         $imageToBeResized: Image(id == $id)
     then
@@ -150,7 +151,7 @@ rule "MAN.0.1: Apply manual resize redaction"
 rule "MAN.1.0: Apply id removals that are valid and not in forced redactions to Entity"
     salience 128
     when
-        $idRemoval: IdRemoval($id: annotationId, !removeFromDictionary, !removeFromAllDossiers, status == AnnotationStatus.APPROVED)
+        $idRemoval: IdRemoval($id: annotationId, !removeFromDictionary, !removeFromAllDossiers)
         $entityToBeRemoved: TextEntity(matchesAnnotationId($id))
     then
         $entityToBeRemoved.getManualOverwrite().addChange($idRemoval);
@@ -162,7 +163,7 @@ rule "MAN.1.0: Apply id removals that are valid and not in forced redactions to
 rule "MAN.1.1: Apply id removals that are valid and not in forced redactions to Image"
     salience 128
     when
-        $idRemoval: IdRemoval($id: annotationId, status == AnnotationStatus.APPROVED)
+        $idRemoval: IdRemoval($id: annotationId)
         $imageEntityToBeRemoved: Image($id == id)
     then
         $imageEntityToBeRemoved.getManualOverwrite().addChange($idRemoval);
@@ -176,7 +177,7 @@ rule "MAN.1.1: Apply id removals that are valid and not in forced redactions to
 rule "MAN.2.0: Apply force redaction"
     salience 128
     when
-        $force: ManualForceRedaction($id: annotationId, status == AnnotationStatus.APPROVED)
+        $force: ManualForceRedaction($id: annotationId)
         $entityToForce: TextEntity(matchesAnnotationId($id))
     then
         $entityToForce.getManualOverwrite().addChange($force);
@@ -188,7 +189,7 @@ rule "MAN.2.0: Apply force redaction"
 rule "MAN.2.1: Apply force redaction to images"
     salience 128
     when
-        $force: ManualForceRedaction($id: annotationId, status == AnnotationStatus.APPROVED)
+        $force: ManualForceRedaction($id: annotationId)
         $imageToForce: Image(id == $id)
     then
         $imageToForce.getManualOverwrite().addChange($force);
@@ -202,23 +203,22 @@ rule "MAN.2.1: Apply force redaction to images"
 rule "MAN.3.0: Apply entity recategorization"
     salience 128
     when
-        $recategorization: ManualRecategorization($id: annotationId, $type: type, status == AnnotationStatus.APPROVED, $requestDate: requestDate)
+        $recategorization: ManualRecategorization($id: annotationId, $type: type, $requestDate: requestDate)
         not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))
-        $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type != $type)
+        $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() != $type)
     then
         $entityToBeRecategorized.getIntersectingNodes().forEach(node -> update(node));
-        manualChangesApplicationService.recategorize($entityToBeRecategorized, $recategorization);
+        $entityToBeRecategorized.getManualOverwrite().addChange($recategorization);
+        update($entityToBeRecategorized);
         retract($recategorization);
-        // Entity is copied and inserted, so the old entity needs to be retracted to avoid duplication.
-        retract($entityToBeRecategorized);
     end
 
 rule "MAN.3.1: Apply entity recategorization of same type"
     salience 128
     when
-        $recategorization: ManualRecategorization($id: annotationId, $type: type, status == AnnotationStatus.APPROVED, $requestDate: requestDate)
+        $recategorization: ManualRecategorization($id: annotationId, $type: type, $requestDate: requestDate)
         not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))
-        $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type == $type)
+        $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() == $type)
     then
         $entityToBeRecategorized.getManualOverwrite().addChange($recategorization);
         retract($recategorization);
@@ -227,7 +227,7 @@ rule "MAN.3.1: Apply entity recategorization of same type"
 rule "MAN.3.2: Apply image recategorization"
     salience 128
     when
-        $recategorization: ManualRecategorization($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate)
+        $recategorization: ManualRecategorization($id: annotationId, $requestDate: requestDate)
         not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))
         $imageToBeRecategorized: Image($id == id)
     then
@@ -237,24 +237,36 @@ rule "MAN.3.2: Apply image recategorization"
         retract($recategorization);
     end
 
+rule "MAN.3.3: Apply recategorization entities by default"
+    salience 128
+    when
+        $entity: TextEntity(getManualOverwrite().getRecategorized().orElse(false), !dictionary.isHint(type()))
+    then
+        $entity.apply("MAN.3.3", "Recategorized entities are applied by default.", $entity.legalBasis());
+    end
+
 
 // Rule unit: MAN.4
 rule "MAN.4.0: Apply legal basis change"
     salience 128
     when
-        $legalbasisChange: ManualLegalBasisChange($id: annotationId, status == AnnotationStatus.APPROVED)
+        $legalBasisChange: ManualLegalBasisChange($id: annotationId)
         $imageToBeRecategorized: Image($id == id)
     then
-        $imageToBeRecategorized.getManualOverwrite().addChange($legalbasisChange);
+        $imageToBeRecategorized.getManualOverwrite().addChange($legalBasisChange);
+        update($imageToBeRecategorized)
+        retract($legalBasisChange)
     end
 
 rule "MAN.4.1: Apply legal basis change"
     salience 128
     when
-        $legalBasisChange: ManualLegalBasisChange($id: annotationId, status == AnnotationStatus.APPROVED)
+        $legalBasisChange: ManualLegalBasisChange($id: annotationId)
         $entityToBeChanged: TextEntity(matchesAnnotationId($id))
     then
         $entityToBeChanged.getManualOverwrite().addChange($legalBasisChange);
+        update($entityToBeChanged)
+        retract($legalBasisChange)
     end
 
 
@@ -264,8 +276,8 @@ rule "MAN.4.1: Apply legal basis change"
 rule "X.0.0: Remove Entity contained by Entity of same type"
     salience 65
     when
-        $larger: TextEntity($type: type, $entityType: entityType, active())
-        $contained: TextEntity(containedBy($larger), type == $type, entityType == $entityType, this != $larger, !hasManualChanges(), active())
+        $larger: TextEntity($type: type(), $entityType: entityType, active())
+        $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges())
     then
         $contained.remove("X.0.0", "remove Entity contained by Entity of same type");
         retract($contained);
@@ -276,8 +288,8 @@ rule "X.0.0: Remove Entity contained by Entity of same type"
 rule "X.1.0: Merge intersecting Entities of same type"
     salience 64
     when
-        $first: TextEntity($type: type, $entityType: entityType, !resized(), active())
-        $second: TextEntity(intersects($first), type == $type, entityType == $entityType, this != $first, !hasManualChanges(), active())
+        $first: TextEntity($type: type(), $entityType: entityType, !resized(), active())
+        $second: TextEntity(intersects($first), type() == $type, entityType == $entityType, this != $first, !hasManualChanges(), active())
     then
         TextEntity mergedEntity = entityCreationService.mergeEntitiesOfSameType(List.of($first, $second), $type, $entityType, document);
         $first.remove("X.1.0", "merge intersecting Entities of same type");
@@ -292,8 +304,8 @@ rule "X.1.0: Merge intersecting Entities of same type"
 rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
     salience 64
     when
-        $falsePositive: TextEntity($type: type, entityType == EntityType.FALSE_POSITIVE, active())
-        $entity: TextEntity(containedBy($falsePositive), type == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges(), active())
+        $falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active())
+        $entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges())
     then
         $entity.getIntersectingNodes().forEach(node -> update(node));
         $entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE");
@@ -305,8 +317,8 @@ rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
 rule "X.3.0: Remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"
     salience 64
     when
-        $falseRecommendation: TextEntity($type: type, entityType == EntityType.FALSE_RECOMMENDATION, active())
-        $recommendation: TextEntity(containedBy($falseRecommendation), type == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
+        $falseRecommendation: TextEntity($type: type(), entityType == EntityType.FALSE_RECOMMENDATION, active())
+        $recommendation: TextEntity(containedBy($falseRecommendation), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges())
     then
         $recommendation.remove("X.3.0", "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION");
         retract($recommendation);
@@ -314,51 +326,61 @@ rule "X.3.0: Remove Entity of type RECOMMENDATION when contained by FALSE_RECOMM
 
 
 // Rule unit: X.4
-rule "X.4.0: Remove Entity of type RECOMMENDATION when intersected by ENTITY with same type"
+rule "X.4.0: Remove Entity of type RECOMMENDATION when text range equals ENTITY with same type"
     salience 256
     when
-        $entity: TextEntity($type: type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
-        $recommendation: TextEntity(intersects($entity), type == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
+        $entity: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
+        $recommendation: TextEntity(getTextRange().equals($entity.getTextRange()), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges())
     then
         $entity.addEngines($recommendation.getEngines());
-        $recommendation.remove("X.4.0", "remove Entity of type RECOMMENDATION when intersected by ENTITY with same type");
+        $recommendation.remove("X.4.0", "remove Entity of type RECOMMENDATION when text range equals ENTITY with same type");
         retract($recommendation);
     end
 
 
 // Rule unit: X.5
-rule "X.5.0: Remove Entity of type RECOMMENDATION when contained by ENTITY"
+rule "X.5.0: Remove Entity of type RECOMMENDATION when intersected by ENTITY"
     salience 256
     when
         $entity: TextEntity((entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
-        $recommendation: TextEntity(containedBy($entity), entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
+        $recommendation: TextEntity(intersects($entity), entityType == EntityType.RECOMMENDATION, !hasManualChanges())
     then
-        $recommendation.remove("X.5.0", "remove Entity of type RECOMMENDATION when contained by ENTITY");
+        $recommendation.remove("X.5.0", "remove Entity of type RECOMMENDATION when intersected by ENTITY");
+        retract($recommendation);
+    end
+
+rule "X.5.1: Remove Entity of type RECOMMENDATION when contained by RECOMMENDATION"
+    salience 256
+    when
+        $entity: TextEntity($type: type(), entityType == EntityType.RECOMMENDATION, active())
+        $recommendation: TextEntity(containedBy($entity), type() != $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges())
+    then
+        $recommendation.remove("X.5.1", "remove Entity of type RECOMMENDATION when contained by RECOMMENDATION");
         retract($recommendation);
     end
 
 
 // Rule unit: X.6
-rule "X.6.0: Remove Entity of lower rank, when contained by by entity of type ENTITY"
+rule "X.6.0: Remove Entity of lower rank, when contained by entity of type ENTITY or HINT"
     salience 32
     when
-        $higherRank: TextEntity($type: type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
-        $lowerRank: TextEntity(containedBy($higherRank), type != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active())
+        $higherRank: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
+        $lowerRank: TextEntity(containedBy($higherRank), type() != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges())
     then
         $lowerRank.getIntersectingNodes().forEach(node -> update(node));
-        $lowerRank.remove("X.6.0", "remove Entity of lower rank, when contained by entity of type ENTITY");
+        $lowerRank.remove("X.6.0", "remove Entity of lower rank, when contained by entity of type ENTITY or HINT");
         retract($lowerRank);
     end
 
-rule "X.6.1: remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity"
+rule "X.6.1: remove Entity, when contained in another entity of type ENTITY or HINT with larger text range"
     salience 32
     when
-        $higherRank: TextEntity($type: type, $value: value, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
-        $lowerRank: TextEntity(intersects($higherRank), type != $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active(), $lowerRank.getValue().length() > $value.length())
+         $outer: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
+         $inner: TextEntity(containedBy($outer), type() != $type, $outer.getTextRange().length > getTextRange().length(), !hasManualChanges())
     then
-        $higherRank.getIntersectingNodes().forEach(node -> update(node));
-        $higherRank.remove("X.6.1", "remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity");
-        retract($higherRank);
+        $inner.getIntersectingNodes().forEach(node -> update(node));
+        $inner.remove("X.6.1", "remove Entity, when contained in another entity of type ENTITY or HINT with larger text range");
+        retract($inner);
     end
 
 
@@ -373,6 +395,50 @@ rule "X.7.0: Remove all images"
     end
 
 
+// Rule unit: X.8
+rule "X.8.0: Remove Entity when text range and type equals to imported Entity"
+    salience 257
+    when
+        $entity: TextEntity($type: type(), engines contains Engine.IMPORTED, active())
+        $other: TextEntity(getTextRange().equals($entity.getTextRange()), this != $entity, type() == $type, engines not contains Engine.IMPORTED)
+    then
+        $other.remove("X.8.0", "remove Entity when text range and type equals to imported Entity");
+        $entity.addEngines($other.getEngines());
+        retract($other);
+    end
+
+rule "X.8.1: Remove Entity when intersected by imported Entity"
+    salience 256
+    when
+        $entity: TextEntity(engines contains Engine.IMPORTED, active())
+        $other: TextEntity(intersects($entity), this != $entity, engines not contains Engine.IMPORTED)
+    then
+        $other.remove("X.8.1", "remove Entity when intersected by imported Entity");
+        retract($other);
+    end
+
+
+// Rule unit: X.9
+rule "X.9.0: Merge mostly contained signatures"
+    when
+        $aiSignature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.AI)
+        $signature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.ALGORITHM, mostlyContains($aiSignature, 0.8))
+    then
+        $aiSignature.remove("X.9.0", "removed because already contained by alogrithm signature");
+        $signature.addEngine(LayoutEngine.AI);
+    end
+
+
+// Rule unit: X.10
+rule "X.10.0: remove false positives of ai"
+    when
+        $anyImage: Image(engines contains LayoutEngine.ALGORITHM)
+        $aiSignature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.AI, !mostlyContainedBy($anyImage, 0.8))
+    then
+        $aiSignature.remove("X.10.0", "Removed because false positive");
+    end
+
+
 //------------------------------------ File attributes rules ------------------------------------
 
 // Rule unit: FA.1

redaction-service-v1/redaction-service-server-v1/src/test/resources/files/migration/test.MANUAL_CHANGES.json

@@ -0,0 +1,513 @@
+{
+  "idsToRemove": [
+    {
+      "removeFromDictionary": false,
+      "removeFromAllDossiers": false,
+      "annotationId": "657b681db1be8118ac305b40758effb8",
+      "fileId": "56595118999b5bf98e47c1c918ada128",
+      "user": "2cb10c0f-e6e9-42de-a4ec-efb3fac5ef61",
+      "requestDate": "2024-02-09T14:03:54.458271Z",
+      "processedDate": "2024-02-09T14:03:54.501Z",
+      "softDeletedTime": null,
+      "approved": true,
+      "status": "APPROVED"
+    },
+    {
+      "removeFromDictionary": false,
+      "removeFromAllDossiers": false,
+      "annotationId": "234bfcab1a40c5346191a0da99e99e7c",
+      "fileId": "56595118999b5bf98e47c1c918ada128",
+      "user": "2cb10c0f-e6e9-42de-a4ec-efb3fac5ef61",
+      "requestDate": "2024-02-09T14:03:34.525304Z",
+      "processedDate": "2024-02-09T14:03:34.558Z",
+      "softDeletedTime": null,
+      "approved": true,
+      "status": "APPROVED"
+    },
+    {
+      "removeFromDictionary": false,
+      "removeFromAllDossiers": false,
+      "annotationId": "c9c3283baf73e327a8ddbdd4d603aad5",
+      "fileId": "56595118999b5bf98e47c1c918ada128",
+      "user": "2cb10c0f-e6e9-42de-a4ec-efb3fac5ef61",
+      "requestDate": "2024-02-09T14:01:01.401478Z",
+      "processedDate": "2024-02-09T14:01:01.432Z",
+      "softDeletedTime": null,
+      "approved": true,
+      "status": "APPROVED"
+    },
+    {
+      "removeFromDictionary": false,
+      "removeFromAllDossiers": false,
+      "annotationId": "aed17872b01a30e019978a99fb1b3c69",
+      "fileId": "56595118999b5bf98e47c1c918ada128",
+      "user": "2cb10c0f-e6e9-42de-a4ec-efb3fac5ef61",
+      "requestDate": "2024-02-09T14:01:12.658542Z",
+      "processedDate": "2024-02-09T14:01:12.7Z",
+      "softDeletedTime": null,
+      "approved": true,
+      "status": "APPROVED"
+    },
+    {
+      "removeFromDictionary": false,
+      "removeFromAllDossiers": false,
+      "annotationId": "5c0ee1942517fa0f8b7862eeb720a7fe",
+      "fileId": "56595118999b5bf98e47c1c918ada128",
+      "user": "2cb10c0f-e6e9-42de-a4ec-efb3fac5ef61",
+      "requestDate": "2024-02-09T14:01:04.622727Z",
+      "processedDate": "2024-02-09T14:01:04.659Z",
+      "softDeletedTime": null,
+      "approved": true,
+      "status": "APPROVED"
+    },
+    {
+      "removeFromDictionary": false,
+      "removeFromAllDossiers": false,
+      "annotationId": "c4587ec065c8e81ad53c34417b8ef866",
+      "fileId": "56595118999b5bf98e47c1c918ada128",
+      "user": "2cb10c0f-e6e9-42de-a4ec-efb3fac5ef61",
+      "requestDate": "2024-02-09T14:03:40.359758Z",
+      "processedDate": "2024-02-09T14:03:40.4Z",
+      "softDeletedTime": null,
+      "approved": true,
+      "status": "APPROVED"
+    }
+  ],
+  "entriesToAdd": [
+    {
+      "type": "PII",
+      "value": "the six desire",
+      "reason": "Dictionary Request",
+      "legalBasis": null,
+      "section": null,
+      "rectangle": false,
+      "addToDictionary": true,
+      "addToDossierDictionary": false,
+      "positions": [
+        {
+          "topLeft": {
+            "x": 112.86,
+            "y": 590.97
+          },
+          "width": 52.68,
+          "height": 12.84,
+          "page": 19,
+          "topLeftX": 112.86,
+          "topLeftY": 590.97
+        }
+      ],
+      "textBefore": null,
+      "textAfter": null,
+      "sourceId": null,
+      "dictionaryEntryType": null,
+      "annotationId": "19578e7d4275e9e68853a6b2988f0681",
+      "fileId": "56595118999b5bf98e47c1c918ada128",
+      "user": "2cb10c0f-e6e9-42de-a4ec-efb3fac5ef61",
+      "requestDate": "2024-02-09T14:04:26.939Z",
+      "processedDate": "2024-02-09T14:04:28.118Z",
+      "softDeletedTime": null,
+      "approved": true,
+      "status": "APPROVED"
+    },
+    {
+      "type": "manual",
+      "value": "Rule",
+      "reason": "personal data (names and addresses) of individuals involved in testing on vertebrate studies or in obtaining toxicological information",
+      "legalBasis": "Article 39(e)(2) of Regulation (EC) No 178/2002",
+      "section": null,
+      "rectangle": false,
+      "addToDictionary": false,
+      "addToDossierDictionary": false,
+      "positions": [
+        {
+          "topLeft": {
+            "x": 59.8,
+            "y": 687.552
+          },
+          "width": 31.968,
+          "height": 20.96,
+          "page": 1,
+          "topLeftX": 59.8,
+          "topLeftY": 687.552
+        }
+      ],
+      "textBefore": null,
+      "textAfter": null,
+      "sourceId": null,
+      "dictionaryEntryType": null,
+      "annotationId": "cb27dd265a1c09418b55317c68be6c12",
+      "fileId": "56595118999b5bf98e47c1c918ada128",
+      "user": "2cb10c0f-e6e9-42de-a4ec-efb3fac5ef61",
+      "requestDate": "2024-02-09T14:00:09.427Z",
+      "processedDate": "2024-02-09T14:00:09.427Z",
+      "softDeletedTime": null,
+      "approved": true,
+      "status": "APPROVED"
+    },
+    {
+      "type": "manual",
+      "value": "Rule CBI.0.1",
+      "reason": "personal data (names and addresses) of individuals involved in testing on vertebrate studies or in obtaining toxicological information",
+      "legalBasis": "Article 39(e)(2) of Regulation (EC) No 178/2002",
+      "section": null,
+      "rectangle": false,
+      "addToDictionary": false,
+      "addToDossierDictionary": false,
+      "positions": [
+        {
+          "topLeft": {
+            "x": 195.59,
+            "y": 597.27
+          },
+          "width": 51.88,
+          "height": 12.83,
+          "page": 1,
+          "topLeftX": 195.59,
+          "topLeftY": 597.27
+        }
+      ],
+      "textBefore": null,
+      "textAfter": null,
+      "sourceId": null,
+      "dictionaryEntryType": null,
+      "annotationId": "7ff0836d041a05a5e7165c0712d85b61",
+      "fileId": "56595118999b5bf98e47c1c918ada128",
+      "user": "2cb10c0f-e6e9-42de-a4ec-efb3fac5ef61",
+      "requestDate": "2024-02-09T14:00:27.216Z",
+      "processedDate": "2024-02-09T14:00:27.216Z",
+      "softDeletedTime": null,
+      "approved": true,
+      "status": "APPROVED"
+    },
+    {
+      "type": "CBI_author",
+      "value": "Pasture he invited mr company shyness. But when shot real her. Chamber her\nobserve visited removal six sending himself boy. At exquisite existence if an oh\ndependent excellent. Are gay head need down draw. Misery wonder enable\nmutual get set oppose the uneasy. End why melancholy estimating her had\nindulgence middletons. Say ferrars demands besides her address. Blind going\nyou merit few fancy their.",
+      "reason": "Dictionary Request",
+      "legalBasis": null,
+      "section": null,
+      "rectangle": false,
+      "addToDictionary": true,
+      "addToDossierDictionary": false,
+      "positions": [
+        {
+          "topLeft": {
+            "x": 56.8,
+            "y": 671.87
+          },
+          "width": 309.71,
+          "height": 12.84,
+          "page": 19,
+          "topLeftX": 56.8,
+          "topLeftY": 671.87
+        },
+        {
+          "topLeft": {
+            "x": 56.8,
+            "y": 660.27
+          },
+          "width": 318.24,
+          "height": 12.84,
+          "page": 19,
+          "topLeftX": 56.8,
+          "topLeftY": 660.27
+        },
+        {
+          "topLeft": {
+            "x": 56.8,
+            "y": 648.77
+          },
+          "width": 299.68,
+          "height": 12.84,
+          "page": 19,
+          "topLeftX": 56.8,
+          "topLeftY": 648.77
+        },
+        {
+          "topLeft": {
+            "x": 56.8,
+            "y": 637.17
+          },
+          "width": 297.4,
+          "height": 12.84,
+          "page": 19,
+          "topLeftX": 56.8,
+          "topLeftY": 637.17
+        },
+        {
+          "topLeft": {
+            "x": 56.8,
+            "y": 625.67
+          },
+          "width": 309.46,
+          "height": 12.84,
+          "page": 19,
+          "topLeftX": 56.8,
+          "topLeftY": 625.67
+        },
+        {
+          "topLeft": {
+            "x": 56.8,
+            "y": 614.07
+          },
+          "width": 103.53,
+          "height": 12.84,
+          "page": 19,
+          "topLeftX": 56.8,
+          "topLeftY": 614.07
+        }
+      ],
+      "textBefore": null,
+      "textAfter": null,
+      "sourceId": null,
+      "dictionaryEntryType": null,
+      "annotationId": "15e0bbae087bf78e872363512de9c0b6",
+      "fileId": "56595118999b5bf98e47c1c918ada128",
+      "user": "2cb10c0f-e6e9-42de-a4ec-efb3fac5ef61",
+      "requestDate": "2024-02-09T14:04:18.561Z",
+      "processedDate": "2024-02-09T14:04:19.459Z",
+      "softDeletedTime": null,
+      "approved": true,
+      "status": "APPROVED"
+    },
+    {
+      "type": "manual",
+      "value": "CBI.0.0/1: Redact CBI Authors",
+      "reason": "personal data (names and addresses) of individuals involved in testing on vertebrate studies or in obtaining toxicological information",
+      "legalBasis": "Article 39(e)(2) of Regulation (EC) No 178/2002",
+      "section": null,
+      "rectangle": false,
+      "addToDictionary": false,
+      "addToDossierDictionary": false,
+      "positions": [
+        {
+          "topLeft": {
+            "x": 56.8,
+            "y": 626.5984
+          },
+          "width": 200.6712,
+          "height": 19.8669,
+          "page": 1,
+          "topLeftX": 56.8,
+          "topLeftY": 626.5984
+        }
+      ],
+      "textBefore": null,
+      "textAfter": null,
+      "sourceId": null,
+      "dictionaryEntryType": null,
+      "annotationId": "2e0b1dd10a032ce27437b5b404a4d561",
+      "fileId": "56595118999b5bf98e47c1c918ada128",
+      "user": "2cb10c0f-e6e9-42de-a4ec-efb3fac5ef61",
+      "requestDate": "2024-02-09T14:00:16.794Z",
+      "processedDate": "2024-02-09T14:00:16.795Z",
+      "softDeletedTime": null,
+      "approved": true,
+      "status": "APPROVED"
+    },
+    {
+      "type": "PII",
+      "value": "Xinyi Y. Tao Possible incidents",
+      "reason": "False Positive",
+      "legalBasis": null,
+      "section": null,
+      "rectangle": false,
+      "addToDictionary": true,
+      "addToDossierDictionary": false,
+      "positions": [
+        {
+          "topLeft": {
+            "x": 56.8,
+            "y": 568.84204
+          },
+          "width": 60.575993,
+          "height": -12.641998,
+          "page": 6,
+          "topLeftX": 56.8,
+          "topLeftY": 568.84204
+        }
+      ],
+      "textBefore": null,
+      "textAfter": null,
+      "sourceId": "3fcff2c472c4cce852815de9cb17a50d",
+      "dictionaryEntryType": "FALSE_POSITIVE",
+      "annotationId": "1abcd854c21c305d357677346d1fc3e2",
+      "fileId": "56595118999b5bf98e47c1c918ada128",
+      "user": "2cb10c0f-e6e9-42de-a4ec-efb3fac5ef61",
+      "requestDate": "2024-02-09T14:01:28.622Z",
+      "processedDate": "2024-02-09T14:01:29.449Z",
+      "softDeletedTime": null,
+      "approved": true,
+      "status": "APPROVED"
+    },
+    {
+      "type": "manual",
+      "value": "non-readable content",
+      "reason": "personal data (names and addresses) of individuals involved in testing on vertebrate studies or in obtaining toxicological information",
+      "legalBasis": "Article 39(e)(2) of Regulation (EC) No 178/2002",
+      "section": null,
+      "rectangle": true,
+      "addToDictionary": false,
+      "addToDossierDictionary": false,
+      "positions": [
+        {
+          "topLeft": {
+            "x": 346.08,
+            "y": 610.87
+          },
+          "width": 128.31,
+          "height": 109.97,
+          "page": 1,
+          "topLeftX": 346.08,
+          "topLeftY": 610.87
+        }
+      ],
+      "textBefore": null,
+      "textAfter": null,
+      "sourceId": null,
+      "dictionaryEntryType": null,
+      "annotationId": "562ef1c464f3739c79fa9cedb9649658",
+      "fileId": "56595118999b5bf98e47c1c918ada128",
+      "user": "2cb10c0f-e6e9-42de-a4ec-efb3fac5ef61",
+      "requestDate": "2024-02-09T14:00:00.762Z",
+      "processedDate": "2024-02-09T14:00:00.762Z",
+      "softDeletedTime": null,
+      "approved": true,
+      "status": "APPROVED"
+    }
+  ],
+  "forceRedactions": [
+    {
+      "legalBasis": "Article 39(e)(2) of Regulation (EC) No 178/2002",
+      "annotationId": "9b94e089cbc76a4eee06fa358d268015",
+      "fileId": "56595118999b5bf98e47c1c918ada128",
+      "user": "2cb10c0f-e6e9-42de-a4ec-efb3fac5ef61",
+      "requestDate": "2024-02-09T14:02:31.479111Z",
+      "processedDate": "2024-02-09T14:02:32.092Z",
+      "softDeletedTime": null,
+      "approved": true,
+      "status": "APPROVED"
+    },
+    {
+      "legalBasis": "Article 39(e)(2) of Regulation (EC) No 178/2002",
+      "annotationId": "81dea577c7e615375e2a7bd011ad1a6b",
+      "fileId": "56595118999b5bf98e47c1c918ada128",
+      "user": "2cb10c0f-e6e9-42de-a4ec-efb3fac5ef61",
+      "requestDate": "2024-02-09T14:03:59.774394Z",
+      "processedDate": "2024-02-09T14:04:00.373Z",
+      "softDeletedTime": null,
+      "approved": true,
+      "status": "APPROVED"
+    },
+    {
+      "legalBasis": "Article 39(e)(2) of Regulation (EC) No 178/2002",
+      "annotationId": "0c0164dfea2be886475482930b403de5",
+      "fileId": "56595118999b5bf98e47c1c918ada128",
+      "user": "2cb10c0f-e6e9-42de-a4ec-efb3fac5ef61",
+      "requestDate": "2024-02-09T14:02:14.953642Z",
+      "processedDate": "2024-02-09T14:02:15.58Z",
+      "softDeletedTime": null,
+      "approved": true,
+      "status": "APPROVED"
+    }
+  ],
+  "recategorizations": [
+    {
+      "type": "formula",
+      "annotationId": "c6df2cf8cca12cad76fa89f5ee0b3214",
+      "fileId": "56595118999b5bf98e47c1c918ada128",
+      "user": "2cb10c0f-e6e9-42de-a4ec-efb3fac5ef61",
+      "requestDate": "2024-02-09T14:04:06.892584Z",
+      "processedDate": "2024-02-09T14:04:07.574Z",
+      "softDeletedTime": null,
+      "approved": true,
+      "status": "APPROVED"
+    }
+  ],
+  "legalBasisChanges": [],
+  "resizeRedactions": [
+    {
+      "value": "for this project are library@outlook.com",
+      "positions": [
+        {
+          "topLeft": {
+            "x": 248.632,
+            "y": 328.464
+          },
+          "width": 194.58,
+          "height": 15.408,
+          "page": 6,
+          "topLeftX": 248.632,
+          "topLeftY": 328.464
+        }
+      ],
+      "textBefore": null,
+      "textAfter": null,
+      "updateDictionary": false,
+      "addToAllDossiers": false,
+      "annotationId": "f2ef0bb904c4e055ce1963d635b41789",
+      "fileId": "56595118999b5bf98e47c1c918ada128",
+      "user": "2cb10c0f-e6e9-42de-a4ec-efb3fac5ef61",
+      "requestDate": "2024-02-09T14:01:49.857167Z",
+      "processedDate": "2024-02-09T14:01:49.857Z",
+      "softDeletedTime": null,
+      "approved": true,
+      "status": "APPROVED"
+    },
+    {
+      "value": "Alfred",
+      "positions": [
+        {
+          "topLeft": {
+            "x": 80.788,
+            "y": 568.364
+          },
+          "width": 31.284,
+          "height": 15.408,
+          "page": 6,
+          "topLeftX": 80.788,
+          "topLeftY": 568.364
+        }
+      ],
+      "textBefore": null,
+      "textAfter": null,
+      "updateDictionary": false,
+      "addToAllDossiers": false,
+      "annotationId": "d251f8afaa84d0f0c939df9ca0eb1b88",
+      "fileId": "56595118999b5bf98e47c1c918ada128",
+      "user": "2cb10c0f-e6e9-42de-a4ec-efb3fac5ef61",
+      "requestDate": "2024-02-09T14:01:21.749389Z",
+      "processedDate": "2024-02-09T14:01:21.749Z",
+      "softDeletedTime": null,
+      "approved": true,
+      "status": "APPROVED"
+    },
+    {
+      "value": "Image",
+      "positions": [
+        {
+          "topLeft": {
+            "x": 287,
+            "y": 613
+          },
+          "width": 278.86,
+          "height": 128.24,
+          "page": 14,
+          "topLeftX": 287,
+          "topLeftY": 613
+        }
+      ],
+      "textBefore": null,
+      "textAfter": null,
+      "updateDictionary": false,
+      "addToAllDossiers": false,
+      "annotationId": "1d7c699be6cfa57e36d26add7f1fd0c5",
+      "fileId": "56595118999b5bf98e47c1c918ada128",
+      "user": "2cb10c0f-e6e9-42de-a4ec-efb3fac5ef61",
+      "requestDate": "2024-02-09T14:03:51.725296Z",
+      "processedDate": "2024-02-09T14:03:51.725Z",
+      "softDeletedTime": null,
+      "approved": true,
+      "status": "APPROVED"
+    }
+  ]
+}

redaction-service-v1/redaction-service-server-v1/src/test/resources/files/migration/test.MIGRATED_IDS.json

@@ -0,0 +1,68 @@
+{
+  "mappings": [
+    {
+      "oldId": "63a758ca88b0e0a6da546b1a8ece7e39",
+      "newId": "0c0164dfea2be886475482930b403de5"
+    },
+    {
+      "oldId": "01fcfb3581723a4d164b4e5ed9e7db90",
+      "newId": "5c0ee1942517fa0f8b7862eeb720a7fe"
+    },
+    {
+      "oldId": "ad5c82acae51a8af70c9141e1ae4efde",
+      "newId": "c9c3283baf73e327a8ddbdd4d603aad5"
+    },
+    {
+      "oldId": "5316fff0ec9ae3f2773378f3cc833079",
+      "newId": "aed17872b01a30e019978a99fb1b3c69"
+    },
+    {
+      "oldId": "af762b495f33a941dc94155ed7db9fa7",
+      "newId": "9b94e089cbc76a4eee06fa358d268015"
+    },
+    {
+      "oldId": "c2cc46b2fd7b3a68639ad12f9ae4c70d",
+      "newId": "c4587ec065c8e81ad53c34417b8ef866"
+    },
+    {
+      "oldId": "1ec000364ca48d676af7e33be13685e4",
+      "newId": "234bfcab1a40c5346191a0da99e99e7c"
+    },
+    {
+      "oldId": "90129d370d60b07d8cb060400338255e",
+      "newId": "d251f8afaa84d0f0c939df9ca0eb1b88"
+    },
+    {
+      "oldId": "30cb40dc6f495193f1b47d0d557fd682",
+      "newId": "f2ef0bb904c4e055ce1963d635b41789"
+    },
+    {
+      "oldId": "a9eb84b047ecc5f11d1fe835a4c814ad",
+      "newId": "2e0b1dd10a032ce27437b5b404a4d561"
+    },
+    {
+      "oldId": "fe41fc0f9f5f1e90a2df6eda2f1ba88f",
+      "newId": "cb27dd265a1c09418b55317c68be6c12"
+    },
+    {
+      "oldId": "52e0c82360238d1a2d4e7023550f5c3c",
+      "newId": "7ff0836d041a05a5e7165c0712d85b61"
+    },
+    {
+      "oldId": "6808af23a9652917b73c1939b481f3e4",
+      "newId": "657b681db1be8118ac305b40758effb8"
+    },
+    {
+      "oldId": "3ed0487b61fdb556c8982450aa39a8db",
+      "newId": "1d7c699be6cfa57e36d26add7f1fd0c5"
+    },
+    {
+      "oldId": "44f684045879e7a6decd34baaae0930f",
+      "newId": "81dea577c7e615375e2a7bd011ad1a6b"
+    },
+    {
+      "oldId": "647a450f5feba4859297886a4263b653",
+      "newId": "c6df2cf8cca12cad76fa89f5ee0b3214"
+    }
+  ]
+}