Merge branch 'RED-8650' into 'master'

RED-8650 - Add support for more date formats

Closes RED-8650

See merge request redactmanager/redaction-service!373

.gitlab-ci.yml

@@ -1,22 +1,34 @@
 variables:
   SONAR_PROJECT_KEY: 'RED_redaction-service'
+  GIT_SUBMODULE_STRATEGY: recursive
+  GIT_SUBMODULE_FORCE_HTTPS: "true"
 include:
   - project: 'gitlab/gitlab'
     ref: 'main'
     file: 'ci-templates/gradle_java.yml'
 
-deploy:
+deploy JavaDoc:
   stage: deploy
   tags:
     - dind
   script:
-    - echo "Building with gradle version ${BUILDVERSION}"
+    - echo "Building JavaDoc with gradle version ${BUILDVERSION}"
     - gradle -Pversion=${BUILDVERSION} publish
-    - gradle bootBuildImage --publishImage -PbuildbootDockerHostNetwork=true -Pversion=${BUILDVERSION}
-    - echo "BUILDVERSION=$BUILDVERSION" >> version.env
+  rules:
+    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
+    - if: $CI_COMMIT_BRANCH =~ /^release/
+    - if: $CI_COMMIT_TAG
+
+generateJavaDoc:
+  stage: build
+  tags:
+    - dind
+  script:
+    - echo "Generating Javadoc..."
+    - gradle generateJavaDoc -PjavadocDestinationDir="javadoc"
   artifacts:
-    reports:
-      dotenv: version.env
+    paths:
+      - redaction-service-v1/redaction-service-server-v1/javadoc/*
   rules:
     - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
     - if: $CI_COMMIT_BRANCH =~ /^release/

.gitmodules

@@ -0,0 +1,8 @@
+[submodule "redaction-service-v1/redaction-service-server-v1/src/test/resources/files/syngenta"]
+	path = redaction-service-v1/redaction-service-server-v1/src/test/resources/files/syngenta
+	url = ssh://git@git.knecon.com:22222/fforesight/documents/syngenta.git
+	update = merge
+[submodule "redaction-service-v1/redaction-service-server-v1/src/test/resources/files/basf"]
+	path = redaction-service-v1/redaction-service-server-v1/src/test/resources/files/basf
+	url = ssh://git@git.knecon.com:22222/fforesight/documents/basf.git
+	update = merge

buildSrc/src/main/kotlin/com.iqser.red.service.java-conventions.gradle.kts

@@ -11,6 +11,10 @@ group = "com.iqser.red.service"
 java.sourceCompatibility = JavaVersion.VERSION_17
 java.targetCompatibility = JavaVersion.VERSION_17
 
+pmd {
+    isConsoleOutput = true
+}
+
 tasks.pmdMain {
     pmd.ruleSetFiles = files("${rootDir}/config/pmd/pmd.xml")
 }

config/pmd/pmd.xml

@@ -9,13 +9,12 @@
     </description>
 
     <rule ref="category/java/errorprone.xml">
-        <exclude name="DataflowAnomalyAnalysis"/>
         <exclude name="MissingSerialVersionUID"/>
-        <exclude name="NullAssignment"/>
         <exclude name="AvoidLiteralsInIfCondition"/>
         <exclude name="AvoidDuplicateLiterals"/>
-        <exclude name="AvoidFieldNameMatchingMethodName"/>
+        <exclude name="NullAssignment"/>
         <exclude name="AssignmentInOperand"/>
+        <exclude name="BeanMembersShouldSerialize"/>
     </rule>
 
-</ruleset>
+</ruleset>

config/pmd/test_pmd.xml

@@ -10,15 +10,13 @@
 
 
     <rule ref="category/java/errorprone.xml">
-        <exclude name="DataflowAnomalyAnalysis"/>
         <exclude name="MissingSerialVersionUID"/>
-        <exclude name="NullAssignment"/>
         <exclude name="AvoidLiteralsInIfCondition"/>
         <exclude name="AvoidDuplicateLiterals"/>
-        <exclude name="AvoidFieldNameMatchingMethodName"/>
-        <exclude name="AvoidFieldNameMatchingTypeName"/>
+        <exclude name="NullAssignment"/>
         <exclude name="AssignmentInOperand"/>
         <exclude name="TestClassWithoutTestCases"/>
+        <exclude name="BeanMembersShouldSerialize"/>
     </rule>
 
-</ruleset>
+</ruleset>

redaction-service-v1/redaction-service-api-v1/build.gradle.kts

@@ -7,7 +7,7 @@ description = "redaction-service-api-v1"
 
 dependencies {
     implementation("org.springframework:spring-web:6.0.12")
-    implementation("com.iqser.red.service:persistence-service-internal-api-v1:2.279.0")
+    implementation("com.iqser.red.service:persistence-service-internal-api-v1:2.395.0")
 }
 
 publishing {

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/AnalyzeResponse.java

@@ -15,4 +15,5 @@ public class AnalyzeResponse {
 
     private String fileId;
     private List<UnprocessedManualEntity> unprocessedManualEntities;
+
 }

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/DroolsBlacklistErrorMessage.java

@@ -0,0 +1,27 @@
+package com.iqser.red.service.redaction.v1.model;
+
+import java.util.List;
+
+import lombok.AccessLevel;
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.EqualsAndHashCode;
+import lombok.NoArgsConstructor;
+import lombok.experimental.FieldDefaults;
+import lombok.experimental.SuperBuilder;
+
+@Data
+@SuperBuilder
+@AllArgsConstructor
+@NoArgsConstructor
+@FieldDefaults(level = AccessLevel.PRIVATE)
+@EqualsAndHashCode(callSuper = true)
+public class DroolsBlacklistErrorMessage extends DroolsValidationMessage {
+
+    List<String> blacklistedKeywords;
+
+    public String getMessage() {
+        return String.format("Blacklisted keywords found in this rule: %s", String.join(", ", blacklistedKeywords));
+    }
+
+}

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/DroolsSyntaxDeprecatedWarnings.java

@@ -0,0 +1,22 @@
+package com.iqser.red.service.redaction.v1.model;
+
+import lombok.AccessLevel;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.EqualsAndHashCode;
+import lombok.NoArgsConstructor;
+import lombok.experimental.FieldDefaults;
+import lombok.experimental.SuperBuilder;
+
+@Data
+@SuperBuilder
+@AllArgsConstructor
+@NoArgsConstructor
+@FieldDefaults(level = AccessLevel.PRIVATE)
+@EqualsAndHashCode(callSuper = true)
+public class DroolsSyntaxDeprecatedWarnings extends DroolsValidationMessage {
+
+    String message;
+
+}

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/DroolsSyntaxErrorMessage.java

@@ -4,17 +4,19 @@ import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;
+import lombok.EqualsAndHashCode;
 import lombok.NoArgsConstructor;
 import lombok.experimental.FieldDefaults;
+import lombok.experimental.SuperBuilder;
 
 @Data
-@Builder
+@SuperBuilder
 @AllArgsConstructor
 @NoArgsConstructor
 @FieldDefaults(level = AccessLevel.PRIVATE)
-public class DroolsSyntaxErrorMessage {
+@EqualsAndHashCode(callSuper = true)
+public class DroolsSyntaxErrorMessage extends DroolsValidationMessage {
 
-    Integer line;
-    Integer column;
     String message;
+
 }

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/DroolsSyntaxValidation.java

@@ -1,34 +0,0 @@
-package com.iqser.red.service.redaction.v1.model;
-
-import java.util.LinkedList;
-import java.util.List;
-
-import lombok.AccessLevel;
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.Data;
-import lombok.NoArgsConstructor;
-import lombok.experimental.FieldDefaults;
-
-@Data
-@Builder
-@NoArgsConstructor
-@AllArgsConstructor
-@FieldDefaults(level = AccessLevel.PRIVATE)
-public class DroolsSyntaxValidation {
-
-    @Builder.Default
-    List<DroolsSyntaxErrorMessage> droolsSyntaxErrorMessages = new LinkedList<>();
-
-
-    public void addErrorMessage(int line, int column, String message) {
-
-        getDroolsSyntaxErrorMessages().add(DroolsSyntaxErrorMessage.builder().line(line).column(column).message(message).build());
-    }
-
-    public boolean isCompiled() {
-
-        return droolsSyntaxErrorMessages.isEmpty();
-    }
-
-}

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/DroolsValidation.java

@@ -0,0 +1,39 @@
+package com.iqser.red.service.redaction.v1.model;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import lombok.AccessLevel;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+import lombok.experimental.FieldDefaults;
+
+@Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
+@FieldDefaults(level = AccessLevel.PRIVATE)
+public class DroolsValidation {
+
+    @Builder.Default
+    List<DroolsSyntaxErrorMessage> syntaxErrorMessages = new ArrayList<>();
+    @Builder.Default
+    List<DroolsSyntaxDeprecatedWarnings> deprecatedWarnings = new ArrayList<>();
+    @Builder.Default
+    List<DroolsBlacklistErrorMessage> blacklistErrorMessages = new ArrayList<>();
+
+
+    public void addErrorMessage(int line, int column, String message) {
+
+        getSyntaxErrorMessages().add(DroolsSyntaxErrorMessage.builder().line(line).column(column).message(message).build());
+    }
+
+
+    public boolean isCompiled() {
+
+        return syntaxErrorMessages.isEmpty() && blacklistErrorMessages.isEmpty();
+    }
+
+}

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/DroolsValidationMessage.java

@@ -0,0 +1,21 @@
+package com.iqser.red.service.redaction.v1.model;
+
+import lombok.AccessLevel;
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.EqualsAndHashCode;
+import lombok.NoArgsConstructor;
+import lombok.experimental.FieldDefaults;
+import lombok.experimental.SuperBuilder;
+
+@Data
+@SuperBuilder
+@AllArgsConstructor
+@NoArgsConstructor
+@FieldDefaults(level = AccessLevel.PRIVATE)
+@EqualsAndHashCode
+public class DroolsValidationMessage {
+
+    Integer line;
+    Integer column;
+}

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/MigrationRequest.java

@@ -1,12 +1,15 @@
 package com.iqser.red.service.redaction.v1.model;
 
+import java.util.Collections;
+import java.util.Set;
+
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog;
 
 import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;
 import lombok.NoArgsConstructor;
+import lombok.NonNull;
 
 @Data
 @Builder
@@ -14,9 +17,18 @@ import lombok.NoArgsConstructor;
 @AllArgsConstructor
 public class MigrationRequest {
 
+    @NonNull
     String dossierTemplateId;
+    @NonNull
     String dossierId;
+    @NonNull
     String fileId;
+
+    boolean fileIsApproved;
+    @NonNull
     ManualRedactions manualRedactions;
+    @NonNull
+    @Builder.Default
+    Set<String> entitiesWithComments = Collections.emptySet();
 
 }

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/UnprocessedManualEntity.java

@@ -24,4 +24,5 @@ public class UnprocessedManualEntity {
     private String section;
     @Builder.Default
     private List<Position> positions = new ArrayList<>();
+
 }

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/resources/RedactionResource.java

@@ -4,12 +4,12 @@ import org.springframework.http.MediaType;
 import org.springframework.web.bind.annotation.PostMapping;
 import org.springframework.web.bind.annotation.RequestBody;
 
-import com.iqser.red.service.redaction.v1.model.DroolsSyntaxValidation;
+import com.iqser.red.service.redaction.v1.model.DroolsValidation;
 import com.iqser.red.service.redaction.v1.model.RuleValidationModel;
 
 public interface RedactionResource {
 
     @PostMapping(value = "/rules/test", consumes = MediaType.APPLICATION_JSON_VALUE)
-    DroolsSyntaxValidation testRules(@RequestBody RuleValidationModel rulesValidationModel);
+    DroolsValidation testRules(@RequestBody RuleValidationModel rulesValidationModel);
 
 }

redaction-service-v1/redaction-service-server-v1/build.gradle.kts

@@ -12,12 +12,14 @@ plugins {
 description = "redaction-service-server-v1"
 
 
-val layoutParserVersion = "0.86.0"
+val layoutParserVersion = "0.107.0"
 val jacksonVersion = "2.15.2"
 val droolsVersion = "9.44.0.Final"
 val pdfBoxVersion = "3.0.0"
-val persistenceServiceVersion = "2.279.0"
+val persistenceServiceVersion = "2.395.0"
 val springBootStarterVersion = "3.1.5"
+val springCloudVersion = "4.0.4"
+val testContainersVersion = "1.19.7"
 
 configurations {
     all {
@@ -31,6 +33,7 @@ dependencies {
 
     implementation(project(":redaction-service-api-v1")) { exclude(group = "com.iqser.red.service", module = "persistence-service-internal-api-v1") }
     implementation("com.iqser.red.service:persistence-service-internal-api-v1:${persistenceServiceVersion}") { exclude(group = "org.springframework.boot") }
+    implementation("com.iqser.red.service:persistence-service-shared-mongo-v1:${persistenceServiceVersion}")
     implementation("com.knecon.fforesight:layoutparser-service-internal-api:${layoutParserVersion}")
 
     implementation("com.iqser.red.commons:spring-commons:6.2.0")
@@ -38,8 +41,8 @@ dependencies {
 
     implementation("com.iqser.red.commons:dictionary-merge-commons:1.5.0")
     implementation("com.iqser.red.commons:storage-commons:2.45.0")
-    implementation("com.knecon.fforesight:tenant-commons:0.19.0")
-    implementation("com.knecon.fforesight:tracing-commons:0.3.0")
+    implementation("com.knecon.fforesight:tenant-commons:0.24.0")
+    implementation("com.knecon.fforesight:tracing-commons:0.5.0")
 
     implementation("com.fasterxml.jackson.module:jackson-module-afterburner:${jacksonVersion}")
     implementation("com.fasterxml.jackson.datatype:jackson-datatype-jsr310:${jacksonVersion}")
@@ -52,7 +55,7 @@ dependencies {
 
     implementation("org.locationtech.jts:jts-core:1.19.0")
 
-    implementation("org.springframework.cloud:spring-cloud-starter-openfeign:4.0.4")
+    implementation("org.springframework.cloud:spring-cloud-starter-openfeign:${springCloudVersion}")
     implementation("org.springframework.boot:spring-boot-starter-amqp:${springBootStarterVersion}")
     implementation("org.springframework.boot:spring-boot-starter-cache:${springBootStarterVersion}")
     implementation("org.springframework.boot:spring-boot-starter-data-redis:${springBootStarterVersion}")
@@ -60,11 +63,17 @@ dependencies {
     implementation("net.logstash.logback:logstash-logback-encoder:7.4")
     implementation("ch.qos.logback:logback-classic")
 
+    implementation("org.reflections:reflections:0.10.2")
+
     testImplementation(project(":rules-management"))
     testImplementation("org.apache.pdfbox:pdfbox:${pdfBoxVersion}")
     testImplementation("org.apache.pdfbox:pdfbox-tools:${pdfBoxVersion}")
 
+    testImplementation("org.testcontainers:testcontainers:${testContainersVersion}")
+    testImplementation("org.testcontainers:junit-jupiter:${testContainersVersion}")
+
     testImplementation("org.springframework.boot:spring-boot-starter-test:${springBootStarterVersion}")
+    testImplementation("com.knecon.fforesight:viewer-doc-processor:${layoutParserVersion}")
     testImplementation("com.knecon.fforesight:layoutparser-service-processor:${layoutParserVersion}") {
         exclude(
             group = "com.iqser.red.service",
@@ -73,13 +82,23 @@ dependencies {
     }
 }
 
+dependencyManagement {
+    imports {
+        mavenBom("org.testcontainers:testcontainers-bom:${testContainersVersion}")
+    }
+}
+
 tasks.test {
     configure<JacocoTaskExtension> {
         excludes = listOf("org/drools/**/*")
     }
+    minHeapSize = "512m"
+    maxHeapSize = "1024m"
 }
 
 
+
+
 tasks.named<BootBuildImage>("bootBuildImage") {
 
     environment.put("BPE_DELIM_JAVA_TOOL_OPTIONS", " ")
@@ -106,3 +125,42 @@ tasks.named<BootBuildImage>("bootBuildImage") {
         tags.set(listOf(dockerTag))
     }
 }
+
+fun parseDroolsImports(droolsFilePath: String): List<String> {
+
+    val imports = mutableListOf<String>()
+    val importPattern = Regex("^import\\s+(com\\.iqser\\.red\\.service\\.redaction\\.v1\\.[\\w.]+);")
+    val desiredPrefix = "com.iqser.red.service.redaction.v1"
+
+    File(droolsFilePath).forEachLine { line ->
+        importPattern.find(line)?.let { matchResult ->
+            val importPath = matchResult.groupValues[1].trim()
+            if (importPath.startsWith(desiredPrefix)) {
+                val formattedPath = importPath.replace('.', '/')
+                imports.add("$formattedPath.java")
+            }
+        }
+    }
+
+    return imports
+}
+
+val droolsImports = parseDroolsImports("redaction-service-v1/redaction-service-server-v1/src/main/resources/drools/all_rules_documine.drl")
+
+tasks.register("generateJavaDoc", Javadoc::class) {
+
+    dependsOn("compileJava")
+    dependsOn("delombok")
+    classpath = project.sourceSets["main"].runtimeClasspath
+    source = fileTree("${buildDir}/generated/sources/delombok/java/main") {
+        include(droolsImports)
+    }
+    destinationDir = file(project.findProperty("javadocDestinationDir")?.toString() ?: "")
+
+    options.memberLevel = JavadocMemberLevel.PUBLIC
+    (options as StandardJavadocDocletOptions).apply {
+        header = "Redaction Service ${project.version}"
+        footer = "Redaction Service ${project.version}"
+        title = "API Documentation for Redaction Service ${project.version}"
+    }
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/Application.java

@@ -4,16 +4,24 @@ import org.springframework.boot.SpringApplication;
 import org.springframework.boot.actuate.autoconfigure.security.servlet.ManagementWebSecurityAutoConfiguration;
 import org.springframework.boot.autoconfigure.ImportAutoConfiguration;
 import org.springframework.boot.autoconfigure.SpringBootApplication;
+import org.springframework.boot.autoconfigure.data.mongo.MongoDataAutoConfiguration;
+import org.springframework.boot.autoconfigure.jdbc.DataSourceAutoConfiguration;
+import org.springframework.boot.autoconfigure.liquibase.LiquibaseAutoConfiguration;
+import org.springframework.boot.autoconfigure.mongo.MongoAutoConfiguration;
 import org.springframework.boot.autoconfigure.security.servlet.SecurityAutoConfiguration;
 import org.springframework.boot.context.properties.EnableConfigurationProperties;
 import org.springframework.cache.annotation.EnableCaching;
 import org.springframework.cloud.openfeign.EnableFeignClients;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Import;
+import org.springframework.data.mongodb.repository.config.EnableMongoRepositories;
 
 import com.iqser.red.service.dictionarymerge.commons.DictionaryMergeService;
+import com.iqser.red.service.persistence.service.v1.api.shared.mongo.SharedMongoAutoConfiguration;
 import com.iqser.red.service.redaction.v1.server.client.RulesClient;
 import com.iqser.red.storage.commons.StorageAutoConfiguration;
+import com.knecon.fforesight.mongo.database.commons.MongoDatabaseCommonsAutoConfiguration;
+import com.knecon.fforesight.mongo.database.commons.liquibase.EnableMongoLiquibase;
 import com.knecon.fforesight.tenantcommons.MultiTenancyAutoConfiguration;
 
 import io.micrometer.core.aop.TimedAspect;
@@ -22,11 +30,13 @@ import io.micrometer.observation.ObservationRegistry;
 import io.micrometer.observation.aop.ObservedAspect;
 
 @EnableCaching
-@ImportAutoConfiguration({MultiTenancyAutoConfiguration.class})
-@Import({MetricsConfiguration.class, StorageAutoConfiguration.class})
+@ImportAutoConfiguration({MultiTenancyAutoConfiguration.class, SharedMongoAutoConfiguration.class})
+@Import({MetricsConfiguration.class, StorageAutoConfiguration.class, MongoDatabaseCommonsAutoConfiguration.class})
 @EnableFeignClients(basePackageClasses = RulesClient.class)
 @EnableConfigurationProperties(RedactionServiceSettings.class)
-@SpringBootApplication(exclude = {SecurityAutoConfiguration.class, ManagementWebSecurityAutoConfiguration.class})
+@EnableMongoRepositories(basePackages = "com.iqser.red.service.persistence")
+@EnableMongoLiquibase
+@SpringBootApplication(exclude = {SecurityAutoConfiguration.class, ManagementWebSecurityAutoConfiguration.class, DataSourceAutoConfiguration.class, LiquibaseAutoConfiguration.class, MongoAutoConfiguration.class, MongoDataAutoConfiguration.class})
 public class Application {
 
     public static void main(String[] args) {
@@ -35,21 +45,32 @@ public class Application {
         SpringApplication.run(Application.class, args);
     }
 
+
     @Bean
     public ObservedAspect observedAspect(ObservationRegistry observationRegistry) {
+
         return new ObservedAspect(observationRegistry);
     }
 
+
     @Bean
     public TimedAspect timedAspect(MeterRegistry registry) {
 
         return new TimedAspect(registry);
     }
 
+
     @Bean
     public DictionaryMergeService dictionaryMergeService() {
 
         return new DictionaryMergeService();
     }
 
+
+    @Bean(initMethod = "findDeprecatedMethods")
+    public DeprecatedElementsFinder deprecatedElementsFinder() {
+
+        return new DeprecatedElementsFinder();
+    }
+
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/DeprecatedElementsFinder.java

@@ -0,0 +1,109 @@
+package com.iqser.red.service.redaction.v1.server;
+
+import static java.util.stream.Collectors.groupingBy;
+
+import java.lang.reflect.Method;
+import java.lang.reflect.Type;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import org.reflections.Reflections;
+import org.reflections.scanners.Scanners;
+import org.reflections.util.ConfigurationBuilder;
+
+import com.iqser.red.service.redaction.v1.server.model.dictionary.SearchImplementation;
+
+import lombok.Getter;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+public class DeprecatedElementsFinder {
+
+    public static final String PACKAGE_NAME = "com.iqser.red.service.redaction.v1.server";
+    private Set<Method> deprecatedMethods;
+    @Getter
+    private Map<String, String> deprecatedMethodsSignaturesMap;
+    private Set<Class<?>> deprecatedClasses;
+    @Getter
+    private SearchImplementation classesSearchImplementation;
+    @Getter
+    private SearchImplementation methodsSearchImplementation;
+
+
+    public void findDeprecatedMethods() {
+
+        deprecatedMethods = new HashSet<>();
+        deprecatedClasses = new HashSet<>();
+        deprecatedMethodsSignaturesMap = new HashMap<>();
+
+        Reflections reflections = new Reflections(new ConfigurationBuilder().forPackage(PACKAGE_NAME)
+                                                          .setExpandSuperTypes(true)
+                                                          .setScanners(Scanners.MethodsAnnotated, Scanners.TypesAnnotated, Scanners.SubTypes));
+
+        deprecatedMethods = reflections.get(Scanners.MethodsAnnotated.with(Deprecated.class).as(Method.class));
+
+        deprecatedClasses = reflections.get(Scanners.TypesAnnotated.with(Deprecated.class).asClass());
+
+        if (!deprecatedClasses.isEmpty()) {
+            deprecatedClasses.forEach(deprecatedType -> deprecatedMethods.addAll(Arrays.stream(deprecatedType.getDeclaredMethods())
+                                                                                         .collect(Collectors.toSet())));
+            classesSearchImplementation = new SearchImplementation(deprecatedClasses.stream()
+                                                                           .map(Class::getName)
+                                                                           .collect(Collectors.toSet()), false);
+        }
+
+        if (!deprecatedMethods.isEmpty()) {
+            methodsSearchImplementation = new SearchImplementation(deprecatedMethods.stream()
+                                                                           .map(Method::getName)
+                                                                           .collect(Collectors.toSet()), false);
+            deprecatedMethodsSignaturesMap = new HashMap<>();
+            Map<String, List<Method>> deprecatedMethodsNameMap = deprecatedMethods.stream()
+                    .collect(groupingBy(Method::getName));
+            for (String deprecatedMethod : deprecatedMethodsNameMap.keySet()) {
+                deprecatedMethodsSignaturesMap.computeIfAbsent(deprecatedMethod,
+                                                               key -> deprecatedMethodsNameMap.get(deprecatedMethod)
+                                                                       .stream()
+                                                                       .map(this::getMethodSignature)
+                                                                       .collect(Collectors.joining("\n")));
+            }
+
+        }
+
+        log.info("For package: {} found deprecated classes {} deprecated methods {}", PACKAGE_NAME, deprecatedClasses.size(), deprecatedMethods.size());
+    }
+
+
+    public Set<Method> getDeprecatedMethods() {
+
+        if (this.deprecatedMethods == null) {
+            findDeprecatedMethods();
+        }
+        return this.deprecatedMethods;
+    }
+
+
+    public Set<Class<?>> getDeprecatedClasses() {
+
+        if (this.deprecatedClasses == null) {
+            findDeprecatedMethods();
+        }
+        return this.deprecatedClasses;
+    }
+
+
+    private String getMethodSignature(Method method) {
+
+        String methodName = method.getName();
+        String parameterTypes = Arrays.stream(method.getGenericParameterTypes())
+                .map(Type::getTypeName)
+                .collect(Collectors.joining(", "));
+
+        return method.getGenericReturnType() + " " + methodName + "(" + parameterTypes + ")";
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/RedactionServiceSettings.java

@@ -30,4 +30,6 @@ public class RedactionServiceSettings {
 
     private int droolsExecutionTimeoutSecs = 300;
 
+    private boolean ruleExecutionSecured = true;
+
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/RedisCachingConfiguration.java

@@ -16,7 +16,7 @@ public class RedisCachingConfiguration {
     public RedisCacheManagerBuilderCustomizer redisCacheManagerBuilderCustomizer() {
 
         return (builder) -> builder.withCacheConfiguration("documentDataCache",
-                RedisCacheConfiguration.defaultCacheConfig().entryTtl(Duration.ofMinutes(30)).disableCachingNullValues());
+                                                           RedisCacheConfiguration.defaultCacheConfig().entryTtl(Duration.ofMinutes(30)).disableCachingNullValues());
     }
 
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/client/model/EntityRecognitionEntity.java

@@ -1,6 +1,5 @@
 package com.iqser.red.service.redaction.v1.server.client.model;
 
-
 import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RedactionController.java

@@ -3,10 +3,10 @@ package com.iqser.red.service.redaction.v1.server.controller;
 import org.springframework.web.bind.annotation.RequestBody;
 import org.springframework.web.bind.annotation.RestController;
 
-import com.iqser.red.service.redaction.v1.model.DroolsSyntaxValidation;
+import com.iqser.red.service.redaction.v1.model.DroolsValidation;
 import com.iqser.red.service.redaction.v1.model.RuleValidationModel;
 import com.iqser.red.service.redaction.v1.resources.RedactionResource;
-import com.iqser.red.service.redaction.v1.server.service.drools.DroolsSyntaxValidationService;
+import com.iqser.red.service.redaction.v1.server.service.drools.DroolsValidationService;
 import com.iqser.red.service.redaction.v1.server.utils.exception.RulesValidationException;
 
 import lombok.RequiredArgsConstructor;
@@ -17,14 +17,14 @@ import lombok.extern.slf4j.Slf4j;
 @RequiredArgsConstructor
 public class RedactionController implements RedactionResource {
 
-    private final DroolsSyntaxValidationService droolsSyntaxValidationService;
+    private final DroolsValidationService droolsValidationService;
 
 
     @Override
-    public DroolsSyntaxValidation testRules(@RequestBody RuleValidationModel rulesValidationModel) {
+    public DroolsValidation testRules(@RequestBody RuleValidationModel rulesValidationModel) {
 
         try {
-            return droolsSyntaxValidationService.testRules(rulesValidationModel);
+            return droolsValidationService.testRules(rulesValidationModel);
         } catch (Exception e) {
             throw new RulesValidationException("Could not test rules: " + e.getMessage(), e);
         }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RuleBuilderController.java

@@ -13,7 +13,6 @@ import lombok.RequiredArgsConstructor;
 @RequiredArgsConstructor
 public class RuleBuilderController implements RuleBuilderResource {
 
-
     @Override
     public RuleBuilderModel getRuleBuilderModel() {
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/migration/LegacyRedactionLogMergeService.java

@@ -12,7 +12,6 @@ import org.springframework.stereotype.Service;
 
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.BaseAnnotation;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange;
@@ -43,10 +42,10 @@ public class LegacyRedactionLogMergeService {
     private final DictionaryService dictionaryService;
 
 
-    public RedactionLog mergeManualChanges(RedactionLog redactionLog, ManualRedactions manualRedactions, String dossierTemplateId) {
+    public RedactionLog addManualAddEntriesAndRemoveSkippedImported(RedactionLog redactionLog, ManualRedactions manualRedactions, String dossierTemplateId) {
 
-        var skippedImportedRedactions = new HashSet<>();
-        log.info("Merging Redaction log with manual redactions");
+        Set<String> skippedImportedRedactions = new HashSet<>();
+        log.info("Adding manual add Entries and removing skipped or imported entries");
         if (manualRedactions != null) {
 
             var manualRedactionLogEntries = addManualAddEntries(manualRedactions.getEntriesToAdd(), redactionLog.getAnalysisNumber());
@@ -57,10 +56,14 @@ public class LegacyRedactionLogMergeService {
 
             for (RedactionLogEntry entry : redactionLog.getRedactionLogEntry()) {
 
-                processRedactionLogEntry(manualRedactionWrappers.stream().filter(mr -> entry.getId().equals(mr.getId())).collect(Collectors.toList()), entry, dossierTemplateId);
-
-                if (entry.isImported() && !entry.isRedacted()) {
-                    skippedImportedRedactions.add(entry.getId());
+                if (entry.isImported()) {
+                    processRedactionLogEntry(manualRedactionWrappers.stream()
+                                                     .filter(ManualRedactionWrapper::isApproved)
+                                                     .filter(mr -> entry.getId().equals(mr.getId()))
+                                                     .collect(Collectors.toList()), entry, dossierTemplateId);
+                    if (!entry.isRedacted()) {
+                        skippedImportedRedactions.add(entry.getId());
+                    }
                 }
 
             }
@@ -69,7 +72,9 @@ public class LegacyRedactionLogMergeService {
 
         Set<String> processedIds = new HashSet<>();
         redactionLog.getRedactionLogEntry().removeIf(entry -> {
-
+            if (entry.isFalsePositive()) {
+                return true;
+            }
             if (entry.getImportedRedactionIntersections() != null) {
                 entry.getImportedRedactionIntersections().removeAll(skippedImportedRedactions);
                 if (!entry.getImportedRedactionIntersections().isEmpty() && (!entry.isImage() || entry.isImage() && !(entry.getType().equals("image") || entry.getType()
@@ -88,39 +93,53 @@ public class LegacyRedactionLogMergeService {
     }
 
 
+    public long getNumberOfAffectedAnnotations(ManualRedactions manualRedactions) {
+
+        return createManualRedactionWrappers(manualRedactions).stream()
+                .map(ManualRedactionWrapper::getId)
+                .distinct()
+                .count();
+    }
+
+
     private List<ManualRedactionWrapper> createManualRedactionWrappers(ManualRedactions manualRedactions) {
 
         List<ManualRedactionWrapper> manualRedactionWrappers = new ArrayList<>();
 
-        manualRedactions.getRecategorizations().stream().filter(BaseAnnotation::isApproved).forEach(item -> {
-            if (item.getSoftDeletedTime() == null) {
-                manualRedactionWrappers.add(new ManualRedactionWrapper(item.getAnnotationId(), item.getRequestDate(), item));
-            }
-        });
+        manualRedactions.getRecategorizations()
+                .forEach(item -> {
+                    if (item.getSoftDeletedTime() == null) {
+                        manualRedactionWrappers.add(new ManualRedactionWrapper(item.getAnnotationId(), item.getRequestDate(), item, item.isApproved()));
+                    }
+                });
 
-        manualRedactions.getIdsToRemove().stream().filter(BaseAnnotation::isApproved).forEach(item -> {
-            if (item.getSoftDeletedTime() == null) {
-                manualRedactionWrappers.add(new ManualRedactionWrapper(item.getAnnotationId(), item.getRequestDate(), item));
-            }
-        });
+        manualRedactions.getIdsToRemove()
+                .forEach(item -> {
+                    if (item.getSoftDeletedTime() == null) {
+                        manualRedactionWrappers.add(new ManualRedactionWrapper(item.getAnnotationId(), item.getRequestDate(), item, item.isApproved()));
+                    }
+                });
 
-        manualRedactions.getForceRedactions().stream().filter(BaseAnnotation::isApproved).forEach(item -> {
-            if (item.getSoftDeletedTime() == null) {
-                manualRedactionWrappers.add(new ManualRedactionWrapper(item.getAnnotationId(), item.getRequestDate(), item));
-            }
-        });
+        manualRedactions.getForceRedactions()
+                .forEach(item -> {
+                    if (item.getSoftDeletedTime() == null) {
+                        manualRedactionWrappers.add(new ManualRedactionWrapper(item.getAnnotationId(), item.getRequestDate(), item, item.isApproved()));
+                    }
+                });
 
-        manualRedactions.getLegalBasisChanges().stream().filter(BaseAnnotation::isApproved).forEach(item -> {
-            if (item.getSoftDeletedTime() == null) {
-                manualRedactionWrappers.add(new ManualRedactionWrapper(item.getAnnotationId(), item.getRequestDate(), item));
-            }
-        });
+        manualRedactions.getLegalBasisChanges()
+                .forEach(item -> {
+                    if (item.getSoftDeletedTime() == null) {
+                        manualRedactionWrappers.add(new ManualRedactionWrapper(item.getAnnotationId(), item.getRequestDate(), item, item.isApproved()));
+                    }
+                });
 
-        manualRedactions.getResizeRedactions().stream().filter(BaseAnnotation::isApproved).forEach(item -> {
-            if (item.getSoftDeletedTime() == null) {
-                manualRedactionWrappers.add(new ManualRedactionWrapper(item.getAnnotationId(), item.getRequestDate(), item));
-            }
-        });
+        manualRedactions.getResizeRedactions()
+                .forEach(item -> {
+                    if (item.getSoftDeletedTime() == null) {
+                        manualRedactionWrappers.add(new ManualRedactionWrapper(item.getAnnotationId(), item.getRequestDate(), item, item.isApproved()));
+                    }
+                });
 
         Collections.sort(manualRedactionWrappers);
 
@@ -183,7 +202,12 @@ public class LegacyRedactionLogMergeService {
         }
 
         redactionLogEntry.getManualChanges()
-                .add(ManualChange.from(imageRecategorization).withManualRedactionType(ManualRedactionType.RECATEGORIZE).withChange("type", imageRecategorization.getType()));
+                .add(ManualChange.from(imageRecategorization)
+                             .withManualRedactionType(ManualRedactionType.RECATEGORIZE)
+                             .withChange("type", imageRecategorization.getType())
+                             .withChange("section", imageRecategorization.getSection())
+                             .withChange("legalBasis", imageRecategorization.getLegalBasis())
+                             .withChange("value", imageRecategorization.getValue()));
     }
 
 
@@ -222,7 +246,7 @@ public class LegacyRedactionLogMergeService {
 
         redactionLogEntry.getManualChanges()
                 .add(ManualChange.from(manualRemoval)
-                        .withManualRedactionType(manualRemoval.isRemoveFromDictionary() ? ManualRedactionType.REMOVE_FROM_DICTIONARY : ManualRedactionType.REMOVE_LOCALLY));
+                             .withManualRedactionType(manualRemoval.isRemoveFromDictionary() ? ManualRedactionType.REMOVE_FROM_DICTIONARY : ManualRedactionType.REMOVE_LOCALLY));
     }
 
 
@@ -355,7 +379,13 @@ public class LegacyRedactionLogMergeService {
     @SuppressWarnings("PMD.UselessParentheses")
     private boolean shouldCreateManualEntry(ManualRedactionEntry manualRedactionEntry) {
 
-        return (!manualRedactionEntry.isAddToDictionary() && !manualRedactionEntry.isAddToDossierDictionary()) || ((manualRedactionEntry.isAddToDictionary() || manualRedactionEntry.isAddToDossierDictionary()) && manualRedactionEntry.getProcessedDate() == null);
+        if (!manualRedactionEntry.isApproved()) {
+            return false;
+        }
+
+        return (!manualRedactionEntry.isAddToDictionary() && !manualRedactionEntry.isAddToDossierDictionary()) || ((manualRedactionEntry.isAddToDictionary()
+                                                                                                                    || manualRedactionEntry.isAddToDossierDictionary())
+                                                                                                                   && manualRedactionEntry.getProcessedDate() == null);
     }
 
 
@@ -394,6 +424,7 @@ public class LegacyRedactionLogMergeService {
         private String id;
         private OffsetDateTime date;
         private Object item;
+        private boolean approved;
 
 
         @Override

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/migration/LegacyVersion0MigrationService.java

@@ -21,7 +21,9 @@ public class LegacyVersion0MigrationService {
     public RedactionLog mergeDuplicateAnnotationIds(RedactionLog redactionLog) {
 
         List<RedactionLogEntry> mergedEntries = new LinkedList<>();
-        Map<String, List<RedactionLogEntry>> entriesById = redactionLog.getRedactionLogEntry().stream().collect(Collectors.groupingBy(RedactionLogEntry::getId));
+        Map<String, List<RedactionLogEntry>> entriesById = redactionLog.getRedactionLogEntry()
+                .stream()
+                .collect(Collectors.groupingBy(RedactionLogEntry::getId));
         for (List<RedactionLogEntry> entries : entriesById.values()) {
 
             if (entries.isEmpty()) {
@@ -33,7 +35,10 @@ public class LegacyVersion0MigrationService {
                 continue;
             }
 
-            List<RedactionLogEntry> sortedEntries = entries.stream().sorted(Comparator.comparing(entry -> entry.getChanges().get(0).getDateTime())).toList();
+            List<RedactionLogEntry> sortedEntries = entries.stream()
+                    .sorted(Comparator.comparing(entry -> entry.getChanges()
+                            .get(0).getDateTime()))
+                    .toList();
 
             RedactionLogEntry initialEntry = sortedEntries.get(0);
             for (RedactionLogEntry entry : sortedEntries.subList(1, sortedEntries.size())) {

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/migration/MigrationMapper.java

@@ -0,0 +1,101 @@
+package com.iqser.red.service.redaction.v1.server.migration;
+
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ChangeType;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualChange;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Change;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ManualRedactionType;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
+
+public class MigrationMapper {
+
+    public static com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Change toEntityLogChanges(Change change) {
+
+        return new com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Change(change.getAnalysisNumber(),
+                                                                                                              toEntityLogType(change.getType()),
+                                                                                                              change.getDateTime());
+    }
+
+
+    public static com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualChange toEntityLogManualChanges(com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ManualChange manualChange) {
+
+        return new com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualChange(toManualRedactionType(manualChange.getManualRedactionType()),
+                                                                                                                    manualChange.getProcessedDate(),
+                                                                                                                    manualChange.getRequestedDate(),
+                                                                                                                    manualChange.getUserId(),
+                                                                                                                    manualChange.getPropertyChanges());
+    }
+
+
+    public static ChangeType toEntityLogType(com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ChangeType type) {
+
+        return switch (type) {
+            case ADDED -> ChangeType.ADDED;
+            case REMOVED -> ChangeType.REMOVED;
+            case CHANGED -> ChangeType.CHANGED;
+        };
+    }
+
+
+    public static com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType toManualRedactionType(ManualRedactionType manualRedactionType) {
+
+        return switch (manualRedactionType) {
+            case ADD_LOCALLY -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.ADD;
+            case ADD_TO_DICTIONARY -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.ADD_TO_DICTIONARY;
+            case REMOVE_LOCALLY -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.REMOVE;
+            case REMOVE_FROM_DICTIONARY -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.REMOVE_FROM_DICTIONARY;
+            case FORCE_REDACT -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.FORCE;
+            case FORCE_HINT -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.FORCE;
+            case RECATEGORIZE -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.RECATEGORIZE;
+            case LEGAL_BASIS_CHANGE -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.LEGAL_BASIS_CHANGE;
+            case RESIZE -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.RESIZE;
+        };
+    }
+
+
+    public static com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine toEntityLogEngine(Engine engine) {
+
+        return switch (engine) {
+            case DICTIONARY -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine.DICTIONARY;
+            case NER -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine.NER;
+            case RULE -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine.RULE;
+        };
+    }
+
+
+    public static Set<com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine> getMigratedEngines(RedactionLogEntry entry) {
+
+        Set<com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine> engines = new HashSet<>();
+
+        if (entry.isImported()) {
+            engines.add(com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine.IMPORTED);
+        }
+
+        if (entry.getEngines() == null) {
+            return engines;
+        }
+        entry.getEngines()
+                .stream()
+                .map(MigrationMapper::toEntityLogEngine)
+                .forEach(engines::add);
+
+        return engines;
+    }
+
+
+    public List<ManualChange> migrateManualChanges(List<com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ManualChange> manualChanges) {
+
+        if (manualChanges == null) {
+            return Collections.emptyList();
+        }
+        return manualChanges.stream()
+                .map(MigrationMapper::toEntityLogManualChanges)
+                .toList();
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/migration/MigrationMessageReceiver.java

@@ -49,7 +49,6 @@ public class MigrationMessageReceiver {
         MigrationRequest migrationRequest = objectMapper.readValue(message.getBody(), MigrationRequest.class);
         log.info("--------------------------------------------------------------------");
         log.info("Starting redactionLog to entityLog migration for dossierId {} and fileId {}", migrationRequest.getDossierId(), migrationRequest.getFileId());
-        // TODO: if an image is not found, try to copy the old one exactly (like with TextEntities)
 
         dictionaryService.updateDictionary(migrationRequest.getDossierTemplateId(), migrationRequest.getDossierId());
 
@@ -58,20 +57,32 @@ public class MigrationMessageReceiver {
 
         if (redactionLog.getAnalysisVersion() == 0) {
             redactionLog = legacyVersion0MigrationService.mergeDuplicateAnnotationIds(redactionLog);
-        } else if (migrationRequest.getManualRedactions() != null) {
-            redactionLog = legacyRedactionLogMergeService.mergeManualChanges(redactionLog, migrationRequest.getManualRedactions(), migrationRequest.getDossierTemplateId());
+        } else {
+            redactionLog = legacyRedactionLogMergeService.addManualAddEntriesAndRemoveSkippedImported(redactionLog,
+                                                                                                      migrationRequest.getManualRedactions(),
+                                                                                                      migrationRequest.getDossierTemplateId());
         }
 
-        MigratedEntityLog migratedEntityLog = redactionLogToEntityLogMigrationService.migrate(redactionLog, document);
+        MigratedEntityLog migratedEntityLog = redactionLogToEntityLogMigrationService.migrate(redactionLog,
+                                                                                              document,
+                                                                                              migrationRequest.getDossierTemplateId(),
+                                                                                              migrationRequest.getManualRedactions(),
+                                                                                              migrationRequest.getFileId(),
+                                                                                              migrationRequest.getEntitiesWithComments(),
+                                                                                              migrationRequest.isFileIsApproved());
+
+        log.info("Storing migrated entityLog and ids to migrate in DB for file {}", migrationRequest.getFileId());
 
         redactionStorageService.storeObject(migrationRequest.getDossierId(), migrationRequest.getFileId(), FileType.ENTITY_LOG, migratedEntityLog.getEntityLog());
         redactionStorageService.storeObject(migrationRequest.getDossierId(), migrationRequest.getFileId(), FileType.MIGRATED_IDS, migratedEntityLog.getMigratedIds());
 
         sendFinished(MigrationResponse.builder().dossierId(migrationRequest.getDossierId()).fileId(migrationRequest.getFileId()).build());
-        log.info("Migrated {} redactionLog entries for dossierId {} and fileId {}",
-                migratedEntityLog.getEntityLog().getEntityLogEntry().size(),
-                migrationRequest.getDossierId(),
-                migrationRequest.getFileId());
+        log.info("Migrated {} redactionLog entries, found {} annotation ids for migration in the db, {} new manual entries, for dossierId {} and fileId {}",
+                 migratedEntityLog.getEntityLog().getEntityLogEntry().size(),
+                 migratedEntityLog.getMigratedIds().getMappings().size(),
+                 migratedEntityLog.getMigratedIds().getManualRedactionEntriesToAdd().size(),
+                 migrationRequest.getDossierId(),
+                 migrationRequest.getFileId());
         log.info("");
     }
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/migration/RedactionLogToEntityLogMigrationService.java

@@ -2,11 +2,13 @@ package com.iqser.red.service.redaction.v1.server.migration;
 
 import java.awt.geom.Rectangle2D;
 import java.util.Collection;
+import java.util.Collections;
 import java.util.Comparator;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
+import java.util.Set;
 import java.util.function.Function;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
@@ -16,24 +18,27 @@ import org.springframework.stereotype.Service;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogLegalBasis;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.migration.MigratedIds;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.BaseAnnotation;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogLegalBasis;
-import com.iqser.red.service.redaction.v1.server.model.ManualEntity;
 import com.iqser.red.service.redaction.v1.server.model.MigratedEntityLog;
 import com.iqser.red.service.redaction.v1.server.model.MigrationEntity;
+import com.iqser.red.service.redaction.v1.server.model.PrecursorEntity;
 import com.iqser.red.service.redaction.v1.server.model.RectangleWithPage;
-import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Image;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.ImageType;
-import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
-import com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService;
-import com.iqser.red.service.redaction.v1.server.service.document.EntityEnrichmentService;
+import com.iqser.red.service.redaction.v1.server.service.DictionaryService;
+import com.iqser.red.service.redaction.v1.server.service.ManualChangesApplicationService;
 import com.iqser.red.service.redaction.v1.server.service.document.EntityFindingUtility;
+import com.iqser.red.service.redaction.v1.server.service.document.EntityFromPrecursorCreationService;
 import com.iqser.red.service.redaction.v1.server.utils.IdBuilder;
 import com.iqser.red.service.redaction.v1.server.utils.MigratedIdsCollector;
 
@@ -51,15 +56,26 @@ public class RedactionLogToEntityLogMigrationService {
 
     private static final double MATCH_THRESHOLD = 10;
     EntityFindingUtility entityFindingUtility;
-    EntityEnrichmentService entityEnrichmentService;
+    DictionaryService dictionaryService;
+    ManualChangesApplicationService manualChangesApplicationService;
 
 
-    public MigratedEntityLog migrate(RedactionLog redactionLog, Document document) {
+    public MigratedEntityLog migrate(RedactionLog redactionLog,
+                                     Document document,
+                                     String dossierTemplateId,
+                                     ManualRedactions manualRedactions,
+                                     String fileId,
+                                     Set<String> entitiesWithComments,
+                                     boolean fileIsApproved) {
 
-        List<MigrationEntity> entitiesToMigrate = calculateMigrationEntitiesFromRedactionLog(redactionLog, document);
-        MigratedIds migratedIds = entitiesToMigrate.stream().collect(new MigratedIdsCollector());
-        MigratedIds idsToMigrateInDb = entitiesToMigrate.stream().filter(MigrationEntity::hasManualChangesOrComments).collect(new MigratedIdsCollector());
+        log.info("Migrating entities for file {}", fileId);
+        List<MigrationEntity> entitiesToMigrate = calculateMigrationEntitiesFromRedactionLog(redactionLog, document, dossierTemplateId, fileId);
 
+        MigratedIds migratedIds = entitiesToMigrate.stream()
+                .collect(new MigratedIdsCollector());
+
+        log.info("applying manual changes to migrated entities for file {}", fileId);
+        applyLocalProcessedManualChanges(entitiesToMigrate, manualRedactions, fileIsApproved);
 
         EntityLog entityLog = new EntityLog();
         entityLog.setAnalysisNumber(redactionLog.getAnalysisNumber());
@@ -68,34 +84,117 @@ public class RedactionLogToEntityLogMigrationService {
         entityLog.setDossierDictionaryVersion(redactionLog.getDossierDictionaryVersion());
         entityLog.setLegalBasisVersion(redactionLog.getLegalBasisVersion());
         entityLog.setAnalysisVersion(redactionLog.getAnalysisVersion());
-        entityLog.setLegalBasis(redactionLog.getLegalBasis().stream().map(RedactionLogToEntityLogMigrationService::toEntityLogLegalBasis).toList());
+        entityLog.setLegalBasis(redactionLog.getLegalBasis()
+                                        .stream()
+                                        .map(RedactionLogToEntityLogMigrationService::toEntityLogLegalBasis)
+                                        .toList());
 
         Map<String, String> oldToNewIDMapping = migratedIds.buildOldToNewMapping();
-        entityLog.setEntityLogEntry(entitiesToMigrate.stream().map(migrationEntity -> migrationEntity.toEntityLogEntry(oldToNewIDMapping)).toList());
 
-        if (getNumberOfApprovedEntries(redactionLog) != entityLog.getEntityLogEntry().size()) {
+        log.info("Writing migrated entities to entityLog for file {}", fileId);
+        entityLog.setEntityLogEntry(entitiesToMigrate.stream()
+                                            .map(migrationEntity -> migrationEntity.toEntityLogEntry(oldToNewIDMapping))
+                                            .toList());
+
+        if (getNumberOfApprovedEntries(redactionLog, document.getNumberOfPages()) != entityLog.getEntityLogEntry().size()) {
             String message = String.format("Not all entities have been found during the migration redactionLog has %d entries and new entityLog %d",
-                    redactionLog.getRedactionLogEntry().size(),
-                    entityLog.getEntityLogEntry().size());
+                                           redactionLog.getRedactionLogEntry().size(),
+                                           entityLog.getEntityLogEntry().size());
             log.error(message);
             throw new AssertionError(message);
         }
 
+        Set<String> entitiesWithUnprocessedChanges = manualRedactions.buildAll()
+                .stream()
+                .filter(manualRedaction -> manualRedaction.getProcessedDate() == null)
+                .map(BaseAnnotation::getAnnotationId)
+                .collect(Collectors.toSet());
+
+        MigratedIds idsToMigrateInDb = entitiesToMigrate.stream()
+                .filter(migrationEntity -> migrationEntity.hasManualChangesOrComments(entitiesWithComments, entitiesWithUnprocessedChanges))
+                .filter(m -> !m.getOldId().equals(m.getNewId()))
+                .collect(new MigratedIdsCollector());
+
+        List<ManualRedactionEntry> manualRedactionEntriesToAdd = entitiesToMigrate.stream()
+                .filter(MigrationEntity::needsManualEntry)
+                .map(MigrationEntity::buildManualRedactionEntry)
+                .toList();
+
+        idsToMigrateInDb.setManualRedactionEntriesToAdd(manualRedactionEntriesToAdd);
+
         return new MigratedEntityLog(idsToMigrateInDb, entityLog);
     }
 
 
-    private static long getNumberOfApprovedEntries(RedactionLog redactionLog) {
+    private void applyLocalProcessedManualChanges(List<MigrationEntity> entitiesToMigrate, ManualRedactions manualRedactions, boolean fileIsApproved) {
+
+        if (manualRedactions == null) {
+            return;
+        }
+        Map<String, List<BaseAnnotation>> manualChangesPerAnnotationId;
+
+        if (fileIsApproved) {
+            manualChangesPerAnnotationId = manualRedactions.buildAll()
+                    .stream()
+                    .filter(manualChange -> (manualChange.getProcessedDate() != null && manualChange.isLocal()) //
+                                            // unprocessed dict change of type IdRemoval or ManualResize must be applied for approved documents
+                                            || (manualChange.getProcessedDate() == null && !manualChange.isLocal() //
+                                                && (manualChange instanceof IdRemoval || manualChange instanceof ManualResizeRedaction)))
+                    .map(this::convertPendingDictChangesToLocal)
+                    .collect(Collectors.groupingBy(BaseAnnotation::getAnnotationId));
+        } else {
+            manualChangesPerAnnotationId = manualRedactions.buildAll()
+                    .stream()
+                    .filter(manualChange -> manualChange.getProcessedDate() != null)
+                    .filter(BaseAnnotation::isLocal)
+                    .collect(Collectors.groupingBy(BaseAnnotation::getAnnotationId));
+        }
+
+        entitiesToMigrate.forEach(migrationEntity -> migrationEntity.applyManualChanges(manualChangesPerAnnotationId.getOrDefault(migrationEntity.getOldId(),
+                                                                                                                                  Collections.emptyList()),
+                                                                                        manualChangesApplicationService));
 
-        return redactionLog.getRedactionLogEntry().size();
     }
 
 
-    private List<MigrationEntity> calculateMigrationEntitiesFromRedactionLog(RedactionLog redactionLog, Document document) {
+    private BaseAnnotation convertPendingDictChangesToLocal(BaseAnnotation baseAnnotation) {
 
-        List<MigrationEntity> images = getImageBasedMigrationEntities(redactionLog, document);
-        List<MigrationEntity> textMigrationEntities = getTextBasedMigrationEntities(redactionLog, document);
-        return Stream.of(textMigrationEntities.stream(), images.stream()).flatMap(Function.identity()).toList();
+        if (baseAnnotation.getProcessedDate() != null) {
+            return baseAnnotation;
+        }
+
+        if (baseAnnotation.isLocal()) {
+            return baseAnnotation;
+        }
+
+        if (baseAnnotation instanceof ManualResizeRedaction manualResizeRedaction) {
+            manualResizeRedaction.setAddToAllDossiers(false);
+            manualResizeRedaction.setUpdateDictionary(false);
+        } else if (baseAnnotation instanceof IdRemoval idRemoval) {
+            idRemoval.setRemoveFromAllDossiers(false);
+            idRemoval.setRemoveFromDictionary(false);
+        }
+
+        return baseAnnotation;
+    }
+
+
+    private long getNumberOfApprovedEntries(RedactionLog redactionLog, int numberOfPages) {
+
+        return redactionLog.getRedactionLogEntry()
+                .stream()
+                .filter(redactionLogEntry -> isOnExistingPage(redactionLogEntry, numberOfPages))
+                .count();
+    }
+
+
+    private List<MigrationEntity> calculateMigrationEntitiesFromRedactionLog(RedactionLog redactionLog, Document document, String dossierTemplateId, String fileId) {
+
+        List<MigrationEntity> images = getImageBasedMigrationEntities(redactionLog, document, fileId, dossierTemplateId);
+        List<MigrationEntity> textMigrationEntities = getTextBasedMigrationEntities(redactionLog, document, dossierTemplateId, fileId);
+        return Stream.of(textMigrationEntities.stream(), images.stream())
+                .flatMap(Function.identity())
+                .toList();
     }
 
 
@@ -105,9 +204,10 @@ public class RedactionLogToEntityLogMigrationService {
     }
 
 
-    private List<MigrationEntity> getImageBasedMigrationEntities(RedactionLog redactionLog, Document document) {
+    private List<MigrationEntity> getImageBasedMigrationEntities(RedactionLog redactionLog, Document document, String fileId, String dossierTemplateId) {
 
-        List<Image> images = document.streamAllImages().collect(Collectors.toList());
+        List<Image> images = document.streamAllImages()
+                .collect(Collectors.toList());
 
         List<RedactionLogEntry> redactionLogImages = redactionLog.getRedactionLogEntry()
                 .stream()
@@ -116,12 +216,16 @@ public class RedactionLogToEntityLogMigrationService {
 
         List<MigrationEntity> migrationEntities = new LinkedList<>();
         for (RedactionLogEntry redactionLogImage : redactionLogImages) {
-            List<RectangleWithPage> imagePositions = redactionLogImage.getPositions().stream().map(RectangleWithPage::fromRedactionLogRectangle).toList();
+            List<RectangleWithPage> imagePositions = redactionLogImage.getPositions()
+                    .stream()
+                    .map(RectangleWithPage::fromRedactionLogRectangle)
+                    .toList();
             assert imagePositions.size() == 1;
             Optional<Image> optionalClosestImage = images.stream()
-                    .filter(image -> image.onPage(redactionLogImage.getPositions().get(0).getPage()))
-                    .min(Comparator.comparingDouble(image -> entityFindingUtility.calculateDistance(image.getPosition(), imagePositions.get(0).rectangle2D())))
-                    .filter(image -> entityFindingUtility.calculateDistance(image.getPosition(), imagePositions.get(0).rectangle2D()) <= MATCH_THRESHOLD);
+                    .filter(image -> image.onPage(redactionLogImage.getPositions()
+                                                          .get(0).getPage()))
+                    .min(Comparator.comparingDouble(image -> EntityFindingUtility.calculateDistance(image.getPosition(), imagePositions.get(0).rectangle2D())))
+                    .filter(image -> EntityFindingUtility.calculateDistance(image.getPosition(), imagePositions.get(0).rectangle2D()) <= MATCH_THRESHOLD);
 
             Image closestImage;
             if (optionalClosestImage.isEmpty()) { // if no fitting image can be found create a new one with the previous values!
@@ -131,15 +235,23 @@ public class RedactionLogToEntityLogMigrationService {
                 images.remove(closestImage);
             }
 
-            String ruleIdentifier = "OLDIMG." + redactionLogImage.getMatchedRule() + ".0";
-            if (redactionLogImage.lastChangeIsRemoved()) {
-                closestImage.remove(ruleIdentifier, redactionLogImage.getReason());
-            } else if (redactionLogImage.isRedacted()) {
-                closestImage.apply(ruleIdentifier, redactionLogImage.getReason(), redactionLogImage.getLegalBasis());
+            String ruleIdentifier;
+            String reason = Optional.ofNullable(redactionLogImage.getReason())
+                    .orElse("");
+            if (redactionLogImage.getMatchedRule().isBlank() || redactionLogImage.getMatchedRule() == null) {
+                ruleIdentifier = "OLDIMG.0.0";
             } else {
-                closestImage.skip(ruleIdentifier, redactionLogImage.getReason());
+                ruleIdentifier = "OLDIMG." + redactionLogImage.getMatchedRule() + ".0";
             }
-            migrationEntities.add(new MigrationEntity(null, redactionLogImage, closestImage, redactionLogImage.getId(), closestImage.getId()));
+
+            if (redactionLogImage.lastChangeIsRemoved()) {
+                closestImage.remove(ruleIdentifier, reason);
+            } else if (redactionLogImage.isRedacted()) {
+                closestImage.apply(ruleIdentifier, reason, redactionLogImage.getLegalBasis());
+            } else {
+                closestImage.skip(ruleIdentifier, reason);
+            }
+            migrationEntities.add(MigrationEntity.fromRedactionLogImage(redactionLogImage, closestImage, fileId, dictionaryService, dossierTemplateId));
         }
         return migrationEntities;
     }
@@ -151,13 +263,24 @@ public class RedactionLogToEntityLogMigrationService {
                 .documentTree(document.getDocumentTree())
                 .imageType(ImageType.fromString(redactionLogImage.getType()))
                 .transparent(redactionLogImage.isImageHasTransparency())
-                .page(document.getPages().stream().filter(p -> p.getNumber() == redactionLogImage.getPositions().get(0).getPage()).findFirst().orElseThrow())
-                .position(toRectangle2D(redactionLogImage.getPositions().get(0)))
+                .page(document.getPages()
+                              .stream()
+                              .filter(p -> p.getNumber() == redactionLogImage.getPositions()
+                                      .get(0).getPage())
+                              .findFirst()
+                              .orElseThrow())
+                .position(toRectangle2D(redactionLogImage.getPositions()
+                                                .get(0)))
                 .build();
 
         List<Integer> treeId = document.getDocumentTree().createNewMainEntryAndReturnId(image);
         image.setTreeId(treeId);
-        image.setId(IdBuilder.buildId(image.getPages(), image.getBBox().values().stream().toList(), "", ""));
+        image.setId(IdBuilder.buildId(image.getPages(),
+                                      image.getBBox().values()
+                                              .stream()
+                                              .toList(),
+                                      "",
+                                      ""));
 
         return image;
     }
@@ -169,62 +292,63 @@ public class RedactionLogToEntityLogMigrationService {
     }
 
 
-    private List<MigrationEntity> getTextBasedMigrationEntities(RedactionLog redactionLog, Document document) {
+    private List<MigrationEntity> getTextBasedMigrationEntities(RedactionLog redactionLog, Document document, String dossierTemplateId, String fileId) {
 
         List<MigrationEntity> entitiesToMigrate = redactionLog.getRedactionLogEntry()
                 .stream()
                 .filter(redactionLogEntry -> !redactionLogEntry.isImage())
-                .map(MigrationEntity::fromRedactionLogEntry)
-                .peek(migrationEntity -> {
-                    if (migrationEntity.getRedactionLogEntry().lastChangeIsRemoved()) {
-                        migrationEntity.getManualEntity().remove(migrationEntity.getManualEntity().getRuleIdentifier(), migrationEntity.getRedactionLogEntry().getReason());
-                    } else if (migrationEntity.getManualEntity().isApplied()) {
-                        migrationEntity.getManualEntity()
-                                .apply(migrationEntity.getManualEntity().getRuleIdentifier(),
-                                        migrationEntity.getManualEntity().getReason(),
-                                        migrationEntity.getManualEntity().getLegalBasis());
-                    } else {
-                        migrationEntity.getManualEntity().skip(migrationEntity.getManualEntity().getRuleIdentifier(), migrationEntity.getManualEntity().getReason());
-                    }
-                })
+                .filter(redactionLogEntry -> isOnExistingPage(redactionLogEntry, document.getNumberOfPages()))
+                .map(entry -> MigrationEntity.fromRedactionLogEntry(entry, fileId, dictionaryService, dossierTemplateId))
                 .toList();
 
-        Map<String, List<TextEntity>> tempEntitiesByValue = entityFindingUtility.findAllPossibleEntitiesAndGroupByValue(document,
-                entitiesToMigrate.stream().map(MigrationEntity::getManualEntity).toList());
+        List<PrecursorEntity> precursorEntities = entitiesToMigrate.stream()
+                .map(MigrationEntity::getPrecursorEntity)
+                .toList();
+
+        log.info("Finding all possible entities");
+        Map<String, List<TextEntity>> tempEntitiesByValue = entityFindingUtility.findAllPossibleEntitiesAndGroupByValue(document, precursorEntities);
 
         for (MigrationEntity migrationEntity : entitiesToMigrate) {
-            Optional<TextEntity> optionalTextEntity = entityFindingUtility.findClosestEntityAndReturnEmptyIfNotFound(migrationEntity.getManualEntity(),
-                    tempEntitiesByValue,
-                    MATCH_THRESHOLD);
+            Optional<TextEntity> optionalTextEntity = entityFindingUtility.findClosestEntityAndReturnEmptyIfNotFound(migrationEntity.getPrecursorEntity(),
+                                                                                                                     tempEntitiesByValue,
+                                                                                                                     MATCH_THRESHOLD);
 
             if (optionalTextEntity.isEmpty()) {
-                migrationEntity.setMigratedEntity(migrationEntity.getManualEntity());
-                migrationEntity.setOldId(migrationEntity.getManualEntity().getId());
-                migrationEntity.setNewId(migrationEntity.getManualEntity().getId());
+                migrationEntity.setMigratedEntity(migrationEntity.getPrecursorEntity());
+                migrationEntity.setOldId(migrationEntity.getPrecursorEntity().getId());
+                migrationEntity.setNewId(migrationEntity.getPrecursorEntity().getId());
                 continue;
             }
 
-            TextEntity entity = createCorrectEntity(migrationEntity.getManualEntity(), document, optionalTextEntity.get().getTextRange());
-            migrationEntity.setMigratedEntity(entity);
-            migrationEntity.setOldId(migrationEntity.getManualEntity().getId());
-            migrationEntity.setNewId(entity.getPositionsOnPagePerPage().get(0).getId()); // Can only be on one page, since redactionLogEntries can only be on one page
+            TextEntity migratedEntity = EntityFromPrecursorCreationService.createCorrectEntity(migrationEntity.getPrecursorEntity(), optionalTextEntity.get(), true);
+
+            migrationEntity.setMigratedEntity(migratedEntity);
+            migrationEntity.setOldId(migrationEntity.getPrecursorEntity().getId());
+            migrationEntity.setNewId(migratedEntity.getId());
         }
 
-        tempEntitiesByValue.values().stream().flatMap(Collection::stream).forEach(TextEntity::removeFromGraph);
+        tempEntitiesByValue.values()
+                .stream()
+                .flatMap(Collection::stream)
+                .forEach(TextEntity::removeFromGraph);
+
         return entitiesToMigrate;
     }
 
 
-    private TextEntity createCorrectEntity(ManualEntity manualEntity, SemanticNode node, TextRange closestTextRange) {
+    private boolean isOnExistingPage(RedactionLogEntry redactionLogEntry, int numberOfPages) {
 
-        EntityCreationService entityCreationService = new EntityCreationService(entityEnrichmentService);
-        TextEntity correctEntity = entityCreationService.forceByTextRange(closestTextRange, manualEntity.getType(), manualEntity.getEntityType(), node);
+        var pages = redactionLogEntry.getPositions()
+                .stream()
+                .map(Rectangle::getPage)
+                .collect(Collectors.toSet());
 
-        correctEntity.addMatchedRules(manualEntity.getMatchedRuleList());
-        correctEntity.setDictionaryEntry(manualEntity.isDictionaryEntry());
-        correctEntity.setDossierDictionaryEntry(manualEntity.isDossierDictionaryEntry());
-        correctEntity.getManualOverwrite().addChanges(manualEntity.getManualOverwrite().getManualChangeLog());
-        return correctEntity;
+        for (int page : pages) {
+            if (page > numberOfPages) {
+                return false;
+            }
+        }
+        return true;
     }
 
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/ClosestEntity.java

@@ -1,6 +1,6 @@
 package com.iqser.red.service.redaction.v1.server.model;
 
-import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
+import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
 
 import lombok.Builder;
 import lombok.Getter;
@@ -12,6 +12,6 @@ import lombok.Setter;
 public class ClosestEntity {
 
     private double distance;
-    private TextRange textRange;
+    private TextEntity textEntity;
 
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/KieWrapper.java

@@ -14,4 +14,5 @@ public record KieWrapper(KieContainer container, long rulesVersion) {
 
         return container != null && rulesVersion >= 0;
     }
+
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/MigratedEntityLog.java

@@ -1,7 +1,10 @@
 package com.iqser.red.service.redaction.v1.server.model;
 
+import java.util.List;
+
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.migration.MigratedIds;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
 
 import lombok.AllArgsConstructor;
 import lombok.Builder;

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/MigrationEntity.java

@@ -1,67 +1,148 @@
 package com.iqser.red.service.redaction.v1.server.model;
 
+import static com.iqser.red.service.redaction.v1.server.service.EntityLogCreatorService.buildEntryState;
+import static com.iqser.red.service.redaction.v1.server.service.EntityLogCreatorService.buildEntryType;
+
+import java.awt.geom.Rectangle2D;
+import java.time.OffsetDateTime;
 import java.util.Collections;
+import java.util.LinkedList;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
+import java.util.Optional;
 import java.util.Set;
 import java.util.stream.Collectors;
 
-import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ChangeType;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryType;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Change;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ManualChange;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.BaseAnnotation;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntryType;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ManualRedactionType;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
+import com.iqser.red.service.redaction.v1.server.migration.MigrationMapper;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.IEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.ManualChangeOverwrite;
-import com.iqser.red.service.redaction.v1.server.model.document.entity.PositionOnPage;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Image;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.ImageType;
+import com.iqser.red.service.redaction.v1.server.service.DictionaryService;
+import com.iqser.red.service.redaction.v1.server.service.ManualChangeFactory;
+import com.iqser.red.service.redaction.v1.server.service.ManualChangesApplicationService;
 
 import lombok.AllArgsConstructor;
+import lombok.Builder;
 import lombok.Data;
 import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
 
+@Slf4j
 @Data
+@Builder
 @AllArgsConstructor
 @RequiredArgsConstructor
 public final class MigrationEntity {
 
-    private final ManualEntity manualEntity;
+    private final PrecursorEntity precursorEntity;
     private final RedactionLogEntry redactionLogEntry;
+    private final DictionaryService dictionaryService;
+    private final String dossierTemplateId;
     private IEntity migratedEntity;
     private String oldId;
     private String newId;
+    private String fileId;
+
+    @Builder.Default
+    List<BaseAnnotation> manualChanges = new LinkedList<>();
 
 
-    public static MigrationEntity fromRedactionLogEntry(RedactionLogEntry redactionLogEntry) {
+    public static MigrationEntity fromRedactionLogEntry(RedactionLogEntry redactionLogEntry, String fileId, DictionaryService dictionaryService, String dossierTemplateId) {
 
-        return new MigrationEntity(createManualEntity(redactionLogEntry), redactionLogEntry);
+        boolean hint = dictionaryService.isHint(redactionLogEntry.getType(), dossierTemplateId);
+        PrecursorEntity precursorEntity = createPrecursorEntity(redactionLogEntry, hint);
+        if (precursorEntity.getEntityType().equals(EntityType.HINT) && !redactionLogEntry.isHint() && !redactionLogEntry.isRedacted()) {
+            precursorEntity.ignore(precursorEntity.getRuleIdentifier(), precursorEntity.getReason());
+        } else if (redactionLogEntry.lastChangeIsRemoved()) {
+            precursorEntity.remove(precursorEntity.getRuleIdentifier(), precursorEntity.getReason());
+        } else if (lastManualChangeIsRemove(redactionLogEntry)) {
+            precursorEntity.ignore(precursorEntity.getRuleIdentifier(), precursorEntity.getReason());
+        } else if (precursorEntity.isApplied() && redactionLogEntry.isRecommendation()) {
+            precursorEntity.skip(precursorEntity.getRuleIdentifier(), precursorEntity.getReason());
+        } else if (precursorEntity.isApplied()) {
+            precursorEntity.apply(precursorEntity.getRuleIdentifier(), precursorEntity.getReason(), precursorEntity.getLegalBasis());
+        } else {
+            precursorEntity.skip(precursorEntity.getRuleIdentifier(), precursorEntity.getReason());
+        }
+
+        return MigrationEntity.builder()
+                .precursorEntity(precursorEntity)
+                .redactionLogEntry(redactionLogEntry)
+                .oldId(redactionLogEntry.getId())
+                .fileId(fileId)
+                .dictionaryService(dictionaryService)
+                .dossierTemplateId(dossierTemplateId)
+                .build();
     }
 
 
-    public static ManualEntity createManualEntity(RedactionLogEntry redactionLogEntry) {
+    public static MigrationEntity fromRedactionLogImage(RedactionLogEntry redactionLogImage,
+                                                        Image image,
+                                                        String fileId,
+                                                        DictionaryService dictionaryService,
+                                                        String dossierTemplateId) {
+
+        return MigrationEntity.builder()
+                .redactionLogEntry(redactionLogImage)
+                .migratedEntity(image)
+                .oldId(redactionLogImage.getId())
+                .newId(image.getId())
+                .fileId(fileId)
+                .dictionaryService(dictionaryService)
+                .dossierTemplateId(dossierTemplateId)
+                .build();
+    }
+
+
+    private static boolean lastManualChangeIsRemove(RedactionLogEntry redactionLogEntry) {
+
+        if (redactionLogEntry.getManualChanges() == null) {
+            return false;
+        }
+
+        return redactionLogEntry.getManualChanges()
+                .stream()
+                .reduce((a, b) -> b)
+                .map(m -> m.getManualRedactionType().equals(ManualRedactionType.REMOVE_LOCALLY))
+                .orElse(false);
+    }
+
+
+    public static PrecursorEntity createPrecursorEntity(RedactionLogEntry redactionLogEntry, boolean hint) {
 
         String ruleIdentifier = buildRuleIdentifier(redactionLogEntry);
-        List<RectangleWithPage> rectangleWithPages = redactionLogEntry.getPositions().stream().map(RectangleWithPage::fromRedactionLogRectangle).toList();
-        EntityType entityType = getEntityType(redactionLogEntry);
-        return ManualEntity.builder()
+        List<RectangleWithPage> rectangleWithPages = redactionLogEntry.getPositions()
+                .stream()
+                .map(RectangleWithPage::fromRedactionLogRectangle)
+                .toList();
+        EntityType entityType = getEntityType(redactionLogEntry, hint);
+        return PrecursorEntity.builder()
                 .id(redactionLogEntry.getId())
                 .value(redactionLogEntry.getValue())
                 .entityPosition(rectangleWithPages)
                 .ruleIdentifier(ruleIdentifier)
-                .reason(redactionLogEntry.getReason())
+                .reason(Optional.ofNullable(redactionLogEntry.getReason())
+                                .orElse(""))
                 .legalBasis(redactionLogEntry.getLegalBasis())
                 .type(redactionLogEntry.getType())
                 .section(redactionLogEntry.getSection())
+                .engines(MigrationMapper.getMigratedEngines(redactionLogEntry))
                 .entityType(entityType)
                 .applied(redactionLogEntry.isRedacted())
                 .isDictionaryEntry(redactionLogEntry.isDictionaryEntry())
@@ -84,74 +165,24 @@ public final class MigrationEntity {
     }
 
 
-    private static EntityType getEntityType(RedactionLogEntry redactionLogEntry) {
+    private static EntityType getEntityType(RedactionLogEntry redactionLogEntry, boolean hint) {
 
-        if (redactionLogEntry.isRecommendation()) {
-            return EntityType.RECOMMENDATION;
+        if (hint) {
+            return EntityType.HINT;
+        }
+        if (redactionLogEntry.isFalsePositive()) {
+            return EntityType.FALSE_POSITIVE;
         }
         if (redactionLogEntry.isHint()) {
             return EntityType.HINT;
         }
+        if (redactionLogEntry.isRecommendation()) {
+            return EntityType.RECOMMENDATION;
+        }
         return EntityType.ENTITY;
     }
 
 
-    private static com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Change toEntityLogChanges(Change change) {
-
-        return new com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Change(change.getAnalysisNumber(),
-                toEntityLogType(change.getType()),
-                change.getDateTime());
-    }
-
-
-    private static EntryType getEntryType(EntityType entityType) {
-
-        return switch (entityType) {
-            case ENTITY -> EntryType.ENTITY;
-            case HINT -> EntryType.HINT;
-            case FALSE_POSITIVE -> EntryType.FALSE_POSITIVE;
-            case RECOMMENDATION -> EntryType.RECOMMENDATION;
-            case FALSE_RECOMMENDATION -> EntryType.FALSE_RECOMMENDATION;
-        };
-    }
-
-
-    private static com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualChange toEntityLogManualChanges(ManualChange manualChange) {
-
-        return new com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualChange(toManualRedactionType(manualChange.getManualRedactionType()),
-                manualChange.getProcessedDate(),
-                manualChange.getRequestedDate(),
-                manualChange.getUserId(),
-                manualChange.getPropertyChanges());
-    }
-
-
-    private static ChangeType toEntityLogType(com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ChangeType type) {
-
-        return switch (type) {
-            case ADDED -> ChangeType.ADDED;
-            case REMOVED -> ChangeType.REMOVED;
-            case CHANGED -> ChangeType.CHANGED;
-        };
-    }
-
-
-    private static com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType toManualRedactionType(ManualRedactionType manualRedactionType) {
-
-        return switch (manualRedactionType) {
-            case ADD_LOCALLY -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.ADD_LOCALLY;
-            case ADD_TO_DICTIONARY -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.ADD_TO_DICTIONARY;
-            case REMOVE_LOCALLY -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.REMOVE_LOCALLY;
-            case REMOVE_FROM_DICTIONARY -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.REMOVE_FROM_DICTIONARY;
-            case FORCE_REDACT -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.FORCE_REDACT;
-            case FORCE_HINT -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.FORCE_HINT;
-            case RECATEGORIZE -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.RECATEGORIZE;
-            case LEGAL_BASIS_CHANGE -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.LEGAL_BASIS_CHANGE;
-            case RESIZE -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.RESIZE;
-        };
-    }
-
-
     public EntityLogEntry toEntityLogEntry(Map<String, String> oldToNewIdMapping) {
 
         EntityLogEntry entityLogEntry;
@@ -159,43 +190,41 @@ public final class MigrationEntity {
             entityLogEntry = createEntityLogEntry(image);
         } else if (migratedEntity instanceof TextEntity textEntity) {
             entityLogEntry = createEntityLogEntry(textEntity);
-        } else if (migratedEntity instanceof ManualEntity entity) {
+        } else if (migratedEntity instanceof PrecursorEntity entity) {
             entityLogEntry = createEntityLogEntry(entity);
         } else {
             throw new UnsupportedOperationException("Unknown subclass " + migratedEntity.getClass());
         }
-        entityLogEntry.setChanges(redactionLogEntry.getChanges().stream().map(MigrationEntity::toEntityLogChanges).toList());
-        entityLogEntry.setManualChanges(migrateManualChanges(redactionLogEntry.getManualChanges()));
+
+        entityLogEntry.setManualChanges(ManualChangeFactory.toManualChangeList(migratedEntity.getManualOverwrite().getManualChangeLog(), redactionLogEntry.isHint()));
         entityLogEntry.setColor(redactionLogEntry.getColor());
+        entityLogEntry.setChanges(redactionLogEntry.getChanges()
+                                          .stream()
+                                          .map(MigrationMapper::toEntityLogChanges)
+                                          .toList());
         entityLogEntry.setReference(migrateSetOfIds(redactionLogEntry.getReference(), oldToNewIdMapping));
         entityLogEntry.setImportedRedactionIntersections(migrateSetOfIds(redactionLogEntry.getImportedRedactionIntersections(), oldToNewIdMapping));
-        entityLogEntry.setEngines(getMigratedEngines(redactionLogEntry));
-        if (redactionLogEntry.getLegalBasis() != null) {
-            entityLogEntry.setLegalBasis(redactionLogEntry.getLegalBasis());
+        entityLogEntry.setEngines(MigrationMapper.getMigratedEngines(redactionLogEntry));
+
+        if (entityLogEntry.getEntryType().equals(EntryType.HINT) && lastManualChangeIsRemoveLocally(entityLogEntry)) {
+            entityLogEntry.setState(EntryState.IGNORED);
+        }
+
+        if (redactionLogEntry.isImported() && redactionLogEntry.getValue() == null) {
+            entityLogEntry.setValue("Imported Redaction");
         }
 
         return entityLogEntry;
     }
 
 
-    private List<com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualChange> migrateManualChanges(List<ManualChange> manualChanges) {
+    private static boolean lastManualChangeIsRemoveLocally(EntityLogEntry entityLogEntry) {
 
-        if (manualChanges == null) {
-            return Collections.emptyList();
-        }
-        return manualChanges.stream()
-                .filter(manualChange -> manualChange.getAnnotationStatus().equals(AnnotationStatus.APPROVED))
-                .map(MigrationEntity::toEntityLogManualChanges)
-                .toList();
-    }
-
-
-    private static Set<com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine> getMigratedEngines(RedactionLogEntry entry) {
-
-        if (entry.getEngines() == null) {
-            return Collections.emptySet();
-        }
-        return entry.getEngines().stream().map(MigrationEntity::toEntityLogEngine).collect(Collectors.toSet());
+        return entityLogEntry.getManualChanges()
+                .stream()
+                .reduce((a, b) -> b)
+                .filter(mc -> mc.getManualRedactionType().equals(com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.REMOVE))
+                .isPresent();
     }
 
 
@@ -204,72 +233,67 @@ public final class MigrationEntity {
         if (ids == null) {
             return Collections.emptySet();
         }
-        return ids.stream().map(oldToNewIdMapping::get).collect(Collectors.toSet());
-    }
-
-
-    private static com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine toEntityLogEngine(Engine engine) {
-
-        return switch (engine) {
-            case DICTIONARY -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine.DICTIONARY;
-            case NER -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine.NER;
-            case RULE -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine.RULE;
-        };
+        return ids.stream()
+                .map(oldToNewIdMapping::get)
+                .collect(Collectors.toSet());
     }
 
 
     public EntityLogEntry createEntityLogEntry(Image image) {
 
         String imageType = image.getImageType().equals(ImageType.OTHER) ? "image" : image.getImageType().toString().toLowerCase(Locale.ENGLISH);
+        List<Position> positions = getPositionsFromOverride(image).orElse(List.of(new Position(image.getPosition(), image.getPage().getNumber())));
         return EntityLogEntry.builder()
                 .id(image.getId())
-                .value(image.value())
+                .value(image.getValue())
                 .type(imageType)
                 .reason(image.buildReasonWithManualChangeDescriptions())
-                .legalBasis(image.legalBasis())
+                .legalBasis(image.getManualOverwrite().getLegalBasis()
+                                    .orElse(redactionLogEntry.getLegalBasis()))
                 .matchedRule(image.getMatchedRule().getRuleIdentifier().toString())
                 .dictionaryEntry(false)
-                .positions(List.of(new Position(image.getPosition(), image.getPage().getNumber())))
+                .positions(positions)
                 .containingNodeId(image.getTreeId())
                 .closestHeadline(image.getHeadline().getTextBlock().getSearchText())
-                .section(redactionLogEntry.getSection())
+                .section(image.getManualOverwrite().getSection()
+                                 .orElse(redactionLogEntry.getSection()))
                 .textAfter(redactionLogEntry.getTextAfter())
                 .textBefore(redactionLogEntry.getTextBefore())
                 .imageHasTransparency(image.isTransparent())
                 .state(buildEntryState(image))
-                .entryType(redactionLogEntry.isHint() ? EntryType.IMAGE_HINT : EntryType.IMAGE)
+                .entryType(dictionaryService.isHint(imageType, dossierTemplateId) ? EntryType.IMAGE_HINT : EntryType.IMAGE)
                 .build();
 
     }
 
 
-    public EntityLogEntry createEntityLogEntry(ManualEntity manualEntity) {
+    public EntityLogEntry createEntityLogEntry(PrecursorEntity precursorEntity) {
 
-        String type = manualEntity.getManualOverwrite().getType().orElse(manualEntity.getType());
         return EntityLogEntry.builder()
-                .id(manualEntity.getId())
-                .reason(manualEntity.buildReasonWithManualChangeDescriptions())
-                .legalBasis(manualEntity.legalBasis())
-                .value(manualEntity.value())
-                .type(type)
-                .state(buildEntryState(manualEntity))
-                .entryType(buildEntryType(manualEntity))
-                .section(redactionLogEntry.getSection())
+                .id(precursorEntity.getId())
+                .reason(precursorEntity.buildReasonWithManualChangeDescriptions())
+                .legalBasis(precursorEntity.getManualOverwrite().getLegalBasis()
+                                    .orElse(redactionLogEntry.getLegalBasis()))
+                .value(precursorEntity.value())
+                .type(precursorEntity.type())
+                .state(buildEntryState(precursorEntity))
+                .entryType(buildEntryType(precursorEntity))
+                .section(precursorEntity.getManualOverwrite().getSection()
+                                 .orElse(redactionLogEntry.getSection()))
                 .textAfter(redactionLogEntry.getTextAfter())
                 .textBefore(redactionLogEntry.getTextBefore())
                 .containingNodeId(Collections.emptyList())
                 .closestHeadline("")
-                .matchedRule(manualEntity.getMatchedRule().getRuleIdentifier().toString())
-                .dictionaryEntry(manualEntity.isDictionaryEntry())
-                .dossierDictionaryEntry(manualEntity.isDossierDictionaryEntry())
+                .matchedRule(precursorEntity.getMatchedRule().getRuleIdentifier().toString())
+                .dictionaryEntry(precursorEntity.isDictionaryEntry())
+                .dossierDictionaryEntry(precursorEntity.isDossierDictionaryEntry())
                 .startOffset(-1)
                 .endOffset(-1)
-                .positions(manualEntity.getManualOverwrite()
-                        .getPositions()
-                        .orElse(manualEntity.getEntityPosition())
-                        .stream()
-                        .map(entityPosition -> new Position(entityPosition.rectangle2D(), entityPosition.pageNumber()))
-                        .toList())
+                .positions(precursorEntity.getManualOverwrite().getPositions()
+                                   .orElse(precursorEntity.getEntityPosition())
+                                   .stream()
+                                   .map(entityPosition -> new Position(entityPosition.rectangle2D(), entityPosition.pageNumber()))
+                                   .toList())
                 .engines(Collections.emptySet())
                 .build();
     }
@@ -278,21 +302,20 @@ public final class MigrationEntity {
     public EntityLogEntry createEntityLogEntry(TextEntity entity) {
 
         assert entity.getPositionsOnPagePerPage().size() == 1;
-        PositionOnPage positionOnPage = entity.getPositionsOnPagePerPage().get(0);
-        List<Position> rectanglesPerLine = positionOnPage.getRectanglePerLine()
-                .stream()
-                .map(rectangle2D -> new Position(rectangle2D, positionOnPage.getPage().getNumber()))
-                .toList();
+        List<Position> rectanglesPerLine = getRectanglesPerLine(entity);
         return EntityLogEntry.builder()
-                .id(positionOnPage.getId())
+                .id(entity.getId())
                 .positions(rectanglesPerLine)
                 .reason(entity.buildReasonWithManualChangeDescriptions())
-                .legalBasis(entity.legalBasis())
-                .value(entity.getManualOverwrite().getValue().orElse(entity.getMatchedRule().isWriteValueWithLineBreaks() ? entity.getValueWithLineBreaks() : entity.getValue()))
-                .type(entity.getType())
-                .section(redactionLogEntry.getSection())
-                .textAfter(redactionLogEntry.getTextAfter())
-                .textBefore(redactionLogEntry.getTextBefore())
+                .legalBasis(entity.getManualOverwrite().getLegalBasis()
+                                    .orElse(redactionLogEntry.getLegalBasis()))
+                .value(entity.getManualOverwrite().getValue()
+                               .orElse(entity.getMatchedRule().isWriteValueWithLineBreaks() ? entity.getValueWithLineBreaks() : entity.getValue()))
+                .type(entity.type())
+                .section(entity.getManualOverwrite().getSection()
+                                 .orElse(redactionLogEntry.getSection()))
+                .textAfter(entity.getTextAfter())
+                .textBefore(entity.getTextBefore())
                 .containingNodeId(entity.getDeepestFullyContainingNode().getTreeId())
                 .closestHeadline(entity.getDeepestFullyContainingNode().getHeadline().getTextBlock().getSearchText())
                 .matchedRule(entity.getMatchedRule().getRuleIdentifier().toString())
@@ -307,40 +330,132 @@ public final class MigrationEntity {
     }
 
 
-    private EntryState buildEntryState(IEntity entity) {
+    private static List<Position> getRectanglesPerLine(TextEntity entity) {
 
-        if (entity.applied() && entity.active()) {
-            return EntryState.APPLIED;
-        } else if (entity.skipped() && entity.active()) {
-            return EntryState.SKIPPED;
-        } else if (entity.ignored()) {
-            return EntryState.IGNORED;
-        } else {
-            return EntryState.REMOVED;
-        }
+        return getPositionsFromOverride(entity).orElse(entity.getPositionsOnPagePerPage()
+                                                               .get(0).getRectanglePerLine()
+                                                               .stream()
+                                                               .map(rectangle2D -> new Position(rectangle2D,
+                                                                                                entity.getPositionsOnPagePerPage()
+                                                                                                        .get(0).getPage().getNumber()))
+                                                               .toList());
     }
 
 
-    private EntryType buildEntryType(IEntity entity) {
+    private static Optional<List<Position>> getPositionsFromOverride(IEntity entity) {
 
-        if (entity instanceof TextEntity textEntity) {
-            return getEntryType(textEntity.getEntityType());
-        } else if (entity instanceof ManualEntity manualEntity) {
-            if (manualEntity.isRectangle()) {
-                return EntryType.AREA;
-            }
-            return getEntryType(manualEntity.getEntityType());
-        } else if (entity instanceof Image) {
-            return EntryType.IMAGE;
-        }
-        throw new UnsupportedOperationException(String.format("Entity subclass %s is not implemented!", entity.getClass()));
+        return entity.getManualOverwrite().getPositions()
+                .map(rects -> rects.stream()
+                        .map(r -> new Position(r.rectangle2D(), r.pageNumber()))
+                        .toList());
     }
 
 
-    public boolean hasManualChangesOrComments() {
+    public boolean hasManualChangesOrComments(Set<String> entitiesWithComments, Set<String> entitiesWithUnprocessedChanges) {
 
         return !(redactionLogEntry.getManualChanges() == null || redactionLogEntry.getManualChanges().isEmpty()) || //
-                !(redactionLogEntry.getComments() == null || redactionLogEntry.getComments().isEmpty());
+               !(redactionLogEntry.getComments() == null || redactionLogEntry.getComments().isEmpty()) //
+               || hasManualChanges() || entitiesWithComments.contains(oldId) || entitiesWithUnprocessedChanges.contains(oldId);
+    }
+
+
+    public boolean hasManualChanges() {
+
+        return !manualChanges.isEmpty();
+    }
+
+
+    public void applyManualChanges(List<BaseAnnotation> manualChangesToApply, ManualChangesApplicationService manualChangesApplicationService) {
+
+        manualChanges.addAll(manualChangesToApply);
+        manualChangesToApply.forEach(manualChange -> {
+            if (manualChange instanceof ManualResizeRedaction manualResizeRedaction && migratedEntity instanceof TextEntity textEntity) {
+                manualResizeRedaction.setAnnotationId(newId);
+                manualChangesApplicationService.resize(textEntity, manualResizeRedaction);
+            } else if (manualChange instanceof ManualRecategorization manualRecategorization && migratedEntity instanceof Image image) {
+                image.setImageType(ImageType.fromString(manualRecategorization.getType()));
+                migratedEntity.getManualOverwrite().addChange(manualChange);
+            } else {
+                migratedEntity.getManualOverwrite().addChange(manualChange);
+            }
+        });
+    }
+
+
+    public ManualRedactionEntry buildManualRedactionEntry() {
+
+        assert hasManualChanges();
+
+        // currently we need to insert a manual redaction entry, whenever an entity has been resized.
+        String user = manualChanges.stream()
+                .filter(mc -> mc instanceof ManualResizeRedaction)
+                .findFirst()
+                .orElse(manualChanges.get(0)).getUser();
+
+        OffsetDateTime requestDate = manualChanges.get(0).getRequestDate();
+
+        return ManualRedactionEntry.builder()
+                .annotationId(newId)
+                .fileId(fileId)
+                .user(user)
+                .requestDate(requestDate)
+                .type(redactionLogEntry.getType())
+                .value(redactionLogEntry.getValue())
+                .reason(redactionLogEntry.getReason())
+                .legalBasis(redactionLogEntry.getLegalBasis())
+                .section(redactionLogEntry.getSection())
+                .rectangle(false)
+                .addToDictionary(false)
+                .addToDossierDictionary(false)
+                .positions(buildPositions(migratedEntity))
+                .textAfter(redactionLogEntry.getTextAfter())
+                .textBefore(redactionLogEntry.getTextBefore())
+                .dictionaryEntryType(DictionaryEntryType.ENTRY)
+                .build();
+    }
+
+
+    private List<Rectangle> buildPositions(IEntity entity) {
+
+        if (entity instanceof TextEntity textEntity) {
+
+            var positionsOnPage = textEntity.getPositionsOnPagePerPage()
+                    .get(0);
+            return positionsOnPage.getRectanglePerLine()
+                    .stream()
+                    .map(p -> new Rectangle((float) p.getX(), (float) p.getY(), (float) p.getWidth(), (float) p.getHeight(), positionsOnPage.getPage().getNumber()))
+                    .toList();
+        }
+        if (entity instanceof PrecursorEntity pEntity) {
+
+            return pEntity.getManualOverwrite().getPositions()
+                    .orElse(pEntity.getEntityPosition())
+                    .stream()
+                    .map(p -> new Rectangle((float) p.rectangle2D().getX(),
+                                            (float) p.rectangle2D().getY(),
+                                            (float) p.rectangle2D().getWidth(),
+                                            (float) p.rectangle2D().getHeight(),
+                                            p.pageNumber()))
+                    .toList();
+        }
+        if (entity instanceof Image image) {
+
+            Rectangle2D position = image.getManualOverwrite().getPositions()
+                    .map(p -> p.get(0).rectangle2D())
+                    .orElse(image.getPosition());
+
+            return List.of(new Rectangle((float) position.getX(), (float) position.getY(), (float) position.getWidth(), (float) position.getHeight(), image.getPage().getNumber()));
+
+        } else {
+            throw new UnsupportedOperationException();
+        }
+    }
+
+
+    public boolean needsManualEntry() {
+
+        return manualChanges.stream()
+                       .anyMatch(mc -> mc instanceof ManualResizeRedaction && !((ManualResizeRedaction) mc).getUpdateDictionary()) && !(migratedEntity instanceof Image);
     }
 
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/NerEntities.java

@@ -11,6 +11,10 @@ import lombok.AllArgsConstructor;
 import lombok.Getter;
 import lombok.experimental.FieldDefaults;
 
+/**
+ * Represents a collection of named entity recognition (NER) entities.
+ * This class provides methods to manage and query NER entities.
+ */
 @Getter
 @AllArgsConstructor
 @FieldDefaults(level = AccessLevel.PRIVATE, makeFinal = true)
@@ -25,18 +29,35 @@ public class NerEntities {
     }
 
 
+    /**
+     * Checks if there are any entities of a specified type.
+     *
+     * @param type The type of entity to check for.
+     * @return true if there is at least one entity of the specified type, false otherwise.
+     */
     public boolean hasEntitiesOfType(String type) {
 
-        return nerEntityList.stream().anyMatch(nerEntity -> nerEntity.type.equals(type));
+        return nerEntityList.stream()
+                .anyMatch(nerEntity -> nerEntity.type.equals(type));
     }
 
 
+    /**
+     * Returns a stream of NER entities of a specified type.
+     *
+     * @param type The type of entities to return.
+     * @return a stream of {@link NerEntity} objects of the specified type.
+     */
     public Stream<NerEntity> streamEntitiesOfType(String type) {
 
-        return nerEntityList.stream().filter(nerEntity -> nerEntity.type().equals(type));
+        return nerEntityList.stream()
+                .filter(nerEntity -> nerEntity.type().equals(type));
     }
 
 
+    /**
+     * Represents a single NER entity with its value, text range, and type.
+     */
     public record NerEntity(String value, TextRange textRange, String type) {
 
     }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/PrecursorEntity.java

@@ -1,11 +1,19 @@
 package com.iqser.red.service.redaction.v1.server.model;
 
-import java.util.List;
-import java.util.PriorityQueue;
+import static com.iqser.red.service.redaction.v1.server.service.NotFoundImportedEntitiesService.IMPORTED_REDACTION_TYPE;
 
+import java.util.List;
+import java.util.Optional;
+import java.util.PriorityQueue;
+import java.util.Set;
+import java.util.UUID;
+
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryType;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedaction;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
 import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.IEntity;
@@ -22,7 +30,8 @@ import lombok.experimental.FieldDefaults;
 @Builder
 @AllArgsConstructor
 @FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
-public class ManualEntity implements IEntity {
+@SuppressWarnings("PMD.AvoidFieldNameMatchingMethodName")
+public class PrecursorEntity implements IEntity {
 
     // The id must be mapped into a TextEntity as is for comments to work correctly
     String id;
@@ -38,19 +47,24 @@ public class ManualEntity implements IEntity {
     boolean isDictionaryEntry;
     boolean isDossierDictionaryEntry;
     boolean rectangle;
+    Set<Engine> engines;
 
     @Builder.Default
     PriorityQueue<MatchedRule> matchedRuleList = new PriorityQueue<>();
     ManualChangeOverwrite manualOverwrite;
 
 
-    public static ManualEntity fromManualRedactionEntry(ManualRedactionEntry manualRedactionEntry, boolean hint) {
+    public static PrecursorEntity fromManualRedactionEntry(ManualRedactionEntry manualRedactionEntry, boolean hint) {
+
+        List<RectangleWithPage> rectangleWithPages = manualRedactionEntry.getPositions()
+                .stream()
+                .map(RectangleWithPage::fromAnnotationRectangle)
+                .toList();
 
-        List<RectangleWithPage> rectangleWithPages = manualRedactionEntry.getPositions().stream().map(RectangleWithPage::fromAnnotationRectangle).toList();
         var entityType = hint ? EntityType.HINT : EntityType.ENTITY;
         ManualChangeOverwrite manualChangeOverwrite = new ManualChangeOverwrite(entityType);
         manualChangeOverwrite.addChange(manualRedactionEntry);
-        return ManualEntity.builder()
+        return PrecursorEntity.builder()
                 .id(manualRedactionEntry.getAnnotationId())
                 .value(manualRedactionEntry.getValue())
                 .entityPosition(rectangleWithPages)
@@ -65,14 +79,19 @@ public class ManualEntity implements IEntity {
                 .isDossierDictionaryEntry(false)
                 .rectangle(manualRedactionEntry.isRectangle())
                 .manualOverwrite(manualChangeOverwrite)
+                .engines(Set.of(Engine.MANUAL))
                 .build();
     }
 
-    public static ManualEntity fromEntityLogEntry(EntityLogEntry entityLogEntry) {
 
-        List<RectangleWithPage> rectangleWithPages = entityLogEntry.getPositions().stream().map(RectangleWithPage::fromEntityLogPosition).toList();
+    public static PrecursorEntity fromEntityLogEntry(EntityLogEntry entityLogEntry) {
+
+        List<RectangleWithPage> rectangleWithPages = entityLogEntry.getPositions()
+                .stream()
+                .map(RectangleWithPage::fromEntityLogPosition)
+                .toList();
         EntityType entityType = getEntityType(entityLogEntry.getEntryType());
-        return ManualEntity.builder()
+        return PrecursorEntity.builder()
                 .id(entityLogEntry.getId())
                 .value(entityLogEntry.getValue())
                 .entityPosition(rectangleWithPages)
@@ -85,6 +104,54 @@ public class ManualEntity implements IEntity {
                 .isDictionaryEntry(entityLogEntry.isDictionaryEntry())
                 .isDossierDictionaryEntry(entityLogEntry.isDossierDictionaryEntry())
                 .manualOverwrite(new ManualChangeOverwrite(entityType))
+                .engines(entityLogEntry.getEngines())
+                .build();
+    }
+
+
+    public static PrecursorEntity fromImportedEntry(ImportedRedaction importedRedaction) {
+
+        List<RectangleWithPage> rectangleWithPages = importedRedaction.getPositions()
+                .stream()
+                .map(RectangleWithPage::fromEntityLogPosition)
+                .toList();
+        EntryType entryType = Optional.ofNullable(importedRedaction.getEntryType())
+                .orElse(EntryType.ENTITY);
+        EntityType entityType = getEntityType(entryType);
+        String value = Optional.ofNullable(importedRedaction.getValue())
+                .orElse("");
+        return PrecursorEntity.builder()
+                .id(importedRedaction.getId())
+                .value(value)
+                .entityPosition(rectangleWithPages)
+                .reason(Optional.ofNullable(importedRedaction.getReason())
+                                .orElse(""))
+                .legalBasis(Optional.ofNullable(importedRedaction.getLegalBasis())
+                                    .orElse(""))
+                .type(Optional.ofNullable(importedRedaction.getType())
+                              .orElse(IMPORTED_REDACTION_TYPE))
+                .entityType(entityType)
+                .isDictionaryEntry(false)
+                .isDossierDictionaryEntry(false)
+                .rectangle(value.isBlank() || entryType.equals(EntryType.IMAGE) || entryType.equals(EntryType.IMAGE_HINT) || entryType.equals(EntryType.AREA))
+                .manualOverwrite(new ManualChangeOverwrite(entityType))
+                .engines(Set.of(Engine.IMPORTED))
+                .build();
+    }
+
+
+    public static PrecursorEntity fromManualResizeRedaction(ManualResizeRedaction manualResizeRedaction) {
+
+        List<RectangleWithPage> rectangleWithPages = manualResizeRedaction.getPositions()
+                .stream()
+                .map(RectangleWithPage::fromAnnotationRectangle)
+                .toList();
+        return PrecursorEntity.builder()
+                .id(UUID.randomUUID().toString())
+                .value(manualResizeRedaction.getValue())
+                .entityPosition(rectangleWithPages)
+                .entityType(EntityType.ENTITY)
+                .manualOverwrite(new ManualChangeOverwrite(EntityType.ENTITY))
                 .build();
     }
 
@@ -99,9 +166,11 @@ public class ManualEntity implements IEntity {
     @Override
     public String type() {
 
-        return getManualOverwrite().getType().orElse(type);
+        return getManualOverwrite().getType()
+                .orElse(type);
     }
 
+
     private static EntityType getEntityType(EntryType entryType) {
 
         switch (entryType) {
@@ -123,8 +192,4 @@ public class ManualEntity implements IEntity {
         }
     }
 
-    private EntityType getEntityType(boolean isHint) {
-        return isHint ? EntityType.HINT : EntityType.ENTITY;
-    }
-
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/component/Entity.java

@@ -1,6 +1,7 @@
 package com.iqser.red.service.redaction.v1.server.model.component;
 
 import java.util.List;
+import java.util.Optional;
 import java.util.Set;
 
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Change;
@@ -87,7 +88,8 @@ public class Entity {
                 .textAfter(e.getTextAfter())
                 .startOffset(e.getStartOffset())
                 .endOffset(e.getEndOffset())
-                .length(e.getValue().length())
+                .length(Optional.ofNullable(e.getValue())
+                                .orElse("").length())
                 .imageHasTransparency(e.isImageHasTransparency())
                 .isDictionaryEntry(e.isDictionaryEntry())
                 .isDossierDictionaryEntry(e.isDossierDictionaryEntry())

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/Dictionary.java

@@ -23,6 +23,9 @@ import com.iqser.red.service.redaction.v1.server.utils.exception.NotFoundExcepti
 import lombok.Data;
 import lombok.Getter;
 
+/**
+ * A class representing a dictionary used for redaction processes, containing various dictionary models and their versions.
+ */
 @Data
 public class Dictionary {
 
@@ -51,9 +54,15 @@ public class Dictionary {
     }
 
 
+    /**
+     * Checks if the dictionary contains local entries.
+     *
+     * @return true if any dictionary model contains local entries, false otherwise.
+     */
     public boolean hasLocalEntries() {
 
-        return dictionaryModels.stream().anyMatch(dm -> !dm.getLocalEntriesWithMatchedRules().isEmpty());
+        return dictionaryModels.stream()
+                .anyMatch(dm -> !dm.getLocalEntriesWithMatchedRules().isEmpty());
     }
 
 
@@ -63,6 +72,13 @@ public class Dictionary {
     }
 
 
+    /**
+     * Retrieves the {@link DictionaryModel} of a specified type.
+     *
+     * @param type The type of dictionary model to retrieve.
+     * @return The {@link DictionaryModel} of the specified type.
+     * @throws NotFoundException If the specified type is not found in the dictionary.
+     */
     public DictionaryModel getType(String type) {
 
         DictionaryModel model = localAccessMap.get(type);
@@ -73,6 +89,12 @@ public class Dictionary {
     }
 
 
+    /**
+     * Checks if the dictionary of a specific type is considered a hint.
+     *
+     * @param type The type of dictionary to check.
+     * @return true if the dictionary model is marked as a hint, false otherwise.
+     */
     public boolean isHint(String type) {
 
         DictionaryModel model = localAccessMap.get(type);
@@ -83,6 +105,12 @@ public class Dictionary {
     }
 
 
+    /**
+     * Checks if the dictionary of a specific type is case-insensitive.
+     *
+     * @param type The type of dictionary to check.
+     * @return true if the dictionary is case-insensitive, false otherwise.
+     */
     public boolean isCaseInsensitiveDictionary(String type) {
 
         DictionaryModel dictionaryModel = localAccessMap.get(type);
@@ -93,6 +121,18 @@ public class Dictionary {
     }
 
 
+    /**
+     * Adds a local dictionary entry of a specific type.
+     *
+     * @param type            The type of dictionary to add the entry to.
+     * @param value           The value of the entry.
+     * @param matchedRules    A collection of {@link MatchedRule} associated with the entry.
+     * @param alsoAddLastname Indicates whether to also add the lastname separately as an entry.
+     * @throws IllegalArgumentException If the specified type does not exist within the dictionary, if the type
+     *                                  does not have any local entries defined, or if the provided value is
+     *                                  blank. This ensures that only valid, non-empty entries
+     *                                  are added to the dictionary.
+     */
     private void addLocalDictionaryEntry(String type, String value, Collection<MatchedRule> matchedRules, boolean alsoAddLastname) {
 
         if (value.isBlank()) {
@@ -116,35 +156,62 @@ public class Dictionary {
         }
         localAccessMap.get(type)
                 .getLocalEntriesWithMatchedRules()
-                .merge(cleanedValue.trim(), matchedRulesSet, (set1, set2) -> Stream.concat(set1.stream(), set2.stream()).collect(Collectors.toSet()));
+                .merge(cleanedValue.trim(),
+                       matchedRulesSet,
+                       (set1, set2) -> Stream.concat(set1.stream(), set2.stream())
+                               .collect(Collectors.toSet()));
         if (alsoAddLastname) {
             String lastname = cleanedValue.split(" ")[0];
             localAccessMap.get(type)
                     .getLocalEntriesWithMatchedRules()
-                    .merge(lastname, matchedRulesSet, (set1, set2) -> Stream.concat(set1.stream(), set2.stream()).collect(Collectors.toSet()));
+                    .merge(lastname,
+                           matchedRulesSet,
+                           (set1, set2) -> Stream.concat(set1.stream(), set2.stream())
+                                   .collect(Collectors.toSet()));
         }
     }
 
 
+    /**
+     * Recommends a text entity for inclusion in every dictionary model without separating the last name.
+     *
+     * @param textEntity The {@link TextEntity} to be recommended.
+     */
     public void recommendEverywhere(TextEntity textEntity) {
 
-        addLocalDictionaryEntry(textEntity.getType(), textEntity.getValue(), textEntity.getMatchedRuleList(), false);
+        addLocalDictionaryEntry(textEntity.type(), textEntity.getValue(), textEntity.getMatchedRuleList(), false);
     }
 
 
+    /**
+     * Recommends a text entity for inclusion in every dictionary model with the last name added separately.
+     *
+     * @param textEntity The {@link TextEntity} to be recommended.
+     */
     public void recommendEverywhereWithLastNameSeparately(TextEntity textEntity) {
 
-        addLocalDictionaryEntry(textEntity.getType(), textEntity.getValue(), textEntity.getMatchedRuleList(), true);
+        addLocalDictionaryEntry(textEntity.type(), textEntity.getValue(), textEntity.getMatchedRuleList(), true);
     }
 
 
+    /**
+     * Adds multiple author names contained within a text entity as recommendations in the dictionary.
+     *
+     * @param textEntity The {@link TextEntity} containing author names to be added.
+     */
     public void addMultipleAuthorsAsRecommendation(TextEntity textEntity) {
 
-        splitIntoAuthorNames(textEntity).forEach(authorName -> addLocalDictionaryEntry(textEntity.getType(), authorName, textEntity.getMatchedRuleList(), true));
+        splitIntoAuthorNames(textEntity).forEach(authorName -> addLocalDictionaryEntry(textEntity.type(), authorName, textEntity.getMatchedRuleList(), true));
 
     }
 
 
+    /**
+     * Splits a {@link TextEntity} into individual author names based on commas or new lines.
+     *
+     * @param textEntity The {@link TextEntity} to split.
+     * @return A list of strings where each string is an author name.
+     */
     public static List<String> splitIntoAuthorNames(TextEntity textEntity) {
 
         List<String> splitAuthorNames;
@@ -153,7 +220,10 @@ public class Dictionary {
         } else {
             splitAuthorNames = Arrays.asList(textEntity.getValueWithLineBreaks().split("\n"));
         }
-        return splitAuthorNames.stream().map(String::trim).filter(authorName -> Patterns.AUTHOR_NAME_PATTERN.matcher(authorName).matches()).toList();
+        return splitAuthorNames.stream()
+                .map(String::trim)
+                .filter(authorName -> Patterns.AUTHOR_NAME_PATTERN.matcher(authorName).matches())
+                .toList();
     }
 
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/DictionaryModel.java

@@ -2,7 +2,6 @@ package com.iqser.red.service.redaction.v1.server.model.dictionary;
 
 import java.io.Serializable;
 import java.util.HashMap;
-import java.util.List;
 import java.util.Locale;
 import java.util.Set;
 import java.util.stream.Collectors;
@@ -11,11 +10,17 @@ import com.iqser.red.service.dictionarymerge.commons.DictionaryEntry;
 import com.iqser.red.service.dictionarymerge.commons.DictionaryEntryModel;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule;
 
-import lombok.AllArgsConstructor;
 import lombok.Data;
+import lombok.extern.slf4j.Slf4j;
 
+/**
+ * Represents a model of a dictionary containing entries for redaction processes.
+ * It includes various types of entries such as standard entries, false positives,
+ * and false recommendations. Additionally, it manages local entries with matched
+ * rules for enhanced search and matching capabilities.
+ */
 @Data
-@AllArgsConstructor
+@Slf4j
 public class DictionaryModel implements Serializable {
 
     private final String type;
@@ -29,6 +34,7 @@ public class DictionaryModel implements Serializable {
     private final Set<DictionaryEntryModel> falseRecommendations;
 
     private transient SearchImplementation entriesSearch;
+    private transient SearchImplementation deletionEntriesSearch;
     private transient SearchImplementation falsePositiveSearch;
     private transient SearchImplementation falseRecommendationsSearch;
 
@@ -36,6 +42,19 @@ public class DictionaryModel implements Serializable {
     private transient SearchImplementation localSearch;
 
 
+    /**
+     * Constructs a new DictionaryModel with specified parameters.
+     *
+     * @param type                 The type of the dictionary model.
+     * @param rank                 The rank order of the dictionary model.
+     * @param color                An array representing the color associated with this model.
+     * @param caseInsensitive      Flag indicating whether the dictionary is case-insensitive.
+     * @param hint                 Flag indicating whether this model should be used as a hint.
+     * @param entries              Set of dictionary entry models representing the entries.
+     * @param falsePositives       Set of dictionary entry models representing false positives.
+     * @param falseRecommendations Set of dictionary entry models representing false recommendations.
+     * @param isDossierDictionary  Flag indicating whether this model is for a dossier dictionary.
+     */
     public DictionaryModel(String type,
                            int rank,
                            float[] color,
@@ -52,23 +71,17 @@ public class DictionaryModel implements Serializable {
         this.caseInsensitive = caseInsensitive;
         this.hint = hint;
         this.isDossierDictionary = isDossierDictionary;
-
         this.entries = entries;
         this.falsePositives = falsePositives;
         this.falseRecommendations = falseRecommendations;
-
-        this.entriesSearch = new SearchImplementation(this.entries.stream().filter(e -> !e.isDeleted()).map(DictionaryEntryModel::getValue).collect(Collectors.toList()),
-                caseInsensitive);
-        this.falsePositiveSearch = new SearchImplementation(this.falsePositives.stream().filter(e -> !e.isDeleted()).map(DictionaryEntryModel::getValue).collect(Collectors.toList()),
-                caseInsensitive);
-        this.falseRecommendationsSearch = new SearchImplementation(this.falseRecommendations.stream()
-                .filter(e -> !e.isDeleted())
-                .map(DictionaryEntry::getValue)
-                .collect(Collectors.toList()), caseInsensitive);
-
     }
 
 
+    /**
+     * Returns the search implementation for local entries.
+     *
+     * @return The {@link SearchImplementation} for local entries.
+     */
     public SearchImplementation getLocalSearch() {
 
         if (this.localSearch == null || this.localSearch.getValues().size() != this.localEntriesWithMatchedRules.size()) {
@@ -78,44 +91,85 @@ public class DictionaryModel implements Serializable {
     }
 
 
+    /**
+     * Returns the search implementation for non-deleted dictionary entries.
+     *
+     * @return The {@link SearchImplementation} for non-deleted dictionary entries.
+     */
     public SearchImplementation getEntriesSearch() {
 
         if (entriesSearch == null) {
-            this.entriesSearch = new SearchImplementation(this.entries.stream().filter(e -> !e.isDeleted()).map(DictionaryEntry::getValue).collect(Collectors.toList()),
-                    caseInsensitive);
+            this.entriesSearch = new SearchImplementation(this.entries.stream()
+                                                                  .filter(e -> !e.isDeleted())
+                                                                  .map(DictionaryEntry::getValue)
+                                                                  .collect(Collectors.toList()), caseInsensitive);
         }
         return entriesSearch;
     }
 
 
+    /**
+     * Returns the search implementation for deleted dictionary entries.
+     *
+     * @return The {@link SearchImplementation} for deleted dictionary entries.
+     */
+    public SearchImplementation getDeletionEntriesSearch() {
+
+        if (deletionEntriesSearch == null) {
+            this.deletionEntriesSearch = new SearchImplementation(this.entries.stream()
+                                                                          .filter(DictionaryEntry::isDeleted)
+                                                                          .map(DictionaryEntry::getValue)
+                                                                          .collect(Collectors.toList()), caseInsensitive);
+        }
+        return deletionEntriesSearch;
+    }
+
+
+    /**
+     * Returns the search implementation for non-deleted false positive entries.
+     *
+     * @return The {@link SearchImplementation} for non-deleted false positive entries.
+     */
     public SearchImplementation getFalsePositiveSearch() {
 
         if (falsePositiveSearch == null) {
             this.falsePositiveSearch = new SearchImplementation(this.falsePositives.stream()
-                    .filter(e -> !e.isDeleted())
-                    .map(DictionaryEntry::getValue)
-                    .collect(Collectors.toList()), caseInsensitive);
+                                                                        .filter(e -> !e.isDeleted())
+                                                                        .map(DictionaryEntry::getValue)
+                                                                        .collect(Collectors.toList()), caseInsensitive);
         }
         return falsePositiveSearch;
     }
 
 
+    /**
+     * Returns the search implementation for non-deleted false recommendation entries.
+     *
+     * @return The {@link SearchImplementation} for non-deleted false recommendation entries.
+     */
     public SearchImplementation getFalseRecommendationsSearch() {
 
         if (falseRecommendationsSearch == null) {
             this.falseRecommendationsSearch = new SearchImplementation(this.falseRecommendations.stream()
-                    .filter(e -> !e.isDeleted())
-                    .map(DictionaryEntry::getValue)
-                    .collect(Collectors.toList()), caseInsensitive);
+                                                                               .filter(e -> !e.isDeleted())
+                                                                               .map(DictionaryEntry::getValue)
+                                                                               .collect(Collectors.toList()), caseInsensitive);
         }
         return falseRecommendationsSearch;
     }
 
+
+    /**
+     * Retrieves the matched rules for a given value from the local dictionary entries.
+     * The value is processed based on the case sensitivity of the dictionary.
+     *
+     * @param value The value for which to retrieve the matched rules.
+     * @return A set of {@link MatchedRule} associated with the given value, or null if no rules are found.
+     */
     public Set<MatchedRule> getMatchedRulesForLocalDictionaryEntry(String value) {
+
         var cleanedValue = isCaseInsensitive() ? value.toLowerCase(Locale.US) : value;
-
         return localEntriesWithMatchedRules.get(cleanedValue);
-
     }
 
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/SearchImplementation.java

@@ -76,7 +76,9 @@ public class SearchImplementation {
             if (ignoreCase) {
                 textToCheck = textToCheck.toLowerCase(Locale.ROOT);
             }
-            return this.pattern.matcher(textToCheck).results().findAny().isPresent();
+            return this.pattern.matcher(textToCheck).results()
+                    .findAny()
+                    .isPresent();
         } else {
             return this.trie.containsMatch(textToCheck);
         }
@@ -89,9 +91,14 @@ public class SearchImplementation {
             return new ArrayList<>();
         }
         if (this.pattern != null) {
-            return this.pattern.matcher(text).results().map(r -> new TextRange(r.start(), r.end())).collect(Collectors.toList());
+            return this.pattern.matcher(text).results()
+                    .map(r -> new TextRange(r.start(), r.end()))
+                    .collect(Collectors.toList());
         } else {
-            return this.trie.parseText(text).stream().map(r -> new TextRange(r.getStart(), r.getEnd() + 1)).collect(Collectors.toList());
+            return this.trie.parseText(text)
+                    .stream()
+                    .map(r -> new TextRange(r.getStart(), r.getEnd() + 1))
+                    .collect(Collectors.toList());
         }
     }
 
@@ -103,9 +110,14 @@ public class SearchImplementation {
         }
         CharSequence subSequence = text.subSequence(region.start(), region.end());
         if (this.pattern != null) {
-            return this.pattern.matcher(subSequence).results().map(r -> new TextRange(r.start() + region.start(), r.end() + region.start())).collect(Collectors.toList());
+            return this.pattern.matcher(subSequence).results()
+                    .map(r -> new TextRange(r.start() + region.start(), r.end() + region.start()))
+                    .collect(Collectors.toList());
         } else {
-            return this.trie.parseText(subSequence).stream().map(r -> new TextRange(r.getStart() + region.start(), r.getEnd() + region.start() + 1)).collect(Collectors.toList());
+            return this.trie.parseText(subSequence)
+                    .stream()
+                    .map(r -> new TextRange(r.getStart() + region.start(), r.getEnd() + region.start() + 1))
+                    .collect(Collectors.toList());
         }
     }
 
@@ -120,9 +132,14 @@ public class SearchImplementation {
             if (ignoreCase) {
                 textToCheck = textToCheck.toLowerCase(Locale.ROOT);
             }
-            return this.pattern.matcher(textToCheck).results().map(r -> new MatchPosition(r.start(), r.end())).collect(Collectors.toList());
+            return this.pattern.matcher(textToCheck).results()
+                    .map(r -> new MatchPosition(r.start(), r.end()))
+                    .collect(Collectors.toList());
         } else {
-            return this.trie.parseText(textToCheck).stream().map(r -> new MatchPosition(r.getStart(), r.getEnd() + 1)).collect(Collectors.toList());
+            return this.trie.parseText(textToCheck)
+                    .stream()
+                    .map(r -> new MatchPosition(r.getStart(), r.getEnd() + 1))
+                    .collect(Collectors.toList());
         }
     }
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/DocumentTree.java

@@ -40,7 +40,10 @@ public class DocumentTree {
 
     public TextBlock buildTextBlock() {
 
-        return allEntriesInOrder().map(Entry::getNode).filter(SemanticNode::isLeaf).map(SemanticNode::getLeafTextBlock).collect(new TextBlockCollector());
+        return allEntriesInOrder().map(Entry::getNode)
+                .filter(SemanticNode::isLeaf)
+                .map(SemanticNode::getLeafTextBlock)
+                .collect(new TextBlockCollector());
     }
 
 
@@ -89,8 +92,8 @@ public class DocumentTree {
         if (treeId.isEmpty()) {
             return root != null;
         }
-        Entry entry = root.children.get(treeId.get(0));
-        for (int id : treeId.subList(1, treeId.size())) {
+        Entry entry = root;
+        for (int id : treeId) {
             if (id >= entry.children.size() || 0 > id) {
                 return false;
             }
@@ -114,13 +117,16 @@ public class DocumentTree {
 
     public Stream<SemanticNode> childNodes(List<Integer> treeId) {
 
-        return getEntryById(treeId).children.stream().map(Entry::getNode);
+        return getEntryById(treeId).children.stream()
+                .map(Entry::getNode);
     }
 
 
     public Stream<SemanticNode> childNodesOfType(List<Integer> treeId, NodeType nodeType) {
 
-        return getEntryById(treeId).children.stream().filter(entry -> entry.node.getType().equals(nodeType)).map(Entry::getNode);
+        return getEntryById(treeId).children.stream()
+                .filter(entry -> entry.node.getType().equals(nodeType))
+                .map(Entry::getNode);
     }
 
 
@@ -199,26 +205,32 @@ public class DocumentTree {
 
     public Stream<Entry> allEntriesInOrder() {
 
-        return Stream.of(root).flatMap(DocumentTree::flatten);
+        return Stream.of(root)
+                .flatMap(DocumentTree::flatten);
     }
 
 
     public Stream<Entry> allSubEntriesInOrder(List<Integer> parentId) {
 
-        return getEntryById(parentId).children.stream().flatMap(DocumentTree::flatten);
+        return getEntryById(parentId).children.stream()
+                .flatMap(DocumentTree::flatten);
     }
 
 
     @Override
     public String toString() {
 
-        return String.join("\n", allEntriesInOrder().map(Entry::toString).toList());
+        return String.join("\n",
+                           allEntriesInOrder().map(Entry::toString)
+                                   .toList());
     }
 
 
     private static Stream<Entry> flatten(Entry entry) {
 
-        return Stream.concat(Stream.of(entry), entry.children.stream().flatMap(DocumentTree::flatten));
+        return Stream.concat(Stream.of(entry),
+                             entry.children.stream()
+                                     .flatMap(DocumentTree::flatten));
     }
 
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/TextRange.java

@@ -11,14 +11,26 @@ import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBl
 import lombok.EqualsAndHashCode;
 import lombok.Setter;
 
+/**
+ * Represents a range of text defined by a start and end index.
+ * Provides functionality to check containment, intersection, and to adjust ranges based on specified conditions.
+ */
 @Setter
 @EqualsAndHashCode
+@SuppressWarnings("PMD.AvoidFieldNameMatchingMethodName")
 public class TextRange implements Comparable<TextRange> {
 
     private int start;
     private int end;
 
 
+    /**
+     * Constructs a TextRange with specified start and end indexes.
+     *
+     * @param start The starting index of the range.
+     * @param end   The ending index of the range.
+     * @throws IllegalArgumentException If start is greater than end.
+     */
     public TextRange(int start, int end) {
 
         if (start > end) {
@@ -29,6 +41,11 @@ public class TextRange implements Comparable<TextRange> {
     }
 
 
+    /**
+     * Returns the length of the text range.
+     *
+     * @return The length of the range.
+     */
     public int length() {
 
         return end - start;
@@ -47,18 +64,38 @@ public class TextRange implements Comparable<TextRange> {
     }
 
 
+    /**
+     * Checks if this {@link TextRange} fully contains another TextRange.
+     *
+     * @param textRange The {@link TextRange} to check.
+     * @return true if this range contains the specified range, false otherwise.
+     */
     public boolean contains(TextRange textRange) {
 
         return start <= textRange.start() && textRange.end() <= end;
     }
 
 
+    /**
+     * Checks if this {@link TextRange} is fully contained by another TextRange.
+     *
+     * @param textRange The {@link TextRange}  to check against.
+     * @return true if this range is contained by the specified range, false otherwise.
+     */
     public boolean containedBy(TextRange textRange) {
 
         return textRange.contains(this);
     }
 
 
+    /**
+     * Checks if this {@link TextRange} contains another range specified by start and end indices.
+     *
+     * @param start The starting index of the range to check.
+     * @param end   The ending index of the range to check.
+     * @return true if this range fully contains the specified range, false otherwise.
+     * @throws IllegalArgumentException If the start index is greater than the end index.
+     */
     public boolean contains(int start, int end) {
 
         if (start > end) {
@@ -68,6 +105,14 @@ public class TextRange implements Comparable<TextRange> {
     }
 
 
+    /**
+     * Checks if this {@link TextRange} is fully contained within another range specified by start and end indices.
+     *
+     * @param start The starting index of the outer range.
+     * @param end   The ending index of the outer range.
+     * @return true if this range is fully contained within the specified range, false otherwise.
+     * @throws IllegalArgumentException If the start index is greater than the end index.
+     */
     public boolean containedBy(int start, int end) {
 
         if (start > end) {
@@ -77,22 +122,46 @@ public class TextRange implements Comparable<TextRange> {
     }
 
 
+    /**
+     * Determines if the specified index is within this {@link TextRange}.
+     *
+     * @param index The index to check.
+     * @return true if the index is within the range (inclusive of the start and exclusive of the end), false otherwise.
+     */
     public boolean contains(int index) {
 
         return start <= index && index < end;
     }
 
 
+    /**
+     * Checks if this {@link TextRange} intersects with another {@link TextRange}.
+     *
+     * @param textRange The {@link TextRange} to check for intersection.
+     * @return true if the ranges intersect, false otherwise.
+     */
     public boolean intersects(TextRange textRange) {
 
         return textRange.start() < this.end && this.start < textRange.end();
     }
 
 
+    /**
+     * Splits this TextRange into multiple ranges based on a list of indices.
+     *
+     * @param splitIndices The indices at which to split the range.
+     * @return A list of TextRanges resulting from the split.
+     * @throws IndexOutOfBoundsException If any split index is outside this TextRange.
+     */
     public List<TextRange> split(List<Integer> splitIndices) {
 
-        if (splitIndices.stream().anyMatch(idx -> !this.contains(idx))) {
-            throw new IndexOutOfBoundsException(format("%s splitting indices are out of range for %s", splitIndices.stream().filter(idx -> !this.contains(idx)).toList(), this));
+        if (splitIndices.stream()
+                .anyMatch(idx -> !this.contains(idx))) {
+            throw new IndexOutOfBoundsException(format("%s splitting indices are out of range for %s",
+                                                       splitIndices.stream()
+                                                               .filter(idx -> !this.contains(idx))
+                                                               .toList(),
+                                                       this));
         }
         List<TextRange> splitBoundaries = new LinkedList<>();
         int previousIndex = start;
@@ -110,10 +179,23 @@ public class TextRange implements Comparable<TextRange> {
     }
 
 
+    /**
+     * Merges a collection of TextRanges into a single Text range encompassing all.
+     *
+     * @param boundaries The collection of TextRanges to merge.
+     * @return A new TextRange covering the entire span of the given ranges.
+     * @throws IllegalArgumentException If boundaries are empty.
+     */
     public static TextRange merge(Collection<TextRange> boundaries) {
 
-        int minStart = boundaries.stream().mapToInt(TextRange::start).min().orElseThrow(IllegalArgumentException::new);
-        int maxEnd = boundaries.stream().mapToInt(TextRange::end).max().orElseThrow(IllegalArgumentException::new);
+        int minStart = boundaries.stream()
+                .mapToInt(TextRange::start)
+                .min()
+                .orElseThrow(IllegalArgumentException::new);
+        int maxEnd = boundaries.stream()
+                .mapToInt(TextRange::end)
+                .max()
+                .orElseThrow(IllegalArgumentException::new);
         return new TextRange(minStart, maxEnd);
     }
 
@@ -140,16 +222,17 @@ public class TextRange implements Comparable<TextRange> {
 
 
     /**
-     * shrinks the boundary, such that textBlock.subSequence(boundary) returns a string without trailing or preceding whitespaces.
+     * Shrinks the boundary, such that textBlock.subSequence(boundary) returns a string without trailing or preceding whitespaces.
      *
      * @param textBlock TextBlock to check whitespaces against
-     * @return trimmed boundary
+     * @return Trimmed boundary
      */
     public TextRange trim(TextBlock textBlock) {
 
         if (this.length() == 0) {
             return this;
         }
+
         int trimmedStart = this.start;
         while (textBlock.containsIndex(trimmedStart) && trimmedStart < end && Character.isWhitespace(textBlock.charAt(trimmedStart))) {
             trimmedStart++;

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/EntityType.java

@@ -5,5 +5,6 @@ public enum EntityType {
     HINT,
     RECOMMENDATION,
     FALSE_POSITIVE,
-    FALSE_RECOMMENDATION
+    FALSE_RECOMMENDATION,
+    DICTIONARY_REMOVAL
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/IEntity.java

@@ -12,82 +12,173 @@ import lombok.NonNull;
 
 public interface IEntity {
 
+    /**
+     * Gets the list of rules matched against this entity.
+     *
+     * @return A priority queue of matched rules.
+     */
     PriorityQueue<MatchedRule> getMatchedRuleList();
 
 
+    /**
+     * Gets the manual overwrite actions applied to this entity, if any.
+     *
+     * @return The manual overwrite details.
+     */
     ManualChangeOverwrite getManualOverwrite();
 
 
+    /**
+     * Gets the value of this entity as a string.
+     *
+     * @return The string value.
+     */
     String getValue();
 
 
+    /**
+     * Gets the range of text in the document associated with this entity.
+     *
+     * @return The text range.
+     */
     TextRange getTextRange();
 
 
+    /**
+     * Gets the type of this entity.
+     *
+     * @return The entity type.
+     */
     String type();
 
 
+    /**
+     * Calculates the length of the entity's value.
+     *
+     * @return The length of the value.
+     */
     default int length() {
 
         return value().length();
     }
 
 
+    /**
+     * Retrieves the value of the entity, considering any manual overwrite.
+     * If no manual overwrite value is found, return the value of the entity or an empty string
+     * if that value is null.
+     *
+     * @return The possibly overwritten value
+     */
     default String value() {
 
-        return getManualOverwrite().getValue().orElse(getValue() == null ? "" : getValue());
+        return getManualOverwrite().getValue()
+                .orElse(getValue() == null ? "" : getValue());
     }
 
 
+    /**
+     * Determines if the entity has been applied, considering manual overwrites.
+     *
+     * @return True if applied, false otherwise.
+     */
     // Don't use default accessor pattern (e.g. isApplied()), as it might lead to errors in drools due to property-specific optimization of the drools planner.
     default boolean applied() {
 
-        return getManualOverwrite().getApplied().orElse(getMatchedRule().isApplied());
+        return getManualOverwrite().getApplied()
+                .orElse(getMatchedRule().isApplied());
     }
 
 
+    /**
+     * Determines if the entity has been skipped, based on its applied status.
+     *
+     * @return True if skipped, false otherwise.
+     */
     default boolean skipped() {
 
         return !applied();
     }
 
 
+    /**
+     * Determines if the entity has been ignored, considering manual overwrites.
+     *
+     * @return True if ignored, false otherwise.
+     */
     default boolean ignored() {
 
-        return getManualOverwrite().getIgnored().orElse(getMatchedRule().isIgnored());
+        return getManualOverwrite().getIgnored()
+                .orElse(getMatchedRule().isIgnored());
     }
 
 
+    /**
+     * Determines if the entity has been removed, considering manual overwrites.
+     *
+     * @return True if removed, false otherwise.
+     */
     default boolean removed() {
 
-        return getManualOverwrite().getRemoved().orElse(getMatchedRule().isRemoved());
+        return getManualOverwrite().getRemoved()
+                .orElse(getMatchedRule().isRemoved());
     }
 
 
+    /**
+     * Checks if the entity has been resized, considering manual overwrites.
+     *
+     * @return True if resized, false otherwise.
+     */
     default boolean resized() {
 
-        return getManualOverwrite().getResized().orElse(false);
+        return getManualOverwrite().getResized()
+                .orElse(false);
     }
 
 
+    /**
+     * Checks if the entity is considered active, based on its removed and ignored status.
+     * An active entry is not removed or ignored.
+     *
+     * @return True if active, false otherwise.
+     */
     default boolean active() {
 
         return !(removed() || ignored());
     }
 
 
+    /**
+     * Checks if there are any manual changes applied to the entity.
+     *
+     * @return True if there are manual changes, false otherwise.
+     */
     default boolean hasManualChanges() {
 
         return !getManualOverwrite().getManualChangeLog().isEmpty();
     }
 
 
+    /**
+     * Retrieves a set of references associated with the entity's matched rule.
+     *
+     * @return A set of references.
+     */
     default Set<TextEntity> references() {
 
         return getMatchedRule().getReferences();
     }
 
 
+    /**
+     * Applies a redaction to the entity with a specified legal basis.
+     *
+     * @param ruleIdentifier The identifier of the rule being applied.
+     * @param reason         The reason for the redaction.
+     * @param legalBasis     The legal basis for the redaction, which must not be blank or empty.
+     * @throws IllegalArgumentException If the legal basis is blank or empty.
+     */
     default void redact(@NonNull String ruleIdentifier, String reason, @NonNull String legalBasis) {
 
         if (legalBasis.isBlank() || legalBasis.isEmpty()) {
@@ -97,78 +188,143 @@ public interface IEntity {
     }
 
 
+    /**
+     * Applies a rule to the entity with an optional legal basis.
+     *
+     * @param ruleIdentifier The identifier of the rule being applied.
+     * @param reason         The reason for applying the rule.
+     * @param legalBasis     The legal basis for the application, can be a default or unspecified value.
+     */
     default void apply(@NonNull String ruleIdentifier, String reason, String legalBasis) {
 
         addMatchedRule(MatchedRule.builder().ruleIdentifier(RuleIdentifier.fromString(ruleIdentifier)).reason(reason).legalBasis(legalBasis).applied(true).build());
     }
 
 
+    /**
+     * Applies a rule to the entity without specifying a legal basis, which will be replaced by "n-a".
+     *
+     * @param ruleIdentifier The identifier of the rule being applied.
+     * @param reason         The reason for applying the rule.
+     */
     default void apply(@NonNull String ruleIdentifier, String reason) {
 
         apply(ruleIdentifier, reason, "n-a");
     }
 
 
+    /**
+     * Marks the entity as skipped according to a specific rule.
+     *
+     * @param ruleIdentifier The identifier of the rule being skipped.
+     * @param reason         The reason for skipping the rule.
+     */
     default void skip(@NonNull String ruleIdentifier, String reason) {
 
         addMatchedRule(MatchedRule.builder().ruleIdentifier(RuleIdentifier.fromString(ruleIdentifier)).reason(reason).build());
     }
 
 
+    /**
+     * Marks the entity as removed according to a specific rule.
+     *
+     * @param ruleIdentifier The identifier of the rule based on which the entity is removed.
+     * @param reason         The reason for the removal.
+     */
     default void remove(String ruleIdentifier, String reason) {
 
         addMatchedRule(MatchedRule.builder().ruleIdentifier(RuleIdentifier.fromString(ruleIdentifier)).reason(reason).removed(true).build());
     }
 
 
+    /**
+     * Marks the entity as ignored according to a specific rule.
+     *
+     * @param ruleIdentifier The identifier of the rule based on which the entity is removed.
+     * @param reason         The reason for the removal.
+     */
     default void ignore(String ruleIdentifier, String reason) {
 
         addMatchedRule(MatchedRule.builder().ruleIdentifier(RuleIdentifier.fromString(ruleIdentifier)).reason(reason).ignored(true).build());
     }
 
 
+    /**
+     * Applies a rule to the entity, indicating that the value should be written with line breaks.
+     *
+     * @param ruleIdentifier The identifier of the rule being applied.
+     * @param reason         The reason for the rule application.
+     * @param legalBasis     The legal basis for the rule, which must not be empty.
+     * @throws IllegalArgumentException If the legal basis is blank or empty.
+     */
     default void applyWithLineBreaks(@NonNull String ruleIdentifier, String reason, @NonNull String legalBasis) {
 
         if (legalBasis.isBlank() || legalBasis.isEmpty()) {
             throw new IllegalArgumentException("legal basis cannot be empty when redacting an entity");
         }
         getMatchedRuleList().add(MatchedRule.builder()
-                .ruleIdentifier(RuleIdentifier.fromString(ruleIdentifier))
-                .reason(reason)
-                .legalBasis(legalBasis)
-                .applied(true)
-                .writeValueWithLineBreaks(true)
-                .build());
+                                         .ruleIdentifier(RuleIdentifier.fromString(ruleIdentifier))
+                                         .reason(reason)
+                                         .legalBasis(legalBasis)
+                                         .applied(true)
+                                         .writeValueWithLineBreaks(true)
+                                         .build());
     }
 
 
+    /**
+     * Applies a rule to the entity with a collection of references.
+     *
+     * @param ruleIdentifier The identifier of the rule being applied.
+     * @param reason         The reason for the rule application.
+     * @param legalBasis     The legal basis for the rule, which must not be empty.
+     * @param references     A collection of text entities that are referenced by this rule application.
+     * @throws IllegalArgumentException If the legal basis is blank or empty.
+     */
     default void applyWithReferences(@NonNull String ruleIdentifier, String reason, @NonNull String legalBasis, Collection<TextEntity> references) {
 
         if (legalBasis.isBlank() || legalBasis.isEmpty()) {
             throw new IllegalArgumentException("legal basis cannot be empty when redacting an entity");
         }
         getMatchedRuleList().add(MatchedRule.builder()
-                .ruleIdentifier(RuleIdentifier.fromString(ruleIdentifier))
-                .reason(reason)
-                .legalBasis(legalBasis)
-                .applied(true)
-                .references(new HashSet<>(references))
-                .build());
+                                         .ruleIdentifier(RuleIdentifier.fromString(ruleIdentifier))
+                                         .reason(reason)
+                                         .legalBasis(legalBasis)
+                                         .applied(true)
+                                         .references(new HashSet<>(references))
+                                         .build());
     }
 
 
+    /**
+     * Marks the entity as skipped for a specific rule and associates a collection of references.
+     *
+     * @param ruleIdentifier The identifier of the rule being skipped.
+     * @param reason         The reason for skipping the rule.
+     * @param references     A collection of text entities that are referenced by the skipped rule.
+     */
     default void skipWithReferences(@NonNull String ruleIdentifier, String reason, Collection<TextEntity> references) {
 
         getMatchedRuleList().add(MatchedRule.builder().ruleIdentifier(RuleIdentifier.fromString(ruleIdentifier)).reason(reason).references(new HashSet<>(references)).build());
     }
 
 
+    /**
+     * Adds a single matched rule to this entity.
+     *
+     * @param matchedRule The matched rule to add.
+     */
     default void addMatchedRule(MatchedRule matchedRule) {
 
         getMatchedRuleList().add(matchedRule);
     }
 
 
+    /**
+     * Adds a collection of matched rules to this entity.
+     *
+     * @param matchedRules The collection of matched rules to add.
+     */
     default void addMatchedRules(Collection<MatchedRule> matchedRules) {
 
         if (getMatchedRuleList().equals(matchedRules)) {
@@ -178,12 +334,22 @@ public interface IEntity {
     }
 
 
+    /**
+     * Retrieves the 'unit' value of the highest priority matched rule.
+     *
+     * @return The unit value of the matched rule.
+     */
     default int getMatchedRuleUnit() {
 
         return getMatchedRule().getRuleIdentifier().unit();
     }
 
 
+    /**
+     * Gets the highest priority matched rule for this entity.
+     *
+     * @return The matched rule.
+     */
     default MatchedRule getMatchedRule() {
 
         if (getMatchedRuleList().isEmpty()) {
@@ -193,6 +359,11 @@ public interface IEntity {
     }
 
 
+    /**
+     * Builds a reason string for this entity, incorporating descriptions from manual changes.
+     *
+     * @return The built reason string.
+     */
     default String buildReasonWithManualChangeDescriptions() {
 
         if (getManualOverwrite().getDescriptions().isEmpty()) {
@@ -205,9 +376,15 @@ public interface IEntity {
     }
 
 
+    /**
+     * Retrieves the legal basis for the action taken on this entity, considering any manual overwrite.
+     *
+     * @return The legal basis.
+     */
     default String legalBasis() {
 
-        return getManualOverwrite().getLegalBasis().orElse(getMatchedRule().getLegalBasis());
+        return getManualOverwrite().getLegalBasis()
+                .orElse(getMatchedRule().getLegalBasis());
     }
 
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/ManualChangeOverwrite.java

@@ -27,12 +27,12 @@ import lombok.experimental.FieldDefaults;
 public class ManualChangeOverwrite {
 
     private static final Map<Class<? extends BaseAnnotation>, String> MANUAL_CHANGE_DESCRIPTIONS = Map.of(//
-            ManualRedactionEntry.class, "created by manual change", //
-            ManualLegalBasisChange.class, "legal basis was manually changed", //
-            ManualResizeRedaction.class, "resized by manual override", //
-            ManualForceRedaction.class, "forced by manual override", //
-            IdRemoval.class, "removed by manual override", //
-            ManualRecategorization.class, "recategorized by manual override");
+                                                                                                          ManualRedactionEntry.class, "created by manual change", //
+                                                                                                          ManualLegalBasisChange.class, "legal basis was manually changed", //
+                                                                                                          ManualResizeRedaction.class, "resized by manual override", //
+                                                                                                          ManualForceRedaction.class, "forced by manual override", //
+                                                                                                          IdRemoval.class, "removed by manual override", //
+                                                                                                          ManualRecategorization.class, "recategorized by manual override");
 
     List<BaseAnnotation> manualChanges = new LinkedList<>();
     boolean changed;
@@ -80,7 +80,8 @@ public class ManualChangeOverwrite {
         manualChanges.sort(Comparator.comparing(BaseAnnotation::getRequestDate));
         updateFields(manualChanges);
         // make list unmodifiable.
-        return manualChanges.stream().toList();
+        return manualChanges.stream()
+                .toList();
     }
 
 
@@ -121,14 +122,20 @@ public class ManualChangeOverwrite {
                 resized = true;
                 // This is only for not found Manual Entities.
                 value = manualResizeRedaction.getValue();
-                positions = manualResizeRedaction.getPositions().stream().map(RectangleWithPage::fromAnnotationRectangle).toList();
+                positions = manualResizeRedaction.getPositions()
+                        .stream()
+                        .map(RectangleWithPage::fromAnnotationRectangle)
+                        .toList();
             }
 
             if (manualChange instanceof ManualRecategorization recategorization) {
-                // recategorization logic happens in ManualChangesApplicationService.
                 recategorized = true;
-                // this is only relevant for ManualEntities. Image and TextEntity is recategorized in the ManualChangesApplicationService.
                 type = recategorization.getType();
+                section = recategorization.getSection();
+                value = recategorization.getValue();
+                if (recategorization.getLegalBasis() != null && !recategorization.getLegalBasis().isEmpty()) {
+                    legalBasis = recategorization.getLegalBasis();
+                }
             }
 
             descriptions.add(MANUAL_CHANGE_DESCRIPTIONS.get(manualChange.getClass()));

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/MatchedRule.java

@@ -15,6 +15,9 @@ import lombok.EqualsAndHashCode;
 import lombok.Getter;
 import lombok.experimental.FieldDefaults;
 
+/**
+ * Represents a rule that has been matched during the document redaction process.
+ */
 @Getter
 @Builder
 @AllArgsConstructor
@@ -24,7 +27,9 @@ public final class MatchedRule implements Comparable<MatchedRule> {
 
     public static final RuleType FINAL_TYPE = RuleType.fromString("FINAL");
     public static final RuleType ELIMINATION_RULE_TYPE = RuleType.fromString("X");
-    private static final List<RuleType> RULE_TYPE_PRIORITIES = List.of(FINAL_TYPE, ELIMINATION_RULE_TYPE);
+    public static final RuleType IMPORTED_TYPE = RuleType.fromString("IMP");
+    public static final RuleType DICTIONARY_TYPE = RuleType.fromString("DICT");
+    private static final List<RuleType> RULE_TYPE_PRIORITIES = List.of(FINAL_TYPE, ELIMINATION_RULE_TYPE, IMPORTED_TYPE, DICTIONARY_TYPE);
 
     RuleIdentifier ruleIdentifier;
     @Builder.Default
@@ -40,18 +45,33 @@ public final class MatchedRule implements Comparable<MatchedRule> {
     Set<TextEntity> references = Collections.emptySet();
 
 
+    /**
+     * Creates an empty instance of {@link MatchedRule}.
+     * This can be used as a placeholder or when no rule is actually matched.
+     *
+     * @return An empty {@link MatchedRule} instance.
+     */
     public static MatchedRule empty() {
 
         return MatchedRule.builder().ruleIdentifier(RuleIdentifier.empty()).build();
     }
 
 
+    /**
+     * Returns a modified instance of {@link MatchedRule} based on its applied status.
+     * If the rule has been applied, it returns a new {@link MatchedRule} instance that retains all properties of the original
+     * except for the 'applied' status, which is set to false.
+     * If the rule has not been applied, it returns the original instance.
+     *
+     * @return A {@link MatchedRule} instance with 'applied' set to false.
+     */
     public MatchedRule asSkippedIfApplied() {
 
         if (!this.isApplied()) {
             return this;
         }
-        return MatchedRule.builder().ruleIdentifier(getRuleIdentifier())
+        return MatchedRule.builder()
+                .ruleIdentifier(getRuleIdentifier())
                 .writeValueWithLineBreaks(this.isWriteValueWithLineBreaks())
                 .legalBasis(this.getLegalBasis())
                 .reason(this.getReason())
@@ -60,6 +80,13 @@ public final class MatchedRule implements Comparable<MatchedRule> {
     }
 
 
+    /**
+     * Compares this rule with another {@link MatchedRule} to establish a priority order.
+     * The comparison is based on the rule type, unit, and ID, in that order.
+     *
+     * @param matchedRule The {@link MatchedRule} to compare against.
+     * @return A negative integer, zero, or a positive integer as this rule is less than, equal to, or greater than the specified rule.
+     */
     @Override
     public int compareTo(MatchedRule matchedRule) {
 
@@ -96,7 +123,19 @@ public final class MatchedRule implements Comparable<MatchedRule> {
     @Override
     public String toString() {
 
-        return "MatchedRule[ruleIdentifier=" + ruleIdentifier + ", reason=" + reason + ", legalBasis=" + legalBasis + ", applied=" + applied + ", writeValueWithLineBreaks=" + writeValueWithLineBreaks + ", references=" + references + ']';
+        return "MatchedRule[ruleIdentifier="
+               + ruleIdentifier
+               + ", reason="
+               + reason
+               + ", legalBasis="
+               + legalBasis
+               + ", applied="
+               + applied
+               + ", writeValueWithLineBreaks="
+               + writeValueWithLineBreaks
+               + ", references="
+               + references
+               + ']';
     }
 
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/TextEntity.java

@@ -1,6 +1,7 @@
 package com.iqser.red.service.redaction.v1.server.model.document.entity;
 
 import java.awt.geom.Rectangle2D;
+import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Comparator;
 import java.util.HashSet;
@@ -28,17 +29,20 @@ import lombok.experimental.FieldDefaults;
 @AllArgsConstructor
 @FieldDefaults(level = AccessLevel.PRIVATE)
 @EqualsAndHashCode(onlyExplicitlyIncluded = true)
+@SuppressWarnings("PMD.AvoidFieldNameMatchingMethodName")
 public class TextEntity implements IEntity {
 
     // primary key
     @EqualsAndHashCode.Include
-    final TextRange textRange;
-    @EqualsAndHashCode.Include
-    final String type;
-    @EqualsAndHashCode.Include
-    final EntityType entityType;
+    final String id;
     // primary key end
 
+    TextRange textRange;
+    @Builder.Default
+    List<TextRange> duplicateTextRanges = new ArrayList<>();
+    String type; // TODO: make final once ManualChangesApplicationService::recategorize is deleted
+    final EntityType entityType;
+
     @Builder.Default
     final PriorityQueue<MatchedRule> matchedRuleList = new PriorityQueue<>();
     final ManualChangeOverwrite manualOverwrite;
@@ -61,33 +65,66 @@ public class TextEntity implements IEntity {
     SemanticNode deepestFullyContainingNode;
 
 
-    public static TextEntity initialEntityNode(TextRange textRange, String type, EntityType entityType) {
+    public static TextEntity initialEntityNode(TextRange textRange, String type, EntityType entityType, SemanticNode node) {
 
-        return TextEntity.builder().type(type).entityType(entityType).textRange(textRange).manualOverwrite(new ManualChangeOverwrite(entityType)).build();
+        return TextEntity.builder()
+                .id(buildId(node, textRange, type, entityType))
+                .type(type)
+                .entityType(entityType)
+                .textRange(textRange)
+                .manualOverwrite(new ManualChangeOverwrite(entityType))
+                .build();
+    }
+
+
+    public static TextEntity initialEntityNode(TextRange textRange, String type, EntityType entityType, String id) {
+
+        return TextEntity.builder().id(id).type(type).entityType(entityType).textRange(textRange).manualOverwrite(new ManualChangeOverwrite(entityType)).build();
+    }
+
+
+    private static String buildId(SemanticNode node, TextRange textRange, String type, EntityType entityType) {
+
+        Map<Page, List<Rectangle2D>> rectanglesPerLinePerPage = node.getPositionsPerPage(textRange);
+        return IdBuilder.buildId(rectanglesPerLinePerPage.keySet(),
+                                 rectanglesPerLinePerPage.values()
+                                         .stream()
+                                         .flatMap(Collection::stream)
+                                         .toList(),
+                                 type,
+                                 entityType.name());
+    }
+
+
+    public void addTextRange(TextRange textRange) {
+
+        duplicateTextRanges.add(textRange);
     }
 
 
     public boolean occursInNodeOfType(Class<? extends SemanticNode> clazz) {
 
-        return intersectingNodes.stream().anyMatch(clazz::isInstance);
+        return intersectingNodes.stream()
+                .anyMatch(clazz::isInstance);
     }
 
 
     public boolean occursInNode(SemanticNode semanticNode) {
 
-        return intersectingNodes.stream().anyMatch(node -> node.equals(semanticNode));
+        return intersectingNodes.stream()
+                .anyMatch(node -> node.equals(semanticNode));
     }
 
 
     public boolean isType(String type) {
 
-        return this.type.equals(type);
+        return type().equals(type);
     }
 
 
     public boolean isAnyType(List<String> types) {
 
-        return types.contains(type);
+        return types.contains(type());
     }
 
 
@@ -124,8 +161,10 @@ public class TextEntity implements IEntity {
                     .min(Comparator.comparingInt(Page::getNumber))
                     .orElseThrow(() -> new RuntimeException("No Positions found on any page!"));
 
-            String id = IdBuilder.buildId(pages, rectanglesPerLinePerPage.values().stream().flatMap(Collection::stream).toList(), type, entityType.name());
-            positionsOnPagePerPage = rectanglesPerLinePerPage.entrySet().stream().map(entry -> buildPositionOnPage(firstPage, id, entry)).toList();
+            positionsOnPagePerPage = rectanglesPerLinePerPage.entrySet()
+                    .stream()
+                    .map(entry -> buildPositionOnPage(firstPage, id, entry))
+                    .toList();
         }
         return positionsOnPagePerPage;
     }
@@ -143,19 +182,37 @@ public class TextEntity implements IEntity {
 
     public boolean containedBy(TextEntity textEntity) {
 
-        return this.textRange.containedBy(textEntity.getTextRange());
+        return this.textRange.containedBy(textEntity.getTextRange()) //
+               || duplicateTextRanges.stream()
+                       .anyMatch(duplicateTextRange -> duplicateTextRange.containedBy(textEntity.textRange)) //
+               || duplicateTextRanges.stream()
+                       .anyMatch(duplicateTextRange -> textEntity.getDuplicateTextRanges()
+                               .stream()
+                               .anyMatch(duplicateTextRange::containedBy));
     }
 
 
     public boolean contains(TextEntity textEntity) {
 
-        return this.textRange.contains(textEntity.getTextRange());
+        return this.textRange.contains(textEntity.getTextRange()) //
+               || duplicateTextRanges.stream()
+                       .anyMatch(duplicateTextRange -> duplicateTextRange.contains(textEntity.textRange)) //
+               || duplicateTextRanges.stream()
+                       .anyMatch(duplicateTextRange -> textEntity.getDuplicateTextRanges()
+                               .stream()
+                               .anyMatch(duplicateTextRange::contains));
     }
 
 
     public boolean intersects(TextEntity textEntity) {
 
-        return this.textRange.intersects(textEntity.getTextRange());
+        return this.textRange.intersects(textEntity.getTextRange()) //
+               || duplicateTextRanges.stream()
+                       .anyMatch(duplicateTextRange -> duplicateTextRange.intersects(textEntity.textRange)) //
+               || duplicateTextRanges.stream()
+                       .anyMatch(duplicateTextRange -> textEntity.getDuplicateTextRanges()
+                               .stream()
+                               .anyMatch(duplicateTextRange::intersects));
     }
 
 
@@ -173,7 +230,8 @@ public class TextEntity implements IEntity {
 
     public boolean matchesAnnotationId(String manualRedactionId) {
 
-        return getPositionsOnPagePerPage().stream().anyMatch(entityPosition -> entityPosition.getId().equals(manualRedactionId));
+        return getPositionsOnPagePerPage().stream()
+                .anyMatch(entityPosition -> entityPosition.getId().equals(manualRedactionId));
     }
 
 
@@ -192,7 +250,7 @@ public class TextEntity implements IEntity {
         });
         sb.delete(sb.length() - 2, sb.length());
         sb.append("], type = \"");
-        sb.append(type);
+        sb.append(type());
         sb.append("\", EntityType.");
         sb.append(entityType);
         sb.append("]");
@@ -203,7 +261,16 @@ public class TextEntity implements IEntity {
     @Override
     public String type() {
 
-        return getManualOverwrite().getType().orElse(type);
+        return getManualOverwrite().getType()
+                .orElse(type);
+    }
+
+
+    @Override
+    public String value() {
+
+        return getManualOverwrite().getValue()
+                .orElse(getMatchedRule().isWriteValueWithLineBreaks() ? getValueWithLineBreaks() : value);
     }
 
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Document.java

@@ -14,21 +14,32 @@ import com.iqser.red.service.redaction.v1.server.model.document.DocumentTree;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlockCollector;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
 
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;
+import lombok.EqualsAndHashCode;
 import lombok.NoArgsConstructor;
 import lombok.experimental.FieldDefaults;
 
+/**
+ * Represents the entire document as a node within the document's semantic structure.
+ */
 @Data
 @Builder
 @AllArgsConstructor
 @NoArgsConstructor
 @FieldDefaults(level = AccessLevel.PRIVATE)
+@EqualsAndHashCode(onlyExplicitlyIncluded = true)
 public class Document implements GenericSemanticNode {
 
+    @EqualsAndHashCode.Include
+    List<Integer> treeId = Collections.emptyList();
+    @Builder.Default
+    Set<LayoutEngine> engines = new HashSet<>(Set.of(LayoutEngine.ALGORITHM));
+
     Set<Page> pages;
     DocumentTree documentTree;
     Integer numberOfPages;
@@ -49,28 +60,32 @@ public class Document implements GenericSemanticNode {
     public TextBlock getTextBlock() {
 
         if (textBlock == null) {
-            textBlock = streamTerminalTextBlocksInOrder().collect(new TextBlockCollector());
+            textBlock = GenericSemanticNode.super.getTextBlock();
         }
         return textBlock;
     }
 
 
+    /**
+     * Gets the main sections of the document as a list.
+     *
+     * @return A list of main sections within the document.
+     */
     public List<Section> getMainSections() {
 
-        return streamChildrenOfType(NodeType.SECTION).map(node -> (Section) node).collect(Collectors.toList());
+        return streamChildrenOfType(NodeType.SECTION).map(node -> (Section) node)
+                .collect(Collectors.toList());
     }
 
 
+    /**
+     * Streams all terminal (leaf) text blocks within the document in their natural order.
+     *
+     * @return A stream of terminal {@link TextBlock}.
+     */
     public Stream<TextBlock> streamTerminalTextBlocksInOrder() {
 
-        return streamAllNodes().filter(SemanticNode::isLeaf).map(SemanticNode::getLeafTextBlock);
-    }
-
-
-    @Override
-    public List<Integer> getTreeId() {
-
-        return Collections.emptyList();
+        return streamAllNodes().filter(SemanticNode::isLeaf).map(SemanticNode::getTextBlock);
     }
 
 
@@ -91,16 +106,29 @@ public class Document implements GenericSemanticNode {
     @Override
     public Headline getHeadline() {
 
-        return streamAllSubNodesOfType(NodeType.HEADLINE).map(node -> (Headline) node).findFirst().orElseGet(Headline::empty);
+        return streamAllSubNodesOfType(NodeType.HEADLINE).map(node -> (Headline) node)
+                .findFirst()
+                .orElseGet(Headline::empty);
     }
 
 
+    /**
+     * Streams all nodes within the document, regardless of type, in their natural order.
+     *
+     * @return A stream of all {@link SemanticNode} within the document.
+     */
     private Stream<SemanticNode> streamAllNodes() {
 
-        return documentTree.allEntriesInOrder().map(DocumentTree.Entry::getNode);
+        return documentTree.allEntriesInOrder()
+                .map(DocumentTree.Entry::getNode);
     }
 
 
+    /**
+     * Streams all image nodes contained within the document.
+     *
+     * @return A stream of {@link Image} nodes.
+     */
     public Stream<Image> streamAllImages() {
 
         return streamAllSubNodesOfType(NodeType.IMAGE).map(node -> (Image) node);

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/DuplicatedParagraph.java

@@ -0,0 +1,34 @@
+package com.iqser.red.service.redaction.v1.server.model.document.nodes;
+
+import java.util.stream.Stream;
+
+import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
+import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlockCollector;
+
+import lombok.Data;
+import lombok.EqualsAndHashCode;
+import lombok.experimental.SuperBuilder;
+
+@Data
+@EqualsAndHashCode(callSuper = true)
+@SuperBuilder
+public class DuplicatedParagraph extends Paragraph {
+
+    TextBlock unsortedLeafTextBlock;
+
+
+    @Override
+    public TextBlock getTextBlock() {
+
+        return Stream.of(leafTextBlock, unsortedLeafTextBlock).collect(new TextBlockCollector());
+
+    }
+
+
+    @Override
+    public String toString() {
+
+        return super.toString();
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Footer.java

@@ -9,6 +9,7 @@ import java.util.Set;
 import com.iqser.red.service.redaction.v1.server.model.document.DocumentTree;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
 
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
@@ -23,22 +24,23 @@ import lombok.experimental.FieldDefaults;
 @AllArgsConstructor
 @NoArgsConstructor
 @FieldDefaults(level = AccessLevel.PRIVATE)
+@EqualsAndHashCode(onlyExplicitlyIncluded = true)
 public class Footer implements GenericSemanticNode {
 
-    @Builder.Default
     final static SectionIdentifier sectionIdentifier = SectionIdentifier.empty();
+    @Builder.Default
+    Set<LayoutEngine> engines = new HashSet<>(Set.of(LayoutEngine.ALGORITHM));
 
+    @EqualsAndHashCode.Include
     List<Integer> treeId;
+
     TextBlock leafTextBlock;
 
-    @EqualsAndHashCode.Exclude
     DocumentTree documentTree;
 
     @Builder.Default
-    @EqualsAndHashCode.Exclude
     Set<TextEntity> entities = new HashSet<>();
 
-    @EqualsAndHashCode.Exclude
     Map<Page, Rectangle2D> bBoxCache;
 
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Header.java

@@ -9,6 +9,7 @@ import java.util.Set;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.DocumentTree;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
 
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
@@ -18,27 +19,31 @@ import lombok.EqualsAndHashCode;
 import lombok.NoArgsConstructor;
 import lombok.experimental.FieldDefaults;
 
+/**
+ * Represents the header part of a document page.
+ */
 @Data
 @Builder
 @AllArgsConstructor
 @NoArgsConstructor
 @FieldDefaults(level = AccessLevel.PRIVATE)
+@EqualsAndHashCode(onlyExplicitlyIncluded = true)
 public class Header implements GenericSemanticNode {
 
-    @Builder.Default
     final static SectionIdentifier sectionIdentifier = SectionIdentifier.empty();
+    @Builder.Default
+    Set<LayoutEngine> engines = new HashSet<>(Set.of(LayoutEngine.ALGORITHM));
 
+    @EqualsAndHashCode.Include
     List<Integer> treeId;
+
     TextBlock leafTextBlock;
 
-    @EqualsAndHashCode.Exclude
     DocumentTree documentTree;
 
     @Builder.Default
-    @EqualsAndHashCode.Exclude
     Set<TextEntity> entities = new HashSet<>();
 
-    @EqualsAndHashCode.Exclude
     Map<Page, Rectangle2D> bBoxCache;
 
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Headline.java

@@ -10,6 +10,7 @@ import com.iqser.red.service.redaction.v1.server.model.document.DocumentTree;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.AtomicTextBlock;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
 
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
@@ -19,25 +20,29 @@ import lombok.EqualsAndHashCode;
 import lombok.NoArgsConstructor;
 import lombok.experimental.FieldDefaults;
 
+/**
+ * Represents a headline in a document.
+ */
 @Data
 @Builder
 @AllArgsConstructor
 @NoArgsConstructor
 @FieldDefaults(level = AccessLevel.PRIVATE)
+@EqualsAndHashCode(onlyExplicitlyIncluded = true)
 public class Headline implements GenericSemanticNode {
 
+    @Builder.Default
+    Set<LayoutEngine> engines = new HashSet<>(Set.of(LayoutEngine.ALGORITHM));
+    @EqualsAndHashCode.Include
     List<Integer> treeId;
     TextBlock leafTextBlock;
     SectionIdentifier sectionIdentifier;
 
-    @EqualsAndHashCode.Exclude
     DocumentTree documentTree;
 
     @Builder.Default
-    @EqualsAndHashCode.Exclude
     Set<TextEntity> entities = new HashSet<>();
 
-    @EqualsAndHashCode.Exclude
     Map<Page, Rectangle2D> bBoxCache;
 
 
@@ -96,15 +101,27 @@ public class Headline implements GenericSemanticNode {
     }
 
 
+    /**
+     * Creates an empty headline with no text content.
+     *
+     * @return An empty {@link Headline} instance.
+     */
     public static Headline empty() {
 
         return Headline.builder().leafTextBlock(AtomicTextBlock.empty(-1L, 0, new Page(), -1, null)).build();
     }
 
 
+    /**
+     * Checks if this headline is associated with any paragraphs within its parent section or node.
+     *
+     * @return True if there are paragraphs associated with this headline, false otherwise.
+     */
     public boolean hasParagraphs() {
 
-        return getParent().streamAllSubNodesOfType(NodeType.PARAGRAPH).findFirst().isPresent();
+        return getParent().streamAllSubNodesOfType(NodeType.PARAGRAPH)
+                .findFirst()
+                .isPresent();
     }
 
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Image.java

@@ -18,6 +18,7 @@ import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRu
 import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlockCollector;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
 
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
@@ -27,13 +28,22 @@ import lombok.EqualsAndHashCode;
 import lombok.NoArgsConstructor;
 import lombok.experimental.FieldDefaults;
 
+/**
+ *
+ Represents an image within the document.
+ */
 @Data
 @Builder
 @AllArgsConstructor
 @NoArgsConstructor
 @FieldDefaults(level = AccessLevel.PRIVATE)
+@EqualsAndHashCode(onlyExplicitlyIncluded = true)
 public class Image implements GenericSemanticNode, IEntity {
 
+    @Builder.Default
+    Set<LayoutEngine> engines = new HashSet<>(Set.of(LayoutEngine.ALGORITHM));
+
+    @EqualsAndHashCode.Include
     List<Integer> treeId;
     String id;
 
@@ -68,7 +78,9 @@ public class Image implements GenericSemanticNode, IEntity {
     @Override
     public TextBlock getTextBlock() {
 
-        return streamAllSubNodes().filter(SemanticNode::isLeaf).map(SemanticNode::getLeafTextBlock).collect(new TextBlockCollector());
+        return streamAllSubNodes().filter(SemanticNode::isLeaf)
+                .map(SemanticNode::getLeafTextBlock)
+                .collect(new TextBlockCollector());
     }
 
 
@@ -89,14 +101,15 @@ public class Image implements GenericSemanticNode, IEntity {
     @Override
     public String type() {
 
-        return getManualOverwrite().getType().orElse(imageType.toString());
+        return getManualOverwrite().getType()
+                .orElse(imageType.toString().toLowerCase(Locale.ENGLISH));
     }
 
 
     @Override
     public String toString() {
 
-        return treeId + ": " + NodeType.IMAGE + ": " + imageType.toString() + " " + position;
+        return treeId + ": " + getValue() + " " + position;
     }
 
 
@@ -122,6 +135,32 @@ public class Image implements GenericSemanticNode, IEntity {
     }
 
 
+    public boolean mostlyContainedBy(Image image, double containmentThreshold) {
+
+        Map<Page, Rectangle2D> bboxImage = image.getBBox();
+        Map<Page, Rectangle2D> bbox = this.getBBox();
+        //image needs to be on the same page
+        if (bboxImage.get(this.page) != null) {
+            Rectangle2D intersection = bboxImage.get(this.page).createIntersection(bbox.get(this.page));
+            double calculatedIntersection = intersection.getWidth() * intersection.getHeight();
+            double area = bbox.get(this.page).getWidth() * bbox.get(this.page).getHeight();
+            return (calculatedIntersection / area) > containmentThreshold;
+        }
+        return false;
+    }
+
+
+    public boolean mostlyContains(Image image, double containmentThreshold) {
+
+        Map<Page, Rectangle2D> bboxImage = image.getBBox();
+        Map<Page, Rectangle2D> bbox = this.getBBox();
+        Rectangle2D intersection = bbox.get(this.page).createIntersection(bboxImage.get(this.page));
+        double calculatedIntersection = intersection.getWidth() * intersection.getHeight();
+        double area = bbox.get(this.page).getWidth() * bbox.get(this.page).getHeight();
+        return (area / calculatedIntersection) > containmentThreshold;
+    }
+
+
     public int length() {
 
         return 0;

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Page.java

@@ -17,38 +17,46 @@ import lombok.NoArgsConstructor;
 import lombok.Setter;
 import lombok.experimental.FieldDefaults;
 
+/**
+ * Represents a single page in a document.
+ */
 @Getter
 @Setter
 @Builder
 @NoArgsConstructor
 @AllArgsConstructor
 @FieldDefaults(level = AccessLevel.PRIVATE)
+@EqualsAndHashCode(onlyExplicitlyIncluded = true)
 public class Page {
 
+    @EqualsAndHashCode.Include
     Integer number;
     Integer height;
     Integer width;
     Integer rotation;
 
-    @EqualsAndHashCode.Exclude
     List<SemanticNode> mainBody;
-    @EqualsAndHashCode.Exclude
     Header header;
-    @EqualsAndHashCode.Exclude
     Footer footer;
 
     @Builder.Default
-    @EqualsAndHashCode.Exclude
     Set<TextEntity> entities = new HashSet<>();
 
     @Builder.Default
-    @EqualsAndHashCode.Exclude
     Set<Image> images = new HashSet<>();
 
 
+    /**
+     * Constructs and returns a {@link TextBlock} representing the concatenated text of all leaf semantic nodes in the main body.
+     *
+     * @return The main body text block.
+     */
     public TextBlock getMainBodyTextBlock() {
 
-        return mainBody.stream().filter(SemanticNode::isLeaf).map(SemanticNode::getLeafTextBlock).collect(new TextBlockCollector());
+        return mainBody.stream()
+                .filter(SemanticNode::isLeaf)
+                .map(SemanticNode::getLeafTextBlock)
+                .collect(new TextBlockCollector());
     }
 
 
@@ -58,18 +66,4 @@ public class Page {
         return String.valueOf(number);
     }
 
-
-    @Override
-    public int hashCode() {
-
-        return number;
-    }
-
-
-    @Override
-    public boolean equals(Object o) {
-
-        return o instanceof Page && o.hashCode() == this.hashCode();
-    }
-
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Paragraph.java

@@ -9,6 +9,7 @@ import java.util.Set;
 import com.iqser.red.service.redaction.v1.server.model.document.DocumentTree;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
 
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
@@ -16,24 +17,30 @@ import lombok.Builder;
 import lombok.Data;
 import lombok.EqualsAndHashCode;
 import lombok.experimental.FieldDefaults;
+import lombok.experimental.SuperBuilder;
 
+/**
+ * Represents a paragraph in the document.
+ */
 @Data
-@Builder
+@SuperBuilder
 @AllArgsConstructor
-@FieldDefaults(level = AccessLevel.PRIVATE)
+@FieldDefaults(level = AccessLevel.PROTECTED)
+@EqualsAndHashCode(onlyExplicitlyIncluded = true)
 public class Paragraph implements GenericSemanticNode {
 
+    @Builder.Default
+    Set<LayoutEngine> engines = new HashSet<>(Set.of(LayoutEngine.ALGORITHM));
+
+    @EqualsAndHashCode.Include
     List<Integer> treeId;
     TextBlock leafTextBlock;
 
-    @EqualsAndHashCode.Exclude
     DocumentTree documentTree;
 
     @Builder.Default
-    @EqualsAndHashCode.Exclude
     Set<TextEntity> entities = new HashSet<>();
 
-    @EqualsAndHashCode.Exclude
     Map<Page, Rectangle2D> bBoxCache;
 
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Section.java

@@ -10,35 +10,42 @@ import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntit
 import com.iqser.red.service.redaction.v1.server.model.document.DocumentTree;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlockCollector;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
 
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;
 import lombok.EqualsAndHashCode;
+import lombok.RequiredArgsConstructor;
 import lombok.experimental.FieldDefaults;
 import lombok.extern.slf4j.Slf4j;
 
+/**
+ * Represents a section within a document, encapsulating both its textual content and semantic structure.
+ */
 @Slf4j
 @Data
 @Builder
 @AllArgsConstructor
 @FieldDefaults(level = AccessLevel.PRIVATE)
+@EqualsAndHashCode(onlyExplicitlyIncluded = true)
 public class Section implements GenericSemanticNode {
 
+    @EqualsAndHashCode.Include
     List<Integer> treeId;
 
     TextBlock textBlock;
-    @EqualsAndHashCode.Exclude
     DocumentTree documentTree;
 
     @Builder.Default
-    @EqualsAndHashCode.Exclude
     Set<TextEntity> entities = new HashSet<>();
 
-    @EqualsAndHashCode.Exclude
     Map<Page, Rectangle2D> bBoxCache;
 
+    @Builder.Default
+    Set<LayoutEngine> engines = new HashSet<>(Set.of(LayoutEngine.ALGORITHM));
+
 
     @Override
     public NodeType getType() {
@@ -47,9 +54,15 @@ public class Section implements GenericSemanticNode {
     }
 
 
+    /**
+     * Checks if this section contains any tables.
+     *
+     * @return True if the section contains at least one table, false otherwise.
+     */
     public boolean hasTables() {
 
-        return streamAllSubNodesOfType(NodeType.TABLE).findAny().isPresent();
+        return streamAllSubNodesOfType(NodeType.TABLE).findAny()
+                .isPresent();
     }
 
 
@@ -64,7 +77,7 @@ public class Section implements GenericSemanticNode {
     public TextBlock getTextBlock() {
 
         if (textBlock == null) {
-            textBlock = streamAllSubNodes().filter(SemanticNode::isLeaf).map(SemanticNode::getLeafTextBlock).collect(new TextBlockCollector());
+            textBlock = GenericSemanticNode.super.getTextBlock();
         }
         return textBlock;
     }
@@ -86,12 +99,24 @@ public class Section implements GenericSemanticNode {
     }
 
 
+    /**
+     * Checks if any headline within this section or its sub-nodes contains a given string.
+     *
+     * @param value The string to search for within headlines, case-sensitive.
+     * @return True if at least one headline contains the specified string, false otherwise.
+     */
     public boolean anyHeadlineContainsString(String value) {
 
         return streamAllSubNodesOfType(NodeType.HEADLINE).anyMatch(h -> h.containsString(value));
     }
 
 
+    /**
+     * Checks if any headline within this section or its sub-nodes contains a given string, case-insensitive.
+     *
+     * @param value The string to search for within headlines, case-insensitive.
+     * @return True if at least one headline contains the specified string, false otherwise.
+     */
     public boolean anyHeadlineContainsStringIgnoreCase(String value) {
 
         return streamAllSubNodesOfType(NodeType.HEADLINE).anyMatch(h -> h.containsStringIgnoreCase(value));

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/SectionIdentifier.java

@@ -10,6 +10,9 @@ import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
 import lombok.experimental.FieldDefaults;
 
+/**
+ * Represents a unique identifier for a section within a document.
+ */
 @AllArgsConstructor
 @FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
 public class SectionIdentifier {
@@ -28,6 +31,12 @@ public class SectionIdentifier {
     boolean asChild;
 
 
+    /**
+     * Generates a SectionIdentifier from the headline text of a section, determining its format and structure.
+     *
+     * @param headline The headline text from which to generate the section identifier.
+     * @return A {@link SectionIdentifier} instance corresponding to the headline text.
+     */
     public static SectionIdentifier fromSearchText(String headline) {
 
         if (headline == null || headline.isEmpty() || headline.isBlank()) {
@@ -43,18 +52,34 @@ public class SectionIdentifier {
     }
 
 
+    /**
+     * Marks the current section identifier as a child of another section.
+     *
+     * @param sectionIdentifier The parent section identifier.
+     * @return A new {@link SectionIdentifier} instance marked as a child.
+     */
     public static SectionIdentifier asChildOf(SectionIdentifier sectionIdentifier) {
 
         return new SectionIdentifier(sectionIdentifier.format, sectionIdentifier.toString(), sectionIdentifier.identifiers, true);
     }
 
 
+    /**
+     * Generates a SectionIdentifier that represents the entire document.
+     *
+     * @return A {@link SectionIdentifier} with a document-wide scope.
+     */
     public static SectionIdentifier document() {
 
         return new SectionIdentifier(Format.DOCUMENT, "document", Collections.emptyList(), false);
     }
 
 
+    /**
+     * Generates an empty SectionIdentifier.
+     *
+     * @return An empty {@link SectionIdentifier} instance.
+     */
     public static SectionIdentifier empty() {
 
         return new SectionIdentifier(Format.EMPTY, "empty", Collections.emptyList(), false);
@@ -72,7 +97,11 @@ public class SectionIdentifier {
             }
             identifiers.add(Integer.parseInt(numericalIdentifier.trim()));
         }
-        return new SectionIdentifier(Format.NUMERICAL, identifierString, identifiers.stream().toList(), false);
+        return new SectionIdentifier(Format.NUMERICAL,
+                                     identifierString,
+                                     identifiers.stream()
+                                             .toList(),
+                                     false);
     }
 
 
@@ -105,6 +134,12 @@ public class SectionIdentifier {
     }
 
 
+    /**
+     * Determines if the current section is a child of the given section, based on their identifiers.
+     *
+     * @param sectionIdentifier The section identifier to compare against.
+     * @return True if the current section is a child of the given section, false otherwise.
+     */
     public boolean isChildOf(SectionIdentifier sectionIdentifier) {
 
         if (this.format.equals(Format.DOCUMENT) || this.format.equals(Format.EMPTY)) {

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/SemanticNode.java

@@ -19,8 +19,11 @@ import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.AtomicTextBlock;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
+import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlockCollector;
+import com.iqser.red.service.redaction.v1.server.service.document.NodeVisitor;
 import com.iqser.red.service.redaction.v1.server.utils.RectangleTransformations;
 import com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
 
 public interface SemanticNode {
 
@@ -39,7 +42,12 @@ public interface SemanticNode {
      *
      * @return TextBlock containing all AtomicTextBlocks that are located under this Node.
      */
-    TextBlock getTextBlock();
+    default TextBlock getTextBlock() {
+
+        return streamAllSubNodes().filter(SemanticNode::isLeaf)
+                .map(SemanticNode::getTextBlock)
+                .collect(new TextBlockCollector());
+    }
 
 
     /**
@@ -69,7 +77,10 @@ public interface SemanticNode {
      */
     default Page getFirstPage() {
 
-        return getTextBlock().getPages().stream().min(Comparator.comparingInt(Page::getNumber)).orElseThrow();
+        return getTextBlock().getPages()
+                .stream()
+                .min(Comparator.comparingInt(Page::getNumber))
+                .orElseThrow();
     }
 
 
@@ -80,8 +91,8 @@ public interface SemanticNode {
      */
     default Set<Page> getPages(TextRange textRange) {
 
-        if (!getTextRange().contains(textRange)) {
-            throw new IllegalArgumentException(format("%s which was used to query for pages is not contained in the %s of this node!", textRange, getTextRange()));
+        if (!getTextRange().intersects(textRange)) {
+            throw new IllegalArgumentException(format("%s which was used to query for pages is not intersected in the %s of this node!", textRange, getTextRange()));
         }
         return getTextBlock().getPages(textRange);
     }
@@ -95,7 +106,8 @@ public interface SemanticNode {
      */
     default boolean onPage(int pageNumber) {
 
-        return getPages().stream().anyMatch(page -> page.getNumber() == pageNumber);
+        return getPages().stream()
+                .anyMatch(page -> page.getNumber() == pageNumber);
     }
 
 
@@ -247,7 +259,9 @@ public interface SemanticNode {
      */
     default boolean hasEntitiesOfType(String type) {
 
-        return getEntities().stream().filter(TextEntity::active).anyMatch(redactionEntity -> redactionEntity.getType().equals(type));
+        return getEntities().stream()
+                .filter(TextEntity::active)
+                .anyMatch(redactionEntity -> redactionEntity.type().equals(type));
     }
 
 
@@ -260,7 +274,10 @@ public interface SemanticNode {
      */
     default boolean hasEntitiesOfAnyType(String... types) {
 
-        return getEntities().stream().filter(TextEntity::active).anyMatch(redactionEntity -> Arrays.stream(types).anyMatch(type -> redactionEntity.getType().equals(type)));
+        return getEntities().stream()
+                .filter(TextEntity::active)
+                .anyMatch(redactionEntity -> Arrays.stream(types)
+                        .anyMatch(type -> redactionEntity.type().equals(type)));
     }
 
 
@@ -273,7 +290,12 @@ public interface SemanticNode {
      */
     default boolean hasEntitiesOfAllTypes(String... types) {
 
-        return getEntities().stream().filter(TextEntity::active).map(TextEntity::getType).collect(Collectors.toUnmodifiableSet()).containsAll(Arrays.stream(types).toList());
+        return getEntities().stream()
+                .filter(TextEntity::active)
+                .map(TextEntity::type)
+                .collect(Collectors.toUnmodifiableSet())
+                .containsAll(Arrays.stream(types)
+                                     .toList());
     }
 
 
@@ -286,7 +308,10 @@ public interface SemanticNode {
      */
     default List<TextEntity> getEntitiesOfType(String type) {
 
-        return getEntities().stream().filter(TextEntity::active).filter(redactionEntity -> redactionEntity.getType().equals(type)).toList();
+        return getEntities().stream()
+                .filter(TextEntity::active)
+                .filter(redactionEntity -> redactionEntity.type().equals(type))
+                .toList();
     }
 
 
@@ -299,7 +324,10 @@ public interface SemanticNode {
      */
     default List<TextEntity> getEntitiesOfType(List<String> types) {
 
-        return getEntities().stream().filter(TextEntity::active).filter(redactionEntity -> redactionEntity.isAnyType(types)).toList();
+        return getEntities().stream()
+                .filter(TextEntity::active)
+                .filter(redactionEntity -> redactionEntity.isAnyType(types))
+                .toList();
     }
 
 
@@ -312,7 +340,11 @@ public interface SemanticNode {
      */
     default List<TextEntity> getEntitiesOfType(String... types) {
 
-        return getEntities().stream().filter(TextEntity::active).filter(redactionEntity -> redactionEntity.isAnyType(Arrays.stream(types).toList())).toList();
+        return getEntities().stream()
+                .filter(TextEntity::active)
+                .filter(redactionEntity -> redactionEntity.isAnyType(Arrays.stream(types)
+                                                                             .toList()))
+                .toList();
     }
 
 
@@ -326,7 +358,8 @@ public interface SemanticNode {
 
         TextBlock textBlock = getTextBlock();
         if (!textBlock.getAtomicTextBlocks().isEmpty()) {
-            return getTextBlock().getAtomicTextBlocks().get(0).getNumberOnPage();
+            return getTextBlock().getAtomicTextBlocks()
+                    .get(0).getNumberOnPage();
         } else {
             return -1;
         }
@@ -356,6 +389,15 @@ public interface SemanticNode {
     }
 
 
+    Set<LayoutEngine> getEngines();
+
+
+    default void addEngine(LayoutEngine engine) {
+
+        getEngines().add(engine);
+    }
+
+
     /**
      * Checks whether this SemanticNode contains all the provided Strings.
      *
@@ -364,7 +406,8 @@ public interface SemanticNode {
      */
     default boolean containsAllStrings(String... strings) {
 
-        return Arrays.stream(strings).allMatch(this::containsString);
+        return Arrays.stream(strings)
+                .allMatch(this::containsString);
     }
 
 
@@ -376,7 +419,8 @@ public interface SemanticNode {
      */
     default boolean containsAnyString(String... strings) {
 
-        return Arrays.stream(strings).anyMatch(this::containsString);
+        return Arrays.stream(strings)
+                .anyMatch(this::containsString);
     }
 
 
@@ -388,15 +432,16 @@ public interface SemanticNode {
      */
     default boolean containsAnyString(List<String> strings) {
 
-        return strings.stream().anyMatch(this::containsString);
+        return strings.stream()
+                .anyMatch(this::containsString);
     }
 
 
     /**
-     * Checks whether this SemanticNode contains all the provided Strings ignoring case.
+     * Checks whether this SemanticNode contains all the provided Strings case-insensitive.
      *
      * @param string A String which the TextBlock might contain
-     * @return true, if this node's TextBlock contains the string ignoring case
+     * @return true, if this node's TextBlock contains the string case-insensitive
      */
     default boolean containsStringIgnoreCase(String string) {
 
@@ -405,26 +450,28 @@ public interface SemanticNode {
 
 
     /**
-     * Checks whether this SemanticNode contains any of the provided Strings ignoring case.
+     * Checks whether this SemanticNode contains any of the provided Strings case-insensitive.
      *
      * @param strings A List of Strings which the TextBlock might contain
      * @return true, if this node's TextBlock contains any of the strings
      */
     default boolean containsAnyStringIgnoreCase(String... strings) {
 
-        return Arrays.stream(strings).anyMatch(this::containsStringIgnoreCase);
+        return Arrays.stream(strings)
+                .anyMatch(this::containsStringIgnoreCase);
     }
 
 
     /**
-     * Checks whether this SemanticNode contains any of the provided Strings ignoring case.
+     * Checks whether this SemanticNode contains any of the provided Strings case-insensitive.
      *
      * @param strings A List of Strings which the TextBlock might contain
      * @return true, if this node's TextBlock contains any of the strings
      */
     default boolean containsAllStringsIgnoreCase(String... strings) {
 
-        return Arrays.stream(strings).allMatch(this::containsStringIgnoreCase);
+        return Arrays.stream(strings)
+                .allMatch(this::containsStringIgnoreCase);
     }
 
 
@@ -436,19 +483,24 @@ public interface SemanticNode {
      */
     default boolean containsWord(String word) {
 
-        return getTextBlock().getWords().stream().anyMatch(s -> s.equals(word));
+        return getTextBlock().getWords()
+                .stream()
+                .anyMatch(s -> s.equals(word));
     }
 
 
     /**
-     * Checks whether this SemanticNode contains exactly the provided String as a word ignoring case.
+     * Checks whether this SemanticNode contains exactly the provided String as a word case-insensitive.
      *
      * @param word - String which the TextBlock might contain
      * @return true, if this node's TextBlock contains string
      */
     default boolean containsWordIgnoreCase(String word) {
 
-        return getTextBlock().getWords().stream().map(String::toLowerCase).anyMatch(s -> s.equals(word.toLowerCase(Locale.ENGLISH)));
+        return getTextBlock().getWords()
+                .stream()
+                .map(String::toLowerCase)
+                .anyMatch(s -> s.equals(word.toLowerCase(Locale.ENGLISH)));
     }
 
 
@@ -460,19 +512,27 @@ public interface SemanticNode {
      */
     default boolean containsAnyWord(String... words) {
 
-        return Arrays.stream(words).anyMatch(word -> getTextBlock().getWords().stream().anyMatch(word::equals));
+        return Arrays.stream(words)
+                .anyMatch(word -> getTextBlock().getWords()
+                        .stream()
+                        .anyMatch(word::equals));
     }
 
 
     /**
-     * Checks whether this SemanticNode contains any of the provided Strings as a word ignoring case.
+     * Checks whether this SemanticNode contains any of the provided Strings as a word case-insensitive.
      *
      * @param words - A List of Strings which the TextBlock might contain
      * @return true, if this node's TextBlock contains any of the provided strings
      */
     default boolean containsAnyWordIgnoreCase(String... words) {
 
-        return Arrays.stream(words).map(String::toLowerCase).anyMatch(word -> getTextBlock().getWords().stream().map(String::toLowerCase).anyMatch(word::equals));
+        return Arrays.stream(words)
+                .map(String::toLowerCase)
+                .anyMatch(word -> getTextBlock().getWords()
+                        .stream()
+                        .map(String::toLowerCase)
+                        .anyMatch(word::equals));
     }
 
 
@@ -484,19 +544,27 @@ public interface SemanticNode {
      */
     default boolean containsAllWords(String... words) {
 
-        return Arrays.stream(words).allMatch(word -> getTextBlock().getWords().stream().anyMatch(word::equals));
+        return Arrays.stream(words)
+                .allMatch(word -> getTextBlock().getWords()
+                        .stream()
+                        .anyMatch(word::equals));
     }
 
 
     /**
-     * Checks whether this SemanticNode contains all the provided Strings as word ignoring case.
+     * Checks whether this SemanticNode contains all the provided Strings as word case-insensitive.
      *
      * @param words - A List of Strings which the TextBlock might contain
      * @return true, if this node's TextBlock contains all the provided strings
      */
     default boolean containsAllWordsIgnoreCase(String... words) {
 
-        return Arrays.stream(words).map(String::toLowerCase).allMatch(word -> getTextBlock().getWords().stream().map(String::toLowerCase).anyMatch(word::equals));
+        return Arrays.stream(words)
+                .map(String::toLowerCase)
+                .allMatch(word -> getTextBlock().getWords()
+                        .stream()
+                        .map(String::toLowerCase)
+                        .anyMatch(word::equals));
     }
 
 
@@ -513,10 +581,10 @@ public interface SemanticNode {
 
 
     /**
-     * Checks whether this SemanticNode matches the provided regex pattern ignoring case.
+     * Checks whether this SemanticNode matches the provided regex pattern case-insensitive.
      *
      * @param regexPattern A String representing a regex pattern, which the TextBlock might contain
-     * @return true, if this node's TextBlock contains the regex pattern ignoring case
+     * @return true, if this node's TextBlock contains the regex pattern case-insensitive
      */
     default boolean matchesRegexIgnoreCase(String regexPattern) {
 
@@ -536,7 +604,11 @@ public interface SemanticNode {
      */
     default boolean intersectsRectangle(int x, int y, int w, int h, int pageNumber) {
 
-        return getBBox().entrySet().stream().filter(entry -> entry.getKey().getNumber() == pageNumber).map(Map.Entry::getValue).anyMatch(rect -> rect.intersects(x, y, w, h));
+        return getBBox().entrySet()
+                .stream()
+                .filter(entry -> entry.getKey().getNumber() == pageNumber)
+                .map(Map.Entry::getValue)
+                .anyMatch(rect -> rect.intersects(x, y, w, h));
     }
 
 
@@ -553,7 +625,6 @@ public interface SemanticNode {
             if (textBlock.containsTextRange(textEntity.getTextRange())) {
                 textEntity.setDeepestFullyContainingNode(this);
             }
-
             textEntity.addIntersectingNode(this);
             streamChildren().filter(semanticNode -> semanticNode.getTextRange().intersects(textEntity.getTextRange()))
                     .forEach(node -> node.addThisToEntityIfIntersects(textEntity));
@@ -590,7 +661,8 @@ public interface SemanticNode {
      */
     default Stream<SemanticNode> streamAllSubNodes() {
 
-        return getDocumentTree().allSubEntriesInOrder(getTreeId()).map(DocumentTree.Entry::getNode);
+        return getDocumentTree().allSubEntriesInOrder(getTreeId())
+                .map(DocumentTree.Entry::getNode);
     }
 
 
@@ -601,7 +673,9 @@ public interface SemanticNode {
      */
     default Stream<SemanticNode> streamAllSubNodesOfType(NodeType nodeType) {
 
-        return getDocumentTree().allSubEntriesInOrder(getTreeId()).filter(entry -> entry.getType().equals(nodeType)).map(DocumentTree.Entry::getNode);
+        return getDocumentTree().allSubEntriesInOrder(getTreeId())
+                .filter(entry -> entry.getType().equals(nodeType))
+                .map(DocumentTree.Entry::getNode);
     }
 
 
@@ -627,6 +701,28 @@ public interface SemanticNode {
     }
 
 
+    /**
+     * For a given TextRange this function returns a List of rectangle around the text in the range.
+     * These Rectangles are split either by a new line or by a large gap in the current line.
+     * This is mainly used to find the positions of TextEntities
+     *
+     * @param textRange A TextRange to calculate the positions for.
+     * @return A Map, where the keys are the pages and the values are a list of rectangles describing the position of words
+     */
+    default Map<Page, List<Rectangle2D>> getPositionsPerPage(TextRange textRange) {
+
+        if (isLeaf()) {
+            return getTextBlock().getPositionsPerPage(textRange);
+        }
+        Optional<SemanticNode> containingChildNode = streamChildren().filter(child -> child.getTextRange().contains(textRange))
+                .findFirst();
+        if (containingChildNode.isEmpty()) {
+            return getTextBlock().getPositionsPerPage(textRange);
+        }
+        return containingChildNode.get().getPositionsPerPage(textRange);
+    }
+
+
     /**
      * If this Node is a Leaf it will calculate the boundingBox of its LeafTextBlock, otherwise it will calculate the Union of the BoundingBoxes of all its Children.
      * If called on the Document, it will return the cropbox of each page
@@ -669,8 +765,12 @@ public interface SemanticNode {
     private Map<Page, Rectangle2D> getBBoxFromChildren() {
 
         Map<Page, Rectangle2D> bBoxPerPage = new HashMap<>();
-        List<Map<Page, Rectangle2D>> childrenBBoxes = streamChildren().map(SemanticNode::getBBox).toList();
-        Set<Page> pages = childrenBBoxes.stream().flatMap(map -> map.keySet().stream()).collect(Collectors.toSet());
+        List<Map<Page, Rectangle2D>> childrenBBoxes = streamChildren().map(SemanticNode::getBBox)
+                .toList();
+        Set<Page> pages = childrenBBoxes.stream()
+                .flatMap(map -> map.keySet()
+                        .stream())
+                .collect(Collectors.toSet());
         for (Page page : pages) {
             Rectangle2D bBoxOnPage = childrenBBoxes.stream()
                     .filter(childBboxPerPage -> childBboxPerPage.containsKey(page))
@@ -688,9 +788,25 @@ public interface SemanticNode {
     private Map<Page, Rectangle2D> getBBoxFromLeafTextBlock() {
 
         Map<Page, Rectangle2D> bBoxPerPage = new HashMap<>();
-        Map<Page, List<AtomicTextBlock>> atomicTextBlockPerPage = getTextBlock().getAtomicTextBlocks().stream().collect(Collectors.groupingBy(AtomicTextBlock::getPage));
+        Map<Page, List<AtomicTextBlock>> atomicTextBlockPerPage = getTextBlock().getAtomicTextBlocks()
+                .stream()
+                .collect(Collectors.groupingBy(AtomicTextBlock::getPage));
         atomicTextBlockPerPage.forEach((page, atomicTextBlocks) -> bBoxPerPage.put(page, RectangleTransformations.atomicTextBlockBBox(atomicTextBlocks)));
         return bBoxPerPage;
     }
 
+
+    /**
+     * Accepts a {@link NodeVisitor} and initiates a depth-first traversal of the semantic tree rooted at this node.
+     * The visitor's {@link NodeVisitor#visit(SemanticNode)} method is invoked for each node encountered during the traversal.
+     *
+     * @param visitor The {@link NodeVisitor} to accept and apply during the traversal.
+     * @see NodeVisitor
+     */
+    default void accept(NodeVisitor visitor) {
+
+        visitor.visit(this);
+        streamChildren().forEach(childNode -> childNode.accept(visitor));
+    }
+
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Table.java

@@ -9,6 +9,7 @@ import java.util.List;
 import java.util.Locale;
 import java.util.Map;
 import java.util.Set;
+import java.util.stream.Collectors;
 import java.util.stream.IntStream;
 import java.util.stream.Stream;
 
@@ -16,6 +17,7 @@ import com.iqser.red.service.redaction.v1.server.model.document.DocumentTree;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlockCollector;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
 
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
@@ -24,12 +26,19 @@ import lombok.Data;
 import lombok.EqualsAndHashCode;
 import lombok.experimental.FieldDefaults;
 
+/**
+ * Represents a table within a document.
+ */
 @Data
 @Builder
 @AllArgsConstructor
 @FieldDefaults(level = AccessLevel.PRIVATE)
+@EqualsAndHashCode(onlyExplicitlyIncluded = true)
 public class Table implements SemanticNode {
 
+    @Builder.Default
+    Set<LayoutEngine> engines = new HashSet<>(Set.of(LayoutEngine.ALGORITHM));
+    @EqualsAndHashCode.Include
     List<Integer> treeId;
     DocumentTree documentTree;
 
@@ -39,10 +48,8 @@ public class Table implements SemanticNode {
     TextBlock textBlock;
 
     @Builder.Default
-    @EqualsAndHashCode.Exclude
     Set<TextEntity> entities = new HashSet<>();
 
-    @EqualsAndHashCode.Exclude
     Map<Page, Rectangle2D> bBoxCache;
 
 
@@ -64,8 +71,7 @@ public class Table implements SemanticNode {
      */
     public Stream<TextEntity> streamEntitiesWhereRowContainsStringsIgnoreCase(List<String> strings) {
 
-        return IntStream.range(0, numberOfRows)
-                .boxed()
+        return IntStream.range(0, numberOfRows).boxed()
                 .filter(row -> rowContainsStringsIgnoreCase(row, strings))
                 .flatMap(this::streamRow)
                 .map(TableCell::getEntities)
@@ -82,8 +88,11 @@ public class Table implements SemanticNode {
      */
     public boolean rowContainsStringsIgnoreCase(Integer row, List<String> strings) {
 
-        String rowText = streamRow(row).map(TableCell::getTextBlock).collect(new TextBlockCollector()).getSearchText().toLowerCase(Locale.ROOT);
-        return strings.stream().map(String::toLowerCase).allMatch(rowText::contains);
+        String rowText = streamRow(row).map(TableCell::getTextBlock)
+                .collect(new TextBlockCollector()).getSearchText().toLowerCase(Locale.ROOT);
+        return strings.stream()
+                .map(String::toLowerCase)
+                .allMatch(rowText::contains);
     }
 
 
@@ -96,9 +105,13 @@ public class Table implements SemanticNode {
      */
     public Stream<TextEntity> streamEntitiesWhereRowHasHeaderAndValue(String header, String value) {
 
-        List<Integer> vertebrateStudyCols = streamHeaders().filter(headerNode -> headerNode.containsString(header)).map(TableCell::getCol).toList();
+        List<Integer> vertebrateStudyCols = streamHeaders().filter(headerNode -> headerNode.containsString(header))
+                .map(TableCell::getCol)
+                .toList();
         return streamTableCells().filter(tableCellNode -> vertebrateStudyCols.stream()
-                .anyMatch(vertebrateStudyCol -> getCell(tableCellNode.getRow(), vertebrateStudyCol).containsString(value))).map(TableCell::getEntities).flatMap(Collection::stream);
+                        .anyMatch(vertebrateStudyCol -> getCell(tableCellNode.getRow(), vertebrateStudyCol).containsString(value)))
+                .map(TableCell::getEntities)
+                .flatMap(Collection::stream);
     }
 
 
@@ -111,9 +124,13 @@ public class Table implements SemanticNode {
      */
     public Stream<TextEntity> streamEntitiesWhereRowHasHeaderAndAnyValue(String header, List<String> values) {
 
-        List<Integer> colsWithHeader = streamHeaders().filter(headerNode -> headerNode.containsString(header)).map(TableCell::getCol).toList();
+        List<Integer> colsWithHeader = streamHeaders().filter(headerNode -> headerNode.containsString(header))
+                .map(TableCell::getCol)
+                .toList();
         return streamTableCells().filter(tableCellNode -> colsWithHeader.stream()
-                .anyMatch(colWithHeader -> getCell(tableCellNode.getRow(), colWithHeader).containsAnyString(values))).map(TableCell::getEntities).flatMap(Collection::stream);
+                        .anyMatch(colWithHeader -> getCell(tableCellNode.getRow(), colWithHeader).containsAnyString(values)))
+                .map(TableCell::getEntities)
+                .flatMap(Collection::stream);
     }
 
 
@@ -126,16 +143,33 @@ public class Table implements SemanticNode {
      */
     public Stream<TextEntity> streamEntitiesWhereRowContainsEntitiesOfType(List<String> types) {
 
-        List<Integer> rowsWithEntityOfType = getEntities().stream()
-                .filter(TextEntity::active)
-                .filter(redactionEntity -> types.stream().anyMatch(type -> type.equals(redactionEntity.getType())))
-                .map(TextEntity::getIntersectingNodes)
-                .filter(node -> node instanceof TableCell)
-                .map(node -> (TableCell) node)
-                .map(TableCell::getRow)
-                .toList();
+        return IntStream.range(0, numberOfRows).boxed()
+                .filter(rowNumber -> streamTextEntitiesInRow(rowNumber).map(TextEntity::type)
+                        .anyMatch(types::contains))
+                .flatMap(this::streamRow)
+                .map(TableCell::getEntities)
+                .flatMap(Collection::stream);
+    }
 
-        return rowsWithEntityOfType.stream().flatMap(this::streamRow).map(TableCell::getEntities).flatMap(Collection::stream);
+
+    /**
+     * Streams all entities in this table, that appear in a row, which contains at least one entity of each of the provided types.
+     * Ignores Entity with ignored == true or removed == true.
+     *
+     * @param types type strings to check whether a row contains an entity like them
+     * @return Stream of all entities in this table, that appear in a row, which contains at least one entity of each of the provided types.
+     */
+    public Stream<TextEntity> streamEntitiesWhereRowContainsEntitiesOfEachType(List<String> types) {
+
+        return IntStream.range(0, numberOfRows).boxed()
+                .filter(rowNumber -> {
+                    Set<String> entityTypes = streamTextEntitiesInRow(rowNumber).map(TextEntity::type)
+                            .collect(Collectors.toSet());
+                    return entityTypes.containsAll(types);
+                })
+                .flatMap(this::streamRow)
+                .map(TableCell::getEntities)
+                .flatMap(Collection::stream);
     }
 
 
@@ -148,18 +182,43 @@ public class Table implements SemanticNode {
      */
     public Stream<TextEntity> streamEntitiesWhereRowContainsNoEntitiesOfType(List<String> types) {
 
-        return IntStream.range(0, numberOfRows)
-                .boxed()
-                .filter(rowNumber -> streamRow(rowNumber).map(TableCell::getEntities)
-                        .flatMap(Collection::stream)
-                        .filter(TextEntity::active)
-                        .noneMatch(entity -> types.contains(entity.getType())))
+        return IntStream.range(0, numberOfRows).boxed()
+                .filter(rowNumber -> streamTextEntitiesInRow(rowNumber).map(TextEntity::type)
+                        .noneMatch(types::contains))
                 .flatMap(this::streamRow)
                 .map(TableCell::getEntities)
                 .flatMap(Collection::stream);
     }
 
 
+    /**
+     * Streams all Entities in the given row.
+     *
+     * @param rowNumber the row number to look for
+     * @return stream of TextEntities occurring in row
+     */
+    public Stream<TextEntity> streamTextEntitiesInRow(int rowNumber) {
+
+        return streamRow(rowNumber).map(TableCell::getEntities)
+                .flatMap(Collection::stream)
+                .filter(TextEntity::active);
+    }
+
+
+    /**
+     * Streams all Entities in the given col.
+     *
+     * @param colNumber the column number to look for
+     * @return stream of TextEntities occurring in row
+     */
+    public Stream<TextEntity> streamTextEntitiesInCol(int colNumber) {
+
+        return streamCol(colNumber).map(TableCell::getEntities)
+                .flatMap(Collection::stream)
+                .filter(TextEntity::active);
+    }
+
+
     /**
      * Returns a TableCell at the provided row and column location.
      *
@@ -173,7 +232,8 @@ public class Table implements SemanticNode {
             throw new IllegalArgumentException(format("row %d, col %d is out of bounds for number of rows of %d and number of cols %d", row, col, numberOfRows, numberOfCols));
         }
         int idx = row * numberOfCols + col;
-        return (TableCell) documentTree.getEntryById(treeId).getChildren().get(idx).getNode();
+        return (TableCell) documentTree.getEntryById(treeId).getChildren()
+                .get(idx).getNode();
     }
 
 
@@ -196,7 +256,7 @@ public class Table implements SemanticNode {
      */
     public Stream<TableCell> streamTableCellsWhichContainType(String type) {
 
-        return streamTableCells().filter(tableCell -> tableCell.getEntities().stream().filter(TextEntity::active).anyMatch(entity -> entity.getType().equals(type)));
+        return streamTableCells().filter(tableCell -> tableCell.hasEntitiesOfType(type));
     }
 
 
@@ -222,7 +282,8 @@ public class Table implements SemanticNode {
      */
     public Stream<TableCell> streamCol(int col) {
 
-        return IntStream.range(0, numberOfRows).boxed().map(row -> getCell(row, col));
+        return IntStream.range(0, numberOfRows).boxed()
+                .map(row -> getCell(row, col));
     }
 
 
@@ -234,7 +295,8 @@ public class Table implements SemanticNode {
      */
     public Stream<TableCell> streamRow(int row) {
 
-        return IntStream.range(0, numberOfCols).boxed().map(col -> getCell(row, col));
+        return IntStream.range(0, numberOfCols).boxed()
+                .map(col -> getCell(row, col));
     }
 
 
@@ -258,7 +320,8 @@ public class Table implements SemanticNode {
      */
     public Stream<TableCell> streamHeadersForCell(int row, int col) {
 
-        return Stream.concat(streamRow(row), streamCol(col)).filter(TableCell::isHeader);
+        return Stream.concat(streamRow(row), streamCol(col))
+                .filter(TableCell::isHeader);
     }
 
 
@@ -348,7 +411,7 @@ public class Table implements SemanticNode {
     public TextBlock getTextBlock() {
 
         if (textBlock == null) {
-            textBlock = streamAllSubNodes().filter(SemanticNode::isLeaf).map(SemanticNode::getLeafTextBlock).collect(new TextBlockCollector());
+            textBlock = SemanticNode.super.getTextBlock();
         }
         return textBlock;
     }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/TableCell.java

@@ -11,6 +11,7 @@ import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntit
 import com.iqser.red.service.redaction.v1.server.model.document.DocumentTree;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlockCollector;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
 
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
@@ -19,12 +20,20 @@ import lombok.Data;
 import lombok.EqualsAndHashCode;
 import lombok.experimental.FieldDefaults;
 
+/**
+ * Represents a single table cell within a table.
+ */
 @Data
 @Builder
 @AllArgsConstructor
 @FieldDefaults(level = AccessLevel.PRIVATE)
+@EqualsAndHashCode(onlyExplicitlyIncluded = true)
 public class TableCell implements GenericSemanticNode {
 
+    @Builder.Default
+    Set<LayoutEngine> engines = new HashSet<>(Set.of(LayoutEngine.ALGORITHM));
+
+    @EqualsAndHashCode.Include
     List<Integer> treeId;
     int row;
     int col;
@@ -36,11 +45,9 @@ public class TableCell implements GenericSemanticNode {
 
     TextBlock textBlock;
 
-    @EqualsAndHashCode.Exclude
     DocumentTree documentTree;
 
     @Builder.Default
-    @EqualsAndHashCode.Exclude
     Set<TextEntity> entities = new HashSet<>();
 
 
@@ -75,7 +82,9 @@ public class TableCell implements GenericSemanticNode {
         }
 
         if (textBlock == null) {
-            textBlock = streamAllSubNodes().filter(SemanticNode::isLeaf).map(SemanticNode::getLeafTextBlock).collect(new TextBlockCollector());
+            textBlock = streamAllSubNodes().filter(SemanticNode::isLeaf)
+                    .map(SemanticNode::getLeafTextBlock)
+                    .collect(new TextBlockCollector());
         }
         return textBlock;
     }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/textblock/AtomicTextBlock.java

@@ -61,6 +61,7 @@ public class AtomicTextBlock implements TextBlock {
         return lineBreaks.size() + 1;
     }
 
+
     public static AtomicTextBlock empty(Long textBlockIdx, int stringOffset, Page page, int numberOnPage, SemanticNode parent) {
 
         return AtomicTextBlock.builder()
@@ -77,10 +78,7 @@ public class AtomicTextBlock implements TextBlock {
     }
 
 
-    public static AtomicTextBlock fromAtomicTextBlockData(DocumentTextData atomicTextBlockData,
-                                                          DocumentPositionData atomicPositionBlockData,
-                                                          SemanticNode parent,
-                                                          Page page) {
+    public static AtomicTextBlock fromAtomicTextBlockData(DocumentTextData atomicTextBlockData, DocumentPositionData atomicPositionBlockData, SemanticNode parent, Page page) {
 
         return AtomicTextBlock.builder()
                 .id(atomicTextBlockData.getId())
@@ -88,8 +86,10 @@ public class AtomicTextBlock implements TextBlock {
                 .page(page)
                 .textRange(new TextRange(atomicTextBlockData.getStart(), atomicTextBlockData.getEnd()))
                 .searchText(atomicTextBlockData.getSearchText())
-                .lineBreaks(Arrays.stream(atomicTextBlockData.getLineBreaks()).boxed().toList())
-                .stringIdxToPositionIdx(Arrays.stream(atomicPositionBlockData.getStringIdxToPositionIdx()).boxed().toList())
+                .lineBreaks(Arrays.stream(atomicTextBlockData.getLineBreaks()).boxed()
+                                    .toList())
+                .stringIdxToPositionIdx(Arrays.stream(atomicPositionBlockData.getStringIdxToPositionIdx()).boxed()
+                                                .toList())
                 .positions(toRectangle2DList(atomicPositionBlockData.getPositions()))
                 .parent(parent)
                 .build();
@@ -98,7 +98,9 @@ public class AtomicTextBlock implements TextBlock {
 
     private static List<Rectangle2D> toRectangle2DList(float[][] positions) {
 
-        return Arrays.stream(positions).map(floatArr -> (Rectangle2D) new Rectangle2D.Float(floatArr[0], floatArr[1], floatArr[2], floatArr[3])).toList();
+        return Arrays.stream(positions)
+                .map(floatArr -> (Rectangle2D) new Rectangle2D.Float(floatArr[0], floatArr[1], floatArr[2], floatArr[3]))
+                .toList();
     }
 
 
@@ -118,6 +120,7 @@ public class AtomicTextBlock implements TextBlock {
         return new TextRange(lineBreaks.get(lineNumber - 1) + textRange.start(), lineBreaks.get(lineNumber) + textRange.start());
     }
 
+
     public List<String> getWords() {
 
         if (words == null) {
@@ -144,9 +147,9 @@ public class AtomicTextBlock implements TextBlock {
     public int getNextLinebreak(int fromIndex) {
 
         return lineBreaks.stream()//
-                .filter(linebreak -> linebreak > fromIndex - textRange.start()) //
-                .findFirst() //
-                .orElse(searchText.length()) + textRange.start();
+                       .filter(linebreak -> linebreak > fromIndex - textRange.start()) //
+                       .findFirst() //
+                       .orElse(searchText.length()) + textRange.start();
     }
 
 
@@ -154,9 +157,9 @@ public class AtomicTextBlock implements TextBlock {
     public int getPreviousLinebreak(int fromIndex) {
 
         return lineBreaks.stream()//
-                .filter(linebreak -> linebreak <= fromIndex - textRange.start())//
-                .reduce((a, b) -> b)//
-                .orElse(0) + textRange.start();
+                       .filter(linebreak -> linebreak <= fromIndex - textRange.start())//
+                       .reduce((a, b) -> b)//
+                       .orElse(0) + textRange.start();
     }
 
 
@@ -209,7 +212,10 @@ public class AtomicTextBlock implements TextBlock {
             return "";
         }
 
-        Set<Integer> lbInBoundary = lineBreaks.stream().map(i -> i + textRange.start()).filter(textRange::contains).collect(Collectors.toSet());
+        Set<Integer> lbInBoundary = lineBreaks.stream()
+                .map(i -> i + textRange.start())
+                .filter(textRange::contains)
+                .collect(Collectors.toSet());
         if (textRange.end() == getTextRange().end()) {
             lbInBoundary.add(getTextRange().end());
         }
@@ -235,7 +241,10 @@ public class AtomicTextBlock implements TextBlock {
 
     private List<Integer> getAllLineBreaksInBoundary(TextRange textRange) {
 
-        return getLineBreaks().stream().map(linebreak -> linebreak + this.textRange.start()).filter(textRange::contains).toList();
+        return getLineBreaks().stream()
+                .map(linebreak -> linebreak + this.textRange.start())
+                .filter(textRange::contains)
+                .toList();
     }
 
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/textblock/ConcatenatedTextBlock.java

@@ -44,7 +44,8 @@ public class ConcatenatedTextBlock implements TextBlock {
         this.atomicTextBlocks.add(firstTextBlock);
         textRange = new TextRange(firstTextBlock.getTextRange().start(), firstTextBlock.getTextRange().end());
 
-        atomicTextBlocks.subList(1, atomicTextBlocks.size()).forEach(this::concat);
+        atomicTextBlocks.subList(1, atomicTextBlocks.size())
+                .forEach(this::concat);
     }
 
 
@@ -65,13 +66,25 @@ public class ConcatenatedTextBlock implements TextBlock {
 
     private AtomicTextBlock getAtomicTextBlockByStringIndex(int stringIdx) {
 
-        return atomicTextBlocks.stream().filter(textBlock -> textBlock.getTextRange().contains(stringIdx)).findAny().orElseThrow(IndexOutOfBoundsException::new);
+        return atomicTextBlocks.stream()
+                .filter(textBlock -> textBlock.getTextRange().contains(stringIdx))
+                .findAny()
+                .orElseThrow(IndexOutOfBoundsException::new);
     }
 
 
     private List<AtomicTextBlock> getAllAtomicTextBlocksPartiallyInStringBoundary(TextRange textRange) {
 
-        return atomicTextBlocks.stream().filter(tb -> tb.getTextRange().intersects(textRange)).toList();
+        List<AtomicTextBlock> intersectingAtomicTextBlocks = new LinkedList<>();
+        for (AtomicTextBlock atomicTextBlock : atomicTextBlocks) {
+            if (atomicTextBlock.getTextRange().start() > textRange.end()) {
+                break; // early stop, following TextBlocks will never intersect
+            }
+            if (atomicTextBlock.getTextRange().intersects(textRange)) {
+                intersectingAtomicTextBlocks.add(atomicTextBlock);
+            }
+        }
+        return intersectingAtomicTextBlocks;
     }
 
 
@@ -90,14 +103,18 @@ public class ConcatenatedTextBlock implements TextBlock {
     @Override
     public List<String> getWords() {
 
-        return atomicTextBlocks.stream().map(AtomicTextBlock::getWords).flatMap(Collection::stream).toList();
+        return atomicTextBlocks.stream()
+                .map(AtomicTextBlock::getWords)
+                .flatMap(Collection::stream)
+                .toList();
     }
 
 
     @Override
     public int numberOfLines() {
 
-        return atomicTextBlocks.stream().mapToInt(AtomicTextBlock::numberOfLines).sum();
+        return atomicTextBlocks.stream()
+                .mapToInt(AtomicTextBlock::numberOfLines).sum();
     }
 
 
@@ -118,17 +135,20 @@ public class ConcatenatedTextBlock implements TextBlock {
     @Override
     public List<Integer> getLineBreaks() {
 
-        return getAtomicTextBlocks().stream().flatMap(atomicTextBlock -> atomicTextBlock.getLineBreaks().stream()).toList();
+        return getAtomicTextBlocks().stream()
+                .flatMap(atomicTextBlock -> atomicTextBlock.getLineBreaks()
+                        .stream())
+                .toList();
     }
 
 
-
     @Override
     public Rectangle2D getPosition(int stringIdx) {
 
         return getAtomicTextBlockByStringIndex(stringIdx).getPosition(stringIdx);
     }
 
+
     public TextRange getLineTextRange(int lineNumber) {
 
         if (atomicTextBlocks.size() == 1) {
@@ -144,10 +164,10 @@ public class ConcatenatedTextBlock implements TextBlock {
         return new TextRange(textRange.start(), textRange.start());
     }
 
+
     @Override
     public List<Rectangle2D> getPositions(TextRange stringTextRange) {
 
-
         List<AtomicTextBlock> textBlocks = getAllAtomicTextBlocksPartiallyInStringBoundary(stringTextRange);
 
         if (textBlocks.isEmpty()) {
@@ -193,7 +213,8 @@ public class ConcatenatedTextBlock implements TextBlock {
 
         AtomicTextBlock lastTextBlock = textBlocks.get(textBlocks.size() - 1);
         rectanglesPerLinePerPage = mergeEntityPositionsWithSamePageNode(rectanglesPerLinePerPage,
-                lastTextBlock.getPositionsPerPage(new TextRange(lastTextBlock.getTextRange().start(), stringTextRange.end())));
+                                                                        lastTextBlock.getPositionsPerPage(new TextRange(lastTextBlock.getTextRange().start(),
+                                                                                                                        stringTextRange.end())));
 
         return rectanglesPerLinePerPage;
     }
@@ -230,7 +251,10 @@ public class ConcatenatedTextBlock implements TextBlock {
     private Map<Page, List<Rectangle2D>> mergeEntityPositionsWithSamePageNode(Map<Page, List<Rectangle2D>> map1, Map<Page, List<Rectangle2D>> map2) {
 
         Map<Page, List<Rectangle2D>> mergedMap = new HashMap<>(map1);
-        map2.forEach((pageNode, rectangles) -> mergedMap.merge(pageNode, rectangles, (l1, l2) -> Stream.concat(l1.stream(), l2.stream()).toList()));
+        map2.forEach((pageNode, rectangles) -> mergedMap.merge(pageNode,
+                                                               rectangles,
+                                                               (l1, l2) -> Stream.concat(l1.stream(), l2.stream())
+                                                                       .toList()));
         return mergedMap;
     }
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/textblock/TextBlock.java

@@ -18,8 +18,10 @@ public interface TextBlock extends CharSequence {
 
     String getSearchText();
 
+
     List<String> getWords();
 
+
     List<AtomicTextBlock> getAtomicTextBlocks();
 
 
@@ -35,7 +37,6 @@ public interface TextBlock extends CharSequence {
     TextRange getLineTextRange(int lineNumber);
 
 
-
     List<Integer> getLineBreaks();
 
 
@@ -71,6 +72,7 @@ public interface TextBlock extends CharSequence {
         return RectangleTransformations.rectangle2DBBox(getLinePositions(lineNumber));
     }
 
+
     default String searchTextWithLineBreaks() {
 
         return subSequenceWithLineBreaks(getTextRange());
@@ -85,7 +87,9 @@ public interface TextBlock extends CharSequence {
 
     default Set<Page> getPages() {
 
-        return getAtomicTextBlocks().stream().map(AtomicTextBlock::getPage).collect(Collectors.toUnmodifiableSet());
+        return getAtomicTextBlocks().stream()
+                .map(AtomicTextBlock::getPage)
+                .collect(Collectors.toUnmodifiableSet());
     }
 
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/drools/RuleClass.java

@@ -9,7 +9,8 @@ public record RuleClass(RuleType ruleType, List<RuleUnit> ruleUnits) {
     public Optional<RuleUnit> findRuleUnitByInteger(Integer unit) {
 
         return ruleUnits.stream()
-                .filter(ruleUnit -> Objects.equals(ruleUnit.unit(), unit)).findFirst();
+                .filter(ruleUnit -> Objects.equals(ruleUnit.unit(), unit))
+                .findFirst();
     }
 
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/drools/RuleFileBluePrint.java

@@ -10,7 +10,7 @@ import java.util.Set;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
-import com.iqser.red.service.redaction.v1.model.DroolsSyntaxValidation;
+import com.iqser.red.service.redaction.v1.model.DroolsValidation;
 
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
@@ -32,18 +32,22 @@ public final class RuleFileBluePrint {
     int globalsLine;
     List<BasicQuery> queries;
     List<RuleClass> ruleClasses;
-    DroolsSyntaxValidation droolsSyntaxValidation;
+    DroolsValidation droolsValidation;
 
 
     public Optional<RuleClass> findRuleClassByType(RuleType ruleType) {
 
-        return ruleClasses.stream().filter(ruleClass -> Objects.equals(ruleClass.ruleType(), ruleType)).findFirst();
+        return ruleClasses.stream()
+                .filter(ruleClass -> Objects.equals(ruleClass.ruleType(), ruleType))
+                .findFirst();
     }
 
 
     public Set<String> getImportSplitByKeyword() {
 
-        return Arrays.stream(imports.replaceAll("\n", "").split("import")).map(String::trim).collect(Collectors.toSet());
+        return Arrays.stream(imports.replaceAll("\n", "").split("import"))
+                .map(String::trim)
+                .collect(Collectors.toSet());
     }
 
 
@@ -53,11 +57,15 @@ public final class RuleFileBluePrint {
             return findRuleClassByType(ruleIdentifier.type()).map(RuleClass::ruleUnits)
                     .orElse(Collections.emptyList())
                     .stream()
-                    .flatMap(ruleUnit -> ruleUnit.rules().stream().filter(rule -> rule.getIdentifier().matches(ruleIdentifier)))
+                    .flatMap(ruleUnit -> ruleUnit.rules()
+                            .stream()
+                            .filter(rule -> rule.getIdentifier().matches(ruleIdentifier)))
                     .toList();
         }
         return findRuleClassByType(ruleIdentifier.type()).flatMap(ruleClass -> ruleClass.findRuleUnitByInteger(ruleIdentifier.unit()))
-                .map(ruleUnit -> ruleUnit.rules().stream().filter(rule -> rule.getIdentifier().matches(ruleIdentifier)))
+                .map(ruleUnit -> ruleUnit.rules()
+                        .stream()
+                        .filter(rule -> rule.getIdentifier().matches(ruleIdentifier)))
                 .orElse(Stream.empty())
                 .toList();
     }
@@ -65,13 +73,18 @@ public final class RuleFileBluePrint {
 
     public List<RuleIdentifier> getAllRuleIdentifiers() {
 
-        return streamAllRules().map(BasicRule::getIdentifier).collect(Collectors.toList());
+        return streamAllRules().map(BasicRule::getIdentifier)
+                .collect(Collectors.toList());
     }
 
 
     public Stream<BasicRule> streamAllRules() {
 
-        return getRuleClasses().stream().map(RuleClass::ruleUnits).flatMap(Collection::stream).map(RuleUnit::rules).flatMap(Collection::stream);
+        return getRuleClasses().stream()
+                .map(RuleClass::ruleUnits)
+                .flatMap(Collection::stream)
+                .map(RuleUnit::rules)
+                .flatMap(Collection::stream);
     }
 
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/drools/RuleIdentifier.java

@@ -42,8 +42,8 @@ public record RuleIdentifier(@NonNull RuleType type, Integer unit, Integer id) {
     public boolean matches(RuleIdentifier ruleIdentifier) {
 
         return ruleIdentifier.type().equals(this.type()) && //
-                (Objects.isNull(ruleIdentifier.unit()) || Objects.isNull(this.unit()) || Objects.equals(this.unit(), ruleIdentifier.unit())) && //
-                (Objects.isNull(ruleIdentifier.id()) || Objects.isNull(this.id()) || Objects.equals(this.id(), ruleIdentifier.id()));
+               (Objects.isNull(ruleIdentifier.unit()) || Objects.isNull(this.unit()) || Objects.equals(this.unit(), ruleIdentifier.unit())) && //
+               (Objects.isNull(ruleIdentifier.id()) || Objects.isNull(this.id()) || Objects.equals(this.id(), ruleIdentifier.id()));
 
     }
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/queue/MessagingConfiguration.java

@@ -70,6 +70,7 @@ public class MessagingConfiguration {
                 .build();
     }
 
+
     @Bean
     public Queue redactionAnalysisResponseQueue() {
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/queue/RedactionMessageReceiver.java

@@ -51,14 +51,14 @@ public class RedactionMessageReceiver {
         // This prevents from endless retries oom errors.
         if (message.getMessageProperties().isRedelivered()) {
             var errorMessage = format("Error during last processing of request with dossierId: %s and fileId: %s, do not retry.",
-                    analyzeRequest.getDossierId(),
-                    analyzeRequest.getFileId());
+                                      analyzeRequest.getDossierId(),
+                                      analyzeRequest.getFileId());
             fileStatusProcessingUpdateClient.analysisFailed(analyzeRequest.getDossierId(),
-                    analyzeRequest.getFileId(),
-                    new FileErrorInfo(errorMessage,
-                            priority ? REDACTION_PRIORITY_QUEUE : REDACTION_QUEUE,
-                            "redaction-service",
-                            OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS)));
+                                                            analyzeRequest.getFileId(),
+                                                            new FileErrorInfo(errorMessage,
+                                                                              priority ? REDACTION_PRIORITY_QUEUE : REDACTION_QUEUE,
+                                                                              "redaction-service",
+                                                                              OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS)));
             throw new AmqpRejectAndDontRequeueException(errorMessage);
         }
 
@@ -84,9 +84,9 @@ public class RedactionMessageReceiver {
                     log.debug(analyzeRequest.getManualRedactions().toString());
                     result = analyzeService.analyze(analyzeRequest);
                     log.info("Successfully analyzed dossier {} file {} took: {} s",
-                            analyzeRequest.getDossierId(),
-                            analyzeRequest.getFileId(),
-                            format("%.2f", result.getDuration() / 1000.0));
+                             analyzeRequest.getDossierId(),
+                             analyzeRequest.getFileId(),
+                             format("%.2f", result.getDuration() / 1000.0));
                     log.info("----------------------------------------------------------------------------------");
                     break;
 
@@ -96,9 +96,9 @@ public class RedactionMessageReceiver {
                     log.debug(analyzeRequest.getManualRedactions().toString());
                     result = analyzeService.reanalyze(analyzeRequest);
                     log.info("Successfully reanalyzed dossier {} file {} took: {} s",
-                            analyzeRequest.getDossierId(),
-                            analyzeRequest.getFileId(),
-                            format("%.2f", result.getDuration() / 1000.0));
+                             analyzeRequest.getDossierId(),
+                             analyzeRequest.getFileId(),
+                             format("%.2f", result.getDuration() / 1000.0));
                     log.info("----------------------------------------------------------------------------------");
                     break;
                 case SURROUNDING_TEXT_ANALYSIS:
@@ -106,9 +106,7 @@ public class RedactionMessageReceiver {
                     log.info("Starting Surrounding Text Analysis for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
                     log.debug(analyzeRequest.getManualRedactions().toString());
                     unprocessedChangesService.analyseSurroundingText(analyzeRequest);
-                    log.info("Successful Surrounding Text Analysis dossier {} file {} ",
-                            analyzeRequest.getDossierId(),
-                            analyzeRequest.getFileId());
+                    log.info("Successful Surrounding Text Analysis dossier {} file {} ", analyzeRequest.getDossierId(), analyzeRequest.getFileId());
                     log.info("-------------------------------------------------------------------------------------------------");
                     shouldRespond = false;
                     break;
@@ -137,8 +135,8 @@ public class RedactionMessageReceiver {
         log.warn("Failed to process analyze request: {}", analyzeRequest, e);
         var timestamp = OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS);
         fileStatusProcessingUpdateClient.analysisFailed(analyzeRequest.getDossierId(),
-                analyzeRequest.getFileId(),
-                new FileErrorInfo(e.getMessage(), priority ? REDACTION_PRIORITY_QUEUE : REDACTION_QUEUE, "redaction-service", timestamp));
+                                                        analyzeRequest.getFileId(),
+                                                        new FileErrorInfo(e.getMessage(), priority ? REDACTION_PRIORITY_QUEUE : REDACTION_QUEUE, "redaction-service", timestamp));
     }
 
 
@@ -153,8 +151,8 @@ public class RedactionMessageReceiver {
         timestamp = timestamp != null ? timestamp : OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS);
         log.info("Failed to process analyze request, errorCause: {}, timestamp: {}", errorCause, timestamp);
         fileStatusProcessingUpdateClient.analysisFailed(analyzeRequest.getDossierId(),
-                analyzeRequest.getFileId(),
-                new FileErrorInfo(errorCause, REDACTION_DQL, "redaction-service", timestamp));
+                                                        analyzeRequest.getFileId(),
+                                                        new FileErrorInfo(errorCause, REDACTION_DQL, "redaction-service", timestamp));
     }
 
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalyzeService.java

@@ -1,11 +1,16 @@
 package com.iqser.red.service.redaction.v1.server.service;
 
+import static com.iqser.red.service.redaction.v1.server.service.document.SectionFinderService.getRelevantManuallyModifiedAnnotationIds;
+
+import java.util.ArrayList;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
 import org.springframework.stereotype.Service;
 import org.springframework.web.bind.annotation.RequestBody;
@@ -18,17 +23,11 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileTyp
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentLog;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogChanges;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedactions;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.legalbasis.LegalBasis;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogChanges;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogLegalBasis;
 import com.iqser.red.service.redaction.v1.server.RedactionServiceSettings;
-import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient;
 import com.iqser.red.service.redaction.v1.server.client.model.NerEntitiesModel;
 import com.iqser.red.service.redaction.v1.server.model.KieWrapper;
-import com.iqser.red.service.redaction.v1.server.model.ManualEntity;
 import com.iqser.red.service.redaction.v1.server.model.NerEntities;
 import com.iqser.red.service.redaction.v1.server.model.component.Component;
 import com.iqser.red.service.redaction.v1.server.model.dictionary.Dictionary;
@@ -37,6 +36,7 @@ import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryVers
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
 import com.iqser.red.service.redaction.v1.server.service.document.DocumentGraphMapper;
+import com.iqser.red.service.redaction.v1.server.service.document.ImportedRedactionEntryService;
 import com.iqser.red.service.redaction.v1.server.service.document.ManualRedactionEntryService;
 import com.iqser.red.service.redaction.v1.server.service.document.NerEntitiesAdapter;
 import com.iqser.red.service.redaction.v1.server.service.document.SectionFinderService;
@@ -65,18 +65,15 @@ public class AnalyzeService {
     ComponentDroolsExecutionService componentDroolsExecutionService;
     KieContainerCreationService kieContainerCreationService;
     DictionarySearchService dictionarySearchService;
-    RedactionLogCreatorService redactionLogCreatorService;
     EntityLogCreatorService entityLogCreatorService;
     ComponentLogCreatorService componentLogCreatorService;
     RedactionStorageService redactionStorageService;
-    RedactionChangeLogService redactionChangeLogService;
-    LegalBasisClient legalBasisClient;
     RedactionServiceSettings redactionServiceSettings;
-    ImportedRedactionService importedRedactionService;
+    NotFoundImportedEntitiesService notFoundImportedEntitiesService;
     SectionFinderService sectionFinderService;
     ManualRedactionEntryService manualRedactionEntryService;
+    ImportedRedactionEntryService importedRedactionEntryService;
     ObservedStorageService observedStorageService;
-
     FunctionTimerValues redactmanagerAnalyzePagewiseValues;
 
 
@@ -86,51 +83,71 @@ public class AnalyzeService {
     public AnalyzeResult reanalyze(@RequestBody AnalyzeRequest analyzeRequest) {
 
         long startTime = System.currentTimeMillis();
-        RedactionLog previousRedactionLog = redactionStorageService.getRedactionLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
-        EntityLog previousEntityLog = redactionStorageService.getEntityLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
+        EntityLog entityLogWithoutEntries = redactionStorageService.getEntityLogWithoutEntries(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
         log.info("Loaded previous entity log for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
 
         Document document = DocumentGraphMapper.toDocumentGraph(observedStorageService.getDocumentData(analyzeRequest.getDossierId(), analyzeRequest.getFileId()));
         log.info("Loaded Document Graph for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
 
+        ImportedRedactions importedRedactions = redactionStorageService.getImportedRedactions(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
+        log.info("Loaded Imported Redactions for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
+
         // not yet ready for reanalysis
-        if (previousEntityLog == null || document == null || document.getNumberOfPages() == 0) {
+        if (entityLogWithoutEntries == null || document == null || document.getNumberOfPages() == 0) {
             return analyze(analyzeRequest);
         }
 
         DictionaryIncrement dictionaryIncrement = dictionaryService.getDictionaryIncrements(analyzeRequest.getDossierTemplateId(),
-                new DictionaryVersion(previousEntityLog.getDictionaryVersion(), previousEntityLog.getDossierDictionaryVersion()),
-                analyzeRequest.getDossierId());
+                                                                                            new DictionaryVersion(entityLogWithoutEntries.getDictionaryVersion(),
+                                                                                                                  entityLogWithoutEntries.getDossierDictionaryVersion()),
+                                                                                            analyzeRequest.getDossierId());
+
+        Set<String> relevantManuallyModifiedAnnotationIds = getRelevantManuallyModifiedAnnotationIds(analyzeRequest.getManualRedactions());
+
+        Set<Integer> sectionsToReanalyseIds = redactionStorageService.findIdsOfSectionsToReanalyse(analyzeRequest.getDossierId(),
+                                                                                                                     analyzeRequest.getFileId(),
+                                                                                                                     relevantManuallyModifiedAnnotationIds);
+        sectionsToReanalyseIds.addAll(getSectionsToReanalyseIds(analyzeRequest,
+                                                                        document,
+                                                                        dictionaryIncrement,
+                                                                        importedRedactions,
+                                                                        relevantManuallyModifiedAnnotationIds));
 
-        Set<Integer> sectionsToReanalyseIds = getSectionsToReanalyseIds(analyzeRequest, previousEntityLog, document, dictionaryIncrement);
         List<SemanticNode> sectionsToReAnalyse = getSectionsToReAnalyse(document, sectionsToReanalyseIds);
         log.info("{} Sections to reanalyze found for file {} in dossier {}", sectionsToReanalyseIds.size(), analyzeRequest.getFileId(), analyzeRequest.getDossierId());
 
         if (sectionsToReAnalyse.isEmpty()) {
 
+            EntityLogChanges entityLogChanges = entityLogCreatorService.updateVersionsAndReturnChanges(entityLogWithoutEntries,
+                                                                                                       dictionaryIncrement.getDictionaryVersion(),
+                                                                                                       analyzeRequest,
+                                                                                                       new ArrayList<>(),
+                                                                                                       new ArrayList<>());
+
             return finalizeAnalysis(analyzeRequest,
-                    startTime,
-                    kieContainerCreationService.getLatestKieContainer(analyzeRequest.getDossierTemplateId(), RuleFileType.COMPONENT),
-                    entityLogCreatorService.updateVersionsAndReturnChanges(previousEntityLog,
-                            dictionaryIncrement.getDictionaryVersion(),
-                            analyzeRequest.getDossierTemplateId(),
-                            false),
-                    document,
-                    previousRedactionLog,
-                    document.getNumberOfPages(),
-                    dictionaryIncrement.getDictionaryVersion(),
-                    true,
-                    Collections.emptySet());
+                                    startTime,
+                                    kieContainerCreationService.getLatestKieContainer(analyzeRequest.getDossierTemplateId(), RuleFileType.COMPONENT),
+                                    entityLogChanges,
+                                    document,
+                                    document.getNumberOfPages(),
+                                    dictionaryIncrement.getDictionaryVersion(),
+                                    true,
+                                    Collections.emptySet());
         }
+
         KieWrapper kieWrapperEntityRules = kieContainerCreationService.getLatestKieContainer(analyzeRequest.getDossierTemplateId(), RuleFileType.ENTITY);
         log.info("Updated entity rules to version {} for file {} in dossier {}", kieWrapperEntityRules.rulesVersion(), analyzeRequest.getFileId(), analyzeRequest.getDossierId());
 
         NerEntities nerEntities = getEntityRecognitionEntitiesFilteredBySectionIds(analyzeRequest, document, sectionsToReanalyseIds);
         log.info("Loaded Ner Entities for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
 
-        List<ManualEntity> notFoundManualRedactionEntries = manualRedactionEntryService.addManualRedactionEntriesAndReturnNotFoundEntries(analyzeRequest,
-                document,
-                analyzeRequest.getDossierTemplateId());
+        var notFoundManualRedactionEntries = manualRedactionEntryService.addManualRedactionEntriesAndReturnNotFoundEntries(analyzeRequest,
+                                                                                                                           document,
+                                                                                                                           analyzeRequest.getDossierTemplateId());
+        var notFoundImportedEntries = importedRedactionEntryService.addImportedEntriesAndReturnNotFoundEntries(analyzeRequest, importedRedactions, document);
+        var notFoundManualOrImportedEntries = Stream.of(notFoundManualRedactionEntries, notFoundImportedEntries)
+                .flatMap(Collection::stream)
+                .collect(Collectors.toList());
 
         Dictionary dictionary = dictionaryService.getDeepCopyDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId());
         log.info("Updated Dictionaries for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
@@ -138,33 +155,34 @@ public class AnalyzeService {
         dictionarySearchService.addDictionaryEntities(dictionary, sectionsToReAnalyse);
         log.info("Finished Dictionary Search for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
 
+        // we could add the imported redactions similar to the manual redactions here as well for additional processing
         List<FileAttribute> allFileAttributes = entityDroolsExecutionService.executeRules(kieWrapperEntityRules.container(),
-                document,
-                sectionsToReAnalyse,
-                dictionary,
-                analyzeRequest.getFileAttributes(),
-                analyzeRequest.getManualRedactions(),
-                nerEntities);
+                                                                                          document,
+                                                                                          sectionsToReAnalyse,
+                                                                                          dictionary,
+                                                                                          analyzeRequest.getFileAttributes(),
+                                                                                          analyzeRequest.getManualRedactions(),
+                                                                                          nerEntities);
         log.info("Finished entity rule execution for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
 
-        RedactionLog redactionLog = updatePreviousRedactionLog(analyzeRequest, document, notFoundManualRedactionEntries, previousRedactionLog, sectionsToReanalyseIds);
         EntityLogChanges entityLogChanges = entityLogCreatorService.updatePreviousEntityLog(analyzeRequest,
-                document,
-                notFoundManualRedactionEntries,
-                previousEntityLog,
-                sectionsToReanalyseIds,
-                dictionary.getVersion());
+                                                                                            document,
+                                                                                            entityLogWithoutEntries,
+                                                                                            notFoundManualOrImportedEntries,
+                                                                                            sectionsToReanalyseIds,
+                                                                                            dictionary.getVersion());
+
+        notFoundImportedEntitiesService.processEntityLog(entityLogChanges.getEntityLog(), analyzeRequest, notFoundImportedEntries);
 
         return finalizeAnalysis(analyzeRequest,
-                startTime,
-                kieContainerCreationService.getLatestKieContainer(analyzeRequest.getDossierTemplateId(), RuleFileType.COMPONENT),
-                entityLogChanges,
-                document,
-                redactionLog,
-                document.getNumberOfPages(),
-                dictionaryIncrement.getDictionaryVersion(),
-                true,
-                new HashSet<>(allFileAttributes));
+                                startTime,
+                                kieContainerCreationService.getLatestKieContainer(analyzeRequest.getDossierTemplateId(), RuleFileType.COMPONENT),
+                                entityLogChanges,
+                                document,
+                                document.getNumberOfPages(),
+                                dictionaryIncrement.getDictionaryVersion(),
+                                true,
+                                new HashSet<>(allFileAttributes));
     }
 
 
@@ -183,6 +201,9 @@ public class AnalyzeService {
         Document document = DocumentGraphMapper.toDocumentGraph(observedStorageService.getDocumentData(analyzeRequest.getDossierId(), analyzeRequest.getFileId()));
         log.info("Loaded Document Graph for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
 
+        ImportedRedactions importedRedactions = redactionStorageService.getImportedRedactions(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
+        log.info("Loaded Imported Redactions for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
+
         NerEntities nerEntities = getEntityRecognitionEntities(analyzeRequest, document);
         log.info("Loaded Ner Entities for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
 
@@ -190,64 +211,43 @@ public class AnalyzeService {
         Dictionary dictionary = dictionaryService.getDeepCopyDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId());
         log.info("Updated Dictionaries for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
 
-        List<ManualEntity> notFoundManualRedactionEntries = manualRedactionEntryService.addManualRedactionEntriesAndReturnNotFoundEntries(analyzeRequest,
-                document,
-                analyzeRequest.getDossierTemplateId());
+        var notFoundManualRedactionEntries = manualRedactionEntryService.addManualRedactionEntriesAndReturnNotFoundEntries(analyzeRequest,
+                                                                                                                           document,
+                                                                                                                           analyzeRequest.getDossierTemplateId());
+        var notFoundImportedEntries = importedRedactionEntryService.addImportedEntriesAndReturnNotFoundEntries(analyzeRequest, importedRedactions, document);
+        var notFoundManualOrImportedEntries = Stream.of(notFoundManualRedactionEntries, notFoundImportedEntries)
+                .flatMap(Collection::stream)
+                .collect(Collectors.toList());
 
         dictionarySearchService.addDictionaryEntities(dictionary, document);
         log.info("Finished Dictionary Search for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
 
+        // we could add the imported redactions similar to the manual redactions here as well for additional processing
         List<FileAttribute> allFileAttributes = entityDroolsExecutionService.executeRules(kieWrapperEntityRules.container(),
-                document,
-                dictionary,
-                analyzeRequest.getFileAttributes(),
-                analyzeRequest.getManualRedactions(),
-                nerEntities);
+                                                                                          document,
+                                                                                          dictionary,
+                                                                                          analyzeRequest.getFileAttributes(),
+                                                                                          analyzeRequest.getManualRedactions(),
+                                                                                          nerEntities);
         log.info("Finished entity rule execution for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
 
-        RedactionLog redactionLog = createRedactionLog(analyzeRequest, document, notFoundManualRedactionEntries, dictionary, kieWrapperEntityRules);
-        EntityLog entityLog = entityLogCreatorService.createInitialEntityLog(analyzeRequest,
-                document,
-                notFoundManualRedactionEntries,
-                dictionary.getVersion(),
-                kieWrapperEntityRules.rulesVersion());
+        EntityLogChanges entityLogChanges = entityLogCreatorService.createInitialEntityLog(analyzeRequest,
+                                                                                           document,
+                                                                                           notFoundManualOrImportedEntries,
+                                                                                           dictionary.getVersion(),
+                                                                                           kieWrapperEntityRules.rulesVersion());
+
+        notFoundImportedEntitiesService.processEntityLog(entityLogChanges.getEntityLog(), analyzeRequest, notFoundImportedEntries);
 
         return finalizeAnalysis(analyzeRequest,
-                startTime,
-                kieWrapperComponentRules,
-                new EntityLogChanges(entityLog, false),
-                document,
-                redactionLog,
-                document.getNumberOfPages(),
-                dictionary.getVersion(),
-                false,
-                new HashSet<>(allFileAttributes));
-    }
-
-
-    @Deprecated(forRemoval = true)
-    private RedactionLog updatePreviousRedactionLog(AnalyzeRequest analyzeRequest,
-                                                    Document document,
-                                                    List<ManualEntity> notFoundManualRedactionEntries,
-                                                    RedactionLog previousRedactionLog,
-                                                    Set<Integer> sectionsToReanalyseIds) {
-
-        List<RedactionLogEntry> newRedactionLogEntries = redactionLogCreatorService.createRedactionLog(document,
-                analyzeRequest.getDossierTemplateId(),
-                notFoundManualRedactionEntries);
-
-        var importedRedactionFilteredEntries = importedRedactionService.processImportedRedactions(analyzeRequest.getDossierTemplateId(),
-                analyzeRequest.getDossierId(),
-                analyzeRequest.getFileId(),
-                newRedactionLogEntries,
-                false);
-
-        previousRedactionLog.getRedactionLogEntry()
-                .removeIf(entry -> sectionsToReanalyseIds.contains(entry.getSectionNumber()) && !entry.getType().equals(ImportedRedactionService.IMPORTED_REDACTION_TYPE));
-
-        previousRedactionLog.getRedactionLogEntry().addAll(importedRedactionFilteredEntries);
-
-        return previousRedactionLog;
+                                startTime,
+                                kieWrapperComponentRules,
+                                entityLogChanges,
+                                document,
+                                document.getNumberOfPages(),
+                                dictionary.getVersion(),
+                                false,
+                                new HashSet<>(allFileAttributes));
     }
 
 
@@ -256,19 +256,31 @@ public class AnalyzeService {
                                            KieWrapper kieWrapperComponentRules,
                                            EntityLogChanges entityLogChanges,
                                            Document document,
-                                           RedactionLog redactionLog,
                                            int numberOfPages,
                                            DictionaryVersion dictionaryVersion,
                                            boolean isReanalysis,
                                            Set<FileAttribute> addedFileAttributes) {
 
-        finalizeRedactionLog(analyzeRequest, redactionLog, dictionaryVersion);
-
         EntityLog entityLog = entityLogChanges.getEntityLog();
-        redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.ENTITY_LOG, entityLogChanges.getEntityLog());
+
+        // as workaround for duplicate key exceptions occurring due to simultaneous analyses and reanalyses save instead of insert is used
+        // also analysis numbers should be incremented in every follow-up request, so checking if the log exists is not needed
+        if (!redactionStorageService.entityLogExists(analyzeRequest.getDossierId(), analyzeRequest.getFileId())) {
+            redactionStorageService.saveEntityLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), entityLog);
+
+        } else {
+            redactionStorageService.updateEntityLogWithoutEntries(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), entityLog);
+
+            if (!entityLogChanges.getNewEntityLogEntries().isEmpty()) {
+                redactionStorageService.saveEntityLogEntries(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), entityLogChanges.getNewEntityLogEntries());
+            }
+            if (!entityLogChanges.getUpdatedEntityLogEntries().isEmpty()) {
+                redactionStorageService.updateEntityLogEntries(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), entityLogChanges.getUpdatedEntityLogEntries());
+            }
+        }
 
         log.info("Created entity log for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
-        if (entityLogChanges.isHasChanges() || !isReanalysis) {
+        if (entityLogChanges.hasChanges() || !isReanalysis) {
             computeComponentsWhenRulesArePresent(analyzeRequest, kieWrapperComponentRules, document, addedFileAttributes, entityLogChanges, dictionaryVersion);
         }
 
@@ -283,7 +295,7 @@ public class AnalyzeService {
                 .fileId(analyzeRequest.getFileId())
                 .duration(duration)
                 .numberOfPages(numberOfPages)
-                .hasUpdates(entityLogChanges.isHasChanges())
+                .hasUpdates(entityLogChanges.hasChanges())
                 .analysisVersion(redactionServiceSettings.getAnalysisVersion())
                 .analysisNumber(analyzeRequest.getAnalysisNumber())
                 .rulesVersion(entityLog.getRulesVersion())
@@ -310,9 +322,10 @@ public class AnalyzeService {
         }
 
         List<Component> components = componentDroolsExecutionService.executeRules(kieWrapperComponentRules.container(),
-                entityLogChanges.getEntityLog(),
-                document,
-                addedFileAttributes.stream().toList());
+                                                                                  entityLogChanges.getEntityLog(),
+                                                                                  document,
+                                                                                  addedFileAttributes.stream()
+                                                                                          .toList());
         log.info("Finished component rule execution for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
 
         ComponentLog componentLog = componentLogCreatorService.buildComponentLog(analyzeRequest.getAnalysisNumber(), components, kieWrapperComponentRules.rulesVersion());
@@ -322,33 +335,26 @@ public class AnalyzeService {
     }
 
 
-    private RedactionLogChanges finalizeRedactionLog(AnalyzeRequest analyzeRequest, RedactionLog redactionLog, DictionaryVersion dictionaryVersion) {
-        // TODO: remove redactionLog related stuff
-
-        RedactionLog previousRedactionLog = redactionStorageService.getRedactionLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
-
-        redactionLog.setDictionaryVersion(dictionaryVersion.getDossierTemplateVersion());
-        redactionLog.setDossierDictionaryVersion(dictionaryVersion.getDossierVersion());
-
-        RedactionLogChanges redactionLogChange = redactionChangeLogService.computeChanges(previousRedactionLog, redactionLog, analyzeRequest.getAnalysisNumber());
-
-        redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.REDACTION_LOG, redactionLogChange.getRedactionLog());
-
-        return redactionLogChange;
-    }
-
-
     private static List<SemanticNode> getSectionsToReAnalyse(Document document, Set<Integer> sectionsToReanalyseIds) {
 
-        return document.streamChildren().filter(section -> sectionsToReanalyseIds.contains(section.getTreeId().get(0))).collect(Collectors.toList());
+        return document.streamChildren()
+                .filter(section -> sectionsToReanalyseIds.contains(section.getTreeId()
+                                                                           .get(0)))
+                .collect(Collectors.toList());
     }
 
 
-    private Set<Integer> getSectionsToReanalyseIds(AnalyzeRequest analyzeRequest, EntityLog entityLog, Document document, DictionaryIncrement dictionaryIncrement) {
+    private Set<Integer> getSectionsToReanalyseIds(AnalyzeRequest analyzeRequest,
+                                                   Document document,
+                                                   DictionaryIncrement dictionaryIncrement,
+                                                   ImportedRedactions importedRedactions,
+                                                   Set<String> relevantManuallyModifiedAnnotationIds) {
 
-        return analyzeRequest.getSectionsToReanalyse().isEmpty() //
-                ? sectionFinderService.findSectionsToReanalyse(dictionaryIncrement, entityLog, document, analyzeRequest) //
-                : analyzeRequest.getSectionsToReanalyse();
+        return sectionFinderService.findSectionsToReanalyse(dictionaryIncrement,
+                                                            document,
+                                                            analyzeRequest,
+                                                            importedRedactions,
+                                                            relevantManuallyModifiedAnnotationIds);
     }
 
 
@@ -368,9 +374,10 @@ public class AnalyzeService {
 
     private static NerEntitiesModel filterNerEntitiesModelBySectionIds(Set<Integer> sectionsToReanalyseIds, NerEntitiesModel nerEntitiesModel) {
 
-        return new NerEntitiesModel(nerEntitiesModel.getData().entrySet().stream() //
-                .filter(entry -> sectionsToReanalyseIds.contains(entry.getKey())) //
-                .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)));
+        return new NerEntitiesModel(nerEntitiesModel.getData().entrySet()
+                                            .stream() //
+                                            .filter(entry -> sectionsToReanalyseIds.contains(entry.getKey())) //
+                                            .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)));
     }
 
 
@@ -385,41 +392,4 @@ public class AnalyzeService {
         return nerEntities;
     }
 
-
-    @Deprecated(forRemoval = true)
-    private RedactionLog createRedactionLog(AnalyzeRequest analyzeRequest,
-                                            Document document,
-                                            List<ManualEntity> notFoundManualRedactionEntries,
-                                            Dictionary dictionary,
-                                            KieWrapper wrapper) {
-
-        List<RedactionLogEntry> redactionLogEntries = redactionLogCreatorService.createRedactionLog(document,
-                analyzeRequest.getDossierTemplateId(),
-                notFoundManualRedactionEntries);
-
-        List<LegalBasis> legalBasis = legalBasisClient.getLegalBasisMapping(analyzeRequest.getDossierTemplateId());
-        RedactionLog redactionLog = new RedactionLog(redactionServiceSettings.getAnalysisVersion(),
-                analyzeRequest.getAnalysisNumber(),
-                redactionLogEntries,
-                toRedactionLogLegalBasis(legalBasis),
-                dictionary.getVersion().getDossierTemplateVersion(),
-                dictionary.getVersion().getDossierVersion(),
-                wrapper.rulesVersion(),
-                legalBasisClient.getVersion(analyzeRequest.getDossierTemplateId()));
-
-        List<RedactionLogEntry> importedRedactionFilteredEntries = importedRedactionService.processImportedRedactions(analyzeRequest.getDossierTemplateId(),
-                analyzeRequest.getDossierId(),
-                analyzeRequest.getFileId(),
-                redactionLog.getRedactionLogEntry(),
-                true);
-        redactionLog.setRedactionLogEntry(importedRedactionFilteredEntries);
-        return redactionLog;
-    }
-
-
-    public List<RedactionLogLegalBasis> toRedactionLogLegalBasis(List<LegalBasis> legalBasis) {
-
-        return legalBasis.stream().map(l -> new RedactionLogLegalBasis(l.getName(), l.getDescription(), l.getReason())).collect(Collectors.toList());
-    }
-
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/ComponentLogCreatorService.java

@@ -23,13 +23,15 @@ public class ComponentLogCreatorService {
     public ComponentLog buildComponentLog(int analysisNumber, List<Component> components, long componentRulesVersion) {
 
         Map<String, List<ComponentLogEntryValue>> map = new HashMap<>();
-        components.stream().sorted(ComponentComparator.first()).forEach(component -> {
-            ComponentLogEntryValue componentLogEntryValue = buildComponentLogEntry(component);
-            map.computeIfAbsent(component.getName(), k -> new ArrayList<>()).add(componentLogEntryValue);
-        });
-        List<ComponentLogEntry> componentLogComponents = map
-                .entrySet()
-                .stream().map(entry -> new ComponentLogEntry(entry.getKey(), entry.getValue()))
+        components.stream()
+                .sorted(ComponentComparator.first())
+                .forEach(component -> {
+                    ComponentLogEntryValue componentLogEntryValue = buildComponentLogEntry(component);
+                    map.computeIfAbsent(component.getName(), k -> new ArrayList<>()).add(componentLogEntryValue);
+                });
+        List<ComponentLogEntry> componentLogComponents = map.entrySet()
+                .stream()
+                .map(entry -> new ComponentLogEntry(entry.getKey(), entry.getValue()))
                 .toList();
         return new ComponentLog(analysisNumber, componentRulesVersion, componentLogComponents);
     }
@@ -38,24 +40,36 @@ public class ComponentLogCreatorService {
     private ComponentLogEntryValue buildComponentLogEntry(Component component) {
 
         return ComponentLogEntryValue.builder()
-                .value(component.getValue()).originalValue(component.getValue())
+                .value(component.getValue())
+                .originalValue(component.getValue())
                 .componentRuleId(component.getMatchedRule().toString())
                 .valueDescription(component.getValueDescription())
-                .componentLogEntityReferences(toComponentEntityReferences(component.getReferences().stream().sorted(EntityComparators.first()).toList()))
+                .componentLogEntityReferences(toComponentEntityReferences(component.getReferences()
+                                                                                  .stream()
+                                                                                  .sorted(EntityComparators.first())
+                                                                                  .toList()))
                 .build();
     }
 
 
     private List<ComponentLogEntityReference> toComponentEntityReferences(List<Entity> references) {
 
-        return references.stream().map(this::toComponentEntityReference).toList();
+        return references.stream()
+                .map(this::toComponentEntityReference)
+                .toList();
     }
 
 
     private ComponentLogEntityReference toComponentEntityReference(Entity entity) {
 
-        return ComponentLogEntityReference.builder().id(entity.getId())
-                .page(entity.getPositions().stream().findFirst().map(Position::getPageNumber).orElse(0)).entityRuleId(entity.getMatchedRule())
+        return ComponentLogEntityReference.builder()
+                .id(entity.getId())
+                .page(entity.getPositions()
+                              .stream()
+                              .findFirst()
+                              .map(Position::getPageNumber)
+                              .orElse(0))
+                .entityRuleId(entity.getMatchedRule())
                 .type(entity.getType())
                 .build();
     }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/DictionarySearchService.java

@@ -1,14 +1,16 @@
 package com.iqser.red.service.redaction.v1.server.service;
 
 import java.util.List;
+import java.util.Set;
 
 import org.springframework.stereotype.Service;
 
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine;
+import com.iqser.red.service.redaction.v1.server.model.dictionary.Dictionary;
+import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryModel;
+import com.iqser.red.service.redaction.v1.server.model.dictionary.SearchImplementation;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
-import com.iqser.red.service.redaction.v1.server.model.dictionary.SearchImplementation;
-import com.iqser.red.service.redaction.v1.server.model.dictionary.Dictionary;
 import com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService;
 import com.iqser.red.service.redaction.v1.server.service.document.EntityEnrichmentService;
 
@@ -28,7 +30,8 @@ public class DictionarySearchService {
 
 
     @Observed(name = "DictionarySearchService", contextualName = "add-dictionary-entries")
-    public void addDictionaryEntities(Dictionary dictionary, List<SemanticNode> semanticNodes){
+    public void addDictionaryEntities(Dictionary dictionary, List<SemanticNode> semanticNodes) {
+
         semanticNodes.forEach(node -> addDictionaryEntities(dictionary, node));
     }
 
@@ -36,10 +39,13 @@ public class DictionarySearchService {
     @Observed(name = "DictionarySearchService", contextualName = "add-dictionary-entries")
     public void addDictionaryEntities(Dictionary dictionary, SemanticNode node) {
 
-        for (var model : dictionary.getDictionaryModels()) {
+        for (DictionaryModel model : dictionary.getDictionaryModels()) {
             bySearchImplementationAsDictionary(model.getEntriesSearch(), model.getType(), model.isHint() ? EntityType.HINT : EntityType.ENTITY, node, model.isDossierDictionary());
             bySearchImplementationAsDictionary(model.getFalsePositiveSearch(), model.getType(), EntityType.FALSE_POSITIVE, node, model.isDossierDictionary());
             bySearchImplementationAsDictionary(model.getFalseRecommendationsSearch(), model.getType(), EntityType.FALSE_RECOMMENDATION, node, model.isDossierDictionary());
+            if (model.isDossierDictionary()) {
+                bySearchImplementationAsDictionary(model.getDeletionEntriesSearch(), model.getType(), EntityType.DICTIONARY_REMOVAL, node, model.isDossierDictionary());
+            }
         }
     }
 
@@ -50,13 +56,16 @@ public class DictionarySearchService {
                                                    SemanticNode node,
                                                    boolean isDossierDictionaryEntry) {
 
+        Set<Engine> engines = isDossierDictionaryEntry ? Set.of(Engine.DOSSIER_DICTIONARY) : Set.of(Engine.DICTIONARY);
         EntityCreationService entityCreationService = new EntityCreationService(entityEnrichmentService);
         searchImplementation.getBoundaries(node.getTextBlock(), node.getTextRange())
-                .stream().filter(boundary -> entityCreationService.isValidEntityTextRange(node.getTextBlock(), boundary))
-                .map(bounds -> entityCreationService.forceByTextRange(bounds, type, entityType, node))
-                .peek(entity -> entity.setDictionaryEntry(true))
-                .peek(entity -> entity.setDossierDictionaryEntry(isDossierDictionaryEntry))
-                .forEach(entity -> entity.addEngine(Engine.DICTIONARY));
+                .stream()
+                .filter(boundary -> entityCreationService.isValidEntityTextRange(node.getTextBlock(), boundary))
+                .forEach(bounds -> entityCreationService.byTextRangeWithEngine(bounds, type, entityType, node, engines)
+                        .ifPresent(entity -> {
+                            entity.setDictionaryEntry(true);
+                            entity.setDossierDictionaryEntry(isDossierDictionaryEntry);
+                        }));
     }
 
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/DictionaryService.java

@@ -4,6 +4,7 @@ import java.awt.Color;
 import java.util.ArrayList;
 import java.util.Comparator;
 import java.util.HashSet;
+import java.util.LinkedList;
 import java.util.List;
 import java.util.Locale;
 import java.util.Optional;
@@ -91,7 +92,8 @@ public class DictionaryService {
             updateDictionaryEntry(dossierTemplateId, dossierDictionaryVersion, getVersion(dossierDictionary), dossierId);
         }
 
-        return DictionaryVersion.builder().dossierTemplateVersion(dossierTemplateDictionaryVersion).dossierVersion(dossierDictionaryVersion).build();
+        return DictionaryVersion.builder().dossierTemplateVersion(dossierTemplateDictionaryVersion).dossierVersion(dossierDictionaryVersion)
+                .build();
     }
 
 
@@ -106,41 +108,47 @@ public class DictionaryService {
         List<DictionaryModel> dictionaryModels = getDossierTemplateDictionary(dossierTemplateId).getDictionary();
 
         dictionaryModels.forEach(dictionaryModel -> {
-            dictionaryModel.getEntries().forEach(dictionaryEntry -> {
-                if (dictionaryEntry.getVersion() > fromVersion.getDossierTemplateVersion()) {
-                    newValues.add(new DictionaryIncrementValue(dictionaryEntry.getValue(), dictionaryModel.isCaseInsensitive()));
-                }
-            });
-            dictionaryModel.getFalsePositives().forEach(dictionaryEntry -> {
-                if (dictionaryEntry.getVersion() > fromVersion.getDossierTemplateVersion()) {
-                    newValues.add(new DictionaryIncrementValue(dictionaryEntry.getValue(), dictionaryModel.isCaseInsensitive()));
-                }
-            });
-            dictionaryModel.getFalseRecommendations().forEach(dictionaryEntry -> {
-                if (dictionaryEntry.getVersion() > fromVersion.getDossierTemplateVersion()) {
-                    newValues.add(new DictionaryIncrementValue(dictionaryEntry.getValue(), dictionaryModel.isCaseInsensitive()));
-                }
-            });
+            dictionaryModel.getEntries()
+                    .forEach(dictionaryEntry -> {
+                        if (dictionaryEntry.getVersion() > fromVersion.getDossierTemplateVersion()) {
+                            newValues.add(new DictionaryIncrementValue(dictionaryEntry.getValue(), dictionaryModel.isCaseInsensitive()));
+                        }
+                    });
+            dictionaryModel.getFalsePositives()
+                    .forEach(dictionaryEntry -> {
+                        if (dictionaryEntry.getVersion() > fromVersion.getDossierTemplateVersion()) {
+                            newValues.add(new DictionaryIncrementValue(dictionaryEntry.getValue(), dictionaryModel.isCaseInsensitive()));
+                        }
+                    });
+            dictionaryModel.getFalseRecommendations()
+                    .forEach(dictionaryEntry -> {
+                        if (dictionaryEntry.getVersion() > fromVersion.getDossierTemplateVersion()) {
+                            newValues.add(new DictionaryIncrementValue(dictionaryEntry.getValue(), dictionaryModel.isCaseInsensitive()));
+                        }
+                    });
         });
 
         if (dossierDictionaryExists(dossierId)) {
             dictionaryModels = getDossierDictionary(dossierId).getDictionary();
             dictionaryModels.forEach(dictionaryModel -> {
-                dictionaryModel.getEntries().forEach(dictionaryEntry -> {
-                    if (dictionaryEntry.getVersion() > fromVersion.getDossierVersion()) {
-                        newValues.add(new DictionaryIncrementValue(dictionaryEntry.getValue(), dictionaryModel.isCaseInsensitive()));
-                    }
-                });
-                dictionaryModel.getFalsePositives().forEach(dictionaryEntry -> {
-                    if (dictionaryEntry.getVersion() > fromVersion.getDossierVersion()) {
-                        newValues.add(new DictionaryIncrementValue(dictionaryEntry.getValue(), dictionaryModel.isCaseInsensitive()));
-                    }
-                });
-                dictionaryModel.getFalseRecommendations().forEach(dictionaryEntry -> {
-                    if (dictionaryEntry.getVersion() > fromVersion.getDossierVersion()) {
-                        newValues.add(new DictionaryIncrementValue(dictionaryEntry.getValue(), dictionaryModel.isCaseInsensitive()));
-                    }
-                });
+                dictionaryModel.getEntries()
+                        .forEach(dictionaryEntry -> {
+                            if (dictionaryEntry.getVersion() > fromVersion.getDossierVersion()) {
+                                newValues.add(new DictionaryIncrementValue(dictionaryEntry.getValue(), dictionaryModel.isCaseInsensitive()));
+                            }
+                        });
+                dictionaryModel.getFalsePositives()
+                        .forEach(dictionaryEntry -> {
+                            if (dictionaryEntry.getVersion() > fromVersion.getDossierVersion()) {
+                                newValues.add(new DictionaryIncrementValue(dictionaryEntry.getValue(), dictionaryModel.isCaseInsensitive()));
+                            }
+                        });
+                dictionaryModel.getFalseRecommendations()
+                        .forEach(dictionaryEntry -> {
+                            if (dictionaryEntry.getVersion() > fromVersion.getDossierVersion()) {
+                                newValues.add(new DictionaryIncrementValue(dictionaryEntry.getValue(), dictionaryModel.isCaseInsensitive()));
+                            }
+                        });
             });
         }
 
@@ -155,59 +163,120 @@ public class DictionaryService {
             DictionaryRepresentation dictionaryRepresentation = new DictionaryRepresentation();
 
             var typeResponse = dossierId == null ? dictionaryClient.getAllTypesForDossierTemplate(dossierTemplateId, true) : dictionaryClient.getAllTypesForDossier(dossierId,
-                    true);
+                                                                                                                                                                    true);
             if (CollectionUtils.isNotEmpty(typeResponse)) {
 
-                List<DictionaryModel> dictionary = typeResponse.stream().map(t -> {
+                List<DictionaryModel> dictionary = typeResponse.stream()
+                        .map(t -> {
 
-                    Optional<DictionaryModel> oldModel;
-                    if (dossierId == null) {
-                        var representation = getDossierTemplateDictionary(dossierTemplateId);
-                        oldModel = representation != null ? representation.getDictionary().stream().filter(f -> f.getType().equals(t.getType())).findAny() : Optional.empty();
-                    } else {
-                        var representation = getDossierDictionary(dossierId);
-                        oldModel = representation != null ? representation.getDictionary().stream().filter(f -> f.getType().equals(t.getType())).findAny() : Optional.empty();
-                    }
+                            Optional<DictionaryModel> optionalOldModel;
+                            if (dossierId == null) {
+                                var representation = getDossierTemplateDictionary(dossierTemplateId);
+                                optionalOldModel = representation != null ? representation.getDictionary()
+                                        .stream()
+                                        .filter(f -> f.getType().equals(t.getType()))
+                                        .findAny() : Optional.empty();
+                            } else {
+                                var representation = getDossierDictionary(dossierId);
+                                optionalOldModel = representation != null ? representation.getDictionary()
+                                        .stream()
+                                        .filter(f -> f.getType().equals(t.getType()))
+                                        .findAny() : Optional.empty();
+                            }
 
-                    Set<DictionaryEntryModel> entries = new HashSet<>();
-                    Set<DictionaryEntryModel> falsePositives = new HashSet<>();
-                    Set<DictionaryEntryModel> falseRecommendations = new HashSet<>();
+                            Set<DictionaryEntryModel> entries = new HashSet<>();
+                            Set<DictionaryEntryModel> falsePositives = new HashSet<>();
+                            Set<DictionaryEntryModel> falseRecommendations = new HashSet<>();
 
-                    DictionaryEntries newEntries = getEntries(t.getId(), currentVersion);
+                            DictionaryEntries newEntries = getEntries(t.getId(), currentVersion);
 
-                    var newValues = newEntries.getEntries().stream().map(DictionaryEntry::getValue).collect(Collectors.toSet());
-                    var newFalsePositivesValues = newEntries.getFalsePositives().stream().map(DictionaryEntry::getValue).collect(Collectors.toSet());
-                    var newFalseRecommendationsValues = newEntries.getFalseRecommendations().stream().map(DictionaryEntry::getValue).collect(Collectors.toSet());
+                            var newValues = newEntries.getEntries()
+                                    .stream()
+                                    .map(DictionaryEntry::getValue)
+                                    .collect(Collectors.toSet());
+                            var newFalsePositivesValues = newEntries.getFalsePositives()
+                                    .stream()
+                                    .map(DictionaryEntry::getValue)
+                                    .collect(Collectors.toSet());
+                            var newFalseRecommendationsValues = newEntries.getFalseRecommendations()
+                                    .stream()
+                                    .map(DictionaryEntry::getValue)
+                                    .collect(Collectors.toSet());
 
-                    oldModel.ifPresent(oldDictionaryModel -> {
+                            optionalOldModel.ifPresent(oldDictionaryModel -> {
 
-                    });
-                    // add old entries from existing DictionaryModel
-                    oldModel.ifPresent(dictionaryModel -> entries.addAll(dictionaryModel.getEntries().stream().filter(f -> !newValues.contains(f.getValue())).toList()));
-                    oldModel.ifPresent(dictionaryModel -> falsePositives.addAll(dictionaryModel.getFalsePositives()
-                            .stream()
-                            .filter(f -> !newFalsePositivesValues.contains(f.getValue()))
-                            .toList()));
-                    oldModel.ifPresent(dictionaryModel -> falseRecommendations.addAll(dictionaryModel.getFalseRecommendations()
-                            .stream()
-                            .filter(f -> !newFalseRecommendationsValues.contains(f.getValue()))
-                            .toList()));
+                            });
+                            if (optionalOldModel.isPresent()) {
+                                var oldModel = optionalOldModel.get();
+                                if (oldModel.isCaseInsensitive() && !t.isCaseInsensitive()) {
+                                    // add old entries from existing DictionaryModel but exclude lower case representation
+                                    entries.addAll(oldModel.getEntries()
+                                                           .stream()
+                                                           .filter(f -> !newValues.stream()
+                                                                   .map(s -> s.toLowerCase(Locale.ROOT))
+                                                                   .toList().contains(f.getValue()))
+                                                           .toList());
+                                    falsePositives.addAll(oldModel.getFalsePositives()
+                                                                  .stream()
+                                                                  .filter(f -> !newFalsePositivesValues.stream()
+                                                                          .map(s -> s.toLowerCase(Locale.ROOT))
+                                                                          .toList().contains(f.getValue()))
+                                                                  .toList());
+                                    falseRecommendations.addAll(oldModel.getFalseRecommendations()
+                                                                        .stream()
+                                                                        .filter(f -> !newFalseRecommendationsValues.stream()
+                                                                                .map(s -> s.toLowerCase(Locale.ROOT))
+                                                                                .toList().contains(f.getValue()))
+                                                                        .toList());
+                                } else if (!oldModel.isCaseInsensitive() && t.isCaseInsensitive()) {
+                                    // add old entries from existing DictionaryModel but exclude upper case representation
+                                    entries.addAll(oldModel.getEntries()
+                                                           .stream()
+                                                           .filter(f -> !newValues.contains(f.getValue().toLowerCase(Locale.ROOT)))
+                                                           .toList());
+                                    falsePositives.addAll(oldModel.getFalsePositives()
+                                                                  .stream()
+                                                                  .filter(f -> !newFalsePositivesValues.contains(f.getValue().toLowerCase(Locale.ROOT)))
+                                                                  .toList());
+                                    falseRecommendations.addAll(oldModel.getFalseRecommendations()
+                                                                        .stream()
+                                                                        .filter(f -> !newFalseRecommendationsValues.contains(f.getValue().toLowerCase(Locale.ROOT)))
+                                                                        .toList());
 
-                    // Add Increments
-                    entries.addAll(newEntries.getEntries());
-                    falsePositives.addAll(newEntries.getFalsePositives());
-                    falseRecommendations.addAll(newEntries.getFalseRecommendations());
+                                } else {
+                                    // add old entries from existing DictionaryModel
+                                    entries.addAll(oldModel.getEntries()
+                                                           .stream()
+                                                           .filter(f -> !newValues.contains(f.getValue()))
+                                                           .toList());
+                                    falsePositives.addAll(oldModel.getFalsePositives()
+                                                                  .stream()
+                                                                  .filter(f -> !newFalsePositivesValues.contains(f.getValue()))
+                                                                  .toList());
+                                    falseRecommendations.addAll(oldModel.getFalseRecommendations()
+                                                                        .stream()
+                                                                        .filter(f -> !newFalseRecommendationsValues.contains(f.getValue()))
+                                                                        .toList());
+                                }
+                            }
 
-                    return new DictionaryModel(t.getType(),
-                            t.getRank(),
-                            convertColor(t.getHexColor()),
-                            t.isCaseInsensitive(),
-                            t.isHint(),
-                            entries,
-                            falsePositives,
-                            falseRecommendations,
-                            dossierId != null);
-                }).sorted(Comparator.comparingInt(DictionaryModel::getRank).reversed()).collect(Collectors.toList());
+                            // Add Increments
+                            entries.addAll(newEntries.getEntries());
+                            falsePositives.addAll(newEntries.getFalsePositives());
+                            falseRecommendations.addAll(newEntries.getFalseRecommendations());
+
+                            return new DictionaryModel(t.getType(),
+                                                       t.getRank(),
+                                                       convertColor(t.getHexColor()),
+                                                       t.isCaseInsensitive(),
+                                                       t.isHint(),
+                                                       entries,
+                                                       falsePositives,
+                                                       falseRecommendations,
+                                                       dossierId != null);
+                        })
+                        .sorted(Comparator.comparingInt(DictionaryModel::getRank).reversed())
+                        .collect(Collectors.toList());
 
                 dictionary.forEach(dm -> dictionaryRepresentation.getLocalAccessMap().put(dm.getType(), dm));
 
@@ -239,17 +308,17 @@ public class DictionaryService {
         var type = dictionaryClient.getDictionaryForType(typeId, fromVersion);
 
         Set<DictionaryEntryModel> entries = type.getEntries() != null ? new HashSet<>(type.getEntries()
-                .stream()
-                .map(DictionaryEntryModel::new)
-                .collect(Collectors.toSet())) : new HashSet<>();
+                                                                                              .stream()
+                                                                                              .map(DictionaryEntryModel::new)
+                                                                                              .collect(Collectors.toSet())) : new HashSet<>();
         Set<DictionaryEntryModel> falsePositives = type.getFalsePositiveEntries() != null ? new HashSet<>(type.getFalsePositiveEntries()
-                .stream()
-                .map(DictionaryEntryModel::new)
-                .collect(Collectors.toSet())) : new HashSet<>();
+                                                                                                                  .stream()
+                                                                                                                  .map(DictionaryEntryModel::new)
+                                                                                                                  .collect(Collectors.toSet())) : new HashSet<>();
         Set<DictionaryEntryModel> falseRecommendations = type.getFalseRecommendationEntries() != null ? new HashSet<>(type.getFalseRecommendationEntries()
-                .stream()
-                .map(DictionaryEntryModel::new)
-                .collect(Collectors.toSet())) : new HashSet<>();
+                                                                                                                              .stream()
+                                                                                                                              .map(DictionaryEntryModel::new)
+                                                                                                                              .collect(Collectors.toSet())) : new HashSet<>();
 
         if (type.isCaseInsensitive()) {
             entries.forEach(entry -> entry.setValue(entry.getValue().toLowerCase(Locale.ROOT)));
@@ -257,10 +326,10 @@ public class DictionaryService {
             falseRecommendations.forEach(entry -> entry.setValue(entry.getValue().toLowerCase(Locale.ROOT)));
         }
         log.debug("Dictionary update returned {} entries {} falsePositives and {} falseRecommendations for type {}",
-                entries.size(),
-                falsePositives.size(),
-                falseRecommendations.size(),
-                typeId);
+                  entries.size(),
+                  falsePositives.size(),
+                  falseRecommendations.size(),
+                  typeId);
         return new DictionaryEntries(entries, falsePositives, falseRecommendations);
     }
 
@@ -275,7 +344,8 @@ public class DictionaryService {
     @SneakyThrows
     public float[] getColor(String type, String dossierTemplateId) {
 
-        DictionaryModel model = getDossierTemplateDictionary(dossierTemplateId).getLocalAccessMap().get(type);
+        DictionaryModel model = getDossierTemplateDictionary(dossierTemplateId).getLocalAccessMap()
+                .get(type);
         if (model != null) {
             return model.getColor();
         }
@@ -286,7 +356,8 @@ public class DictionaryService {
     @SneakyThrows
     public boolean isHint(String type, String dossierTemplateId) {
 
-        DictionaryModel model = getDossierTemplateDictionary(dossierTemplateId).getLocalAccessMap().get(type);
+        DictionaryModel model = getDossierTemplateDictionary(dossierTemplateId).getLocalAccessMap()
+                .get(type);
         if (model != null) {
             return model.isHint();
         }
@@ -299,26 +370,33 @@ public class DictionaryService {
     @Observed(name = "DictionaryService", contextualName = "deep-copy-dictionary")
     public Dictionary getDeepCopyDictionary(String dossierTemplateId, String dossierId) {
 
-        List<DictionaryModel> mergedDictionaries;
+        List<DictionaryModel> mergedDictionaries = new LinkedList<>();
 
-        var dossierTemplateRepresentation = getDossierTemplateDictionary(dossierTemplateId);
-        var dossierTemplateDictionaries = dossierTemplateRepresentation.getDictionary();
+        DictionaryRepresentation dossierTemplateRepresentation = getDossierTemplateDictionary(dossierTemplateId);
+        List<DictionaryModel> dossierTemplateDictionaries = dossierTemplateRepresentation.getDictionary();
+        dossierTemplateDictionaries.forEach(dm -> mergedDictionaries.add(SerializationUtils.clone(dm)));
 
-        // merge dictionaries if they have same names
+        // add dossier
         long dossierDictionaryVersion = -1;
         if (dossierDictionaryExists(dossierId)) {
-            var dossierRepresentation = getDossierDictionary(dossierId);
-            var dossierDictionaries = dossierRepresentation.getDictionary();
-            mergedDictionaries = convertCommonsDictionaryModel(dictionaryMergeService.getMergedDictionary(convertDictionaryModel(dossierTemplateDictionaries),
-                    convertDictionaryModel(dossierDictionaries)));
-            dossierDictionaryVersion = dossierRepresentation.getDictionaryVersion();
+            DictionaryRepresentation dossierRepresentation = getDossierDictionary(dossierId);
+            List<DictionaryModel> dossierDictionaries = dossierRepresentation.getDictionary();
+            dossierDictionaries.forEach(dm -> mergedDictionaries.add(SerializationUtils.clone(dm)));
+            return getDictionary(mergedDictionaries, dossierTemplateRepresentation, dossierRepresentation.getDictionaryVersion());
         } else {
-            mergedDictionaries = new ArrayList<>();
-            dossierTemplateDictionaries.forEach(dm -> mergedDictionaries.add(SerializationUtils.clone(dm)));
+            return getDictionary(mergedDictionaries, dossierTemplateRepresentation, dossierDictionaryVersion);
         }
 
-        return new Dictionary(mergedDictionaries.stream().sorted(Comparator.comparingInt(DictionaryModel::getRank).reversed()).collect(Collectors.toList()),
-                DictionaryVersion.builder().dossierTemplateVersion(dossierTemplateRepresentation.getDictionaryVersion()).dossierVersion(dossierDictionaryVersion).build());
+    }
+
+
+    private Dictionary getDictionary(List<DictionaryModel> mergedDictionaries, DictionaryRepresentation dossierTemplateRepresentation, long dossierDictionaryVersion) {
+
+        return new Dictionary(mergedDictionaries.stream()
+                                      .sorted(Comparator.comparingInt(DictionaryModel::getRank).reversed())
+                                      .collect(Collectors.toList()),
+                              DictionaryVersion.builder().dossierTemplateVersion(dossierTemplateRepresentation.getDictionaryVersion()).dossierVersion(dossierDictionaryVersion)
+                                      .build());
     }
 
 
@@ -346,14 +424,16 @@ public class DictionaryService {
     @SneakyThrows
     private DictionaryRepresentation getDossierTemplateDictionary(String dossierTemplateId) {
 
-        return tenantDictionaryCache.get(TenantContext.getTenantId()).getDictionariesByDossierTemplate().get(dossierTemplateId);
+        return tenantDictionaryCache.get(TenantContext.getTenantId()).getDictionariesByDossierTemplate()
+                .get(dossierTemplateId);
     }
 
 
     @SneakyThrows
     private DictionaryRepresentation getDossierDictionary(String dossierId) {
 
-        return tenantDictionaryCache.get(TenantContext.getTenantId()).getDictionariesByDossier().get(dossierId);
+        return tenantDictionaryCache.get(TenantContext.getTenantId()).getDictionariesByDossier()
+                .get(dossierId);
     }
 
 
@@ -396,14 +476,14 @@ public class DictionaryService {
 
         return commonsDictionaries.stream()
                 .map(cd -> new DictionaryModel(cd.getType(),
-                        cd.getRank(),
-                        cd.getColor(),
-                        cd.isCaseInsensitive(),
-                        cd.isHint(),
-                        cd.getEntries(),
-                        cd.getFalsePositives(),
-                        cd.getFalseRecommendations(),
-                        cd.isDossierDictionary()))
+                                               cd.getRank(),
+                                               cd.getColor(),
+                                               cd.isCaseInsensitive(),
+                                               cd.isHint(),
+                                               cd.getEntries(),
+                                               cd.getFalsePositives(),
+                                               cd.getFalseRecommendations(),
+                                               cd.isDossierDictionary()))
                 .collect(Collectors.toList());
     }
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityChangeLogService.java

@@ -1,6 +1,7 @@
 package com.iqser.red.service.redaction.v1.server.service;
 
 import java.time.OffsetDateTime;
+import java.util.ArrayList;
 import java.util.List;
 import java.util.Optional;
 import java.util.Set;
@@ -14,58 +15,69 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState;
 
 import io.micrometer.core.annotation.Timed;
+import lombok.AccessLevel;
 import lombok.RequiredArgsConstructor;
+import lombok.experimental.FieldDefaults;
 import lombok.extern.slf4j.Slf4j;
 
 @Slf4j
 @Service
 @RequiredArgsConstructor
+@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
 public class EntityChangeLogService {
 
-
     @Timed("redactmanager_computeChanges")
-    public boolean computeChanges(List<EntityLogEntry> previousEntityLogEntries, List<EntityLogEntry> newEntityLogEntries, int analysisNumber) {
+    public EntryChanges computeChanges(List<EntityLogEntry> previousEntityLogEntries, List<EntityLogEntry> newEntityLogEntries, int analysisNumber) {
 
         var now = OffsetDateTime.now();
         if (previousEntityLogEntries.isEmpty()) {
             newEntityLogEntries.forEach(entry -> entry.getChanges().add(new Change(analysisNumber, ChangeType.ADDED, now)));
-            return true;
+            return new EntryChanges(newEntityLogEntries, new ArrayList<>());
         }
 
-        boolean hasChanges = false;
-
+        List<EntityLogEntry> toInsert = new ArrayList<>();
+        List<EntityLogEntry> toUpdate = new ArrayList<>();
         for (EntityLogEntry entityLogEntry : newEntityLogEntries) {
-            Optional<EntityLogEntry> optionalPreviousEntity = previousEntityLogEntries.stream().filter(entry -> entry.getId().equals(entityLogEntry.getId())).findAny();
+            Optional<EntityLogEntry> optionalPreviousEntity = previousEntityLogEntries.stream()
+                    .filter(entry -> entry.getId().equals(entityLogEntry.getId()))
+                    .findAny();
             if (optionalPreviousEntity.isEmpty()) {
-                hasChanges = true;
                 entityLogEntry.getChanges().add(new Change(analysisNumber, ChangeType.ADDED, now));
+                toInsert.add(entityLogEntry);
                 continue;
             }
-            var previousEntity = optionalPreviousEntity.get();
+
+            EntityLogEntry previousEntity = optionalPreviousEntity.get();
             entityLogEntry.getChanges().addAll(previousEntity.getChanges());
             if (!previousEntity.getState().equals(entityLogEntry.getState())) {
-                hasChanges = true;
                 ChangeType changeType = calculateChangeType(entityLogEntry.getState(), previousEntity.getState());
-                if (changeType != null) {
-                    entityLogEntry.getChanges().add(new Change(analysisNumber, changeType, now));
-                }
+                entityLogEntry.getChanges().add(new Change(analysisNumber, changeType, now));
+                toUpdate.add(entityLogEntry);
             }
         }
-        addRemovedEntriesAsRemoved(previousEntityLogEntries, newEntityLogEntries, analysisNumber, now);
-        return hasChanges;
+
+        toUpdate.addAll(addRemovedEntriesAsRemoved(previousEntityLogEntries, newEntityLogEntries, analysisNumber, now));
+        return new EntryChanges(toInsert, toUpdate);
     }
 
 
-    private static void addRemovedEntriesAsRemoved(List<EntityLogEntry> previousEntityLogEntries,
-                                                   List<EntityLogEntry> newEntityLogEntries,
-                                                   int analysisNumber,
-                                                   OffsetDateTime now) {
+    private List<EntityLogEntry> addRemovedEntriesAsRemoved(List<EntityLogEntry> previousEntityLogEntries,
+                                                            List<EntityLogEntry> newEntityLogEntries,
+                                                            int analysisNumber,
+                                                            OffsetDateTime now) {
 
-        Set<String> existingIds = newEntityLogEntries.stream().map(EntityLogEntry::getId).collect(Collectors.toSet());
-        List<EntityLogEntry> removedEntries = previousEntityLogEntries.stream().filter(entry -> !existingIds.contains(entry.getId())).toList();
-        removedEntries.forEach(entry -> entry.getChanges().add(new Change(analysisNumber, ChangeType.REMOVED, now)));
-        removedEntries.forEach(entry -> entry.setState(EntryState.REMOVED));
+        Set<String> existingIds = newEntityLogEntries.stream()
+                .map(EntityLogEntry::getId)
+                .collect(Collectors.toSet());
+        List<EntityLogEntry> removedEntries = previousEntityLogEntries.stream()
+                .filter(entry -> !existingIds.contains(entry.getId()))
+                .toList();
+        removedEntries.stream()
+                .filter(entry -> !entry.getState().equals(EntryState.REMOVED))
+                .peek(entry -> entry.getChanges().add(new Change(analysisNumber, ChangeType.REMOVED, now)))
+                .forEach(entry -> entry.setState(EntryState.REMOVED));
         newEntityLogEntries.addAll(removedEntries);
+        return removedEntries;
     }
 
 
@@ -90,4 +102,8 @@ public class EntityChangeLogService {
     }
 
 
+    public record EntryChanges(List<EntityLogEntry> inserted, List<EntityLogEntry> updated) {
+
+    }
+
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java

@@ -1,7 +1,5 @@
 package com.iqser.red.service.redaction.v1.server.service;
 
-import static com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryType.IMAGE;
-
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashSet;
@@ -13,6 +11,7 @@ import java.util.stream.Collectors;
 import org.springframework.stereotype.Service;
 
 import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogChanges;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
@@ -23,15 +22,17 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.legalbasis.LegalBasis;
 import com.iqser.red.service.redaction.v1.server.RedactionServiceSettings;
 import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient;
-import com.iqser.red.service.redaction.v1.server.model.ManualEntity;
+import com.iqser.red.service.redaction.v1.server.model.PrecursorEntity;
 import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryVersion;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.IEntity;
+import com.iqser.red.service.redaction.v1.server.model.document.entity.ManualChangeOverwrite;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.PositionOnPage;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Image;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.ImageType;
+import com.iqser.red.service.redaction.v1.server.service.EntityChangeLogService.EntryChanges;
 import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
 
 import lombok.AccessLevel;
@@ -46,48 +47,46 @@ import lombok.extern.slf4j.Slf4j;
 public class EntityLogCreatorService {
 
     DictionaryService dictionaryService;
-    ManualChangeFactory manualChangeFactory;
-    ImportedRedactionService importedRedactionService;
     RedactionServiceSettings redactionServiceSettings;
     LegalBasisClient legalBasisClient;
     EntityChangeLogService entityChangeLogService;
     RedactionStorageService redactionStorageService;
 
 
-    private static boolean notFalsePositiveOrFalseRecommendation(TextEntity textEntity) {
+    private static boolean notFalsePositiveOrFalseRecommendationOrRemoval(TextEntity textEntity) {
 
-        return !(textEntity.getEntityType().equals(EntityType.FALSE_POSITIVE) || textEntity.getEntityType().equals(EntityType.FALSE_RECOMMENDATION));
+        return !(textEntity.getEntityType().equals(EntityType.FALSE_POSITIVE) //
+                 || textEntity.getEntityType().equals(EntityType.FALSE_RECOMMENDATION) //
+                 || textEntity.getEntityType().equals(EntityType.DICTIONARY_REMOVAL));
     }
 
 
-    public EntityLog createInitialEntityLog(AnalyzeRequest analyzeRequest,
-                                            Document document,
-                                            List<ManualEntity> notFoundManualEntities,
-                                            DictionaryVersion dictionaryVersion,
-                                            long rulesVersion) {
+    public EntityLogChanges createInitialEntityLog(AnalyzeRequest analyzeRequest,
+                                                   Document document,
+                                                   List<PrecursorEntity> notFoundEntities,
+                                                   DictionaryVersion dictionaryVersion,
+                                                   long rulesVersion) {
 
-        List<EntityLogEntry> entityLogEntries = createEntityLogEntries(document, analyzeRequest.getDossierTemplateId(), notFoundManualEntities);
+        List<EntityLogEntry> entityLogEntries = createEntityLogEntries(document, analyzeRequest, notFoundEntities);
 
         List<LegalBasis> legalBasis = legalBasisClient.getLegalBasisMapping(analyzeRequest.getDossierTemplateId());
-        EntityLog entityLog = new EntityLog(redactionServiceSettings.getAnalysisVersion(),
-                analyzeRequest.getAnalysisNumber(),
-                entityLogEntries,
-                toEntityLogLegalBasis(legalBasis),
-                dictionaryVersion.getDossierTemplateVersion(),
-                dictionaryVersion.getDossierVersion(),
-                rulesVersion,
-                legalBasisClient.getVersion(analyzeRequest.getDossierTemplateId()));
 
-        List<EntityLogEntry> importedRedactionFilteredEntries = importedRedactionService.processImportedEntities(analyzeRequest.getDossierTemplateId(),
-                analyzeRequest.getDossierId(),
-                analyzeRequest.getFileId(),
-                entityLog.getEntityLogEntry(),
-                true);
-
-        entityLog.setEntityLogEntry(importedRedactionFilteredEntries);
         List<EntityLogEntry> previousExistingEntityLogEntries = getPreviousEntityLogEntries(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
-        entityChangeLogService.computeChanges(previousExistingEntityLogEntries, entityLogEntries, analyzeRequest.getAnalysisNumber());
-        return entityLog;
+
+        EntryChanges entryChanges = entityChangeLogService.computeChanges(previousExistingEntityLogEntries, entityLogEntries, analyzeRequest.getAnalysisNumber());
+
+        return EntityLogChanges.builder()
+                .entityLog(new EntityLog(redactionServiceSettings.getAnalysisVersion(),
+                                         analyzeRequest.getAnalysisNumber(),
+                                         entityLogEntries,
+                                         toEntityLogLegalBasis(legalBasis),
+                                         dictionaryVersion.getDossierTemplateVersion(),
+                                         dictionaryVersion.getDossierVersion(),
+                                         rulesVersion,
+                                         legalBasisClient.getVersion(analyzeRequest.getDossierTemplateId())))
+                .updatedEntityLogEntries(entryChanges.updated())
+                .newEntityLogEntries(entryChanges.inserted())
+                .build();
     }
 
 
@@ -101,90 +100,88 @@ public class EntityLogCreatorService {
     }
 
 
-    public EntityLogChanges updateVersionsAndReturnChanges(EntityLog entityLog, DictionaryVersion dictionaryVersion, String dossierTemplateId, boolean hasChanges) {
+    public EntityLogChanges updateVersionsAndReturnChanges(EntityLog entityLog,
+                                                           DictionaryVersion dictionaryVersion,
+                                                           AnalyzeRequest analyzeRequest,
+                                                           List<EntityLogEntry> newEntries,
+                                                           List<EntityLogEntry> updatedEntries) {
 
-        List<LegalBasis> legalBasis = legalBasisClient.getLegalBasisMapping(dossierTemplateId);
-        entityLog.setLegalBasisVersion(legalBasisClient.getVersion(dossierTemplateId));
+        List<LegalBasis> legalBasis = legalBasisClient.getLegalBasisMapping(analyzeRequest.getDossierTemplateId());
+        entityLog.setLegalBasisVersion(legalBasisClient.getVersion(analyzeRequest.getDossierTemplateId()));
         entityLog.setLegalBasis(toEntityLogLegalBasis(legalBasis));
         entityLog.setDictionaryVersion(dictionaryVersion.getDossierTemplateVersion());
         entityLog.setDossierDictionaryVersion(dictionaryVersion.getDossierVersion());
+        entityLog.setAnalysisNumber(analyzeRequest.getAnalysisNumber());
 
-        return new EntityLogChanges(entityLog, hasChanges);
+        return EntityLogChanges.builder().entityLog(entityLog).newEntityLogEntries(newEntries).updatedEntityLogEntries(updatedEntries).build();
     }
 
 
     public EntityLogChanges updatePreviousEntityLog(AnalyzeRequest analyzeRequest,
                                                     Document document,
-                                                    List<ManualEntity> notFoundManualRedactionEntries,
-                                                    EntityLog previousEntityLog,
+                                                    EntityLog entityLogWithoutEntries,
+                                                    List<PrecursorEntity> notFoundEntries,
                                                     Set<Integer> sectionsToReanalyseIds,
                                                     DictionaryVersion dictionaryVersion) {
 
-        List<EntityLogEntry> newEntityLogEntries = createEntityLogEntries(document, analyzeRequest.getDossierTemplateId(), notFoundManualRedactionEntries);
-        Set<String> newEntityIds = newEntityLogEntries.stream().map(EntityLogEntry::getId).collect(Collectors.toSet());
+        List<EntityLogEntry> newEntityLogEntries = createEntityLogEntries(document, analyzeRequest, notFoundEntries).stream()
+                .filter(entry -> entry.getContainingNodeId().isEmpty() || sectionsToReanalyseIds.contains(entry.getContainingNodeId()
+                                                                                                                  .get(0)))
+                .collect(Collectors.toList());
 
-        List<EntityLogEntry> previousEntriesFromReAnalyzedSections = previousEntityLog.getEntityLogEntry()
-                .stream()
-                .filter(entry -> !entry.getType().equals(ImportedRedactionService.IMPORTED_REDACTION_TYPE) && (newEntityIds.contains(entry.getId()) || entry.getContainingNodeId()
-                        .isEmpty() || sectionsToReanalyseIds.contains(entry.getContainingNodeId().get(0))))
-                .toList();
-        previousEntityLog.getEntityLogEntry().removeAll(previousEntriesFromReAnalyzedSections);
+        List<EntityLogEntry> previousEntriesFromReAnalyzedSections = redactionStorageService.findEntriesContainedBySectionsOrNotContained(analyzeRequest.getDossierId(),
+                                                                                                                                          analyzeRequest.getFileId(),
+                                                                                                                                          sectionsToReanalyseIds);
 
-        boolean hasChanges = entityChangeLogService.computeChanges(previousEntriesFromReAnalyzedSections, newEntityLogEntries, analyzeRequest.getAnalysisNumber());
+        EntryChanges entryChanges = entityChangeLogService.computeChanges(previousEntriesFromReAnalyzedSections, newEntityLogEntries, analyzeRequest.getAnalysisNumber());
 
-        var newEntityLogWithImportedEntities = importedRedactionService.processImportedEntities(analyzeRequest.getDossierTemplateId(),
-                analyzeRequest.getDossierId(),
-                analyzeRequest.getFileId(),
-                newEntityLogEntries,
-                false);
-
-        previousEntityLog.getEntityLogEntry().addAll(newEntityLogWithImportedEntities);
-
-        return updateVersionsAndReturnChanges(previousEntityLog, dictionaryVersion, analyzeRequest.getDossierTemplateId(), hasChanges);
+        return updateVersionsAndReturnChanges(entityLogWithoutEntries, dictionaryVersion, analyzeRequest, entryChanges.inserted(), entryChanges.updated());
     }
 
 
-    private List<EntityLogEntry> createEntityLogEntries(Document document, String dossierTemplateId, List<ManualEntity> notFoundManualRedactionEntries) {
+    private List<EntityLogEntry> createEntityLogEntries(Document document, AnalyzeRequest analyzeRequest, List<PrecursorEntity> notFoundPrecursorEntries) {
+
+        String dossierTemplateId = analyzeRequest.getDossierTemplateId();
 
         List<EntityLogEntry> entries = new ArrayList<>();
-        Set<String> processedIds = new HashSet<>();
+
         document.getEntities()
                 .stream()
                 .filter(entity -> !entity.getValue().isEmpty())
-                .filter(EntityLogCreatorService::notFalsePositiveOrFalseRecommendation)
+                .filter(EntityLogCreatorService::notFalsePositiveOrFalseRecommendationOrRemoval)
                 .filter(entity -> !entity.removed())
-                .forEach(entityNode -> entries.addAll(toEntityLogEntries(entityNode, dossierTemplateId, processedIds)));
-        document.streamAllImages().filter(entity -> !entity.removed()).forEach(imageNode -> entries.add(createEntityLogEntry(imageNode, dossierTemplateId)));
-        notFoundManualRedactionEntries.stream().filter(entity -> !entity.removed()).forEach(manualEntity -> entries.add(createEntityLogEntry(manualEntity, dossierTemplateId)));
+                .forEach(entityNode -> entries.addAll(toEntityLogEntries(entityNode)));
+        document.streamAllImages()
+                .filter(entity -> !entity.removed())
+                .forEach(imageNode -> entries.add(createEntityLogEntry(imageNode, dossierTemplateId)));
+        notFoundPrecursorEntries.stream()
+                .filter(entity -> !entity.removed())
+                .forEach(precursorEntity -> entries.add(createEntityLogEntry(precursorEntity, dossierTemplateId)));
         return entries;
     }
 
 
-    private List<EntityLogEntry> toEntityLogEntries(TextEntity textEntity, String dossierTemplateId, Set<String> processedIds) {
+    private List<EntityLogEntry> toEntityLogEntries(TextEntity textEntity) {
 
-        List<EntityLogEntry> redactionLogEntities = new ArrayList<>();
+        List<EntityLogEntry> entityLogEntries = new ArrayList<>();
 
+        // split entity into multiple entries if it occurs on multiple pages, since FE can't handle multi page entities
         for (PositionOnPage positionOnPage : textEntity.getPositionsOnPagePerPage()) {
 
-            // Duplicates should be removed. They might exist due to table extraction duplicating cells spanning multiple columns/rows.
-            if (processedIds.contains(positionOnPage.getId())) {
-                continue;
-            }
-            processedIds.add(positionOnPage.getId());
-
-            EntityLogEntry entityLogEntries = createEntityLogEntry(textEntity, dossierTemplateId);
-            entityLogEntries.setId(positionOnPage.getId());
+            EntityLogEntry entityLogEntry = createEntityLogEntry(textEntity);
 
             List<Position> rectanglesPerLine = positionOnPage.getRectanglePerLine()
                     .stream()
                     .map(rectangle2D -> new Position(rectangle2D, positionOnPage.getPage().getNumber()))
                     .toList();
 
-            entityLogEntries.setPositions(rectanglesPerLine);
-            redactionLogEntities.add(entityLogEntries);
+            // set the ID from the positions, since it might contain a "-" with the page number if the entity is split across multiple pages
+            entityLogEntry.setId(positionOnPage.getId());
+            entityLogEntry.setPositions(rectanglesPerLine);
+            entityLogEntries.add(entityLogEntry);
         }
 
-        return redactionLogEntities;
+        return entityLogEntries;
     }
 
 
@@ -194,9 +191,7 @@ public class EntityLogCreatorService {
         boolean isHint = dictionaryService.isHint(imageType, dossierTemplateId);
         return EntityLogEntry.builder()
                 .id(image.getId())
-                .value(image.value())
-                .color(getColor(imageType, dossierTemplateId, image.applied(), isHint))
-                .value(image.value())
+                .value(image.getValue())
                 .type(imageType)
                 .reason(image.buildReasonWithManualChangeDescriptions())
                 .legalBasis(image.legalBasis())
@@ -205,64 +200,74 @@ public class EntityLogCreatorService {
                 .positions(List.of(new Position(image.getPosition(), image.getPage().getNumber())))
                 .containingNodeId(image.getTreeId())
                 .closestHeadline(image.getHeadline().getTextBlock().getSearchText())
-                .section(image.getManualOverwrite().getSection().orElse(image.getParent().toString()))
+                .section(image.getManualOverwrite().getSection()
+                                 .orElse(image.getParent().toString()))
                 .imageHasTransparency(image.isTransparent())
-                .manualChanges(manualChangeFactory.toManualChangeList(image.getManualOverwrite().getManualChangeLog(), isHint))
+                .manualChanges(ManualChangeFactory.toManualChangeList(image.getManualOverwrite().getManualChangeLog(), isHint))
                 .state(buildEntryState(image))
                 .entryType(isHint ? EntryType.IMAGE_HINT : EntryType.IMAGE)
+                .engines(getEngines(null, image.getManualOverwrite()))
                 .build();
 
     }
 
 
-    private EntityLogEntry createEntityLogEntry(ManualEntity manualEntity, String dossierTemplateId) {
+    private EntityLogEntry createEntityLogEntry(PrecursorEntity precursorEntity, String dossierTemplateId) {
 
-        String type = manualEntity.getManualOverwrite().getType().orElse(manualEntity.getType());
-        boolean isHint = isHint(manualEntity.getEntityType());
+        String type = precursorEntity.getManualOverwrite().getType()
+                .orElse(precursorEntity.getType());
+        boolean isHint = isHint(precursorEntity.getEntityType());
         return EntityLogEntry.builder()
-                .id(manualEntity.getId())
-                .color(getColor(type, dossierTemplateId, manualEntity.applied(), isHint))
-                .reason(manualEntity.buildReasonWithManualChangeDescriptions())
-                .legalBasis(manualEntity.legalBasis())
-                .value(manualEntity.value())
+                .id(precursorEntity.getId())
+                .reason(precursorEntity.buildReasonWithManualChangeDescriptions())
+                .legalBasis(precursorEntity.legalBasis())
+                .value(precursorEntity.value())
                 .type(type)
-                .state(buildEntryState(manualEntity))
-                .entryType(buildEntryType(manualEntity))
-                .section(manualEntity.getManualOverwrite().getSection().orElse(manualEntity.getSection()))
+                .state(buildEntryState(precursorEntity))
+                .entryType(buildEntryType(precursorEntity))
+                .section(precursorEntity.getManualOverwrite().getSection()
+                                 .orElse(precursorEntity.getSection()))
                 .containingNodeId(Collections.emptyList())
                 .closestHeadline("")
-                .matchedRule(manualEntity.getMatchedRule().getRuleIdentifier().toString())
-                .dictionaryEntry(manualEntity.isDictionaryEntry())
-                .dossierDictionaryEntry(manualEntity.isDossierDictionaryEntry())
+                .matchedRule(precursorEntity.getMatchedRule().getRuleIdentifier().toString())
+                .dictionaryEntry(precursorEntity.isDictionaryEntry())
+                .dossierDictionaryEntry(precursorEntity.isDossierDictionaryEntry())
                 .textAfter("")
                 .textBefore("")
                 .startOffset(-1)
                 .endOffset(-1)
-                .positions(manualEntity.getManualOverwrite()
-                        .getPositions()
-                        .orElse(manualEntity.getEntityPosition())
-                        .stream()
-                        .map(entityPosition -> new Position(entityPosition.rectangle2D(), entityPosition.pageNumber()))
-                        .toList())
-                .engines(Collections.emptySet())
+                .positions(precursorEntity.getManualOverwrite().getPositions()
+                                   .orElse(precursorEntity.getEntityPosition())
+                                   .stream()
+                                   .map(entityPosition -> new Position(entityPosition.rectangle2D(), entityPosition.pageNumber()))
+                                   .toList())
+                .engines(getEngines(precursorEntity.getEngines(), precursorEntity.getManualOverwrite()))
+                //imported is no longer used, frontend should check engines
+                //(was .imported(precursorEntity.getEngines() != null && precursorEntity.getEngines().contains(Engine.IMPORTED)))
+                .imported(false)
                 .reference(Collections.emptySet())
-                .manualChanges(manualChangeFactory.toManualChangeList(manualEntity.getManualOverwrite().getManualChangeLog(), isHint))
+                .manualChanges(ManualChangeFactory.toManualChangeList(precursorEntity.getManualOverwrite().getManualChangeLog(), isHint))
                 .build();
     }
 
 
-    private EntityLogEntry createEntityLogEntry(TextEntity entity, String dossierTemplateId) {
+    private EntityLogEntry createEntityLogEntry(TextEntity entity) {
 
         Set<String> referenceIds = new HashSet<>();
-        entity.references().stream().filter(TextEntity::active).forEach(ref -> ref.getPositionsOnPagePerPage().forEach(pos -> referenceIds.add(pos.getId())));
+        entity.references()
+                .stream()
+                .filter(TextEntity::active)
+                .forEach(ref -> ref.getPositionsOnPagePerPage()
+                        .forEach(pos -> referenceIds.add(pos.getId())));
         boolean isHint = isHint(entity.getEntityType());
         return EntityLogEntry.builder()
-                .color(getColor(entity.getType(), dossierTemplateId, entity.applied(), isHint))
                 .reason(entity.buildReasonWithManualChangeDescriptions())
                 .legalBasis(entity.legalBasis())
-                .value(entity.getManualOverwrite().getValue().orElse(entity.getMatchedRule().isWriteValueWithLineBreaks() ? entity.getValueWithLineBreaks() : entity.getValue()))
-                .type(entity.getType())
-                .section(entity.getManualOverwrite().getSection().orElse(entity.getDeepestFullyContainingNode().toString()))
+                .value(entity.getManualOverwrite().getValue()
+                               .orElse(entity.getMatchedRule().isWriteValueWithLineBreaks() ? entity.getValueWithLineBreaks() : entity.getValue()))
+                .type(entity.type())
+                .section(entity.getManualOverwrite().getSection()
+                                 .orElse(entity.getDeepestFullyContainingNode().toString()))
                 .containingNodeId(entity.getDeepestFullyContainingNode().getTreeId())
                 .closestHeadline(entity.getDeepestFullyContainingNode().getHeadline().getTextBlock().getSearchText())
                 .matchedRule(entity.getMatchedRule().getRuleIdentifier().toString())
@@ -272,32 +277,36 @@ public class EntityLogCreatorService {
                 .startOffset(entity.getTextRange().start())
                 .endOffset(entity.getTextRange().end())
                 .dossierDictionaryEntry(entity.isDossierDictionaryEntry())
-                .engines(entity.getEngines() != null ? entity.getEngines() : Collections.emptySet())
+                .engines(getEngines(entity.getEngines(), entity.getManualOverwrite()))
+                //imported is no longer used, frontend should check engines
+                //(was .imported(entity.getEngines() != null && entity.getEngines().contains(Engine.IMPORTED)))
+                .imported(false)
                 .reference(referenceIds)
-                .manualChanges(manualChangeFactory.toManualChangeList(entity.getManualOverwrite().getManualChangeLog(), isHint))
+                .manualChanges(ManualChangeFactory.toManualChangeList(entity.getManualOverwrite().getManualChangeLog(), isHint))
                 .state(buildEntryState(entity))
                 .entryType(buildEntryType(entity))
                 .build();
     }
 
 
+    private Set<Engine> getEngines(Set<Engine> currentEngines, ManualChangeOverwrite manualChangeOverwrite) {
+
+        Set<Engine> engines = currentEngines != null ? new HashSet<>(currentEngines) : new HashSet<>();
+
+        if (manualChangeOverwrite != null && !manualChangeOverwrite.getManualChangeLog().isEmpty()) {
+            engines.add(Engine.MANUAL);
+        }
+        return engines;
+    }
+
+
     private boolean isHint(EntityType entityType) {
 
         return entityType.equals(EntityType.HINT);
     }
 
 
-    private float[] getColor(String type, String dossierTemplateId, boolean isApplied, boolean isHint) {
-
-        if (!isApplied && !isHint) {
-            return dictionaryService.getNotRedactedColor(dossierTemplateId);
-        }
-        return dictionaryService.getColor(type, dossierTemplateId);
-
-    }
-
-
-    private EntryState buildEntryState(IEntity entity) {
+    public static EntryState buildEntryState(IEntity entity) {
 
         if (entity.applied() && entity.active()) {
             return EntryState.APPLIED;
@@ -311,15 +320,17 @@ public class EntityLogCreatorService {
     }
 
 
-    private EntryType buildEntryType(IEntity entity) {
+    public static EntryType buildEntryType(IEntity entity) {
 
         if (entity instanceof TextEntity textEntity) {
             return getEntryType(textEntity.getEntityType());
-        } else if (entity instanceof ManualEntity manualEntity) {
-            if (manualEntity.isRectangle()) {
+        } else if (entity instanceof PrecursorEntity precursorEntity) {
+            if (precursorEntity.isRectangle()) {
                 return EntryType.AREA;
             }
-            return getEntryType(manualEntity.getEntityType());
+            return getEntryType(precursorEntity.getEntityType());
+        } else if (entity instanceof Image) {
+            return EntryType.IMAGE;
         }
         throw new UnsupportedOperationException(String.format("Entity subclass %s is not implemented!", entity.getClass()));
     }
@@ -333,13 +344,16 @@ public class EntityLogCreatorService {
             case FALSE_POSITIVE -> EntryType.FALSE_POSITIVE;
             case RECOMMENDATION -> EntryType.RECOMMENDATION;
             case FALSE_RECOMMENDATION -> EntryType.FALSE_RECOMMENDATION;
+            case DICTIONARY_REMOVAL -> EntryType.FALSE_POSITIVE;
         };
     }
 
 
     private List<EntityLogLegalBasis> toEntityLogLegalBasis(List<LegalBasis> legalBasis) {
 
-        return legalBasis.stream().map(l -> new EntityLogLegalBasis(l.getName(), l.getDescription(), l.getReason())).collect(Collectors.toList());
+        return legalBasis.stream()
+                .map(l -> new EntityLogLegalBasis(l.getName(), l.getDescription(), l.getReason()))
+                .collect(Collectors.toList());
     }
 
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/ImportedRedactionService.java

@@ -1,272 +0,0 @@
-package com.iqser.red.service.redaction.v1.server.service;
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.List;
-import java.util.stream.Collectors;
-
-import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedaction;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedactions;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Point;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
-import org.springframework.stereotype.Service;
-
-import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryType;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position;
-
-
-import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
-
-import io.micrometer.core.annotation.Timed;
-import lombok.RequiredArgsConstructor;
-
-@Service
-@RequiredArgsConstructor
-public class ImportedRedactionService {
-
-    public static final String IMPORTED_REDACTION_TYPE = "imported_redaction";
-
-    private final DictionaryService dictionaryService;
-    private final RedactionStorageService redactionStorageService;
-
-
-    @Deprecated(forRemoval = true)
-    @Timed("redactmanager_processImportedRedactions")
-    public List<RedactionLogEntry> processImportedRedactions(String dossierTemplateId,
-                                                             String dossierId,
-                                                             String fileId,
-                                                             List<RedactionLogEntry> redactionLogEntries,
-                                                             boolean addImportedRedactions) {
-
-        var importedRedactions = redactionStorageService.getImportedRedactions(dossierId, fileId);
-        if (importedRedactions == null) {
-            return redactionLogEntries;
-        }
-
-        redactionLogEntries.forEach(redactionLogEntry -> addIntersections(redactionLogEntry, importedRedactions));
-
-        if (addImportedRedactions) {
-            return addImportedRedactionsRedactionLogEntries(dossierTemplateId, redactionLogEntries, importedRedactions);
-        }
-
-        return redactionLogEntries;
-    }
-
-
-    @Timed("redactmanager_processImportedRedactions")
-    public List<EntityLogEntry> processImportedEntities(String dossierTemplateId,
-                                                        String dossierId,
-                                                        String fileId,
-                                                        List<EntityLogEntry> entityLogEntries,
-                                                        boolean addImportedRedactions) {
-
-        ImportedRedactions importedRedactions = redactionStorageService.getImportedRedactions(dossierId, fileId);
-        if (importedRedactions == null) {
-            return entityLogEntries;
-        }
-
-        entityLogEntries.forEach(redactionLogEntry -> addIntersections(redactionLogEntry, importedRedactions));
-
-        if (addImportedRedactions) {
-            return addImportedRedactionsEntityLogEntries(dossierTemplateId, entityLogEntries, importedRedactions);
-        }
-
-        return entityLogEntries;
-    }
-
-
-    @Deprecated(forRemoval = true)
-    private List<RedactionLogEntry> addImportedRedactionsRedactionLogEntries(String dossierTemplateId,
-                                                                             List<RedactionLogEntry> redactionLogEntries,
-                                                                             ImportedRedactions importedRedactions) {
-
-        for (var importedRedactionsValues : importedRedactions.getImportedRedactions().values()) {
-            for (var importedRedaction : importedRedactionsValues) {
-                RedactionLogEntry redactionLogEntry = RedactionLogEntry.builder()
-                        .id(importedRedaction.getId())
-                        .type(IMPORTED_REDACTION_TYPE)
-                        .imported(true)
-                        .redacted(true)
-                        .positions(importedRedaction.getPositions().stream().map(this::normalize).collect(Collectors.toList()))
-                        .color(getColor(IMPORTED_REDACTION_TYPE, dossierTemplateId))
-                        .build();
-
-                redactionLogEntries.add(redactionLogEntry);
-            }
-        }
-
-        return redactionLogEntries;
-    }
-
-
-    private List<EntityLogEntry> addImportedRedactionsEntityLogEntries(String dossierTemplateId, List<EntityLogEntry> entityLogEntries, ImportedRedactions importedRedactions) {
-
-        for (var importedRedactionsValues : importedRedactions.getImportedRedactions().values()) {
-            for (var importedRedaction : importedRedactionsValues) {
-                EntityLogEntry redactionLogEntry = EntityLogEntry.builder()
-                        .id(importedRedaction.getId())
-                        .type(IMPORTED_REDACTION_TYPE)
-                        .entryType(EntryType.ENTITY)
-                        .imported(true)
-                        .state(EntryState.APPLIED)
-                        .positions(importedRedaction.getPositions())
-                        .color(getColor(IMPORTED_REDACTION_TYPE, dossierTemplateId))
-                        .containingNodeId(Collections.emptyList())
-                        .value("")
-                        .build();
-
-                entityLogEntries.add(redactionLogEntry);
-            }
-        }
-
-        return entityLogEntries;
-    }
-
-
-    private List<Position> toPositions(List<Rectangle> positions) {
-
-        return positions.stream().map(this::toPosition).toList();
-    }
-
-
-    private Position toPosition(Rectangle rectangle) {
-
-        return new Position(new float[]{rectangle.getTopLeft().getX(), rectangle.getTopLeft().getY(), rectangle.getWidth(), rectangle.getHeight()}, rectangle.getPage());
-    }
-
-
-    @Deprecated(forRemoval = true)
-    private void addIntersections(RedactionLogEntry redactionLogEntry, ImportedRedactions importedRedactions) {
-
-        for (Rectangle rectangle : redactionLogEntry.getPositions()) {
-            var normalizedRectangle = normalize(rectangle);
-            if (importedRedactions.getImportedRedactions().containsKey(rectangle.getPage())) {
-                var importedRedactionsOnPage = importedRedactions.getImportedRedactions().get(rectangle.getPage());
-                for (ImportedRedaction importedRedaction : importedRedactionsOnPage) {
-                    for (Position importedRedactionPosition : importedRedaction.getPositions()) {
-                        if (rectOverlap(normalizedRectangle, normalize(importedRedactionPosition))) {
-                            if (redactionLogEntry.getImportedRedactionIntersections() == null) {
-                                redactionLogEntry.setImportedRedactionIntersections(new HashSet<>());
-                            }
-                            redactionLogEntry.getImportedRedactionIntersections().add(importedRedaction.getId());
-                        }
-                    }
-                }
-            }
-        }
-    }
-
-
-    private void addIntersections(EntityLogEntry entityLogEntry, ImportedRedactions importedRedactions) {
-
-        for (Position rectangle : entityLogEntry.getPositions()) {
-            var normalizedRectangle = normalize(rectangle);
-            if (importedRedactions.getImportedRedactions().containsKey(rectangle.getPageNumber())) {
-                var importedRedactionsOnPage = importedRedactions.getImportedRedactions().get(rectangle.getPageNumber());
-                for (ImportedRedaction importedRedaction : importedRedactionsOnPage) {
-                    for (Position importedRedactionPosition : importedRedaction.getPositions()) {
-                        if (rectOverlap(normalizedRectangle, normalize(importedRedactionPosition))) {
-                            if (entityLogEntry.getImportedRedactionIntersections() == null) {
-                                entityLogEntry.setImportedRedactionIntersections(new HashSet<>());
-                            }
-                            entityLogEntry.getImportedRedactionIntersections().add(importedRedaction.getId());
-                        }
-                    }
-                }
-            }
-        }
-    }
-
-
-    boolean valueInRange(float value, float min, float max) {
-
-        return round(value) >= round(min) && round(value) <= round(max);
-    }
-
-
-    boolean rectOverlap(Rectangle a, Rectangle b) {
-
-        boolean xOverlap = valueInRange(a.getTopLeft().getX(), b.getTopLeft().getX(), b.getTopLeft().getX() + b.getWidth()) || valueInRange(b.getTopLeft().getX(),
-                a.getTopLeft().getX(),
-                a.getTopLeft().getX() + a.getWidth());
-
-        boolean yOverlap = valueInRange(a.getTopLeft().getY(), b.getTopLeft().getY(), b.getTopLeft().getY() + b.getHeight()) || valueInRange(b.getTopLeft().getY(),
-                a.getTopLeft().getY(),
-                a.getTopLeft().getY() + a.getHeight());
-
-        return xOverlap && yOverlap;
-    }
-
-
-    @Deprecated(forRemoval = true)
-    private Rectangle normalize(Rectangle rectangle) {
-
-        Rectangle r = new Rectangle();
-        Point p = new Point();
-
-        if (rectangle.getWidth() < 0) {
-            p.setX(rectangle.getTopLeft().getX() + rectangle.getWidth());
-            r.setWidth(Math.abs(rectangle.getWidth()));
-        } else {
-            p.setX(rectangle.getTopLeft().getX());
-            r.setWidth(rectangle.getWidth());
-        }
-
-        if (rectangle.getHeight() < 0) {
-            p.setY(rectangle.getTopLeft().getY() + rectangle.getHeight());
-            r.setHeight(Math.abs(rectangle.getHeight()));
-        } else {
-            p.setY(rectangle.getTopLeft().getY());
-            r.setHeight(rectangle.getHeight());
-        }
-
-        r.setTopLeft(p);
-        r.setPage(rectangle.getPage());
-        return r;
-    }
-
-
-    private Rectangle normalize(Position position) {
-
-        Rectangle r = new Rectangle();
-        Point p = new Point();
-
-        if (position.getRectangle()[2] < 0) {
-            p.setX(position.getRectangle()[0] + position.getRectangle()[2]);
-            r.setWidth(Math.abs(position.getRectangle()[2]));
-        } else {
-            p.setX(position.getRectangle()[0]);
-            r.setWidth(position.getRectangle()[2]);
-        }
-
-        if (position.getRectangle()[3] < 0) {
-            p.setY(position.getRectangle()[1] + position.getRectangle()[3]);
-            r.setHeight(Math.abs(position.getRectangle()[3]));
-        } else {
-            p.setY(position.getRectangle()[1]);
-            r.setHeight(position.getRectangle()[3]);
-        }
-
-        r.setTopLeft(p);
-        r.setPage(position.getPageNumber());
-        return r;
-    }
-
-
-    private float round(float value) {
-
-        double d = Math.pow(10, 0);
-        return (float) (Math.round(value * d) / d);
-    }
-
-
-    private float[] getColor(String type, String dossierTemplateId) {
-
-        return dictionaryService.getColor(type, dossierTemplateId);
-    }
-
-}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/ManualChangeFactory.java

@@ -2,9 +2,10 @@ package com.iqser.red.service.redaction.v1.server.service;
 
 import java.time.OffsetDateTime;
 import java.util.List;
+import java.util.stream.Collectors;
 
-import org.springframework.stereotype.Service;
-
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualChange;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.BaseAnnotation;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction;
@@ -12,31 +13,38 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualChange;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType;
 
-@Service
+import lombok.experimental.UtilityClass;
+
+@UtilityClass
 public class ManualChangeFactory {
 
     public List<ManualChange> toManualChangeList(List<BaseAnnotation> manualChanges, boolean isHint) {
 
-        return manualChanges.stream().map(baseAnnotation -> toManualChange(baseAnnotation, isHint)).toList();
+        return manualChanges.stream()
+                .map(baseAnnotation -> toManualChange(baseAnnotation, isHint))
+                .collect(Collectors.toList());
     }
 
 
-    private ManualChange toManualChange(BaseAnnotation baseAnnotation, boolean isHint) {
+    public ManualChange toManualChange(BaseAnnotation baseAnnotation, boolean isHint) {
 
         ManualChange manualChange = ManualChange.from(baseAnnotation);
-        if (baseAnnotation instanceof ManualRecategorization imageRecategorization) {
-            manualChange.withManualRedactionType(ManualRedactionType.RECATEGORIZE).withChange("type", imageRecategorization.getType());
+        if (baseAnnotation instanceof ManualRecategorization recategorization) {
+            manualChange.withManualRedactionType(ManualRedactionType.RECATEGORIZE)
+                    .withChange("type", recategorization.getType())
+                    .withChange("section", recategorization.getSection())
+                    .withChange("legalBasis", recategorization.getLegalBasis())
+                    .withChange("value", recategorization.getValue());
         } else if (baseAnnotation instanceof IdRemoval manualRemoval) {
-            manualChange.withManualRedactionType(manualRemoval.isRemoveFromDictionary() ? ManualRedactionType.REMOVE_FROM_DICTIONARY : ManualRedactionType.REMOVE_LOCALLY);
-        } else if (baseAnnotation instanceof ManualForceRedaction) {
-            manualChange.withManualRedactionType(isHint ? ManualRedactionType.FORCE_HINT : ManualRedactionType.FORCE_REDACT);
+            manualChange.withManualRedactionType(manualRemoval.isRemoveFromDictionary() ? ManualRedactionType.REMOVE_FROM_DICTIONARY : ManualRedactionType.REMOVE);
+        } else if (baseAnnotation instanceof ManualForceRedaction manualForceRedaction) {
+            manualChange.withManualRedactionType(ManualRedactionType.FORCE).withChange("legalBasis", manualForceRedaction.getLegalBasis());
         } else if (baseAnnotation instanceof ManualResizeRedaction manualResizeRedact) {
-            manualChange.withManualRedactionType(ManualRedactionType.RESIZE).withChange("value", manualResizeRedact.getValue());
+            manualChange.withManualRedactionType(manualResizeRedact.getUpdateDictionary() ? ManualRedactionType.RESIZE_IN_DICTIONARY : ManualRedactionType.RESIZE)
+                    .withChange("value", manualResizeRedact.getValue());
         } else if (baseAnnotation instanceof ManualRedactionEntry manualRedactionEntry) {
-            manualChange.withManualRedactionType(manualRedactionEntry.isAddToDictionary() ? ManualRedactionType.ADD_TO_DICTIONARY : ManualRedactionType.ADD_LOCALLY)
+            manualChange.withManualRedactionType(manualRedactionEntry.isAddToDictionary() ? ManualRedactionType.ADD_TO_DICTIONARY : ManualRedactionType.ADD)
                     .withChange("value", manualRedactionEntry.getValue());
         } else if (baseAnnotation instanceof ManualLegalBasisChange manualLegalBasisChange) {
             manualChange.withManualRedactionType(ManualRedactionType.LEGAL_BASIS_CHANGE)

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/ManualChangesApplicationService.java

@@ -1,54 +1,76 @@
 package com.iqser.red.service.redaction.v1.server.service;
 
 import java.awt.geom.Rectangle2D;
+import java.util.ArrayList;
+import java.util.Collection;
 import java.util.HashSet;
-import java.util.LinkedList;
 import java.util.List;
+import java.util.Map;
 import java.util.NoSuchElementException;
+import java.util.Optional;
+import java.util.Set;
 import java.util.stream.Collectors;
 
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle;
+import org.springframework.stereotype.Service;
+
+import com.google.common.collect.Sets;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
-import com.iqser.red.service.redaction.v1.server.model.ClosestEntity;
-import com.iqser.red.service.redaction.v1.server.model.dictionary.SearchImplementation;
-import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
-import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
+import com.iqser.red.service.redaction.v1.server.model.PrecursorEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.IEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.PositionOnPage;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Image;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.ImageType;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.Page;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
-import com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService;
+import com.iqser.red.service.redaction.v1.server.service.document.EntityFindingUtility;
 import com.iqser.red.service.redaction.v1.server.utils.RectangleTransformations;
 
 import lombok.AccessLevel;
 import lombok.RequiredArgsConstructor;
 import lombok.experimental.FieldDefaults;
 
+@Service
 @RequiredArgsConstructor
 @FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
 public class ManualChangesApplicationService {
 
-    EntityCreationService entityCreationService;
+    static double MATCH_THRESHOLD = 10;
 
-    public void recategorize(IEntity IEntityToBeReCategorized, ManualRecategorization manualRecategorization) {
+    EntityFindingUtility entityFindingUtility;
 
-        if (IEntityToBeReCategorized instanceof Image image) {
+
+    @Deprecated
+    public void recategorize(IEntity entityToBeReCategorized, ManualRecategorization manualRecategorization) {
+
+        entityToBeReCategorized.getMatchedRuleList().clear();
+        entityToBeReCategorized.getManualOverwrite().addChange(manualRecategorization);
+
+        if (entityToBeReCategorized instanceof Image image) {
             image.setImageType(ImageType.fromString(manualRecategorization.getType()));
             return;
         }
-        // need to create a new entity and copy over all values, since type is part of the primary key for entities and should never be changed!
-        if (IEntityToBeReCategorized instanceof TextEntity textEntity) {
-            TextEntity recategorizedEntity = entityCreationService.copyEntityWithoutRules(textEntity, manualRecategorization.getType(), textEntity.getEntityType(), textEntity.getDeepestFullyContainingNode());
-            recategorizedEntity.setPositionsOnPagePerPage(textEntity.getPositionsOnPagePerPage());
-            recategorizedEntity.getManualOverwrite().addChange(manualRecategorization);
-            textEntity.remove("FINAL.0.0", "removed by manual recategorization");
+
+        if (entityToBeReCategorized instanceof TextEntity textEntity) {
+            textEntity.setType(manualRecategorization.getType());
         }
     }
 
 
+    /**
+     * Resizes a text entity based on manual resize redaction details.
+     *
+     * @param entityToBeResized     The entity to resize.
+     * @param manualResizeRedaction The details of the resize operation.
+     */
+    public void resize(TextEntity entityToBeResized, ManualResizeRedaction manualResizeRedaction) {
+
+        resizeEntityAndReinsert(entityToBeResized, manualResizeRedaction);
+    }
+
+
+    @Deprecated
     public void resizeEntityAndReinsert(TextEntity entityToBeResized, ManualResizeRedaction manualResizeRedaction) {
 
         PositionOnPage positionOnPageToBeResized = entityToBeResized.getPositionsOnPagePerPage()
@@ -57,102 +79,88 @@ public class ManualChangesApplicationService {
                 .findFirst()
                 .orElseThrow(() -> new NoSuchElementException("No redaction position with matching annotation id found!"));
 
-        positionOnPageToBeResized.setRectanglePerLine(manualResizeRedaction.getPositions().stream().map(ManualChangesApplicationService::toRectangle2D).collect(Collectors.toList()));
+        positionOnPageToBeResized.setRectanglePerLine(manualResizeRedaction.getPositions()
+                                                              .stream()
+                                                              .map(ManualChangesApplicationService::toRectangle2D)
+                                                              .collect(Collectors.toList()));
+
+        entityToBeResized.getManualOverwrite().addChange(manualResizeRedaction);
 
-        String value = manualResizeRedaction.getValue();
-        int newStartOffset = -1;
         SemanticNode node = entityToBeResized.getDeepestFullyContainingNode();
-        ClosestEntity closestEntity = ClosestEntity.builder().distance(100).textRange(null).build();
-
+        PrecursorEntity searchEntity = PrecursorEntity.fromManualResizeRedaction(manualResizeRedaction);
         // Loop through nodes starting from the deepest fully containing node all the way to the document node
         while (node != null) {
-            if (node.containsString(value)) {
-                SearchImplementation searchImplementation = new SearchImplementation(value, false);
-                List<TextRange> textRanges = searchImplementation.getBoundaries(node.getTextBlock(), node.getTextRange());
 
-                for (TextRange textRange : textRanges) {
-                    SemanticNode finalNode = node;
+            Map<String, List<TextEntity>> possibleEntities = entityFindingUtility.findAllPossibleEntitiesAndGroupByValue(node, List.of(searchEntity));
+            Optional<TextEntity> closestEntity = entityFindingUtility.findClosestEntityAndReturnEmptyIfNotFound(searchEntity, possibleEntities, MATCH_THRESHOLD);
 
-                    List<TextEntity> tempEntities = searchImplementation.getBoundaries(node.getTextBlock(), textRange)
-                            .stream()
-                            .map(boundary -> entityCreationService.forceByTextRange(boundary, "temp", EntityType.ENTITY, finalNode))
-                            .collect(Collectors.toList());
-
-                    // If a value appears multiple times in a section after resizing, we need to make sure we select the surrounding text for the correct one.
-                    determineCorrectEntity(manualResizeRedaction, textRange, tempEntities, closestEntity);
-
-                    // Remove all temp entities from the graph
-                    tempEntities.forEach(TextEntity::removeFromGraph);
-                }
-                break;
+            if (closestEntity.isPresent()) {
+                copyValuesFromClosestEntity(entityToBeResized, manualResizeRedaction, closestEntity.get());
+                possibleEntities.values()
+                        .stream()
+                        .flatMap(Collection::stream)
+                        .forEach(TextEntity::removeFromGraph);
+                return;
             }
 
-            // If the current node is the document node then it does not have a parent, meaning we could not find the value anywhere.
+            possibleEntities.values()
+                    .stream()
+                    .flatMap(Collection::stream)
+                    .forEach(TextEntity::removeFromGraph);
+
             if (node.hasParent()) {
                 node = node.getParent();
             } else {
                 break;
             }
         }
-
-        if (closestEntity.getTextRange() != null) {
-            newStartOffset = closestEntity.getTextRange().start();
-        }
-
-        // need to reinsert the entity, due to the boundary having changed.
-        if (newStartOffset > -1) {
-            removeAndUpdateAndReInsertEntity(entityToBeResized, manualResizeRedaction, newStartOffset);
-        }
-        entityToBeResized.getManualOverwrite().addChange(manualResizeRedaction);
     }
 
 
-    public static void determineCorrectEntity(ManualResizeRedaction manualResizeRedaction,
-                                        TextRange textRange,
-                                        List<TextEntity> tempEntities,
-                                        ClosestEntity closestEntity) {
+    private static void copyValuesFromClosestEntity(TextEntity entityToBeResized, ManualResizeRedaction manualResizeRedaction, TextEntity closestEntity) {
 
-        double currentDistance = calculateClosest(manualResizeRedaction.getPositions().get(0), tempEntities.get(0).getPositionsOnPagePerPage());
-        if (currentDistance < closestEntity.getDistance()) {
-            closestEntity.setDistance(currentDistance);
-            closestEntity.setTextRange(textRange);
-        }
-    }
+        Set<SemanticNode> currentIntersectingNodes = new HashSet<>(entityToBeResized.getIntersectingNodes());
+        Set<SemanticNode> newIntersectingNodes = new HashSet<>(closestEntity.getIntersectingNodes());
 
+        Sets.difference(currentIntersectingNodes, newIntersectingNodes)
+                .forEach(removedNode -> removedNode.getEntities().remove(entityToBeResized));
+        Sets.difference(newIntersectingNodes, currentIntersectingNodes)
+                .forEach(addedNode -> addedNode.getEntities().add(entityToBeResized));
 
-    private static double calculateClosest(Rectangle position, List<PositionOnPage> positionOnPages) {
+        Set<Page> currentIntersectingPages = new HashSet<>(entityToBeResized.getPages());
+        Set<Page> newIntersectingPages = new HashSet<>(closestEntity.getPages());
 
-        Rectangle2D rectangle2D = positionOnPages.get(0).getRectanglePerLine().get(0);
-        double difference = 0;
-        difference +=  position.getTopLeftX() > rectangle2D.getX() ? (position.getTopLeftX() - rectangle2D.getX()) : (rectangle2D.getX() - position.getTopLeftX());
-        difference +=  position.getTopLeftY() > rectangle2D.getY() ? (position.getTopLeftY() - rectangle2D.getY()) : (rectangle2D.getY() - position.getTopLeftY());
-        difference +=  position.getWidth() > rectangle2D.getWidth() ? (position.getWidth() - rectangle2D.getWidth()) : (rectangle2D.getWidth() - position.getWidth());
-        difference +=  position.getHeight() > rectangle2D.getHeight() ? (position.getHeight() - rectangle2D.getHeight()) : (rectangle2D.getHeight() - position.getHeight());
-        return difference;
-    }
+        Sets.difference(currentIntersectingPages, newIntersectingPages)
+                .forEach(removedPage -> removedPage.getEntities().remove(entityToBeResized));
+        Sets.difference(newIntersectingPages, currentIntersectingPages)
+                .forEach(addedPage -> addedPage.getEntities().add(entityToBeResized));
 
-
-    private void removeAndUpdateAndReInsertEntity(TextEntity entityToBeResized, ManualResizeRedaction manualResizeRedaction, int newStartOffset) {
-
-        SemanticNode nodeToInsertInto = entityToBeResized.getDeepestFullyContainingNode().getDocumentTree().getRoot().getNode();
-        entityToBeResized.getIntersectingNodes().forEach(node -> node.getEntities().remove(this));
-        entityToBeResized.getPages().forEach(page -> page.getEntities().remove(this));
-        entityToBeResized.setIntersectingNodes(new LinkedList<>());
-        entityToBeResized.setDeepestFullyContainingNode(null);
-        entityToBeResized.setPages(new HashSet<>());
-        entityToBeResized.getTextRange().setStart(newStartOffset);
-        entityToBeResized.getTextRange().setEnd(newStartOffset + manualResizeRedaction.getValue().length());
-
-        entityCreationService.addEntityToGraph(entityToBeResized, nodeToInsertInto);
+        entityToBeResized.setDeepestFullyContainingNode(closestEntity.getDeepestFullyContainingNode());
+        entityToBeResized.setIntersectingNodes(new ArrayList<>(newIntersectingNodes));
+        entityToBeResized.setTextRange(closestEntity.getTextRange());
+        entityToBeResized.setTextAfter(closestEntity.getTextAfter());
+        entityToBeResized.setTextBefore(closestEntity.getTextBefore());
+        entityToBeResized.setDuplicateTextRanges(new ArrayList<>(closestEntity.getDuplicateTextRanges()));
+        entityToBeResized.setValue(closestEntity.getValue());
+        entityToBeResized.setPages(newIntersectingPages);
     }
 
 
+    /**
+     * Resizes an image entity based on manual resize redaction instructions.
+     *
+     * @param image                 The image to resize.
+     * @param manualResizeRedaction The details of the resize operation.
+     */
     public void resizeImage(Image image, ManualResizeRedaction manualResizeRedaction) {
 
         if (manualResizeRedaction.getPositions().isEmpty() || manualResizeRedaction.getPositions() == null) {
             return;
         }
-        var bBox = RectangleTransformations.rectangle2DBBox(manualResizeRedaction.getPositions().stream().map(ManualChangesApplicationService::toRectangle2D).toList());
+        var bBox = RectangleTransformations.rectangle2DBBox(manualResizeRedaction.getPositions()
+                                                                    .stream()
+                                                                    .map(ManualChangesApplicationService::toRectangle2D)
+                                                                    .toList());
         image.setPosition(bBox);
         image.getManualOverwrite().addChange(manualResizeRedaction);
     }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/NotFoundImportedEntitiesService.java

@@ -0,0 +1,108 @@
+package com.iqser.red.service.redaction.v1.server.service;
+
+import java.time.OffsetDateTime;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Change;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ChangeType;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position;
+import com.iqser.red.service.redaction.v1.server.model.PrecursorEntity;
+import com.iqser.red.service.redaction.v1.server.model.RectangleWithPage;
+import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
+
+import org.springframework.stereotype.Service;
+
+import io.micrometer.core.annotation.Timed;
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+
+@Service
+@Slf4j
+@RequiredArgsConstructor
+public class NotFoundImportedEntitiesService {
+
+    public static final String IMPORTED_REDACTION_TYPE = "imported_redaction";
+
+
+    @Timed("redactmanager_processEntityLog")
+    public void processEntityLog(EntityLog entityLog, AnalyzeRequest analyzeRequest, List<PrecursorEntity> notFoundEntities) {
+        // recreate imported redactions with manual changes
+        if (notFoundEntities == null || notFoundEntities.isEmpty()) {
+            log.debug("checkForImportedRedactions - no not found imported entities");
+            return;
+        }
+
+        // remove all  intersections and update imported entities
+        calculateIntersectionsForNotFound(notFoundEntities, entityLog, analyzeRequest.getAnalysisNumber());
+    }
+
+
+    private void calculateIntersectionsForNotFound(List<PrecursorEntity> notFoundEntities, EntityLog entityLog, int analysisNumber) {
+
+        if (!notFoundEntities.isEmpty()) {
+            // imported redactions present, intersections must be added with merged imported redactions
+            Map<Integer, List<PrecursorEntity>> importedRedactionsMap = mapImportedRedactionsOnPage(notFoundEntities);
+            entityLog.getEntityLogEntry()
+                    .stream()
+                    .filter(entry -> !entry.getEngines().contains(Engine.IMPORTED))
+                    .forEach(redactionLogEntry -> {
+                        redactionLogEntry.setImportedRedactionIntersections(new HashSet<>());
+                        addIntersections(redactionLogEntry, importedRedactionsMap, analysisNumber);
+                    });
+        }
+    }
+
+
+    public Map<Integer, List<PrecursorEntity>> mapImportedRedactionsOnPage(List<PrecursorEntity> importedEntities) {
+
+        Map<Integer, List<PrecursorEntity>> importedRedactionsMap = new HashMap<>();
+        Set<Integer> pageNumbers = importedEntities.stream()
+                .map(PrecursorEntity::getEntityPosition)
+                .flatMap(Collection::stream)
+                .map(RectangleWithPage::pageNumber)
+                .collect(Collectors.toSet());
+        pageNumbers.forEach(pageNumber -> importedRedactionsMap.put(pageNumber,
+                                                                    importedEntities.stream()
+                                                                            .filter(i -> pageNumber == i.getEntityPosition()
+                                                                                    .get(0).pageNumber())
+                                                                            .collect(Collectors.toList())));
+        return importedRedactionsMap;
+    }
+
+
+    public void addIntersections(EntityLogEntry entityLogEntry, Map<Integer, List<PrecursorEntity>> importedEntitiesMap, int analysisNumber) {
+
+        for (Position rectangle : entityLogEntry.getPositions()) {
+            if (importedEntitiesMap.containsKey(rectangle.getPageNumber())) {
+                var importedEntitiesOnPage = importedEntitiesMap.get(rectangle.getPageNumber());
+                for (PrecursorEntity precursorEntity : importedEntitiesOnPage) {
+                    for (RectangleWithPage rectangleWithPage : precursorEntity.getEntityPosition()) {
+                        if (rectangle.toRectangle2D().intersects(rectangleWithPage.rectangle2D())) {
+                            if (entityLogEntry.getImportedRedactionIntersections() == null) {
+                                entityLogEntry.setImportedRedactionIntersections(new HashSet<>());
+                            }
+                            entityLogEntry.getImportedRedactionIntersections().add(precursorEntity.getId());
+
+                            if (entityLogEntry.getState() != EntryState.REMOVED) {
+                                entityLogEntry.setState(EntryState.REMOVED);
+                                entityLogEntry.getChanges().add(new Change(analysisNumber, ChangeType.REMOVED, OffsetDateTime.now()));
+                            }
+                        }
+                    }
+                }
+            }
+        }
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RedactionChangeLogService.java

@@ -1,125 +0,0 @@
-package com.iqser.red.service.redaction.v1.server.service;
-
-import java.time.OffsetDateTime;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.stream.Collectors;
-
-import org.springframework.stereotype.Service;
-
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Change;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ChangeType;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ManualRedactionType;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogChanges;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
-import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
-
-import io.micrometer.core.annotation.Timed;
-import lombok.RequiredArgsConstructor;
-import lombok.extern.slf4j.Slf4j;
-
-@Deprecated(forRemoval = true)
-@Slf4j
-@Service
-@RequiredArgsConstructor
-public class RedactionChangeLogService {
-
-    private final RedactionStorageService redactionStorageService;
-
-
-    @Timed("redactmanager_computeChanges")
-    public RedactionLogChanges computeChanges(RedactionLog previousRedactionLog, RedactionLog currentRedactionLog, int analysisNumber) {
-
-        long start = System.currentTimeMillis();
-
-        if (previousRedactionLog == null) {
-            currentRedactionLog.getRedactionLogEntry().forEach(entry -> {
-                entry.getChanges().add(new Change(analysisNumber, ChangeType.ADDED, OffsetDateTime.now()));
-            });
-            return new RedactionLogChanges(currentRedactionLog, false);
-        }
-
-        List<RedactionLogEntry> previouslyExistingEntries = previousRedactionLog.getRedactionLogEntry().stream().filter(entry -> !entry.lastChangeIsRemoved()).toList();
-
-        Map<String, RedactionLogEntry> addedEntryIds = getEntriesThatExistInCurrentButNotInPreviousRedactionLog(currentRedactionLog, previouslyExistingEntries);
-        Set<String> removedIds = getEntryIdsThatExistInPreviousButNotInCurrentRedactionLog(currentRedactionLog, previouslyExistingEntries);
-
-        List<RedactionLogEntry> newRedactionLogEntries = previousRedactionLog.getRedactionLogEntry();
-
-        List<RedactionLogEntry> toRemove = new ArrayList<>();
-        newRedactionLogEntries.forEach(entry -> {
-            if (removedIds.contains(entry.getId()) && addedEntryIds.containsKey(entry.getId())) {
-                List<Change> changes = entry.getChanges();
-                changes.add(new Change(analysisNumber, ChangeType.CHANGED, OffsetDateTime.now()));
-                var newEntry = addedEntryIds.get(entry.getId());
-                newEntry.setChanges(changes);
-                addedEntryIds.put(entry.getId(), newEntry);
-                toRemove.add(entry);
-            } else if (removedIds.contains(entry.getId())) {
-                entry.getChanges().add(new Change(analysisNumber, ChangeType.REMOVED, OffsetDateTime.now()));
-            } else if (addedEntryIds.containsKey(entry.getId())) {
-                List<Change> changes = entry.getChanges();
-                changes.add(new Change(analysisNumber, ChangeType.ADDED, OffsetDateTime.now()));
-                var newEntry = addedEntryIds.get(entry.getId());
-                newEntry.setChanges(changes);
-                addedEntryIds.put(entry.getId(), newEntry);
-                toRemove.add(entry);
-            }
-        });
-
-        newRedactionLogEntries.removeAll(toRemove);
-
-        addedEntryIds.forEach((k, v) -> {
-            if (v.getChanges().isEmpty()) {
-                v.getChanges().add(new Change(analysisNumber, ChangeType.ADDED, OffsetDateTime.now()));
-            }
-            newRedactionLogEntries.add(v);
-        });
-
-        currentRedactionLog.setRedactionLogEntry(newRedactionLogEntries);
-
-        log.debug("Change computation took: {}", System.currentTimeMillis() - start);
-        return new RedactionLogChanges(currentRedactionLog, !addedEntryIds.isEmpty() || !removedIds.isEmpty());
-    }
-
-
-    private static Set<String> getEntryIdsThatExistInPreviousButNotInCurrentRedactionLog(RedactionLog currentRedactionLog, List<RedactionLogEntry> previouslyExistingEntries) {
-
-        Set<RedactionLogEntry> removed = new HashSet<>(previouslyExistingEntries);
-        currentRedactionLog.getRedactionLogEntry().forEach(removed::remove);
-
-        Set<String> removedIds = removed.stream().map(RedactionLogEntry::getId).collect(Collectors.toSet());
-        return removedIds;
-    }
-
-
-    private static Map<String, RedactionLogEntry> getEntriesThatExistInCurrentButNotInPreviousRedactionLog(RedactionLog currentRedactionLog,
-                                                                                                           List<RedactionLogEntry> previouslyExistingEntries) {
-
-        Set<RedactionLogEntry> currentExistingEntries = currentRedactionLog.getRedactionLogEntry()
-                .stream()
-                .filter(entry -> (entry.getChanges().isEmpty() || !entry.lastChangeIsRemoved()) && !isLastManualChangeRemove(entry))
-                .collect(Collectors.toSet());
-
-        previouslyExistingEntries.forEach(currentExistingEntries::remove);
-        Map<String, RedactionLogEntry> addedIds = new HashMap<>();
-        currentExistingEntries.forEach(entry -> {
-            addedIds.put(entry.getId(), entry);
-        });
-        return addedIds;
-    }
-
-
-    private static boolean isLastManualChangeRemove(RedactionLogEntry redactionLogEntry){
-        if(redactionLogEntry.getManualChanges() == null || redactionLogEntry.getManualChanges().isEmpty()){
-            return false;
-        }
-        return redactionLogEntry.getManualChanges().get(redactionLogEntry.getManualChanges().size() -1).getManualRedactionType() == ManualRedactionType.REMOVE_LOCALLY;
-    }
-
-}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RedactionLogCreatorService.java

@@ -1,259 +0,0 @@
-package com.iqser.red.service.redaction.v1.server.service;
-
-import java.awt.geom.Rectangle2D;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Locale;
-import java.util.Objects;
-import java.util.Set;
-import java.util.stream.Collectors;
-
-import org.springframework.beans.BeanUtils;
-import org.springframework.stereotype.Service;
-
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ManualChange;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ManualRedactionType;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Point;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
-import com.iqser.red.service.redaction.v1.server.model.ManualEntity;
-import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
-import com.iqser.red.service.redaction.v1.server.model.document.entity.IEntity;
-import com.iqser.red.service.redaction.v1.server.model.document.entity.ManualChangeOverwrite;
-import com.iqser.red.service.redaction.v1.server.model.document.entity.PositionOnPage;
-import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
-import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
-import com.iqser.red.service.redaction.v1.server.model.document.nodes.Image;
-import com.iqser.red.service.redaction.v1.server.model.document.nodes.ImageType;
-
-import lombok.RequiredArgsConstructor;
-import lombok.extern.slf4j.Slf4j;
-
-@Deprecated(forRemoval = true)
-@Service
-@Slf4j
-@RequiredArgsConstructor
-public class RedactionLogCreatorService {
-
-    private final DictionaryService dictionaryService;
-    private final ManualChangeFactory manualChangeFactory;
-
-
-    public List<RedactionLogEntry> createRedactionLog(Document document, String dossierTemplateId, List<ManualEntity> notFoundManualRedactionEntries) {
-
-        List<RedactionLogEntry> entries = new ArrayList<>();
-        Set<String> processIds = new HashSet<>();
-        document.getEntities()
-                .stream()
-                .filter(entity -> !entity.getValue().isEmpty())
-                .filter(RedactionLogCreatorService::notFalsePositiveOrFalseRecommendation)
-                .filter(IEntity::active)
-                .forEach(entityNode -> entries.addAll(toRedactionLogEntries(entityNode, dossierTemplateId, processIds)));
-        document.streamAllImages().filter(image -> !image.removed()).forEach(imageNode -> entries.add(createRedactionLogEntry(imageNode, dossierTemplateId)));
-        notFoundManualRedactionEntries.forEach(entityIdentifier -> entries.add(createRedactionLogEntry(entityIdentifier, dossierTemplateId)));
-        return entries;
-    }
-
-
-    private static boolean notFalsePositiveOrFalseRecommendation(TextEntity textEntity) {
-
-        return !(textEntity.getEntityType() == EntityType.FALSE_POSITIVE || textEntity.getEntityType() == EntityType.FALSE_RECOMMENDATION);
-    }
-
-
-    private List<RedactionLogEntry> toRedactionLogEntries(TextEntity textEntity, String dossierTemplateId, Set<String> processedIds) {
-
-        List<RedactionLogEntry> redactionLogEntities = new ArrayList<>();
-
-        for (PositionOnPage positionOnPage : textEntity.getPositionsOnPagePerPage()) {
-
-            // Duplicates should be removed. They might exist due to table extraction duplicating cells spanning multiple columns/rows.
-            if (processedIds.contains(positionOnPage.getId())) {
-                continue;
-            }
-            processedIds.add(positionOnPage.getId());
-
-            RedactionLogEntry redactionLogEntry = createRedactionLogEntry(textEntity, dossierTemplateId);
-            redactionLogEntry.setId(positionOnPage.getId());
-
-            List<Rectangle> rectanglesPerLine = positionOnPage.getRectanglePerLine()
-                    .stream()
-                    .map(rectangle2D -> toRedactionLogRectangle(rectangle2D, positionOnPage.getPage().getNumber()))
-                    .toList();
-
-            redactionLogEntry.setPositions(rectanglesPerLine);
-            redactionLogEntities.add(redactionLogEntry);
-        }
-
-        return redactionLogEntities;
-    }
-
-
-    private RedactionLogEntry createRedactionLogEntry(TextEntity entity, String dossierTemplateId) {
-
-        Set<String> referenceIds = new HashSet<>();
-        entity.references().stream().filter(TextEntity::active).forEach(ref -> ref.getPositionsOnPagePerPage().forEach(pos -> referenceIds.add(pos.getId())));
-        int sectionNumber = entity.getDeepestFullyContainingNode().getTreeId().isEmpty() ? 0 : entity.getDeepestFullyContainingNode().getTreeId().get(0);
-        boolean isHint = isHint(entity.getEntityType());
-        return RedactionLogEntry.builder()
-                .color(getColor(entity.getType(), dossierTemplateId, entity.applied(), isHint))
-                .reason(entity.buildReasonWithManualChangeDescriptions())
-                .legalBasis(entity.legalBasis())
-                .value(entity.getManualOverwrite().getValue().orElse(entity.getMatchedRule().isWriteValueWithLineBreaks() ? entity.getValueWithLineBreaks() : entity.getValue()))
-                .type(entity.getType())
-                .redacted(entity.applied())
-                .isHint(isHint)
-                .isRecommendation(entity.getEntityType().equals(EntityType.RECOMMENDATION))
-                .isFalsePositive(entity.getEntityType().equals(EntityType.FALSE_POSITIVE) || entity.getEntityType().equals(EntityType.FALSE_RECOMMENDATION))
-                .section(entity.getManualOverwrite().getSection().orElse(entity.getDeepestFullyContainingNode().toString()))
-                .sectionNumber(sectionNumber)
-                .matchedRule(entity.getMatchedRule().getRuleIdentifier().toString())
-                .isDictionaryEntry(entity.isDictionaryEntry())
-                .textAfter(entity.getTextAfter())
-                .textBefore(entity.getTextBefore())
-                .startOffset(entity.getTextRange().start())
-                .endOffset(entity.getTextRange().end())
-                .isDossierDictionaryEntry(entity.isDossierDictionaryEntry())
-                .engines(getEngines(entity))
-                .reference(referenceIds)
-                .manualChanges(mapManualChanges(entity.getManualOverwrite(), isHint))
-                .build();
-    }
-
-
-    private Set<Engine> getEngines(TextEntity entity) {
-
-        return entity.getEngines() != null ? mapToEngines(entity.getEngines()) : Collections.emptySet();
-    }
-
-
-    private Set<Engine> mapToEngines(Set<com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine> engines) {
-
-        return engines.stream().map(engine -> switch (engine) {
-            case NER -> Engine.NER;
-            case RULE -> Engine.RULE;
-            case DICTIONARY -> Engine.DICTIONARY;
-            default -> null;
-        }).filter(Objects::nonNull).collect(Collectors.toSet());
-    }
-
-
-    private boolean isHint(EntityType entityType) {
-
-        return entityType.equals(EntityType.HINT);
-    }
-
-
-    private List<ManualChange> mapManualChanges(ManualChangeOverwrite manualEntity, boolean isHint) {
-
-        return manualChangeFactory.toManualChangeList(manualEntity.getManualChangeLog(), isHint).stream().map(this::mapManualChange).toList();
-    }
-
-
-    private ManualChange mapManualChange(com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualChange manualChange) {
-
-        ManualChange manualChange1 = new ManualChange();
-        BeanUtils.copyProperties(manualChange, manualChange1);
-        manualChange1.setManualRedactionType(switch (manualChange.getManualRedactionType()) {
-            case ADD_LOCALLY -> ManualRedactionType.ADD_LOCALLY;
-            case ADD_TO_DICTIONARY -> ManualRedactionType.ADD_TO_DICTIONARY;
-            case REMOVE_LOCALLY -> ManualRedactionType.REMOVE_LOCALLY;
-            case REMOVE_FROM_DICTIONARY -> ManualRedactionType.REMOVE_FROM_DICTIONARY;
-            case FORCE_REDACT -> ManualRedactionType.FORCE_REDACT;
-            case FORCE_HINT -> ManualRedactionType.FORCE_HINT;
-            case RECATEGORIZE -> ManualRedactionType.RECATEGORIZE;
-            case LEGAL_BASIS_CHANGE -> ManualRedactionType.LEGAL_BASIS_CHANGE;
-            case RESIZE -> ManualRedactionType.RESIZE;
-        });
-        manualChange1.setProcessedDate(manualChange.getProcessedDate());
-        manualChange1.setRequestedDate(manualChange.getRequestedDate());
-        manualChange1.setPropertyChanges(manualChange.getPropertyChanges());
-        manualChange1.setAnnotationStatus(AnnotationStatus.APPROVED);
-        return manualChange1;
-    }
-
-
-    private float[] getColor(String type, String dossierTemplateId, boolean isRedaction, boolean isHint) {
-
-        if (!isRedaction && isHint) {
-            return dictionaryService.getNotRedactedColor(dossierTemplateId);
-        }
-        return dictionaryService.getColor(type, dossierTemplateId);
-    }
-
-
-    public RedactionLogEntry createRedactionLogEntry(Image image, String dossierTemplateId) {
-
-        String imageType = image.getImageType().equals(ImageType.OTHER) ? "image" : image.getImageType().toString().toLowerCase(Locale.ENGLISH);
-        boolean isHint = dictionaryService.isHint(imageType, dossierTemplateId);
-        return RedactionLogEntry.builder()
-                .id(image.getId())
-                .color(getColor(imageType, dossierTemplateId, image.applied(), isHint))
-                .isImage(true)
-                .value(image.value())
-                .type(imageType)
-                .redacted(image.applied())
-                .reason(image.buildReasonWithManualChangeDescriptions())
-                .legalBasis(image.legalBasis())
-                .matchedRule(image.getMatchedRule().getRuleIdentifier().toString())
-                .isHint(isHint)
-                .isDictionaryEntry(false)
-                .isRecommendation(false)
-                .positions(List.of(toRedactionLogRectangle(image.getPosition(), image.getPage().getNumber())))
-                .sectionNumber(image.getTreeId().get(0))
-                .section(image.getManualOverwrite().getSection().orElse(image.getParent().toString()))
-                .imageHasTransparency(image.isTransparent())
-                .manualChanges(mapManualChanges(image.getManualOverwrite(), isHint))
-                .build();
-
-    }
-
-
-    private Rectangle toRedactionLogRectangle(Rectangle2D rectangle2D, int pageNumber) {
-
-        return new Rectangle(new Point((float) rectangle2D.getMinX(), (float) rectangle2D.getMinY()), (float) rectangle2D.getWidth(), (float) rectangle2D.getHeight(), pageNumber);
-    }
-
-
-    public RedactionLogEntry createRedactionLogEntry(ManualEntity manualEntity, String dossierTemplateId) {
-
-        String type = manualEntity.getManualOverwrite().getType().orElse(manualEntity.getType());
-        boolean isHint = isHint(manualEntity.getEntityType());
-        return RedactionLogEntry.builder()
-                .id(manualEntity.getId())
-                .color(getColor(type, dossierTemplateId, manualEntity.applied(), isHint))
-                .reason(manualEntity.buildReasonWithManualChangeDescriptions())
-                .legalBasis(manualEntity.legalBasis())
-                .value(manualEntity.value())
-                .type(type)
-                .redacted(manualEntity.applied())
-                .isHint(isHint)
-                .isRecommendation(manualEntity.getEntityType().equals(EntityType.RECOMMENDATION))
-                .isFalsePositive(manualEntity.getEntityType().equals(EntityType.FALSE_POSITIVE) || manualEntity.getEntityType().equals(EntityType.FALSE_RECOMMENDATION))
-                .section(manualEntity.getManualOverwrite().getSection().orElse(manualEntity.getSection()))
-                .sectionNumber(0)
-                .matchedRule(manualEntity.getMatchedRule().getRuleIdentifier().toString())
-                .rectangle(manualEntity.isRectangle())
-                .isDictionaryEntry(manualEntity.isDictionaryEntry())
-                .isDossierDictionaryEntry(manualEntity.isDossierDictionaryEntry())
-                .textAfter("")
-                .textBefore("")
-                .startOffset(-1)
-                .endOffset(-1)
-                .positions(manualEntity.getManualOverwrite()
-                        .getPositions()
-                        .orElse(manualEntity.getEntityPosition())
-                        .stream()
-                        .map(entityPosition -> toRedactionLogRectangle(entityPosition.rectangle2D(), entityPosition.pageNumber()))
-                        .toList())
-                .engines(Collections.emptySet())
-                .reference(Collections.emptySet())
-                .manualChanges(mapManualChanges(manualEntity.getManualOverwrite(), isHint))
-                .build();
-    }
-
-}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/UnprocessedChangesService.java

@@ -3,10 +3,8 @@ package com.iqser.red.service.redaction.v1.server.service;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.HashSet;
-import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
-import java.util.NoSuchElementException;
 import java.util.Optional;
 import java.util.Set;
 import java.util.stream.Collectors;
@@ -18,30 +16,22 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequ
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.BaseAnnotation;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
 import com.iqser.red.service.redaction.v1.model.AnalyzeResponse;
 import com.iqser.red.service.redaction.v1.model.QueueNames;
 import com.iqser.red.service.redaction.v1.model.UnprocessedManualEntity;
-import com.iqser.red.service.redaction.v1.server.model.ClosestEntity;
-import com.iqser.red.service.redaction.v1.server.model.ManualEntity;
-import com.iqser.red.service.redaction.v1.server.model.dictionary.SearchImplementation;
-import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
-import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
-import com.iqser.red.service.redaction.v1.server.model.document.entity.PositionOnPage;
+import com.iqser.red.service.redaction.v1.server.model.PrecursorEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
-import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
 import com.iqser.red.service.redaction.v1.server.service.document.DocumentGraphMapper;
-import com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService;
-import com.iqser.red.service.redaction.v1.server.service.document.EntityEnrichmentService;
 import com.iqser.red.service.redaction.v1.server.service.document.EntityFindingUtility;
-import com.iqser.red.service.redaction.v1.server.service.document.ManualEntityCreationService;
+import com.iqser.red.service.redaction.v1.server.service.document.EntityFromPrecursorCreationService;
 import com.iqser.red.service.redaction.v1.server.storage.ObservedStorageService;
 import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
 
 import io.micrometer.observation.annotation.Observed;
-import jakarta.annotation.PostConstruct;
 import lombok.AccessLevel;
 import lombok.RequiredArgsConstructor;
 import lombok.experimental.FieldDefaults;
@@ -59,17 +49,9 @@ public class UnprocessedChangesService {
     final ObservedStorageService observedStorageService;
     final EntityFindingUtility entityFindingUtility;
     final RedactionStorageService redactionStorageService;
-    final EntityEnrichmentService entityEnrichmentService;
-    final ManualEntityCreationService manualEntityCreationService;
+    final EntityFromPrecursorCreationService entityFromPrecursorCreationService;
     final DictionaryService dictionaryService;
-    EntityCreationService entityCreationService;
-
-
-    @PostConstruct
-    public void initEntityCreationService() {
-
-        entityCreationService = new EntityCreationService(entityEnrichmentService);
-    }
+    final ManualChangesApplicationService manualChangesApplicationService;
 
 
     @Observed(name = "UnprocessedChangesService", contextualName = "analyse-surrounding-text")
@@ -82,201 +64,134 @@ public class UnprocessedChangesService {
         EntityLog previousEntityLog = redactionStorageService.getEntityLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
         Document document = DocumentGraphMapper.toDocumentGraph(observedStorageService.getDocumentData(analyzeRequest.getDossierId(), analyzeRequest.getFileId()));
 
-        Set<String> allAnnotationIds = analyzeRequest.getManualRedactions().getEntriesToAdd().stream().map(ManualRedactionEntry::getAnnotationId).collect(Collectors.toSet());
-        Set<String> resizeIds = analyzeRequest.getManualRedactions().getResizeRedactions().stream().map(ManualResizeRedaction::getAnnotationId).collect(Collectors.toSet());
+        Set<String> allAnnotationIds = analyzeRequest.getManualRedactions().getEntriesToAdd()
+                .stream()
+                .map(ManualRedactionEntry::getAnnotationId)
+                .collect(Collectors.toSet());
+        Set<String> resizeIds = analyzeRequest.getManualRedactions().getResizeRedactions()
+                .stream()
+                .map(ManualResizeRedaction::getAnnotationId)
+                .collect(Collectors.toSet());
         allAnnotationIds.addAll(resizeIds);
 
-        List<ManualResizeRedaction> manualResizeRedactions = analyzeRequest.getManualRedactions().getResizeRedactions().stream().toList();
-        List<ManualEntity> manualEntitiesToBeResized = previousEntityLog.getEntityLogEntry()
-                .stream().filter(entityLogEntry -> resizeIds.contains(entityLogEntry.getId())).toList()
-                .stream().map(ManualEntity::fromEntityLogEntry).toList();
+        List<ManualResizeRedaction> manualResizeRedactions = analyzeRequest.getManualRedactions().getResizeRedactions()
+                .stream()
+                .toList();
+        List<PrecursorEntity> manualEntitiesToBeResized = previousEntityLog.getEntityLogEntry()
+                .stream()
+                .filter(entityLogEntry -> resizeIds.contains(entityLogEntry.getId()))
+                .toList()
+                .stream()
+                .map(PrecursorEntity::fromEntityLogEntry)
+                .toList();
 
         if (!manualResizeRedactions.isEmpty()) {
             processManualResizeRedactions(document, manualEntitiesToBeResized, unprocessedManualEntities, manualResizeRedactions);
         }
 
-        List<ManualEntity> notFoundManualEntities = new ArrayList<>();
-        List<ManualEntity> manualEntities = manualEntitiesConverter(analyzeRequest.getManualRedactions(), analyzeRequest.getDossierTemplateId());
+        List<PrecursorEntity> notFoundManualEntities = new ArrayList<>();
+        List<PrecursorEntity> manualEntities = manualEntitiesConverter(analyzeRequest.getManualRedactions(), analyzeRequest.getDossierTemplateId());
         if (!manualEntities.isEmpty()) {
-            notFoundManualEntities = manualEntityCreationService.toTextEntity(manualEntities, document);
+            notFoundManualEntities = entityFromPrecursorCreationService.toTextEntity(manualEntities, document);
         }
 
-        document.getEntities().forEach(textEntity -> {
-            Set<String> processedIds = new HashSet<>();
-            for (var positionsOnPerPage : textEntity.getPositionsOnPagePerPage()) {
-                if (processedIds.contains(positionsOnPerPage.getId())) {
-                    continue;
-                }
-                processedIds.add(positionsOnPerPage.getId());
-                List<Position> positions = positionsOnPerPage.getRectanglePerLine()
-                        .stream()
-                        .map(rectangle2D -> new Position(rectangle2D, positionsOnPerPage.getPage().getNumber()))
-                        .collect(Collectors.toList());
-                unprocessedManualEntities.add(UnprocessedManualEntity.builder()
-                        .annotationId(allAnnotationIds.stream().filter(textEntity::matchesAnnotationId).findFirst().orElse(""))
-                        .textBefore(textEntity.getTextBefore())
-                        .textAfter(textEntity.getTextAfter())
-                        .section(textEntity.getManualOverwrite().getSection().orElse(textEntity.getDeepestFullyContainingNode().toString()))
-                        .positions(positions)
-                        .build());
-            }
-        });
+        document.getEntities()
+                .forEach(textEntity -> {
+                    Set<String> processedIds = new HashSet<>();
+                    for (var positionsOnPerPage : textEntity.getPositionsOnPagePerPage()) {
+                        if (processedIds.contains(positionsOnPerPage.getId())) {
+                            continue;
+                        }
+                        processedIds.add(positionsOnPerPage.getId());
+                        List<Position> positions = positionsOnPerPage.getRectanglePerLine()
+                                .stream()
+                                .map(rectangle2D -> new Position(rectangle2D, positionsOnPerPage.getPage().getNumber()))
+                                .collect(Collectors.toList());
+                        unprocessedManualEntities.add(UnprocessedManualEntity.builder()
+                                                              .annotationId(allAnnotationIds.stream()
+                                                                                    .filter(textEntity::matchesAnnotationId)
+                                                                                    .findFirst()
+                                                                                    .orElse(""))
+                                                              .textBefore(textEntity.getTextBefore())
+                                                              .textAfter(textEntity.getTextAfter())
+                                                              .section(textEntity.getManualOverwrite().getSection()
+                                                                               .orElse(textEntity.getDeepestFullyContainingNode().toString()))
+                                                              .positions(positions)
+                                                              .build());
+                    }
+                });
 
         notFoundManualEntities.forEach(manualEntity -> unprocessedManualEntities.add(builDefaultUnprocessedManualEntity(manualEntity)));
 
         rabbitTemplate.convertAndSend(QueueNames.REDACTION_ANALYSIS_RESPONSE_QUEUE,
-                AnalyzeResponse.builder().fileId(analyzeRequest.getFileId()).unprocessedManualEntities(unprocessedManualEntities).build());
+                                      AnalyzeResponse.builder().fileId(analyzeRequest.getFileId()).unprocessedManualEntities(unprocessedManualEntities).build());
     }
 
 
     private void processManualResizeRedactions(Document document,
-                                               List<ManualEntity> manualEntities,
+                                               List<PrecursorEntity> manualEntities,
                                                List<UnprocessedManualEntity> unprocessedManualEntities,
                                                List<ManualResizeRedaction> manualResizeRedactions) {
 
         Map<String, List<TextEntity>> tempEntities = entityFindingUtility.findAllPossibleEntitiesAndGroupByValue(document, manualEntities);
 
-        for (ManualEntity manualEntity : manualEntities) {
+        for (PrecursorEntity precursorEntity : manualEntities) {
 
-            Optional<TextEntity> optionalTextEntity = entityFindingUtility.findClosestEntityAndReturnEmptyIfNotFound(manualEntity, tempEntities, THRESHOLD);
+            Optional<TextEntity> optionalTextEntity = entityFindingUtility.findClosestEntityAndReturnEmptyIfNotFound(precursorEntity, tempEntities, THRESHOLD);
 
             if (optionalTextEntity.isEmpty()) {
-                unprocessedManualEntities.add(builDefaultUnprocessedManualEntity(manualEntity));
+                unprocessedManualEntities.add(builDefaultUnprocessedManualEntity(precursorEntity));
                 continue;
             }
 
-            TextEntity correctEntity = createCorrectEntity(manualEntity, document, optionalTextEntity.get().getTextRange());
-            resizeEntityAndReinsert(correctEntity, manualResizeRedactions.stream().filter(manualResizeRedaction -> manualResizeRedaction.getAnnotationId().equals(manualEntity.getId())).findFirst());
+            TextEntity correctEntity = EntityFromPrecursorCreationService.createCorrectEntity(precursorEntity, optionalTextEntity.get());
+            Optional<ManualResizeRedaction> optionalManualResizeRedaction = manualResizeRedactions.stream()
+                    .filter(manualResizeRedaction -> manualResizeRedaction.getAnnotationId().equals(precursorEntity.getId()))
+                    .findFirst();
+            if (optionalManualResizeRedaction.isPresent()) {
+                ManualResizeRedaction manualResizeRedaction = optionalManualResizeRedaction.get();
+                manualChangesApplicationService.resize(correctEntity, manualResizeRedaction);
+
+                // If the entity's value is not the same as the manual resize request's value it means we didn't find it anywhere and we want to remove it
+                // from the graph, so it does not get processed and sent back to persistence-service to update its value.
+                if (!correctEntity.getValue().equals(manualResizeRedaction.getValue())) {
+                    correctEntity.removeFromGraph();
+                }
+            }
         }
 
         // remove all temp entities from the graph
-        tempEntities.values().stream().flatMap(Collection::stream).forEach(TextEntity::removeFromGraph);
-    }
-
-
-    public void resizeEntityAndReinsert(TextEntity entityToBeResized, Optional<ManualResizeRedaction> optionalManualResizeRedaction) {
-
-        if (optionalManualResizeRedaction.isEmpty()) {
-            return;
-        }
-
-        ManualResizeRedaction manualResizeRedaction = optionalManualResizeRedaction.get();
-
-        PositionOnPage positionOnPageToBeResized = entityToBeResized.getPositionsOnPagePerPage()
+        tempEntities.values()
                 .stream()
-                .filter(redactionPosition -> redactionPosition.getId().equals(manualResizeRedaction.getAnnotationId()))
-                .findFirst()
-                .orElseThrow(() -> new NoSuchElementException("No redaction position with matching annotation id found!"));
-
-        positionOnPageToBeResized.setRectanglePerLine(manualResizeRedaction.getPositions().stream().map(ManualChangesApplicationService::toRectangle2D).collect(Collectors.toList()));
-
-        String value = manualResizeRedaction.getValue();
-        int newStartOffset = -1;
-        SemanticNode node = entityToBeResized.getDeepestFullyContainingNode();
-        ClosestEntity closestEntity = ClosestEntity.builder().distance(100).textRange(null).build();
-
-        // Loop through nodes starting from the deepest fully containing node all the way to the document node
-        while (node != null) {
-            if (node.containsString(value)) {
-                SearchImplementation searchImplementation = new SearchImplementation(value, false);
-                List<TextRange> textRanges = searchImplementation.getBoundaries(node.getTextBlock(), node.getTextRange());
-
-                for (TextRange textRange : textRanges) {
-                    SemanticNode finalNode = node;
-
-                    List<TextEntity> tempEntities = searchImplementation.getBoundaries(node.getTextBlock(), textRange)
-                            .stream()
-                            .map(boundary -> entityCreationService.forceByTextRange(boundary, "temp", EntityType.ENTITY, finalNode))
-                            .collect(Collectors.toList());
-
-                    // If a value appears multiple times in a section after resizing, we need to make sure we select the surrounding text for the correct one.
-                    ManualChangesApplicationService.determineCorrectEntity(manualResizeRedaction, textRange, tempEntities, closestEntity);
-
-                    // Remove all temp entities from the graph
-                    tempEntities.forEach(TextEntity::removeFromGraph);
-                }
-                break;
-            }
-
-            // If the current node is the document node then it does not have a parent, meaning we could not find the value anywhere.
-            if (node.hasParent()) {
-                node = node.getParent();
-            } else {
-                break;
-            }
-        }
-
-        if (closestEntity.getTextRange() != null) {
-            newStartOffset = closestEntity.getTextRange().start();
-        }
-
-        // need to reinsert the entity, due to the boundary having changed.
-        removeAndUpdateAndReInsertEntity(entityToBeResized, manualResizeRedaction, newStartOffset);
-        entityToBeResized.getManualOverwrite().addChange(manualResizeRedaction);
+                .flatMap(Collection::stream)
+                .forEach(TextEntity::removeFromGraph);
     }
 
 
-    private void removeAndUpdateAndReInsertEntity(TextEntity entityToBeResized, ManualResizeRedaction manualResizeRedaction, int newStartOffset) {
-
-        SemanticNode nodeToInsertInto = entityToBeResized.getDeepestFullyContainingNode().getDocumentTree().getRoot().getNode();
-        entityToBeResized.removeFromGraph();
-        entityToBeResized.getIntersectingNodes().forEach(node -> node.getEntities().remove(this));
-        entityToBeResized.getPages().forEach(page -> page.getEntities().remove(this));
-        entityToBeResized.setIntersectingNodes(new LinkedList<>());
-        entityToBeResized.setDeepestFullyContainingNode(null);
-        entityToBeResized.setPages(new HashSet<>());
-        entityToBeResized.getTextRange().setStart(newStartOffset);
-        entityToBeResized.getTextRange().setEnd(newStartOffset + manualResizeRedaction.getValue().length());
-        // Don't insert into the graph if newStartOffset is -1 because it means nothing was found.
-        if (newStartOffset > -1) {
-            entityCreationService.addEntityToGraph(entityToBeResized, nodeToInsertInto);
-        }
-    }
-
-
-    private TextEntity createCorrectEntity(ManualEntity manualEntity, SemanticNode node, TextRange closestTextRange) {
-
-        TextEntity correctEntity = entityCreationService.forceByTextRange(closestTextRange, manualEntity.getType(), manualEntity.getEntityType(), node);
-
-        correctEntity.addMatchedRules(manualEntity.getMatchedRuleList());
-        correctEntity.setDictionaryEntry(manualEntity.isDictionaryEntry());
-        correctEntity.setDossierDictionaryEntry(manualEntity.isDossierDictionaryEntry());
-        correctEntity.getManualOverwrite().addChanges(manualEntity.getManualOverwrite().getManualChangeLog());
-
-        List<PositionOnPage> redactionPositionsWithIdOfManualOnPage = new ArrayList<>(correctEntity.getPositionsOnPagePerPage().size());
-        for (PositionOnPage positionOnPage : correctEntity.getPositionsOnPagePerPage()) {
-            redactionPositionsWithIdOfManualOnPage.add(new PositionOnPage(manualEntity.getId(), positionOnPage.getPage(), positionOnPage.getRectanglePerLine()));
-        }
-        correctEntity.setPositionsOnPagePerPage(redactionPositionsWithIdOfManualOnPage);
-
-        return correctEntity;
-    }
-
-
-    private UnprocessedManualEntity builDefaultUnprocessedManualEntity(ManualEntity manualEntity) {
+    private UnprocessedManualEntity builDefaultUnprocessedManualEntity(PrecursorEntity precursorEntity) {
 
         return UnprocessedManualEntity.builder()
-                .annotationId(manualEntity.getId())
+                .annotationId(precursorEntity.getId())
                 .textAfter("")
                 .textBefore("")
                 .section("")
-                .positions(manualEntity.getManualOverwrite()
-                        .getPositions()
-                        .orElse(manualEntity.getEntityPosition())
-                        .stream()
-                        .map(entityPosition -> new Position(entityPosition.rectangle2D(), entityPosition.pageNumber()))
-                        .toList())
+                .positions(precursorEntity.getManualOverwrite().getPositions()
+                                   .orElse(precursorEntity.getEntityPosition())
+                                   .stream()
+                                   .map(entityPosition -> new Position(entityPosition.rectangle2D(), entityPosition.pageNumber()))
+                                   .toList())
                 .build();
     }
 
 
-    private List<ManualEntity> manualEntitiesConverter(ManualRedactions manualRedactions, String dossierTemplateId) {
+    private List<PrecursorEntity> manualEntitiesConverter(ManualRedactions manualRedactions, String dossierTemplateId) {
 
         return manualRedactions.getEntriesToAdd()
                 .stream()
                 .filter(manualRedactionEntry -> manualRedactionEntry.getPositions() != null && !manualRedactionEntry.getPositions().isEmpty())
-                .map(manualRedactionEntry -> ManualEntity.fromManualRedactionEntry(manualRedactionEntry,
-                        dictionaryService.isHint(manualRedactionEntry.getType(), dossierTemplateId)))
+                .filter(BaseAnnotation::isLocal)
+                .map(manualRedactionEntry -> PrecursorEntity.fromManualRedactionEntry(manualRedactionEntry,
+                                                                                      dictionaryService.isHint(manualRedactionEntry.getType(), dossierTemplateId)))
                 .toList();
     }
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/ComponentComparator.java

@@ -15,8 +15,12 @@ public class ComponentComparator implements Comparator<Component> {
     @Override
     public int compare(Component component1, Component component2) {
 
-        var firstEntity1 = component1.getReferences().stream().min(EntityComparators.first());
-        var firstEntity2 = component2.getReferences().stream().min(EntityComparators.first());
+        var firstEntity1 = component1.getReferences()
+                .stream()
+                .min(EntityComparators.first());
+        var firstEntity2 = component2.getReferences()
+                .stream()
+                .min(EntityComparators.first());
         if (firstEntity1.isEmpty() && firstEntity2.isEmpty()) {
             return 0;
         }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/ComponentCreationService.java

@@ -40,7 +40,8 @@ public class ComponentCreationService {
 
     private static List<Entity> findEntitiesFromLongestSection(Collection<Entity> entities) {
 
-        var entitiesBySection = entities.stream().collect(Collectors.groupingBy(entity -> entity.getContainingNode().getHighestParent()));
+        var entitiesBySection = entities.stream()
+                .collect(Collectors.groupingBy(entity -> entity.getContainingNode().getHighestParent()));
         Optional<SemanticNode> longestSection = entitiesBySection.entrySet()
                 .stream()
                 .sorted(Comparator.comparingInt(ComponentCreationService::getTotalLengthOfEntities).reversed())
@@ -79,14 +80,20 @@ public class ComponentCreationService {
     public void firstOrElse(String ruleIdentifier, String name, Collection<Entity> entities, String fallback) {
 
         String valueDescription = String.format("First found value of type %s or else '%s'", joinTypes(entities), fallback);
-        String value = entities.stream().min(EntityComparators.first()).map(Entity::getValue).orElse(fallback);
+        String value = entities.stream()
+                .min(EntityComparators.first())
+                .map(Entity::getValue)
+                .orElse(fallback);
         create(ruleIdentifier, name, value, valueDescription, entities);
     }
 
 
     private static String joinTypes(Collection<Entity> entities) {
 
-        return entities.stream().map(Entity::getType).distinct().collect(Collectors.joining(", "));
+        return entities.stream()
+                .map(Entity::getType)
+                .distinct()
+                .collect(Collectors.joining(", "));
     }
 
 
@@ -104,12 +111,12 @@ public class ComponentCreationService {
         referencedEntities.addAll(references);
 
         kieSession.insert(Component.builder()
-                .matchedRule(RuleIdentifier.fromString(ruleIdentifier))
-                .name(name)
-                .value(value)
-                .valueDescription(valueDescription)
-                .references(new LinkedList<>(references))
-                .build());
+                                  .matchedRule(RuleIdentifier.fromString(ruleIdentifier))
+                                  .name(name)
+                                  .value(value)
+                                  .valueDescription(valueDescription)
+                                  .references(new LinkedList<>(references))
+                                  .build());
     }
 
 
@@ -142,8 +149,11 @@ public class ComponentCreationService {
 
     private static List<Entity> findEntitiesFromFirstSection(Collection<Entity> entities) {
 
-        var entitiesBySection = entities.stream().collect(Collectors.groupingBy(entity -> entity.getContainingNode().getHighestParent()));
-        Optional<SemanticNode> firstSection = entitiesBySection.keySet().stream().min(SemanticNodeComparators.first());
+        var entitiesBySection = entities.stream()
+                .collect(Collectors.groupingBy(entity -> entity.getContainingNode().getHighestParent()));
+        Optional<SemanticNode> firstSection = entitiesBySection.keySet()
+                .stream()
+                .min(SemanticNodeComparators.first());
         if (firstSection.isEmpty()) {
             return Collections.emptyList();
         }
@@ -188,7 +198,10 @@ public class ComponentCreationService {
     public void joining(String ruleIdentifier, String name, Collection<Entity> entities, String delimiter) {
 
         String valueDescription = String.format("Joining all values of type %s with '%s'", joinTypes(entities), delimiter);
-        String value = entities.stream().sorted(EntityComparators.first()).map(Entity::getValue).collect(Collectors.joining(delimiter));
+        String value = entities.stream()
+                .sorted(EntityComparators.first())
+                .map(Entity::getValue)
+                .collect(Collectors.joining(delimiter));
         create(ruleIdentifier, name, value, valueDescription, entities);
     }
 
@@ -231,14 +244,20 @@ public class ComponentCreationService {
     public void joiningUnique(String ruleIdentifier, String name, Collection<Entity> entities, String delimiter) {
 
         String valueDescription = String.format("Joining all unique values of type %s with '%s'", joinTypes(entities), delimiter);
-        String value = entities.stream().sorted(EntityComparators.first()).map(Entity::getValue).distinct().collect(Collectors.joining(delimiter));
+        String value = entities.stream()
+                .sorted(EntityComparators.first())
+                .map(Entity::getValue)
+                .distinct()
+                .collect(Collectors.joining(delimiter));
         create(ruleIdentifier, name, value, valueDescription, entities);
     }
 
 
     private static int getTotalLengthOfEntities(Map.Entry<SemanticNode, List<Entity>> entry) {
 
-        return entry.getValue().stream().mapToInt(Entity::getLength).sum();
+        return entry.getValue()
+                .stream()
+                .mapToInt(Entity::getLength).sum();
     }
 
 
@@ -293,7 +312,10 @@ public class ComponentCreationService {
      */
     public void uniqueValueCount(String ruleIdentifier, String name, Collection<Entity> entities) {
 
-        long count = entities.stream().map(Entity::getValue).distinct().count();
+        long count = entities.stream()
+                .map(Entity::getValue)
+                .distinct()
+                .count();
         create(ruleIdentifier, name, String.valueOf(count), "Number of unique values in the entity references", entities);
     }
 
@@ -307,18 +329,20 @@ public class ComponentCreationService {
      */
     public void rowValueCount(String ruleIdentifier, String name, Collection<Entity> entities) {
 
-        entities.stream().collect(Collectors.groupingBy(this::getFirstTable)).forEach((optionalTable, groupedEntities) -> {
+        entities.stream()
+                .collect(Collectors.groupingBy(this::getFirstTable))
+                .forEach((optionalTable, groupedEntities) -> {
 
-            if (optionalTable.isEmpty()) {
-                return;
-            }
+                    if (optionalTable.isEmpty()) {
+                        return;
+                    }
 
-            long count = groupedEntities.stream()
-                    .collect(Collectors.groupingBy(entity -> getFirstTableCell(entity).map(TableCell::getRow).orElse(-1)))
-                    .size();
+                    long count = groupedEntities.stream()
+                            .collect(Collectors.groupingBy(entity -> getFirstTableCell(entity).map(TableCell::getRow)
+                                    .orElse(-1))).size();
 
-            create(ruleIdentifier, name, String.valueOf(count), "Count rows with values in the entity references in same table", entities);
-        });
+                    create(ruleIdentifier, name, String.valueOf(count), "Count rows with values in the entity references in same table", entities);
+                });
     }
 
 
@@ -334,18 +358,20 @@ public class ComponentCreationService {
         if (entities.isEmpty()) {
             return;
         }
-        entities.stream().sorted(EntityComparators.first()).forEach(entity -> {
-            BreakIterator iterator = BreakIterator.getSentenceInstance(Locale.ENGLISH);
-            iterator.setText(entity.getValue());
-            int start = iterator.first();
-            for (int end = iterator.next(); end != BreakIterator.DONE; start = end, end = iterator.next()) {
-                create(ruleIdentifier,
-                        name,
-                        entity.getValue().substring(start, end).replaceAll("\\n", "").trim(),
-                        String.format("Values of type '%s' as sentences", entity.getType()),
-                        entity);
-            }
-        });
+        entities.stream()
+                .sorted(EntityComparators.first())
+                .forEach(entity -> {
+                    BreakIterator iterator = BreakIterator.getSentenceInstance(Locale.ENGLISH);
+                    iterator.setText(entity.getValue());
+                    int start = iterator.first();
+                    for (int end = iterator.next(); end != BreakIterator.DONE; start = end, end = iterator.next()) {
+                        create(ruleIdentifier,
+                               name,
+                               entity.getValue().substring(start, end).replaceAll("\\n", "").trim(),
+                               String.format("Values of type '%s' as sentences", entity.getType()),
+                               entity);
+                    }
+                });
     }
 
 
@@ -366,12 +392,12 @@ public class ComponentCreationService {
         List<Entity> referenceList = new LinkedList<>();
         referenceList.add(reference);
         kieSession.insert(Component.builder()
-                .matchedRule(RuleIdentifier.fromString(ruleIdentifier))
-                .name(name)
-                .value(value)
-                .valueDescription(valueDescription)
-                .references(referenceList)
-                .build());
+                                  .matchedRule(RuleIdentifier.fromString(ruleIdentifier))
+                                  .name(name)
+                                  .value(value)
+                                  .valueDescription(valueDescription)
+                                  .references(referenceList)
+                                  .build());
     }
 
 
@@ -428,8 +454,10 @@ public class ComponentCreationService {
         }
 
         String formattedDateStrings = Stream.concat(//
-                        dates.stream().sorted().map(date -> DateConverter.convertDate(date, resultFormat)), //
-                        unparsedDates.stream())//
+                                                    dates.stream()
+                                                            .sorted()
+                                                            .map(date -> DateConverter.convertDate(date, resultFormat)), //
+                                                    unparsedDates.stream())//
                 .collect(Collectors.joining(", "));
 
         create(ruleIdentifier, name, formattedDateStrings, valueDescription, entities);
@@ -437,7 +465,7 @@ public class ComponentCreationService {
 
 
     /**
-     * Joins values from entities that are in the same table row. If entities are not in a table cell they are added as a single component.
+     * Joins values from entities that are in the same table row. They are sorted from left to right. If entities are not in a table cell they are added as a single component.
      *
      * @param ruleIdentifier the identifier of the rule
      * @param name           the name of the entity
@@ -445,26 +473,34 @@ public class ComponentCreationService {
      */
     public void joiningFromSameTableRow(String ruleIdentifier, String name, Collection<Entity> entities) {
 
-        String types = entities.stream().map(Entity::getType).sorted(Comparator.reverseOrder()).distinct().collect(Collectors.joining(", "));
+        String types = entities.stream()
+                .map(Entity::getType)
+                .sorted(Comparator.reverseOrder())
+                .distinct()
+                .collect(Collectors.joining(", "));
         String valueDescription = String.format("Combine values of %s that are in same table row", types);
-        entities.stream().collect(Collectors.groupingBy(this::getFirstTable)).forEach((optionalTable, groupedEntities) -> {
-            if (optionalTable.isEmpty()) {
-                groupedEntities.forEach(entity -> create(ruleIdentifier, name, entity.getValue(), valueDescription, entity));
-            }
+        entities.stream()
+                .collect(Collectors.groupingBy(this::getFirstTable))
+                .forEach((optionalTable, groupedEntities) -> {
+                    if (optionalTable.isEmpty()) {
+                        groupedEntities.forEach(entity -> create(ruleIdentifier, name, entity.getValue(), valueDescription, entity));
+                    }
 
-            groupedEntities.stream()
-                    .filter(entity -> entity.getContainingNode() instanceof TableCell)
-                    .collect(Collectors.groupingBy(entity -> ((TableCell) entity.getContainingNode()).getRow()))
-                    .entrySet()
-                    .stream()
-                    .sorted(Comparator.comparingInt(Map.Entry::getKey))
-                    .map(Map.Entry::getValue)
-                    .forEach(entitiesInSameRow -> create(ruleIdentifier,
-                            name,
-                            entitiesInSameRow.stream().sorted(EntityComparators.first()).map(Entity::getValue).collect(Collectors.joining(", ")),
-                            valueDescription,
-                            entitiesInSameRow));
-        });
+                    groupedEntities.stream()
+                            .filter(entity -> entity.getContainingNode() instanceof TableCell)
+                            .collect(Collectors.groupingBy(entity -> ((TableCell) entity.getContainingNode()).getRow())).entrySet()
+                            .stream()
+                            .sorted(Comparator.comparingInt(Map.Entry::getKey))
+                            .map(Map.Entry::getValue)
+                            .forEach(entitiesInSameRow -> create(ruleIdentifier,
+                                                                 name,
+                                                                 entitiesInSameRow.stream()
+                                                                         .sorted(EntityComparators.first())
+                                                                         .map(Entity::getValue)
+                                                                         .collect(Collectors.joining(", ")),
+                                                                 valueDescription,
+                                                                 entitiesInSameRow));
+                });
     }
 
 
@@ -521,12 +557,12 @@ public class ComponentCreationService {
     public void create(String ruleIdentifier, String name, String value) {
 
         kieSession.insert(Component.builder()
-                .matchedRule(RuleIdentifier.fromString(ruleIdentifier))
-                .name(name)
-                .value(value)
-                .valueDescription("")
-                .references(Collections.emptyList())
-                .build());
+                                  .matchedRule(RuleIdentifier.fromString(ruleIdentifier))
+                                  .name(name)
+                                  .value(value)
+                                  .valueDescription("")
+                                  .references(Collections.emptyList())
+                                  .build());
     }
 
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/DocumentGraphMapper.java

@@ -6,8 +6,10 @@ import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
 import java.util.NoSuchElementException;
+import java.util.Set;
 
 import com.iqser.red.service.redaction.v1.server.model.document.DocumentTree;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.DuplicatedParagraph;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Footer;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Header;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Image;
@@ -39,7 +41,9 @@ public class DocumentGraphMapper {
         DocumentTree documentTree = new DocumentTree(document);
         Context context = new Context(documentData, documentTree);
 
-        context.pageData.addAll(Arrays.stream(documentData.getDocumentPages()).map(DocumentGraphMapper::buildPage).toList());
+        context.pageData.addAll(Arrays.stream(documentData.getDocumentPages())
+                                        .map(DocumentGraphMapper::buildPage)
+                                        .toList());
 
         context.documentTree.getRoot().getChildren().addAll(buildEntries(documentData.getDocumentStructure().getRoot().getChildren(), context));
 
@@ -57,11 +61,13 @@ public class DocumentGraphMapper {
         List<DocumentTree.Entry> newEntries = new LinkedList<>();
         for (DocumentStructure.EntryData entryData : entries) {
 
-            List<Page> pages = Arrays.stream(entryData.getPageNumbers()).map(pageNumber -> getPage(pageNumber, context)).toList();
+            List<Page> pages = Arrays.stream(entryData.getPageNumbers())
+                    .map(pageNumber -> getPage(pageNumber, context))
+                    .toList();
 
             SemanticNode node = switch (entryData.getType()) {
                 case SECTION -> buildSection(context);
-                case PARAGRAPH -> buildParagraph(context);
+                case PARAGRAPH -> buildParagraph(context, entryData.getProperties());
                 case HEADLINE -> buildHeadline(context);
                 case HEADER -> buildHeader(context);
                 case FOOTER -> buildFooter(context);
@@ -75,7 +81,10 @@ public class DocumentGraphMapper {
                 TextBlock textBlock = toTextBlock(entryData.getAtomicBlockIds(), context, node);
                 node.setLeafTextBlock(textBlock);
             }
-            List<Integer> treeId = Arrays.stream(entryData.getTreeId()).boxed().toList();
+            List<Integer> treeId = Arrays.stream(entryData.getTreeId()).boxed()
+                    .toList();
+            entryData.getEngines()
+                    .forEach(engine -> node.addEngine(engine));
             node.setTreeId(treeId);
 
             switch (entryData.getType()) {
@@ -140,24 +149,35 @@ public class DocumentGraphMapper {
     }
 
 
-    private Paragraph buildParagraph(Context context) {
+    private Paragraph buildParagraph(Context context, Map<String, String> properties) {
+
+        if (PropertiesMapper.isDuplicateParagraph(properties)) {
+
+            DuplicatedParagraph duplicatedParagraph = DuplicatedParagraph.builder().documentTree(context.documentTree).build();
+
+            Long[] unsortedTextblockIds = PropertiesMapper.getUnsortedTextblockIds(properties);
+            duplicatedParagraph.setUnsortedLeafTextBlock(toTextBlock(unsortedTextblockIds, context, duplicatedParagraph));
+            return duplicatedParagraph;
+        }
 
         return Paragraph.builder().documentTree(context.documentTree).build();
     }
 
 
-    private TextBlock toTextBlock(Long[] atomicTextBlockIds, Context context, SemanticNode parent) {
+        private TextBlock toTextBlock(Long[] atomicTextBlockIds, Context context, SemanticNode parent) {
 
-        return Arrays.stream(atomicTextBlockIds).map(atomicTextBlockId -> getAtomicTextBlock(context, parent, atomicTextBlockId)).collect(new TextBlockCollector());
+        return Arrays.stream(atomicTextBlockIds)
+                .map(atomicTextBlockId -> getAtomicTextBlock(context, parent, atomicTextBlockId))
+                .collect(new TextBlockCollector());
     }
 
 
     private AtomicTextBlock getAtomicTextBlock(Context context, SemanticNode parent, Long atomicTextBlockId) {
 
         return AtomicTextBlock.fromAtomicTextBlockData(context.documentTextData.get(Math.toIntExact(atomicTextBlockId)),
-                context.documentPositionData.get(Math.toIntExact(atomicTextBlockId)),
-                parent,
-                getPage(context.documentTextData.get(Math.toIntExact(atomicTextBlockId)).getPage(), context));
+                                                       context.documentPositionData.get(Math.toIntExact(atomicTextBlockId)),
+                                                       parent,
+                                                       getPage(context.documentTextData.get(Math.toIntExact(atomicTextBlockId)).getPage(), context));
     }
 
 
@@ -188,8 +208,10 @@ public class DocumentGraphMapper {
 
             this.documentTree = documentTree;
             this.pageData = new LinkedList<>();
-            this.documentTextData = Arrays.stream(documentData.getDocumentTextData()).toList();
-            this.documentPositionData = Arrays.stream(documentData.getDocumentPositionData()).toList();
+            this.documentTextData = Arrays.stream(documentData.getDocumentTextData())
+                    .toList();
+            this.documentPositionData = Arrays.stream(documentData.getDocumentPositionData())
+                    .toList();
 
         }
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityComparators.java

@@ -11,6 +11,7 @@ public abstract class EntityComparators implements Comparator<Entity> {
         return new FirstEntity();
     }
 
+
     public static class LongestEntity implements Comparator<Entity> {
 
         @Override
@@ -27,6 +28,7 @@ public abstract class EntityComparators implements Comparator<Entity> {
         return new LongestEntity();
     }
 
+
     public static class FirstEntity implements Comparator<Entity> {
 
         @Override

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityCreationUtility.java

@@ -0,0 +1,91 @@
+package com.iqser.red.service.redaction.v1.server.service.document;
+
+import java.util.List;
+import java.util.Set;
+
+import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
+import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.Page;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
+import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
+
+public class EntityCreationUtility {
+
+    public static void checkIfBothStartAndEndAreEmpty(String start, String end) {
+
+        checkIfBothStartAndEndAreEmpty(List.of(start), List.of(end));
+    }
+
+
+    public static <T> void checkIfBothStartAndEndAreEmpty(List<T> start, List<T> end) {
+
+        if ((start == null || start.isEmpty()) && (end == null || end.isEmpty())) {
+            throw new IllegalArgumentException("Start and end values are empty!");
+        }
+    }
+
+
+    public static int truncateEndIfLineBreakIsBetween(int end, int expandedEnd, TextBlock textBlock) {
+
+        if (textBlock.getNextLinebreak(end) < expandedEnd) {
+            return end;
+        }
+        return expandedEnd;
+    }
+
+
+    public static Set<SemanticNode> findIntersectingSubNodes(SemanticNode initialIntersectingNode, TextRange textRange) {
+
+        IntersectingNodeVisitor visitor = new IntersectingNodeVisitor(textRange);
+
+        if (initialIntersectingNode.getTextRange().intersects(textRange)) {
+            initialIntersectingNode.accept(visitor);
+        }
+
+        return visitor.getIntersectingNodes();
+    }
+
+
+    public static void addToPages(TextEntity entity) {
+
+        Set<Page> pages = entity.getDeepestFullyContainingNode().getPages(entity.getTextRange());
+        entity.getPages().addAll(pages);
+        pages.forEach(page -> page.getEntities().add(entity));
+    }
+
+
+    public static void addEntityToNodeEntitySets(TextEntity entity) {
+
+        entity.getIntersectingNodes()
+                .forEach(node -> node.getEntities().add(entity));
+    }
+
+
+    public static boolean allEntitiesIntersectAndHaveSameTypes(List<TextEntity> entitiesToMerge) {
+
+        if (entitiesToMerge.isEmpty()) {
+            return true;
+        }
+        TextEntity previousEntity = entitiesToMerge.get(0);
+        for (TextEntity textEntity : entitiesToMerge.subList(1, entitiesToMerge.size())) {
+            boolean typeMatches = textEntity.type().equals(previousEntity.type());
+            boolean entityTypeMatches = textEntity.getEntityType().equals(previousEntity.getEntityType());
+            boolean intersects = textEntity.intersects(previousEntity);
+            if (!typeMatches || !entityTypeMatches || !intersects) {
+                return false;
+            }
+        }
+        return true;
+    }
+
+
+    public static TextRange toLineAfterTextRange(TextBlock textBlock, TextRange textRange) {
+
+        if (textBlock.getTextRange().end() == textRange.end()) {
+            return new TextRange(textRange.end(), textRange.end());
+        }
+
+        return new TextRange(textRange.end(), textBlock.getNextLinebreak(textRange.end())).trim(textBlock);
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityEnrichmentService.java

@@ -59,7 +59,9 @@ public class EntityEnrichmentService {
 
     private static List<String> splitToWordsAndRemoveEmptyWords(String textAfter) {
 
-        return Arrays.stream(textAfter.split(" ")).filter(word -> !Objects.equals("", word)).toList();
+        return Arrays.stream(textAfter.split(" "))
+                .filter(word -> !Objects.equals("", word))
+                .toList();
     }
 
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityFindingUtility.java

@@ -5,6 +5,7 @@ import static java.util.stream.Collectors.groupingBy;
 
 import java.awt.geom.Rectangle2D;
 import java.util.Collection;
+import java.util.Collections;
 import java.util.Comparator;
 import java.util.List;
 import java.util.Locale;
@@ -17,7 +18,8 @@ import java.util.stream.Collectors;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
 
-import com.iqser.red.service.redaction.v1.server.model.ManualEntity;
+import com.iqser.red.service.redaction.v1.server.model.ClosestEntity;
+import com.iqser.red.service.redaction.v1.server.model.PrecursorEntity;
 import com.iqser.red.service.redaction.v1.server.model.RectangleWithPage;
 import com.iqser.red.service.redaction.v1.server.model.dictionary.SearchImplementation;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
@@ -45,41 +47,47 @@ public class EntityFindingUtility {
     }
 
 
-    public Optional<TextEntity> findClosestEntityAndReturnEmptyIfNotFound(ManualEntity manualEntity, Map<String, List<TextEntity>> entitiesWithSameValue, double matchThreshold) {
+    public Optional<TextEntity> findClosestEntityAndReturnEmptyIfNotFound(PrecursorEntity precursorEntity,
+                                                                          Map<String, List<TextEntity>> entitiesWithSameValue,
+                                                                          double matchThreshold) {
 
-        if (manualEntity.getValue() == null) {
+        if (precursorEntity.getValue() == null) {
             return Optional.empty();
         }
 
-        List<TextEntity> possibleEntities = entitiesWithSameValue.get(manualEntity.getValue().toLowerCase(Locale.ENGLISH));
+        List<TextEntity> possibleEntities = entitiesWithSameValue.get(precursorEntity.getValue().toLowerCase(Locale.ENGLISH));
 
         if (entityIdentifierValueNotFound(possibleEntities)) {
-            log.warn("Entity could not be created with manualEntity: {}, due to the value {} not being found anywhere.", manualEntity, manualEntity.getValue());
+            log.info("Entity could not be created with precursorEntity: {}, due to the value {} not being found anywhere.", precursorEntity, precursorEntity.getValue());
             return Optional.empty();
         }
 
-        Optional<TextEntity> optionalClosestEntity = possibleEntities.stream()
-                .filter(entity -> pagesMatch(entity, manualEntity.getEntityPosition()))
-                .min(Comparator.comparingDouble(entity -> calculateMinDistance(manualEntity.getEntityPosition(), entity)));
+        Optional<ClosestEntity> optionalClosestEntity = possibleEntities.stream()
+                .filter(entity -> pagesMatch(entity, precursorEntity.getEntityPosition()))
+                .map(entity -> ClosestEntity.builder().distance(calculateMinDistance(precursorEntity.getEntityPosition(), entity)).textEntity(entity).build())
+                .min(Comparator.comparingDouble(ClosestEntity::getDistance));
 
         if (optionalClosestEntity.isEmpty()) {
-            log.warn("No Entity with value {} found on page {}", manualEntity.getValue(), manualEntity.getEntityPosition());
+            log.info("No Entity with value {} found on page {}", precursorEntity.getValue(), precursorEntity.getEntityPosition());
             return Optional.empty();
         }
 
-        TextEntity closestEntity = optionalClosestEntity.get();
-        double distance = calculateMinDistance(manualEntity.getEntityPosition(), closestEntity);
-        if (distance > matchThreshold) {
-            log.warn("For entity {} on page {} with positions {} distance to closest found entity is {} and therefore higher than the threshold of {}",
-                    manualEntity.getValue(),
-                    manualEntity.getEntityPosition().get(0).pageNumber(),
-                    manualEntity.getEntityPosition().stream().map(RectangleWithPage::rectangle2D).toList(),
-                    distance,
-                    matchThreshold);
+        ClosestEntity closestEntity = optionalClosestEntity.get();
+        if (closestEntity.getDistance() > matchThreshold) {
+            log.info("For entity {} on page {} with positions {} distance to closest found entity is {} and therefore higher than the threshold of {}",
+                     precursorEntity.getValue(),
+                     precursorEntity.getEntityPosition()
+                             .get(0).pageNumber(),
+                     precursorEntity.getEntityPosition()
+                             .stream()
+                             .map(RectangleWithPage::rectangle2D)
+                             .toList(),
+                     closestEntity.getDistance(),
+                     matchThreshold);
             return Optional.empty();
         }
 
-        return Optional.of(closestEntity);
+        return Optional.of(closestEntity.getTextEntity());
     }
 
 
@@ -91,31 +99,38 @@ public class EntityFindingUtility {
 
     private static boolean pagesMatch(TextEntity entity, List<RectangleWithPage> originalPositions) {
 
-        Set<Integer> entityPageNumbers = entity.getPositionsOnPagePerPage().stream().map(PositionOnPage::getPage).map(Page::getNumber).collect(Collectors.toSet());
-        Set<Integer> originalPageNumbers = originalPositions.stream().map(RectangleWithPage::pageNumber).collect(Collectors.toSet());
+        Set<Integer> entityPageNumbers = entity.getPositionsOnPagePerPage()
+                .stream()
+                .map(PositionOnPage::getPage)
+                .map(Page::getNumber)
+                .collect(Collectors.toSet());
+        Set<Integer> originalPageNumbers = originalPositions.stream()
+                .map(RectangleWithPage::pageNumber)
+                .collect(Collectors.toSet());
         return entityPageNumbers.containsAll(originalPageNumbers);
     }
 
 
-    private double calculateMinDistance(List<RectangleWithPage> originalPositions, TextEntity entity) {
+    public static double calculateMinDistance(List<RectangleWithPage> originalPositions, TextEntity entity) {
 
         if (originalPositions.size() != countRectangles(entity)) {
             return Double.MAX_VALUE;
         }
         return originalPositions.stream()
-                .mapToDouble(rectangleWithPage -> calculateMinDistancePerRectangle(entity, rectangleWithPage.pageNumber(), rectangleWithPage.rectangle2D()))
-                .average()
+                .mapToDouble(rectangleWithPage -> calculateMinDistancePerRectangle(entity, rectangleWithPage.pageNumber(), rectangleWithPage.rectangle2D())).average()
                 .orElse(Double.MAX_VALUE);
     }
 
 
     private static long countRectangles(TextEntity entity) {
 
-        return entity.getPositionsOnPagePerPage().stream().mapToLong(redactionPosition -> redactionPosition.getRectanglePerLine().size()).sum();
+        return entity.getPositionsOnPagePerPage()
+                .stream()
+                .mapToLong(redactionPosition -> redactionPosition.getRectanglePerLine().size()).sum();
     }
 
 
-    private double calculateMinDistancePerRectangle(TextEntity entity, int pageNumber, Rectangle2D originalRectangle) {
+    private static double calculateMinDistancePerRectangle(TextEntity entity, int pageNumber, Rectangle2D originalRectangle) {
 
         return entity.getPositionsOnPagePerPage()
                 .stream()
@@ -128,7 +143,7 @@ public class EntityFindingUtility {
     }
 
 
-    public double calculateDistance(Rectangle2D rectangle1, Rectangle2D rectangle2) {
+    public static double calculateDistance(Rectangle2D rectangle1, Rectangle2D rectangle2) {
 
         // mirrored coordinates safe comparison
         double minX1 = Math.min(rectangle1.getMinX(), rectangle1.getMaxX());
@@ -142,28 +157,43 @@ public class EntityFindingUtility {
         double maxY2 = Math.max(rectangle2.getMinY(), rectangle2.getMaxY());
 
         return Math.abs(minX1 - minX2) //
-                + Math.abs(minY1 - minY2) //
-                + Math.abs(maxX1 - maxX2) //
-                + Math.abs(maxY1 - maxY2);
+               + Math.abs(minY1 - minY2) //
+               + Math.abs(maxX1 - maxX2) //
+               + Math.abs(maxY1 - maxY2);
     }
 
 
-    public Map<String, List<TextEntity>> findAllPossibleEntitiesAndGroupByValue(SemanticNode node, List<ManualEntity> manualEntities) {
+    public Map<String, List<TextEntity>> findAllPossibleEntitiesAndGroupByValue(SemanticNode node, List<PrecursorEntity> manualEntities) {
 
-        Set<Integer> pageNumbers = manualEntities.stream().flatMap(entry -> entry.getEntityPosition().stream().map(RectangleWithPage::pageNumber)).collect(Collectors.toSet());
-        Set<String> entryValues = manualEntities.stream().map(ManualEntity::getValue).filter(Objects::nonNull).map(String::toLowerCase).collect(Collectors.toSet());
+        Set<Integer> pageNumbers = manualEntities.stream()
+                .flatMap(entry -> entry.getEntityPosition()
+                        .stream()
+                        .map(RectangleWithPage::pageNumber))
+                .collect(Collectors.toSet());
+        Set<String> entryValues = manualEntities.stream()
+                .map(PrecursorEntity::getValue)
+                .filter(Objects::nonNull)
+                .map(String::toLowerCase)
+                .collect(Collectors.toSet());
 
-        if (!pageNumbers.stream().allMatch(node::onPage)) {
+        if (!pageNumbers.stream()
+                .allMatch(node::onPage)) {
             throw new IllegalArgumentException(format("SemanticNode \"%s\" does not contain these pages %s, it has pages: %s",
-                    node,
-                    pageNumbers.stream().filter(pageNumber -> !node.onPage(pageNumber)).toList(),
-                    node.getPages()));
+                                                      node,
+                                                      pageNumbers.stream()
+                                                              .filter(pageNumber -> !node.onPage(pageNumber))
+                                                              .toList(),
+                                                      node.getPages()));
         }
-        SearchImplementation searchImplementation = new SearchImplementation(entryValues, true);
+
+        SearchImplementation searchImplementation = new SearchImplementation(entryValues.stream().map(String::trim).collect(Collectors.toSet()), true);
 
         return searchImplementation.getBoundaries(node.getTextBlock(), node.getTextRange())
                 .stream()
-                .map(boundary -> entityCreationService.forceByTextRange(boundary, "temp", EntityType.ENTITY, node))
+                .map(boundary -> entityCreationService.byTextRangeWithEngine(boundary, "temp", EntityType.ENTITY, node, Collections.emptySet()))
+                .filter(Optional::isPresent)
+                .map(Optional::get)
+                .distinct()
                 .collect(groupingBy(entity -> entity.getValue().toLowerCase(Locale.ROOT)));
     }
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityFromPrecursorCreationService.java

@@ -0,0 +1,152 @@
+package com.iqser.red.service.redaction.v1.server.service.document;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import org.springframework.stereotype.Service;
+
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedactions;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.BaseAnnotation;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;
+import com.iqser.red.service.redaction.v1.server.model.PrecursorEntity;
+import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
+import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
+import com.iqser.red.service.redaction.v1.server.service.DictionaryService;
+
+import lombok.AccessLevel;
+import lombok.RequiredArgsConstructor;
+import lombok.experimental.FieldDefaults;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+@Service
+@RequiredArgsConstructor
+@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
+public class EntityFromPrecursorCreationService {
+
+    static double MATCH_THRESHOLD = 10; // Is compared to the average sum of distances in pdf coordinates for each corner of the bounding box of the entities
+    EntityFindingUtility entityFindingUtility;
+    DictionaryService dictionaryService;
+
+
+    public List<PrecursorEntity> createEntitiesIfFoundAndReturnNotFoundEntries(ManualRedactions manualRedactions, SemanticNode node, String dossierTemplateId) {
+
+        Set<IdRemoval> idRemovals = manualRedactions.getIdsToRemove();
+        List<PrecursorEntity> manualEntities = manualRedactions.getEntriesToAdd()
+                .stream()
+                .filter(BaseAnnotation::isLocal)
+                .filter(manualRedactionEntry -> idRemovals.stream()
+                        .filter(idRemoval -> idRemoval.getAnnotationId().equals(manualRedactionEntry.getAnnotationId()))
+                        .filter(idRemoval -> idRemoval.getRequestDate().isBefore(manualRedactionEntry.getRequestDate()))
+                        .findAny()//
+                        .isEmpty())
+                .map(manualRedactionEntry -> //
+                             PrecursorEntity.fromManualRedactionEntry(manualRedactionEntry, dictionaryService.isHint(manualRedactionEntry.getType(), dossierTemplateId)))
+                .peek(manualEntity -> {
+                    if (manualEntity.getEntityType().equals(EntityType.HINT)) {
+                        manualEntity.skip("MAN.5.1", "manual hint is skipped by default");
+                    } else {
+                        manualEntity.apply("MAN.5.0", "manual entries are applied by default", manualEntity.getLegalBasis());
+                    }
+                })
+                .toList();
+        return toTextEntity(manualEntities, node);
+    }
+
+
+    public List<PrecursorEntity> createEntitiesIfFoundAndReturnNotFoundEntries(ImportedRedactions importedRedactions, SemanticNode node) {
+
+        List<PrecursorEntity> importedEntities = importedRedactions.getImportedRedactions()
+                .stream()
+                .map(PrecursorEntity::fromImportedEntry)
+                .peek(importedEntity -> importedEntity.apply("IMP.0.0", importedEntity.getReason(), importedEntity.getLegalBasis()))
+                .toList();
+        //we could add image handling here as well and create not only text entities
+        return toTextEntity(importedEntities, node);
+    }
+
+
+    public List<PrecursorEntity> toTextEntity(List<PrecursorEntity> precursorEntities, SemanticNode node) {
+
+        var notFoundEntities = precursorEntities.stream()
+                .filter(PrecursorEntity::isRectangle)
+                .collect(Collectors.toList());
+
+        var findableEntities = precursorEntities.stream()
+                .filter(precursorEntity -> !precursorEntity.isRectangle())
+                .toList();
+
+        Map<String, List<TextEntity>> tempEntitiesByValue = entityFindingUtility.findAllPossibleEntitiesAndGroupByValue(node, findableEntities);
+
+        for (PrecursorEntity precursorEntity : findableEntities) {
+            Optional<TextEntity> optionalClosestEntity = entityFindingUtility.findClosestEntityAndReturnEmptyIfNotFound(precursorEntity, tempEntitiesByValue, MATCH_THRESHOLD);
+            if (optionalClosestEntity.isEmpty()) {
+                notFoundEntities.add(precursorEntity);
+                continue;
+            }
+            createCorrectEntity(precursorEntity, optionalClosestEntity.get());
+        }
+
+        tempEntitiesByValue.values()
+                .stream()
+                .flatMap(Collection::stream)
+                .forEach(TextEntity::removeFromGraph);
+
+        return notFoundEntities;
+    }
+
+
+    /**
+     * Deletes the temp Entity and creates a RedactionEntity with correct values, based on the given parameters.
+     *
+     * @param precursorEntity The entity identifier for the RedactionEntity.
+     * @param closestEntity   The closest Boundary to the RedactionEntity.
+     */
+    public static TextEntity createCorrectEntity(PrecursorEntity precursorEntity, TextEntity closestEntity) {
+
+        return createCorrectEntity(precursorEntity, closestEntity, false);
+    }
+
+
+    public static TextEntity createCorrectEntity(PrecursorEntity precursorEntity, TextEntity closestEntity, boolean generateId) {
+
+        TextEntity correctEntity;
+        if (generateId) {
+            correctEntity = TextEntity.initialEntityNode(closestEntity.getTextRange(),
+                                                         precursorEntity.type(),
+                                                         precursorEntity.getEntityType(),
+                                                         closestEntity.getDeepestFullyContainingNode());
+        } else {
+            correctEntity = TextEntity.initialEntityNode(closestEntity.getTextRange(), precursorEntity.type(), precursorEntity.getEntityType(), precursorEntity.getId());
+        }
+        correctEntity.setDeepestFullyContainingNode(closestEntity.getDeepestFullyContainingNode());
+        correctEntity.setIntersectingNodes(new ArrayList<>(closestEntity.getIntersectingNodes()));
+        correctEntity.setDuplicateTextRanges(new ArrayList<>(closestEntity.getDuplicateTextRanges()));
+        correctEntity.setPages(new HashSet<>(closestEntity.getPages()));
+
+        correctEntity.setValue(closestEntity.getValue());
+        correctEntity.setTextAfter(closestEntity.getTextAfter());
+        correctEntity.setTextBefore(closestEntity.getTextBefore());
+
+        correctEntity.getIntersectingNodes()
+                .forEach(n -> n.getEntities().add(correctEntity));
+        correctEntity.getPages()
+                .forEach(page -> page.getEntities().add(correctEntity));
+
+        correctEntity.addMatchedRules(precursorEntity.getMatchedRuleList());
+        correctEntity.setDictionaryEntry(precursorEntity.isDictionaryEntry());
+        correctEntity.setDossierDictionaryEntry(precursorEntity.isDossierDictionaryEntry());
+        correctEntity.getManualOverwrite().addChanges(precursorEntity.getManualOverwrite().getManualChangeLog());
+        correctEntity.addEngines(precursorEntity.getEngines());
+        return correctEntity;
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/ImportedRedactionEntryService.java

@@ -0,0 +1,64 @@
+package com.iqser.red.service.redaction.v1.server.service.document;
+
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.stream.Stream;
+
+import org.springframework.stereotype.Service;
+
+import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedactions;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.BaseAnnotation;
+import com.iqser.red.service.redaction.v1.server.model.PrecursorEntity;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
+
+import io.micrometer.observation.annotation.Observed;
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+@Service
+@RequiredArgsConstructor
+public class ImportedRedactionEntryService {
+
+    private final EntityFromPrecursorCreationService entityFromPrecursorCreationService;
+
+
+    @Observed(name = "ImportedRedactionEntryService", contextualName = "add-imported-redaction-entries")
+    public List<PrecursorEntity> addImportedEntriesAndReturnNotFoundEntries(AnalyzeRequest analyzeRequest, ImportedRedactions importedRedactions, Document document) {
+
+        List<PrecursorEntity> notFoundImportedEntries = Collections.emptyList();
+        if (!importedRedactions.getImportedRedactions().isEmpty()) {
+            notFoundImportedEntries = entityFromPrecursorCreationService.createEntitiesIfFoundAndReturnNotFoundEntries(importedRedactions, document);
+            log.info("Added Imported entries for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
+        }
+
+        if (notFoundImportedEntries.isEmpty()) {
+            return Collections.emptyList();
+        }
+        List<BaseAnnotation> manualChanges = allManualChangesExceptAdd(analyzeRequest.getManualRedactions());
+        for (PrecursorEntity notFoundManualRedactionEntry : notFoundImportedEntries) {
+            manualChanges.stream()
+                    .filter(change -> change.getAnnotationId().equals(notFoundManualRedactionEntry.getId()))
+                    .forEach(change -> notFoundManualRedactionEntry.getManualOverwrite().addChange(change));
+        }
+
+        return notFoundImportedEntries;
+    }
+
+
+    private List<BaseAnnotation> allManualChangesExceptAdd(ManualRedactions manualRedactions) {
+
+        return Stream.of(manualRedactions.getForceRedactions(),
+                         manualRedactions.getResizeRedactions(),
+                         manualRedactions.getRecategorizations(),
+                         manualRedactions.getIdsToRemove(),
+                         manualRedactions.getLegalBasisChanges())
+                .flatMap(Collection::stream)
+                .map(baseAnnotation -> (BaseAnnotation) baseAnnotation)
+                .toList();
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/IntersectingNodeVisitor.java

@@ -0,0 +1,33 @@
+package com.iqser.red.service.redaction.v1.server.service.document;
+
+import java.util.HashSet;
+import java.util.Set;
+
+import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
+
+import lombok.Getter;
+
+public class IntersectingNodeVisitor implements NodeVisitor {
+
+    @Getter
+    private Set<SemanticNode> intersectingNodes;
+    private final TextRange textRange;
+
+
+    public IntersectingNodeVisitor(TextRange textRange) {
+
+        this.textRange = textRange;
+        this.intersectingNodes = new HashSet<>();
+    }
+
+
+    @Override
+    public void visit(SemanticNode node) {
+
+        if (node.getTextRange().intersects(textRange)) {
+            intersectingNodes.add(node);
+        }
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/ManualEntityCreationService.java

@@ -1,115 +0,0 @@
-package com.iqser.red.service.redaction.v1.server.service.document;
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Optional;
-import java.util.Set;
-
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.stereotype.Service;
-
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.BaseAnnotation;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
-import com.iqser.red.service.redaction.v1.server.model.ManualEntity;
-import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
-import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
-import com.iqser.red.service.redaction.v1.server.model.document.entity.PositionOnPage;
-import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
-import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
-import com.iqser.red.service.redaction.v1.server.service.DictionaryService;
-
-import lombok.AccessLevel;
-import lombok.experimental.FieldDefaults;
-import lombok.extern.slf4j.Slf4j;
-
-@Slf4j
-@Service
-@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
-public class ManualEntityCreationService {
-
-    static double MATCH_THRESHOLD = 10; // Is compared to the average sum of distances in pdf coordinates for each corner of the bounding box of the entities
-    EntityFindingUtility entityFindingUtility;
-    EntityCreationService entityCreationService;
-    DictionaryService dictionaryService;
-
-
-    @Autowired
-    public ManualEntityCreationService(EntityEnrichmentService entityEnrichmentService, DictionaryService dictionaryService, EntityFindingUtility entityFindingUtility) {
-
-        this.entityFindingUtility = entityFindingUtility;
-        entityCreationService = new EntityCreationService(entityEnrichmentService);
-        this.dictionaryService = dictionaryService;
-    }
-
-
-    public List<ManualEntity> createRedactionEntitiesIfFoundAndReturnNotFoundEntries(ManualRedactions manualRedactions,
-                                                                                     SemanticNode node,
-                                                                                     String dossierTemplateId) {
-
-        Set<IdRemoval> idRemovals = manualRedactions.getIdsToRemove();
-        List<ManualEntity> manualEntities = manualRedactions.getEntriesToAdd().stream()
-                .filter(manualRedactionEntry -> !(idRemovals.stream().map(BaseAnnotation::getAnnotationId).toList().contains(manualRedactionEntry.getAnnotationId()) &&
-                        manualRedactionEntry.getRequestDate().isBefore(idRemovals.stream().filter(idRemoval -> idRemoval.getAnnotationId().equals(manualRedactionEntry.getAnnotationId())).findFirst().get().getRequestDate())))
-                .filter(manualRedactionEntry -> !(manualRedactionEntry.isAddToDictionary() || manualRedactionEntry.isAddToDossierDictionary()))
-                .map(manualRedactionEntry -> ManualEntity.fromManualRedactionEntry(manualRedactionEntry,
-                        dictionaryService.isHint(manualRedactionEntry.getType(), dossierTemplateId))).peek(manualEntity -> {
-                    if (manualEntity.getEntityType().equals(EntityType.HINT)) {
-                        manualEntity.skip("MAN.5.1", "manual hint is skipped by default");
-                    } else {
-                        manualEntity.apply("MAN.5.0", "manual entries are applied by default", manualEntity.getLegalBasis());
-                    }
-                })
-                .toList();
-
-        return toTextEntity(manualEntities, node);
-    }
-
-
-    public List<ManualEntity> toTextEntity(List<ManualEntity> manualEntities, SemanticNode node) {
-
-        Map<String, List<TextEntity>> tempEntitiesByValue = entityFindingUtility.findAllPossibleEntitiesAndGroupByValue(node, manualEntities);
-
-        List<ManualEntity> notFoundManualEntities = new LinkedList<>();
-        for (ManualEntity manualEntity : manualEntities) {
-            Optional<TextEntity> optionalRedactionEntity = entityFindingUtility.findClosestEntityAndReturnEmptyIfNotFound(manualEntity, tempEntitiesByValue, MATCH_THRESHOLD);
-            if (optionalRedactionEntity.isEmpty()) {
-                notFoundManualEntities.add(manualEntity);
-                continue;
-            }
-            createCorrectEntity(manualEntity, node, optionalRedactionEntity.get().getTextRange());
-        }
-        tempEntitiesByValue.values().stream().flatMap(Collection::stream).forEach(TextEntity::removeFromGraph);
-        return notFoundManualEntities;
-    }
-
-
-    /**
-     * Deletes the temp Entity and creates a RedactionEntity with correct values, based on the given parameters.
-     *
-     * @param manualEntity     The entity identifier for the RedactionEntity.
-     * @param node             The SemanticNode associated with the RedactionEntity.
-     * @param closestTextRange The closest Boundary to the RedactionEntity.
-     */
-    private void createCorrectEntity(ManualEntity manualEntity, SemanticNode node, TextRange closestTextRange) {
-
-        TextEntity correctEntity = entityCreationService.forceByTextRange(closestTextRange, manualEntity.getType(), manualEntity.getEntityType(), node);
-
-        correctEntity.addMatchedRules(manualEntity.getMatchedRuleList());
-        correctEntity.setDictionaryEntry(manualEntity.isDictionaryEntry());
-        correctEntity.setDossierDictionaryEntry(manualEntity.isDossierDictionaryEntry());
-        correctEntity.getManualOverwrite().addChanges(manualEntity.getManualOverwrite().getManualChangeLog());
-
-        // AnnotationIds must match the IDs in the add requests, or comments break. Maybe think about migrating IDs on the fly!
-        List<PositionOnPage> redactionPositionsWithIdOfManualOnPage = new ArrayList<>(correctEntity.getPositionsOnPagePerPage().size());
-        for (PositionOnPage positionOnPage : correctEntity.getPositionsOnPagePerPage()) {
-            redactionPositionsWithIdOfManualOnPage.add(new PositionOnPage(manualEntity.getId(), positionOnPage.getPage(), positionOnPage.getRectanglePerLine()));
-        }
-        correctEntity.setPositionsOnPagePerPage(redactionPositionsWithIdOfManualOnPage);
-    }
-
-}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/ManualRedactionEntryService.java

@@ -11,7 +11,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequ
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.BaseAnnotation;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
-import com.iqser.red.service.redaction.v1.server.model.ManualEntity;
+import com.iqser.red.service.redaction.v1.server.model.PrecursorEntity;
 
 import io.micrometer.observation.annotation.Observed;
 import lombok.RequiredArgsConstructor;
@@ -22,23 +22,25 @@ import lombok.extern.slf4j.Slf4j;
 @RequiredArgsConstructor
 public class ManualRedactionEntryService {
 
-    private final ManualEntityCreationService manualEntityCreationService;
+    private final EntityFromPrecursorCreationService entityFromPrecursorCreationService;
 
 
     @Observed(name = "ManualRedactionEntryService", contextualName = "add-manual-redaction-entries")
-    public List<ManualEntity> addManualRedactionEntriesAndReturnNotFoundEntries(AnalyzeRequest analyzeRequest, Document document, String dossierTemplateId) {
+    public List<PrecursorEntity> addManualRedactionEntriesAndReturnNotFoundEntries(AnalyzeRequest analyzeRequest, Document document, String dossierTemplateId) {
 
-        List<ManualEntity> notFoundManualRedactionEntries = Collections.emptyList();
+        List<PrecursorEntity> notFoundManualRedactionEntries = Collections.emptyList();
         if (analyzeRequest.getManualRedactions() != null) {
-            notFoundManualRedactionEntries = manualEntityCreationService.createRedactionEntitiesIfFoundAndReturnNotFoundEntries(analyzeRequest.getManualRedactions(),
-                    document, dossierTemplateId);
+            notFoundManualRedactionEntries = entityFromPrecursorCreationService.createEntitiesIfFoundAndReturnNotFoundEntries(analyzeRequest.getManualRedactions(),
+                                                                                                                              document,
+                                                                                                                              dossierTemplateId);
             log.info("Added Manual redaction entries for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
         }
+
         if (notFoundManualRedactionEntries.isEmpty()) {
             return Collections.emptyList();
         }
         List<BaseAnnotation> manualChanges = allManualChangesExceptAdd(analyzeRequest.getManualRedactions());
-        for (ManualEntity notFoundManualRedactionEntry : notFoundManualRedactionEntries) {
+        for (PrecursorEntity notFoundManualRedactionEntry : notFoundManualRedactionEntries) {
             manualChanges.stream()
                     .filter(change -> change.getAnnotationId().equals(notFoundManualRedactionEntry.getId()))
                     .forEach(change -> notFoundManualRedactionEntry.getManualOverwrite().addChange(change));
@@ -50,10 +52,13 @@ public class ManualRedactionEntryService {
     private List<BaseAnnotation> allManualChangesExceptAdd(ManualRedactions manualRedactions) {
 
         return Stream.of(manualRedactions.getForceRedactions(),
-                manualRedactions.getResizeRedactions(),
-                manualRedactions.getRecategorizations(),
-                manualRedactions.getIdsToRemove(),
-                manualRedactions.getLegalBasisChanges()).flatMap(Collection::stream).map(baseAnnotation -> (BaseAnnotation) baseAnnotation).toList();
+                         manualRedactions.getResizeRedactions(),
+                         manualRedactions.getRecategorizations(),
+                         manualRedactions.getIdsToRemove(),
+                         manualRedactions.getLegalBasisChanges())
+                .flatMap(Collection::stream)
+                .map(baseAnnotation -> (BaseAnnotation) baseAnnotation)
+                .toList();
     }
 
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/NerEntitiesAdapter.java

@@ -44,9 +44,11 @@ public class NerEntitiesAdapter {
     public NerEntities toNerEntities(NerEntitiesModel nerEntitiesModel, Document document) {
 
         return new NerEntities(addOffsetsAndFlatten(getStringStartOffsetsForMainSections(document),
-                nerEntitiesModel).map(nerEntityModel -> new NerEntities.NerEntity(nerEntityModel.getValue(),
-                new TextRange(nerEntityModel.getStartOffset(), nerEntityModel.getEndOffset()),
-                nerEntityModel.getType())).toList());
+                                                    nerEntitiesModel).map(nerEntityModel -> new NerEntities.NerEntity(nerEntityModel.getValue(),
+                                                                                                                      new TextRange(nerEntityModel.getStartOffset(),
+                                                                                                                                    nerEntityModel.getEndOffset()),
+                                                                                                                      nerEntityModel.getType()))
+                                       .toList());
     }
 
 
@@ -83,7 +85,9 @@ public class NerEntitiesAdapter {
 
         List<List<NerEntities.NerEntity>> entityClusters = new LinkedList<>();
 
-        List<NerEntities.NerEntity> startEntitiesOfEssentialType = sortedEntities.stream().filter(e -> essentialTypes.contains(e.type())).toList();
+        List<NerEntities.NerEntity> startEntitiesOfEssentialType = sortedEntities.stream()
+                .filter(e -> essentialTypes.contains(e.type()))
+                .toList();
         for (NerEntities.NerEntity startEntity : startEntitiesOfEssentialType) {
             List<NerEntities.NerEntity> currentCluster = new LinkedList<>();
             entityClusters.add(currentCluster);
@@ -105,7 +109,10 @@ public class NerEntitiesAdapter {
             }
         }
 
-        return entityClusters.stream().filter(cluster -> cluster.size() >= minPartsToCombine).map(NerEntitiesAdapter::toContainingBoundary).distinct();
+        return entityClusters.stream()
+                .filter(cluster -> cluster.size() >= minPartsToCombine)
+                .map(NerEntitiesAdapter::toContainingBoundary)
+                .distinct();
     }
 
 
@@ -124,17 +131,18 @@ public class NerEntitiesAdapter {
     public Stream<TextRange> combineNerEntitiesToCbiAddressDefaults(NerEntities entityRecognitionEntities) {
 
         return combineNerEntities(entityRecognitionEntities,
-                CBI_ADDRESS_ESSENTIAL_TYPES,
-                CBI_ADDRESS_TYPES_TO_COMBINE,
-                MAX_DISTANCE_BETWEEN_PARTS,
-                MIN_PARTS_TO_COMBINE,
-                ALLOW_DUPLICATES);
+                                  CBI_ADDRESS_ESSENTIAL_TYPES,
+                                  CBI_ADDRESS_TYPES_TO_COMBINE,
+                                  MAX_DISTANCE_BETWEEN_PARTS,
+                                  MIN_PARTS_TO_COMBINE,
+                                  ALLOW_DUPLICATES);
     }
 
 
     private static boolean isDuplicate(List<NerEntities.NerEntity> currentCluster, NerEntities.NerEntity entity, boolean allowDuplicates) {
 
-        return allowDuplicates || currentCluster.stream().anyMatch(e -> e.type().equals(entity.type()));
+        return allowDuplicates || currentCluster.stream()
+                .anyMatch(e -> e.type().equals(entity.type()));
     }
 
 
@@ -146,24 +154,34 @@ public class NerEntitiesAdapter {
 
     private static TextRange toContainingBoundary(List<NerEntities.NerEntity> nerEntities) {
 
-        return TextRange.merge(nerEntities.stream().map(NerEntities.NerEntity::textRange).toList());
+        return TextRange.merge(nerEntities.stream()
+                                       .map(NerEntities.NerEntity::textRange)
+                                       .toList());
     }
 
 
     private static Stream<EntityRecognitionEntity> addOffsetsAndFlatten(List<Integer> stringOffsetsForMainSections, NerEntitiesModel nerEntitiesModel) {
 
-        nerEntitiesModel.getData().forEach((sectionNumber, listOfNerEntities) -> listOfNerEntities.forEach(entityRecognitionEntity -> {
-            int newStartOffset = entityRecognitionEntity.getStartOffset() + stringOffsetsForMainSections.get(sectionNumber);
-            entityRecognitionEntity.setStartOffset(newStartOffset);
-            entityRecognitionEntity.setEndOffset(newStartOffset + entityRecognitionEntity.getValue().length());
-        }));
-        return nerEntitiesModel.getData().values().stream().flatMap(Collection::stream);
+        nerEntitiesModel.getData()
+                .forEach((sectionNumber, listOfNerEntities) -> listOfNerEntities.forEach(entityRecognitionEntity -> {
+                    int newStartOffset = entityRecognitionEntity.getStartOffset() + stringOffsetsForMainSections.get(sectionNumber);
+                    entityRecognitionEntity.setStartOffset(newStartOffset);
+                    entityRecognitionEntity.setEndOffset(newStartOffset + entityRecognitionEntity.getValue().length());
+                }));
+        return nerEntitiesModel.getData().values()
+                .stream()
+                .flatMap(Collection::stream);
     }
 
 
     private static List<Integer> getStringStartOffsetsForMainSections(Document document) {
 
-        return document.getMainSections().stream().map(Section::getTextBlock).map(TextBlock::getTextRange).map(TextRange::start).toList();
+        return document.getMainSections()
+                .stream()
+                .map(Section::getTextBlock)
+                .map(TextBlock::getTextRange)
+                .map(TextRange::start)
+                .toList();
     }
 
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/NodeVisitor.java

@@ -0,0 +1,9 @@
+package com.iqser.red.service.redaction.v1.server.service.document;
+
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
+
+public interface NodeVisitor {
+
+    void visit(SemanticNode node);
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/PropertiesMapper.java

@@ -43,8 +43,29 @@ public class PropertiesMapper {
 
     private Rectangle2D parseRectangle2D(String bBox) {
 
-        List<Float> floats = Arrays.stream(bBox.split(DocumentStructure.RECTANGLE_DELIMITER)).map(Float::parseFloat).toList();
+        List<Float> floats = Arrays.stream(bBox.split(DocumentStructure.RECTANGLE_DELIMITER))
+                .map(Float::parseFloat)
+                .toList();
         return new Rectangle2D.Float(floats.get(0), floats.get(1), floats.get(2), floats.get(3));
     }
 
+
+    public static boolean isDuplicateParagraph(Map<String, String> properties) {
+
+        return properties.containsKey(DocumentStructure.DuplicateParagraphProperties.UNSORTED_TEXTBLOCK_ID);
+    }
+
+
+    public static Long[] getUnsortedTextblockIds(Map<String, String> properties) {
+
+        return toLongArray(properties.get(DocumentStructure.DuplicateParagraphProperties.UNSORTED_TEXTBLOCK_ID));
+    }
+
+
+    public static Long[] toLongArray(String ids) {
+
+        return Arrays.stream(ids.substring(1, ids.length() - 1).trim().split(",")).map(Long::valueOf).toArray(Long[]::new);
+    }
+
+
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/SectionFinderService.java

@@ -11,8 +11,10 @@ import java.util.stream.Stream;
 import org.springframework.stereotype.Service;
 
 import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedaction;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedactions;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;
@@ -40,30 +42,31 @@ import lombok.extern.slf4j.Slf4j;
 public class SectionFinderService {
 
     @Timed("redactmanager_findSectionsToReanalyse")
-    public Set<Integer> findSectionsToReanalyse(DictionaryIncrement dictionaryIncrement, EntityLog entityLog, Document document, AnalyzeRequest analyzeRequest) {
+    public Set<Integer> findSectionsToReanalyse(DictionaryIncrement dictionaryIncrement,
+                                                Document document,
+                                                AnalyzeRequest analyzeRequest,
+                                                ImportedRedactions importedRedactions,
+                                                Set<String> relevantManuallyModifiedAnnotationIds) {
 
         long start = System.currentTimeMillis();
-        Set<String> relevantManuallyModifiedAnnotationIds = getRelevantManuallyModifiedAnnotationIds(analyzeRequest.getManualRedactions());
         Set<Integer> sectionsToReanalyse = new HashSet<>();
-        for (EntityLogEntry entry : entityLog.getEntityLogEntry()) {
-            if (relevantManuallyModifiedAnnotationIds.contains(entry.getId())) {
-                if (entry.getContainingNodeId().isEmpty()) {
-                    continue; // Empty list means either Entity has not been found or it is between main sections. Thus, this might lead to wrong reanalysis.
-                }
-                sectionsToReanalyse.add(entry.getContainingNodeId().get(0));
-            }
-        }
 
-        var dictionaryIncrementsSearch = new SearchImplementation(dictionaryIncrement.getValues().stream().map(DictionaryIncrementValue::getValue).collect(Collectors.toList()),
-                true);
 
-        document.streamChildren().forEach(mainNode -> {
-            if (dictionaryIncrementsSearch.atLeastOneMatches(mainNode.getTextBlock().getSearchText())) {
-                sectionsToReanalyse.add(mainNode.getTreeId().get(0));
-            }
-        });
+        var dictionaryIncrementsSearch = new SearchImplementation(dictionaryIncrement.getValues()
+                                                                          .stream()
+                                                                          .map(DictionaryIncrementValue::getValue)
+                                                                          .toList(), true);
+
+        document.streamChildren()
+                .forEach(mainNode -> {
+                    if (dictionaryIncrementsSearch.atLeastOneMatches(mainNode.getTextBlock().getSearchText())) {
+                        sectionsToReanalyse.add(mainNode.getTreeId()
+                                                        .get(0));
+                    }
+                });
 
         Set<Integer> relevantPagesForReanalysis = getRelevantPageNumbersForAddRedactions(analyzeRequest);
+        relevantPagesForReanalysis.addAll(getRelevantPageNumbersForImportedRedactions(importedRedactions, relevantManuallyModifiedAnnotationIds));
         if (!relevantPagesForReanalysis.isEmpty()) {
             sectionsToReanalyse.addAll(getSectionNumbersOnPages(document, relevantPagesForReanalysis));
         }
@@ -79,8 +82,11 @@ public class SectionFinderService {
         return document.getPages()
                 .stream()
                 .filter(page -> relevantPagesForReanalysis.contains(page.getNumber()))
-                .flatMap(page -> Stream.concat(page.getMainBody().stream().filter(node -> node.getType().equals(NodeType.SECTION)), Stream.of(page.getHeader(), page.getFooter())))
-                .map(node -> node.getTreeId().get(0))
+                .flatMap(page -> Stream.concat(page.getMainBody()
+                                                       .stream()
+                                                       .filter(node -> node.getType().equals(NodeType.SECTION)), Stream.of(page.getHeader(), page.getFooter())))
+                .map(node -> node.getTreeId()
+                        .get(0))
                 .toList();
     }
 
@@ -91,8 +97,7 @@ public class SectionFinderService {
             return Collections.emptySet();
         }
 
-        return analyzeRequest.getManualRedactions()
-                .getEntriesToAdd()
+        return analyzeRequest.getManualRedactions().getEntriesToAdd()
                 .stream()
                 .filter(addRedaction -> !(addRedaction.isAddToDictionary() || addRedaction.isAddToDossierDictionary()))
                 .map(ManualRedactionEntry::getPositions)
@@ -102,17 +107,44 @@ public class SectionFinderService {
     }
 
 
-    private static Set<String> getRelevantManuallyModifiedAnnotationIds(ManualRedactions manualRedactions) {
+    private static Set<Integer> getRelevantPageNumbersForImportedRedactions(ImportedRedactions importedRedactions, Set<String> idsWithUnprocessedManualChanges) {
+
+        if (importedRedactions == null || importedRedactions.getImportedRedactions() == null) {
+            return Collections.emptySet();
+        }
+
+        return importedRedactions.getImportedRedactions()
+                .stream()
+                .filter(importedRedaction -> idsWithUnprocessedManualChanges.contains(importedRedaction.getId()))
+                .map(ImportedRedaction::getPositions)
+                .flatMap(Collection::stream)
+                .map(Position::getPageNumber)
+                .collect(Collectors.toSet());
+    }
+
+
+    public static Set<String> getRelevantManuallyModifiedAnnotationIds(ManualRedactions manualRedactions) {
 
         if (manualRedactions == null) {
             return new HashSet<>();
         }
 
-        return Stream.concat(manualRedactions.getResizeRedactions().stream().map(ManualResizeRedaction::getAnnotationId),
-                Stream.concat(manualRedactions.getLegalBasisChanges().stream().map(ManualLegalBasisChange::getAnnotationId),
-                        Stream.concat(manualRedactions.getRecategorizations().stream().map(ManualRecategorization::getAnnotationId),
-                                Stream.concat(manualRedactions.getIdsToRemove().stream().map(IdRemoval::getAnnotationId),
-                                        manualRedactions.getForceRedactions().stream().map(ManualForceRedaction::getAnnotationId))))).collect(Collectors.toSet());
+        return Stream.concat(manualRedactions.getResizeRedactions()
+                                     .stream()
+                                     .map(ManualResizeRedaction::getAnnotationId),
+                             Stream.concat(manualRedactions.getLegalBasisChanges()
+                                                   .stream()
+                                                   .map(ManualLegalBasisChange::getAnnotationId),
+                                           Stream.concat(manualRedactions.getRecategorizations()
+                                                                 .stream()
+                                                                 .map(ManualRecategorization::getAnnotationId),
+                                                         Stream.concat(manualRedactions.getIdsToRemove()
+                                                                               .stream()
+                                                                               .map(IdRemoval::getAnnotationId),
+                                                                       manualRedactions.getForceRedactions()
+                                                                               .stream()
+                                                                               .map(ManualForceRedaction::getAnnotationId)))))
+                .collect(Collectors.toSet());
     }
 
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/SemanticNodeComparators.java

@@ -12,6 +12,7 @@ public abstract class SemanticNodeComparators implements Comparator<SemanticNode
         return new FirstSemanticNode();
     }
 
+
     public static class FirstSemanticNode extends SemanticNodeComparators {
 
         @Override

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/ComponentDroolsExecutionService.java

@@ -50,7 +50,9 @@ public class ComponentDroolsExecutionService {
                 .filter(entityLogEntry -> entityLogEntry.getState().equals(EntryState.APPLIED))
                 .map(entry -> Entity.fromEntityLogEntry(entry, document))
                 .forEach(kieSession::insert);
-        fileAttributes.stream().filter(f -> f.getValue() != null).forEach(kieSession::insert);
+        fileAttributes.stream()
+                .filter(f -> f.getValue() != null)
+                .forEach(kieSession::insert);
 
         CompletableFuture<Void> completableFuture = CompletableFuture.supplyAsync(() -> {
             kieSession.fireAllRules();
@@ -58,7 +60,8 @@ public class ComponentDroolsExecutionService {
         });
 
         try {
-            completableFuture.orTimeout(settings.getDroolsExecutionTimeoutSecs(), TimeUnit.SECONDS).get();
+            completableFuture.orTimeout(settings.getDroolsExecutionTimeoutSecs(), TimeUnit.SECONDS)
+                    .get();
         } catch (ExecutionException e) {
             kieSession.dispose();
             if (e.getCause() instanceof TimeoutException) {
@@ -71,7 +74,9 @@ public class ComponentDroolsExecutionService {
         }
 
         List<FileAttribute> resultingFileAttributes = getFileAttributes(kieSession);
-        List<Component> components = getComponents(kieSession).stream().sorted(ComponentComparator.first()).toList();
+        List<Component> components = getComponents(kieSession).stream()
+                .sorted(ComponentComparator.first())
+                .toList();
         kieSession.dispose();
         return components;
     }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/DroolsSyntaxValidationService.java

@@ -1,147 +0,0 @@
-package com.iqser.red.service.redaction.v1.server.service.drools;
-
-import java.util.List;
-import java.util.Set;
-import java.util.regex.Pattern;
-import java.util.stream.Collectors;
-
-import org.drools.drl.parser.DroolsParserException;
-import org.kie.api.builder.KieBuilder;
-import org.kie.api.builder.Message;
-import org.springframework.stereotype.Service;
-
-import com.google.common.collect.Sets;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType;
-import com.iqser.red.service.redaction.v1.model.DroolsSyntaxErrorMessage;
-import com.iqser.red.service.redaction.v1.model.DroolsSyntaxValidation;
-import com.iqser.red.service.redaction.v1.model.RuleValidationModel;
-import com.iqser.red.service.redaction.v1.server.model.drools.BasicQuery;
-import com.iqser.red.service.redaction.v1.server.model.drools.RuleFileBluePrint;
-import com.iqser.red.service.redaction.v1.server.storage.RuleManagementResources;
-
-import lombok.RequiredArgsConstructor;
-import lombok.SneakyThrows;
-
-@Service
-@RequiredArgsConstructor
-public class DroolsSyntaxValidationService {
-    
-    private static final Pattern allowedImportsPattern = Pattern.compile("^(?:import\\s+static\\s+)?(?:import\\s+)?(?:com\\.knecon\\.fforesight|com\\.iqser\\.red)\\..*;$");
-
-    private final KieContainerCreationService kieContainerCreationService;
-
-
-    @SneakyThrows
-    public DroolsSyntaxValidation testRules(RuleValidationModel rules) {
-
-        DroolsSyntaxValidation customDroolsSyntaxValidation;
-        try {
-            customDroolsSyntaxValidation = buildCustomDroolsSyntaxValidation(rules.getRulesString(), RuleFileType.valueOf(rules.getRuleFileType()));
-        } catch (DroolsParserException e) {
-            // this means the parser could not parse the file at all. In this case use drools compiler only as it will return useful error messages.
-            customDroolsSyntaxValidation = new DroolsSyntaxValidation();
-        }
-        DroolsSyntaxValidation droolsCompilerSyntaxValidation = buildDroolsCompilerSyntaxValidation(rules);
-        droolsCompilerSyntaxValidation.getDroolsSyntaxErrorMessages().addAll(customDroolsSyntaxValidation.getDroolsSyntaxErrorMessages());
-        return droolsCompilerSyntaxValidation;
-    }
-
-
-    private DroolsSyntaxValidation buildCustomDroolsSyntaxValidation(String ruleString, RuleFileType ruleFileType) throws DroolsParserException {
-
-        RuleFileBluePrint ruleFileBluePrint = RuleFileParser.buildBluePrintFromRulesString(ruleString);
-        DroolsSyntaxValidation customSyntaxValidation = ruleFileBluePrint.getDroolsSyntaxValidation();
-
-        RuleFileBluePrint baseRuleFileBluePrint = switch (ruleFileType) {
-            case ENTITY -> RuleFileParser.buildBluePrintFromRulesString(RuleManagementResources.getBaseRuleFileString());
-            case COMPONENT -> RuleFileParser.buildBluePrintFromRulesString(RuleManagementResources.getBaseComponentRuleFileString());
-        };
-
-        if (!importsAreValid(baseRuleFileBluePrint, ruleFileBluePrint)) {
-            customSyntaxValidation.getDroolsSyntaxErrorMessages()
-                    .add(DroolsSyntaxErrorMessage.builder()
-                            .line(ruleFileBluePrint.getImportLine())
-                            .column(0)
-                            .message(String.format("Changing the imports is not allowed! Must be: %n%s", baseRuleFileBluePrint.getImports()))
-                            .build());
-        }
-        if (!ruleFileBluePrint.getGlobals().equals(baseRuleFileBluePrint.getGlobals())) {
-            customSyntaxValidation.getDroolsSyntaxErrorMessages()
-                    .add(DroolsSyntaxErrorMessage.builder()
-                            .line(ruleFileBluePrint.getGlobalsLine())
-                            .column(0)
-                            .message(String.format("Changing the globals is not allowed! Must be: %n%s", baseRuleFileBluePrint.getGlobals()))
-                            .build());
-        }
-        baseRuleFileBluePrint.getQueries().forEach(basicQuery -> {
-            if (!validateQueryIsPresent(basicQuery, ruleFileBluePrint)) {
-                customSyntaxValidation.getDroolsSyntaxErrorMessages()
-                        .add(DroolsSyntaxErrorMessage.builder()
-                                .line(basicQuery.getLine())
-                                .column(0)
-                                .message(String.format("Changing or removing the query %s is not allowed! Must be: %n%s", basicQuery.getName(), basicQuery.getCode()))
-                                .build());
-            }
-        });
-        if (ruleFileType.equals(RuleFileType.ENTITY)) {
-            String requiredAgendaGroup = "LOCAL_DICTIONARY_ADDS";
-            if (!validateAgendaGroupIsPresent(ruleFileBluePrint, requiredAgendaGroup)) {
-                customSyntaxValidation.getDroolsSyntaxErrorMessages().add(DroolsSyntaxErrorMessage.builder().line(0)
-                                .column(0).message(String.format("At least one rule with Agenda-Group '%s' required!", requiredAgendaGroup))
-                                .build());
-            }
-        }
-        return customSyntaxValidation;
-    }
-
-
-    private boolean validateAgendaGroupIsPresent(RuleFileBluePrint ruleFileBluePrint, String agendaGroupName) {
-
-        return ruleFileBluePrint.streamAllRules().anyMatch(basicRule -> basicRule.getAgendaGroup().equals(agendaGroupName));
-    }
-
-
-    private boolean importsAreValid(RuleFileBluePrint baseRuleFileBluePrint, RuleFileBluePrint ruleFileBluePrint) {
-
-        // imports may shrink, but not add anything new!
-        Set<String> baseImports = baseRuleFileBluePrint.getImportSplitByKeyword();
-        Set<String> imports = ruleFileBluePrint.getImportSplitByKeyword();
-
-        Set<String> additionalImports = Sets.difference(imports, baseImports);
-
-        return additionalImports.stream().allMatch(additionalImport -> allowedImportsPattern.matcher(additionalImport).matches());
-    }
-
-
-    private static boolean validateQueryIsPresent(BasicQuery queryToCheckFor, RuleFileBluePrint ruleFileBluePrint) {
-
-        return ruleFileBluePrint.getQueries().stream().anyMatch(query -> query.getName().equals(queryToCheckFor.getName()) && query.getCode().equals(queryToCheckFor.getCode()));
-    }
-
-
-    private DroolsSyntaxValidation buildDroolsCompilerSyntaxValidation(RuleValidationModel rules) {
-
-        var versionId = System.currentTimeMillis();
-        var testRules = "test-rules";
-        KieBuilder kieBuilder = kieContainerCreationService.registerNewKieContainerVersion(testRules,
-                versionId,
-                rules.getRulesString(),
-                RuleFileType.valueOf(rules.getRuleFileType()));
-        return buildDroolsCompilerSyntaxValidation(kieBuilder);
-    }
-
-
-    private DroolsSyntaxValidation buildDroolsCompilerSyntaxValidation(KieBuilder kieBuilder) {
-
-        List<Message> errorMessages = kieBuilder.getResults().getMessages(Message.Level.ERROR);
-        List<DroolsSyntaxErrorMessage> droolsSyntaxErrorMessages = errorMessages.stream().map(this::buildDroolsSyntaxErrorMessage).collect(Collectors.toList());
-        return DroolsSyntaxValidation.builder().droolsSyntaxErrorMessages(droolsSyntaxErrorMessages).build();
-    }
-
-
-    private DroolsSyntaxErrorMessage buildDroolsSyntaxErrorMessage(Message message) {
-
-        return DroolsSyntaxErrorMessage.builder().line(message.getLine()).column(message.getColumn()).message(message.getText()).build();
-    }
-
-}