DM-285: Entity to component mappings with new ruleset

* refactoring structure
* added custom syntax validation, such that rule files may not be corrupted
* added component drools preparations
* deprecated redactionLog
* integrated entityLog
* integrated new getRules with ruleFileType
* renamed service.graph package -> service.document
* fixed tests
* update layout parser version
* initial component drools file - wip
* disabled some tests due to OOM Error

Signed-off-by: Kilian Schuettler <kilian.schuettler@knecon.com>

.gitlab-ci.yml

@@ -12,7 +12,7 @@ deploy:
   script:
     - echo "Building with gradle version ${BUILDVERSION}"
     - gradle -Pversion=${BUILDVERSION} publish
-    - gradle bootBuildImage --publishImage -PbuildbootDockerHostNetwork=true -Pversion=${BUILDVERSION}
+    - gradle bootBuildImage --cleanCache --publishImage -PbuildbootDockerHostNetwork=true -Pversion=${BUILDVERSION}
     - echo "BUILDVERSION=$BUILDVERSION" >> version.env
   artifacts:
     reports:
@@ -21,16 +21,3 @@ deploy:
     - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
     - if: $CI_COMMIT_BRANCH =~ /^release/
     - if: $CI_COMMIT_TAG
-
-pages:
-  stage: build
-  tags:
-    - dind
-  script:
-    - echo "Generating Javadoc..."
-    - gradle generateJavaDoc -PjavadocDestinationDir="javadoc"
-    - mkdir public
-    - mv redaction-service-v1/redaction-service-server-v1/javadoc/* public/
-  artifacts:
-    paths:
-      - public

buildSrc/src/main/kotlin/com.iqser.red.service.java-conventions.gradle.kts

@@ -11,10 +11,6 @@ group = "com.iqser.red.service"
 java.sourceCompatibility = JavaVersion.VERSION_17
 java.targetCompatibility = JavaVersion.VERSION_17
 
-pmd {
-    isConsoleOutput = true
-}
-
 tasks.pmdMain {
     pmd.ruleSetFiles = files("${rootDir}/config/pmd/pmd.xml")
 }

config/pmd/pmd.xml

@@ -9,12 +9,13 @@
     </description>
 
     <rule ref="category/java/errorprone.xml">
+        <exclude name="DataflowAnomalyAnalysis"/>
         <exclude name="MissingSerialVersionUID"/>
+        <exclude name="NullAssignment"/>
         <exclude name="AvoidLiteralsInIfCondition"/>
         <exclude name="AvoidDuplicateLiterals"/>
-        <exclude name="NullAssignment"/>
+        <exclude name="AvoidFieldNameMatchingMethodName"/>
         <exclude name="AssignmentInOperand"/>
-        <exclude name="BeanMembersShouldSerialize"/>
     </rule>
 
-</ruleset>
+</ruleset>

config/pmd/test_pmd.xml

@@ -10,13 +10,15 @@
 
 
     <rule ref="category/java/errorprone.xml">
+        <exclude name="DataflowAnomalyAnalysis"/>
         <exclude name="MissingSerialVersionUID"/>
+        <exclude name="NullAssignment"/>
         <exclude name="AvoidLiteralsInIfCondition"/>
         <exclude name="AvoidDuplicateLiterals"/>
-        <exclude name="NullAssignment"/>
+        <exclude name="AvoidFieldNameMatchingMethodName"/>
+        <exclude name="AvoidFieldNameMatchingTypeName"/>
         <exclude name="AssignmentInOperand"/>
         <exclude name="TestClassWithoutTestCases"/>
-        <exclude name="BeanMembersShouldSerialize"/>
     </rule>
 
-</ruleset>
+</ruleset>

publish-custom-image.sh

@@ -9,7 +9,7 @@ branch=$(git rev-parse --abbrev-ref HEAD)
 commit_hash=$(git rev-parse --short=5 HEAD)
 
 # Combine branch and commit hash
-buildName="${USER}-${branch}-${commit_hash}"
+buildName="${branch}-${commit_hash}"
 
 gradle bootBuildImage --cleanCache --publishImage -PbuildbootDockerHostNetwork=true -Pversion=$buildName
 echo "nexus.knecon.com:5001/red/${dir}-server-v1:$buildName"

redaction-service-v1/redaction-service-api-v1/build.gradle.kts

@@ -1,13 +1,13 @@
 plugins {
     id("com.iqser.red.service.java-conventions")
-    id("io.freefair.lombok") version "8.4"
+    id("io.freefair.lombok") version "8.2.2"
 }
 
 description = "redaction-service-api-v1"
 
 dependencies {
-    implementation("org.springframework:spring-web:6.0.12")
-    implementation("com.iqser.red.service:persistence-service-internal-api-v1:2.351.0")
+    implementation("org.springframework:spring-web:6.0.11")
+    implementation("com.iqser.red.service:persistence-service-internal-api-v1:RED-6725")
 }
 
 publishing {

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/AnalyzeResponse.java

@@ -1,18 +0,0 @@
-package com.iqser.red.service.redaction.v1.model;
-
-import java.util.List;
-
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.Data;
-import lombok.NoArgsConstructor;
-
-@Data
-@Builder
-@NoArgsConstructor
-@AllArgsConstructor
-public class AnalyzeResponse {
-
-    private String fileId;
-    private List<UnprocessedManualEntity> unprocessedManualEntities;
-}

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/DroolsSyntaxDeprecatedWarnings.java

@@ -1,20 +0,0 @@
-package com.iqser.red.service.redaction.v1.model;
-
-import lombok.AccessLevel;
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.Data;
-import lombok.NoArgsConstructor;
-import lombok.experimental.FieldDefaults;
-
-@Data
-@Builder
-@AllArgsConstructor
-@NoArgsConstructor
-@FieldDefaults(level = AccessLevel.PRIVATE)
-public class DroolsSyntaxDeprecatedWarnings {
-
-    Integer line;
-    Integer column;
-    String message;
-}

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/DroolsSyntaxValidation.java

@@ -19,8 +19,6 @@ public class DroolsSyntaxValidation {
 
     @Builder.Default
     List<DroolsSyntaxErrorMessage> droolsSyntaxErrorMessages = new LinkedList<>();
-    @Builder.Default
-    List<DroolsSyntaxDeprecatedWarnings> droolsSyntaxDeprecatedWarnings = new LinkedList<>();
 
 
     public void addErrorMessage(int line, int column, String message) {

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/MigrationRequest.java

@@ -1,21 +0,0 @@
-package com.iqser.red.service.redaction.v1.model;
-
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
-
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.Data;
-import lombok.NoArgsConstructor;
-
-@Data
-@Builder
-@NoArgsConstructor
-@AllArgsConstructor
-public class MigrationRequest {
-
-    String dossierTemplateId;
-    String dossierId;
-    String fileId;
-    ManualRedactions manualRedactions;
-
-}

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/MigrationResponse.java

@@ -1,17 +0,0 @@
-package com.iqser.red.service.redaction.v1.model;
-
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.Data;
-import lombok.NoArgsConstructor;
-
-@Data
-@Builder
-@NoArgsConstructor
-@AllArgsConstructor
-public class MigrationResponse {
-
-    String dossierId;
-    String fileId;
-
-}

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/QueueNames.java

@@ -1,14 +0,0 @@
-package com.iqser.red.service.redaction.v1.model;
-
-public class QueueNames {
-
-    public static final String REDACTION_QUEUE = "redactionQueue";
-    public static final String REDACTION_PRIORITY_QUEUE = "redactionPriorityQueue";
-    public static final String REDACTION_ANALYSIS_RESPONSE_QUEUE = "redactionAnalysisResponseQueue";
-    public static final String REDACTION_DQL = "redactionDQL";
-
-    public static final String MIGRATION_QUEUE = "migrationQueue";
-    public static final String MIGRATION_RESPONSE_QUEUE = "migrationResponseQueue";
-    public static final String MIGRATION_DLQ = "migrationDLQ";
-
-}

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RuleValidationModel.java

@@ -1,18 +1,14 @@
 package com.iqser.red.service.redaction.v1.model;
 
-import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
 import lombok.Data;
 import lombok.NoArgsConstructor;
-import lombok.experimental.FieldDefaults;
 
 @Data
 @AllArgsConstructor
 @NoArgsConstructor
-@FieldDefaults(level = AccessLevel.PRIVATE)
 public class RuleValidationModel {
 
     String ruleFileType;
     String rulesString;
-
 }

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/UnprocessedManualEntity.java

@@ -1,27 +0,0 @@
-package com.iqser.red.service.redaction.v1.model;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position;
-
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.Data;
-import lombok.EqualsAndHashCode;
-import lombok.NoArgsConstructor;
-
-@Data
-@AllArgsConstructor
-@NoArgsConstructor
-@Builder
-@EqualsAndHashCode
-public class UnprocessedManualEntity {
-
-    private String annotationId;
-    private String textBefore;
-    private String textAfter;
-    private String section;
-    @Builder.Default
-    private List<Position> positions = new ArrayList<>();
-}

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/resources/RedactionResource.java

@@ -10,6 +10,6 @@ import com.iqser.red.service.redaction.v1.model.RuleValidationModel;
 public interface RedactionResource {
 
     @PostMapping(value = "/rules/test", consumes = MediaType.APPLICATION_JSON_VALUE)
-    DroolsSyntaxValidation testRules(@RequestBody RuleValidationModel rulesValidationModel);
+    DroolsSyntaxValidation testRules(@RequestBody RuleValidationModel rules);
 
 }

redaction-service-v1/redaction-service-server-v1/build.gradle.kts

@@ -3,27 +3,24 @@ import org.springframework.boot.gradle.tasks.bundling.BootBuildImage
 plugins {
     application
     id("com.iqser.red.service.java-conventions")
-    id("org.springframework.boot") version "3.1.5"
+    id("org.springframework.boot") version "3.1.2"
     id("io.spring.dependency-management") version "1.1.3"
-    id("org.sonarqube") version "4.4.1.3373"
-    id("io.freefair.lombok") version "8.4"
+    id("org.sonarqube") version "4.3.0.3225"
+    id("io.freefair.lombok") version "8.2.2"
 }
 
 description = "redaction-service-server-v1"
 
 
-val layoutParserVersion = "0.94.0"
+val layoutParserVersion = "0.70.0"
 val jacksonVersion = "2.15.2"
-val droolsVersion = "9.44.0.Final"
+val droolsVersion = "8.43.0.Final"
 val pdfBoxVersion = "3.0.0"
-val persistenceServiceVersion = "2.360.0"
-val springBootStarterVersion = "3.1.5"
+val persistenceServiceVersion = "DM-285-5"
 
 configurations {
     all {
-        exclude(group = "commons-logging", module = "commons-logging")
-        exclude(group = "org.springframework.boot", module = "spring-boot-starter-log4j2")
-        exclude(group = "com.iqser.red.commons", module = "logging-commons")
+        exclude(group = "org.springframework.boot", module = "spring-boot-starter-logging")
     }
 }
 
@@ -33,13 +30,12 @@ dependencies {
     implementation("com.iqser.red.service:persistence-service-internal-api-v1:${persistenceServiceVersion}") { exclude(group = "org.springframework.boot") }
     implementation("com.knecon.fforesight:layoutparser-service-internal-api:${layoutParserVersion}")
 
-    implementation("com.iqser.red.commons:spring-commons:6.2.0")
+    implementation("com.iqser.red.commons:spring-commons:2.7.0")
     implementation("com.iqser.red.commons:metric-commons:2.3.0")
 
     implementation("com.iqser.red.commons:dictionary-merge-commons:1.5.0")
-    implementation("com.iqser.red.commons:storage-commons:2.45.0")
-    implementation("com.knecon.fforesight:tenant-commons:0.21.0")
-    implementation("com.knecon.fforesight:tracing-commons:0.5.0")
+    implementation("com.iqser.red.commons:storage-commons:2.43.0")
+    implementation("com.knecon.fforesight:tenant-commons:0.13.0")
 
     implementation("com.fasterxml.jackson.module:jackson-module-afterburner:${jacksonVersion}")
     implementation("com.fasterxml.jackson.datatype:jackson-datatype-jsr310:${jacksonVersion}")
@@ -53,21 +49,12 @@ dependencies {
     implementation("org.locationtech.jts:jts-core:1.19.0")
 
     implementation("org.springframework.cloud:spring-cloud-starter-openfeign:4.0.4")
-    implementation("org.springframework.boot:spring-boot-starter-amqp:${springBootStarterVersion}")
-    implementation("org.springframework.boot:spring-boot-starter-cache:${springBootStarterVersion}")
-    implementation("org.springframework.boot:spring-boot-starter-data-redis:${springBootStarterVersion}")
+    implementation("org.springframework.boot:spring-boot-starter-amqp:3.1.2")
 
-    implementation("net.logstash.logback:logstash-logback-encoder:7.4")
-    implementation("ch.qos.logback:logback-classic")
-
-    implementation("org.reflections:reflections:0.10.2")
-
-    testImplementation(project(":rules-management"))
     testImplementation("org.apache.pdfbox:pdfbox:${pdfBoxVersion}")
     testImplementation("org.apache.pdfbox:pdfbox-tools:${pdfBoxVersion}")
 
-    testImplementation("org.springframework.boot:spring-boot-starter-test:${springBootStarterVersion}")
-    testImplementation("com.knecon.fforesight:viewer-doc-processor:${layoutParserVersion}")
+    testImplementation("org.springframework.boot:spring-boot-starter-test:3.1.2")
     testImplementation("com.knecon.fforesight:layoutparser-service-processor:${layoutParserVersion}") {
         exclude(
             group = "com.iqser.red.service",
@@ -80,16 +67,15 @@ tasks.test {
     configure<JacocoTaskExtension> {
         excludes = listOf("org/drools/**/*")
     }
-    minHeapSize = "512m"
-    maxHeapSize = "1024m"
 }
 
+
 tasks.named<BootBuildImage>("bootBuildImage") {
 
     environment.put("BPE_DELIM_JAVA_TOOL_OPTIONS", " ")
-    environment.put("BPE_APPEND_JAVA_TOOL_OPTIONS", "-XX:MaxMetaspaceSize=512m -Dfile.encoding=UTF-8")
+    environment.put("BPE_APPEND_JAVA_TOOL_OPTIONS", "-Dfile.encoding=UTF-8")
 
-    imageName.set("nexus.knecon.com:5001/red/${project.name}")// must build image with same name always, otherwise the builder will not know which image to use as cache. DO NOT CHANGE!
+    imageName.set("nexus.knecon.com:5001/red/${project.name}:${project.version}")
     if (project.hasProperty("buildbootDockerHostNetwork")) {
         network.set("host")
     }
@@ -105,45 +91,5 @@ tasks.named<BootBuildImage>("bootBuildImage") {
             email.set(providers.gradleProperty("mavenEmail").getOrNull())
             url.set("https://nexus.knecon.com:5001/")
         }
-
-        val dockerTag = "nexus.knecon.com:5001/red/${project.name}:${project.version}"
-        tags.set(listOf(dockerTag))
-    }
-}
-
-fun parseDroolsImports(droolsFilePath: String): List<String> {
-    val imports = mutableListOf<String>()
-    val importPattern = Regex("^import\\s+(com\\.iqser\\.red\\.service\\.redaction\\.v1\\.[\\w.]+);")
-    val desiredPrefix = "com.iqser.red.service.redaction.v1"
-
-    File(droolsFilePath).forEachLine { line ->
-        importPattern.find(line)?.let { matchResult ->
-            val importPath = matchResult.groupValues[1].trim()
-            if (importPath.startsWith(desiredPrefix)) {
-                val formattedPath = importPath.replace('.', '/')
-                imports.add("$formattedPath.java")
-            }
-        }
-    }
-
-    return imports
-}
-
-val droolsImports = parseDroolsImports("redaction-service-v1/redaction-service-server-v1/src/main/resources/drools/all_rules_documine.drl")
-
-tasks.register("generateJavaDoc", Javadoc::class) {
-    dependsOn("compileJava")
-    dependsOn("delombok")
-    classpath = project.sourceSets["main"].runtimeClasspath
-    source = fileTree("${buildDir}/generated/sources/delombok/java/main") {
-        include(droolsImports)
-    }
-    destinationDir = file(project.findProperty("javadocDestinationDir")?.toString() ?: "")
-
-    options.memberLevel = JavadocMemberLevel.PUBLIC
-    (options as StandardJavadocDocletOptions).apply {
-        header = "Redaction Service ${project.version}"
-        footer = "Redaction Service ${project.version}"
-        title = "API Documentation for Redaction Service ${project.version}"
     }
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/Application.java

@@ -6,7 +6,6 @@ import org.springframework.boot.autoconfigure.ImportAutoConfiguration;
 import org.springframework.boot.autoconfigure.SpringBootApplication;
 import org.springframework.boot.autoconfigure.security.servlet.SecurityAutoConfiguration;
 import org.springframework.boot.context.properties.EnableConfigurationProperties;
-import org.springframework.cache.annotation.EnableCaching;
 import org.springframework.cloud.openfeign.EnableFeignClients;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Import;
@@ -18,10 +17,7 @@ import com.knecon.fforesight.tenantcommons.MultiTenancyAutoConfiguration;
 
 import io.micrometer.core.aop.TimedAspect;
 import io.micrometer.core.instrument.MeterRegistry;
-import io.micrometer.observation.ObservationRegistry;
-import io.micrometer.observation.aop.ObservedAspect;
 
-@EnableCaching
 @ImportAutoConfiguration({MultiTenancyAutoConfiguration.class})
 @Import({MetricsConfiguration.class, StorageAutoConfiguration.class})
 @EnableFeignClients(basePackageClasses = RulesClient.class)
@@ -35,10 +31,6 @@ public class Application {
         SpringApplication.run(Application.class, args);
     }
 
-    @Bean
-    public ObservedAspect observedAspect(ObservationRegistry observationRegistry) {
-        return new ObservedAspect(observationRegistry);
-    }
 
     @Bean
     public TimedAspect timedAspect(MeterRegistry registry) {
@@ -46,18 +38,10 @@ public class Application {
         return new TimedAspect(registry);
     }
 
-
     @Bean
     public DictionaryMergeService dictionaryMergeService() {
 
         return new DictionaryMergeService();
     }
 
-
-    @Bean(initMethod = "findDeprecatedMethods")
-    public DeprecatedElementsFinder deprecatedElementsFinder() {
-
-        return new DeprecatedElementsFinder();
-    }
-
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/DeprecatedElementsFinder.java

@@ -1,108 +0,0 @@
-package com.iqser.red.service.redaction.v1.server;
-
-import static java.util.stream.Collectors.groupingBy;
-
-import java.lang.reflect.Method;
-import java.lang.reflect.Type;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.stream.Collectors;
-
-import org.reflections.Reflections;
-import org.reflections.scanners.Scanners;
-import org.reflections.util.ConfigurationBuilder;
-
-import com.iqser.red.service.redaction.v1.server.model.dictionary.SearchImplementation;
-
-import lombok.Getter;
-import lombok.extern.slf4j.Slf4j;
-
-@Slf4j
-public class DeprecatedElementsFinder {
-
-    public static final String PACKAGE_NAME = "com.iqser.red.service.redaction.v1.server";
-    private Set<Method> deprecatedMethods;
-    @Getter
-    private Map<String, String> deprecatedMethodsSignaturesMap;
-    private Set<Class<?>> deprecatedClasses;
-    @Getter
-    private SearchImplementation classesSearchImplementation;
-    @Getter
-    private SearchImplementation methodsSearchImplementation;
-
-
-    public void findDeprecatedMethods() {
-
-        deprecatedMethods = new HashSet<>();
-        deprecatedClasses = new HashSet<>();
-        deprecatedMethodsSignaturesMap = new HashMap<>();
-
-        Reflections reflections = new Reflections(new ConfigurationBuilder().forPackage(PACKAGE_NAME)
-                                                          .setExpandSuperTypes(true)
-                                                          .setScanners(Scanners.MethodsAnnotated, Scanners.TypesAnnotated, Scanners.SubTypes));
-
-        deprecatedMethods = reflections.get(Scanners.MethodsAnnotated.with(Deprecated.class).as(Method.class));
-
-        deprecatedClasses = reflections.get(Scanners.TypesAnnotated.with(Deprecated.class).asClass());
-
-        if (!deprecatedClasses.isEmpty()) {
-            deprecatedClasses.forEach(deprecatedType -> deprecatedMethods.addAll(Arrays.stream(deprecatedType.getDeclaredMethods())
-                                                                                         .collect(Collectors.toSet())));
-            classesSearchImplementation = new SearchImplementation(deprecatedClasses.stream()
-                                                                           .map(Class::getName)
-                                                                           .collect(Collectors.toSet()), false);
-        }
-
-        if (!deprecatedMethods.isEmpty()) {
-            methodsSearchImplementation = new SearchImplementation(deprecatedMethods.stream()
-                                                                           .map(Method::getName)
-                                                                           .collect(Collectors.toSet()), false);
-            deprecatedMethodsSignaturesMap = new HashMap<>();
-            Map<String, List<Method>> deprecatedMethodsNameMap = deprecatedMethods.stream()
-                    .collect(groupingBy(Method::getName));
-            for (String deprecatedMethod : deprecatedMethodsNameMap.keySet()) {
-                deprecatedMethodsSignaturesMap.computeIfAbsent(deprecatedMethod,
-                                                               key -> deprecatedMethodsNameMap.get(deprecatedMethod)
-                                                                       .stream()
-                                                                       .map(this::getMethodSignature)
-                                                                       .collect(Collectors.joining("\n")));
-            }
-
-        }
-
-        log.info("For package: {} found deprecated classes {} deprecated methods {}", PACKAGE_NAME, deprecatedClasses.size(), deprecatedMethods.size());
-    }
-
-
-    public Set<Method> getDeprecatedMethods() {
-
-        if (this.deprecatedMethods == null) {
-            findDeprecatedMethods();
-        }
-        return this.deprecatedMethods;
-    }
-
-
-    public Set<Class<?>> getDeprecatedClasses() {
-
-        if (this.deprecatedClasses == null) {
-            findDeprecatedMethods();
-        }
-        return this.deprecatedClasses;
-    }
-
-    private String getMethodSignature(Method method) {
-
-        String methodName = method.getName();
-        String parameterTypes = Arrays.stream(method.getGenericParameterTypes())
-                .map(Type::getTypeName)
-                .collect(Collectors.joining(", "));
-
-        return method.getGenericReturnType() + " " + methodName + "(" + parameterTypes + ")";
-    }
-
-}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/RedisCachingConfiguration.java

@@ -1,31 +0,0 @@
-package com.iqser.red.service.redaction.v1.server;
-
-import java.time.Duration;
-
-import org.springframework.boot.autoconfigure.cache.RedisCacheManagerBuilderCustomizer;
-import org.springframework.context.annotation.Bean;
-import org.springframework.context.annotation.Configuration;
-import org.springframework.data.redis.cache.RedisCacheConfiguration;
-import org.springframework.data.redis.connection.RedisConnectionFactory;
-import org.springframework.data.redis.core.RedisTemplate;
-
-@Configuration
-public class RedisCachingConfiguration {
-
-    @Bean
-    public RedisCacheManagerBuilderCustomizer redisCacheManagerBuilderCustomizer() {
-
-        return (builder) -> builder.withCacheConfiguration("documentDataCache",
-                RedisCacheConfiguration.defaultCacheConfig().entryTtl(Duration.ofMinutes(30)).disableCachingNullValues());
-    }
-
-
-    @Bean
-    public RedisTemplate<String, Object> redisTemplate(RedisConnectionFactory connectionFactory) {
-
-        RedisTemplate<String, Object> template = new RedisTemplate<>();
-        template.setConnectionFactory(connectionFactory);
-        return template;
-    }
-
-}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RedactionController.java

@@ -21,10 +21,10 @@ public class RedactionController implements RedactionResource {
 
 
     @Override
-    public DroolsSyntaxValidation testRules(@RequestBody RuleValidationModel rulesValidationModel) {
+    public DroolsSyntaxValidation testRules(@RequestBody RuleValidationModel rules) {
 
         try {
-            return droolsSyntaxValidationService.testRules(rulesValidationModel);
+            return droolsSyntaxValidationService.testRules(rules);
         } catch (Exception e) {
             throw new RulesValidationException("Could not test rules: " + e.getMessage(), e);
         }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/migration/LegacyRedactionLogMergeService.java

@@ -1,428 +0,0 @@
-package com.iqser.red.service.redaction.v1.server.migration;
-
-import java.time.OffsetDateTime;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-import java.util.stream.Collectors;
-
-import org.springframework.stereotype.Service;
-
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Change;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ChangeType;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ManualChange;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ManualRedactionType;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Point;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
-import com.iqser.red.service.redaction.v1.server.service.DictionaryService;
-
-import lombok.AllArgsConstructor;
-import lombok.Data;
-import lombok.RequiredArgsConstructor;
-import lombok.extern.slf4j.Slf4j;
-
-@Slf4j
-@Service
-@RequiredArgsConstructor
-public class LegacyRedactionLogMergeService {
-
-    private final DictionaryService dictionaryService;
-
-
-    public RedactionLog addManualAddEntriesAndRemoveSkippedImported(RedactionLog redactionLog, ManualRedactions manualRedactions, String dossierTemplateId) {
-
-        Set<String> skippedImportedRedactions = new HashSet<>();
-        log.info("Adding manual add Entries and removing skipped or imported entries");
-        if (manualRedactions != null) {
-
-            var manualRedactionLogEntries = addManualAddEntries(manualRedactions.getEntriesToAdd(), redactionLog.getAnalysisNumber());
-
-            redactionLog.getRedactionLogEntry().addAll(manualRedactionLogEntries);
-
-            var manualRedactionWrappers = createManualRedactionWrappers(manualRedactions);
-
-            for (RedactionLogEntry entry : redactionLog.getRedactionLogEntry()) {
-
-                if (entry.isImported()) {
-                    processRedactionLogEntry(manualRedactionWrappers.stream()
-                                                     .filter(ManualRedactionWrapper::isApproved)
-                                                     .filter(mr -> entry.getId().equals(mr.getId()))
-                                                     .collect(Collectors.toList()), entry, dossierTemplateId);
-                    if (!entry.isRedacted()) {
-                        skippedImportedRedactions.add(entry.getId());
-                    }
-                }
-
-            }
-
-        }
-
-        Set<String> processedIds = new HashSet<>();
-        redactionLog.getRedactionLogEntry().removeIf(entry -> {
-            if (entry.isFalsePositive()) {
-                return true;
-            }
-            if (entry.getImportedRedactionIntersections() != null) {
-                entry.getImportedRedactionIntersections().removeAll(skippedImportedRedactions);
-                if (!entry.getImportedRedactionIntersections().isEmpty() && (!entry.isImage() || entry.isImage() && !(entry.getType().equals("image") || entry.getType()
-                        .equals("ocr")))) {
-                    return true;
-                }
-            }
-            if (processedIds.contains(entry.getId())) {
-                log.info("Duplicate annotation found with id {}", entry.getId());
-                return true;
-            }
-            processedIds.add(entry.getId());
-            return false;
-        });
-        return redactionLog;
-    }
-
-    public long getNumberOfAffectedAnnotations(ManualRedactions manualRedactions) {
-
-        return createManualRedactionWrappers(manualRedactions).stream().map(ManualRedactionWrapper::getId).distinct().count();
-    }
-
-    private List<ManualRedactionWrapper> createManualRedactionWrappers(ManualRedactions manualRedactions) {
-
-        List<ManualRedactionWrapper> manualRedactionWrappers = new ArrayList<>();
-
-        manualRedactions.getRecategorizations()
-                .forEach(item -> {
-                    if (item.getSoftDeletedTime() == null) {
-                        manualRedactionWrappers.add(new ManualRedactionWrapper(item.getAnnotationId(), item.getRequestDate(), item, item.isApproved()));
-                    }
-                });
-
-        manualRedactions.getIdsToRemove()
-                .forEach(item -> {
-                    if (item.getSoftDeletedTime() == null) {
-                        manualRedactionWrappers.add(new ManualRedactionWrapper(item.getAnnotationId(), item.getRequestDate(), item, item.isApproved()));
-                    }
-                });
-
-        manualRedactions.getForceRedactions()
-                .forEach(item -> {
-                    if (item.getSoftDeletedTime() == null) {
-                        manualRedactionWrappers.add(new ManualRedactionWrapper(item.getAnnotationId(), item.getRequestDate(), item, item.isApproved()));
-                    }
-                });
-
-        manualRedactions.getLegalBasisChanges()
-                .forEach(item -> {
-                    if (item.getSoftDeletedTime() == null) {
-                        manualRedactionWrappers.add(new ManualRedactionWrapper(item.getAnnotationId(), item.getRequestDate(), item, item.isApproved()));
-                    }
-                });
-
-        manualRedactions.getResizeRedactions()
-                .forEach(item -> {
-                    if (item.getSoftDeletedTime() == null) {
-                        manualRedactionWrappers.add(new ManualRedactionWrapper(item.getAnnotationId(), item.getRequestDate(), item, item.isApproved()));
-                    }
-                });
-
-        Collections.sort(manualRedactionWrappers);
-
-        return manualRedactionWrappers;
-    }
-
-
-    private void processRedactionLogEntry(List<ManualRedactionWrapper> manualRedactionWrappers, RedactionLogEntry redactionLogEntry, String dossierTemplateId) {
-
-        manualRedactionWrappers.forEach(mrw -> {
-
-            Object item = mrw.getItem();
-            if (item instanceof ManualRecategorization imageRecategorization) {
-                processManualImageRecategorization(redactionLogEntry, dossierTemplateId, imageRecategorization);
-            }
-
-            if (item instanceof IdRemoval manualRemoval) {
-                processIdRemoval(redactionLogEntry, manualRemoval);
-            }
-
-            if (item instanceof ManualForceRedaction manualForceRedact) {
-                processManualForceRedaction(redactionLogEntry, dossierTemplateId, manualForceRedact);
-            }
-
-            if (item instanceof ManualLegalBasisChange manualLegalBasisChange) {
-                processManualLegalBasisChange(redactionLogEntry, manualLegalBasisChange);
-            }
-
-            if (item instanceof ManualResizeRedaction manualResizeRedact) {
-                processManualResizeRedaction(redactionLogEntry, manualResizeRedact);
-            }
-
-        });
-
-    }
-
-
-    private void processManualImageRecategorization(RedactionLogEntry redactionLogEntry, String dossierTemplateId, ManualRecategorization imageRecategorization) {
-
-        String manualOverrideReason = null;
-        if (imageRecategorization.getStatus().equals(AnnotationStatus.APPROVED)) {
-
-            redactionLogEntry.setType(imageRecategorization.getType());
-            redactionLogEntry.setSection("Image:" + redactionLogEntry.getType());
-
-            if (dictionaryService.isHint(imageRecategorization.getType(), dossierTemplateId)) {
-                redactionLogEntry.setRedacted(false);
-                redactionLogEntry.setHint(true);
-            } else {
-                redactionLogEntry.setHint(false);
-            }
-
-            manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", recategorized by manual override");
-        } else if (imageRecategorization.getStatus().equals(AnnotationStatus.REQUESTED)) {
-            manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", requested to recategorize");
-        }
-
-        if (manualOverrideReason != null) {
-            redactionLogEntry.setReason(manualOverrideReason);
-        }
-
-        redactionLogEntry.getManualChanges()
-                .add(ManualChange.from(imageRecategorization).withManualRedactionType(ManualRedactionType.RECATEGORIZE).withChange("type", imageRecategorization.getType()));
-    }
-
-
-    private String mergeReasonIfNecessary(String currentReason, String addition) {
-
-        if (currentReason != null) {
-            if (!currentReason.contains(addition)) {
-                return currentReason + addition;
-            }
-            return currentReason;
-        } else {
-            return "";
-        }
-    }
-
-
-    private void processIdRemoval(RedactionLogEntry redactionLogEntry, IdRemoval manualRemoval) {
-
-        boolean isApprovedRedaction = manualRemoval.getStatus().equals(AnnotationStatus.APPROVED);
-        if (isApprovedRedaction && manualRemoval.isRemoveFromDictionary() && isBasedOnDictionaryOnly(redactionLogEntry)) {
-            log.debug("Skipping merge for dictionary-modifying entry");
-        } else {
-            String manualOverrideReason = null;
-            if (isApprovedRedaction) {
-                redactionLogEntry.setRedacted(false);
-                manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", removed by manual override");
-                redactionLogEntry.setHint(false);
-            } else if (manualRemoval.getStatus().equals(AnnotationStatus.REQUESTED)) {
-                manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", requested to remove");
-            }
-
-            if (manualOverrideReason != null) {
-                redactionLogEntry.setReason(manualOverrideReason);
-            }
-        }
-
-        redactionLogEntry.getManualChanges()
-                .add(ManualChange.from(manualRemoval)
-                             .withManualRedactionType(manualRemoval.isRemoveFromDictionary() ? ManualRedactionType.REMOVE_FROM_DICTIONARY : ManualRedactionType.REMOVE_LOCALLY));
-    }
-
-
-    private boolean isBasedOnDictionaryOnly(RedactionLogEntry redactionLogEntry) {
-
-        return redactionLogEntry.getEngines().contains(Engine.DICTIONARY) && redactionLogEntry.getEngines().size() == 1;
-    }
-
-
-    private void processManualForceRedaction(RedactionLogEntry redactionLogEntry, String dossierTemplateId, ManualForceRedaction manualForceRedact) {
-
-        String manualOverrideReason = null;
-        var dictionaryIsHint = dictionaryService.isHint(redactionLogEntry.getType(), dossierTemplateId);
-        if (manualForceRedact.getStatus().equals(AnnotationStatus.APPROVED)) {
-            // Forcing a skipped hint should result in a hint
-            if (dictionaryIsHint) {
-                redactionLogEntry.setHint(true);
-            } else {
-                redactionLogEntry.setRedacted(true);
-            }
-            manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", forced by manual override");
-            redactionLogEntry.setLegalBasis(manualForceRedact.getLegalBasis());
-        } else if (manualForceRedact.getStatus().equals(AnnotationStatus.REQUESTED)) {
-            manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", requested to force " + (dictionaryIsHint ? "hint" : "redact"));
-            redactionLogEntry.setLegalBasis(manualForceRedact.getLegalBasis());
-        }
-
-        if (manualOverrideReason != null) {
-            redactionLogEntry.setReason(manualOverrideReason);
-        }
-
-        var manualChange = ManualChange.from(manualForceRedact).withManualRedactionType(dictionaryIsHint ? ManualRedactionType.FORCE_HINT : ManualRedactionType.FORCE_REDACT);
-
-        redactionLogEntry.getManualChanges().add(manualChange);
-    }
-
-
-    private void processManualLegalBasisChange(RedactionLogEntry redactionLogEntry, ManualLegalBasisChange manualLegalBasisChange) {
-
-        String manualOverrideReason = null;
-        if (manualLegalBasisChange.getStatus().equals(AnnotationStatus.APPROVED)) {
-            manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", legal basis was manually changed");
-            redactionLogEntry.setLegalBasis(manualLegalBasisChange.getLegalBasis());
-            redactionLogEntry.setRedacted(true);
-            if (manualLegalBasisChange.getSection() != null) {
-                redactionLogEntry.setSection(manualLegalBasisChange.getSection());
-            }
-            if (redactionLogEntry.isRectangle() && manualLegalBasisChange.getValue() != null) {
-                redactionLogEntry.setValue(manualLegalBasisChange.getValue());
-            }
-        } else if (manualLegalBasisChange.getStatus().equals(AnnotationStatus.REQUESTED)) {
-            manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", legal basis change requested");
-        }
-
-        if (manualOverrideReason != null) {
-            redactionLogEntry.setReason(manualOverrideReason);
-        }
-
-        var manualChange = ManualChange.from(manualLegalBasisChange).withManualRedactionType(ManualRedactionType.LEGAL_BASIS_CHANGE);
-        manualChange.withChange("legalBasis", manualLegalBasisChange.getLegalBasis());
-        if (manualLegalBasisChange.getSection() != null) {
-            manualChange.withChange("section", manualLegalBasisChange.getSection());
-        }
-        if (redactionLogEntry.isRectangle() && manualLegalBasisChange.getValue() != null) {
-            manualChange.withChange("value", manualLegalBasisChange.getValue());
-        }
-        redactionLogEntry.getManualChanges().add(manualChange);
-    }
-
-
-    private void processManualResizeRedaction(RedactionLogEntry redactionLogEntry, ManualResizeRedaction manualResizeRedact) {
-
-        String manualOverrideReason = null;
-        if (manualResizeRedact.getStatus().equals(AnnotationStatus.APPROVED)) {
-            redactionLogEntry.setPositions(convertPositions(manualResizeRedact.getPositions()));
-            if (!"signature".equalsIgnoreCase(redactionLogEntry.getType()) && !"logo".equalsIgnoreCase(redactionLogEntry.getType())) {
-                redactionLogEntry.setValue(manualResizeRedact.getValue());
-            }
-            // This is for backwards compatibility, now the text after/before is calculated during reanalysis because we need to find dict entries on positions where entries are resized to smaller.
-            if (manualResizeRedact.getTextBefore() != null || manualResizeRedact.getTextAfter() != null) {
-                redactionLogEntry.setTextBefore(manualResizeRedact.getTextBefore());
-                redactionLogEntry.setTextAfter(manualResizeRedact.getTextAfter());
-            }
-            manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", resized by manual override");
-        } else if (manualResizeRedact.getStatus().equals(AnnotationStatus.REQUESTED)) {
-            manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", requested to resize redact");
-            redactionLogEntry.setPositions(convertPositions(manualResizeRedact.getPositions()));
-
-            // This is for backwards compatibility, now the text after/before is calculated during reanalysis because we need to find dict entries on positions where entries are resized to smaller.
-            if (manualResizeRedact.getTextBefore() != null || manualResizeRedact.getTextAfter() != null) {
-                redactionLogEntry.setTextBefore(manualResizeRedact.getTextBefore());
-                redactionLogEntry.setTextAfter(manualResizeRedact.getTextAfter());
-            }
-        }
-
-        redactionLogEntry.setReason(manualOverrideReason);
-        redactionLogEntry.getManualChanges()
-                .add(ManualChange.from(manualResizeRedact).withManualRedactionType(ManualRedactionType.RESIZE).withChange("value", manualResizeRedact.getValue()));
-    }
-
-
-    public List<RedactionLogEntry> addManualAddEntries(Set<ManualRedactionEntry> manualAdds, int analysisNumber) {
-
-        List<RedactionLogEntry> redactionLogEntries = new ArrayList<>();
-
-        for (ManualRedactionEntry manualRedactionEntry : manualAdds) {
-
-            if (shouldCreateManualEntry(manualRedactionEntry)) {
-                RedactionLogEntry redactionLogEntry = createRedactionLogEntry(manualRedactionEntry, manualRedactionEntry.getAnnotationId(), analysisNumber);
-                redactionLogEntry.setPositions(convertPositions(manualRedactionEntry.getPositions()));
-                redactionLogEntry.setTextBefore(manualRedactionEntry.getTextBefore());
-                redactionLogEntry.setTextAfter(manualRedactionEntry.getTextAfter());
-
-                redactionLogEntries.add(redactionLogEntry);
-            }
-        }
-
-        return redactionLogEntries;
-    }
-
-
-    private List<Rectangle> convertPositions(List<com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle> positions) {
-
-        return positions.stream()
-                .map(pos -> new Rectangle(new Point(pos.getTopLeftX(), pos.getTopLeftY()), pos.getWidth(), pos.getHeight(), pos.getPage()))
-                .collect(Collectors.toList());
-    }
-
-
-    @SuppressWarnings("PMD.UselessParentheses")
-    private boolean shouldCreateManualEntry(ManualRedactionEntry manualRedactionEntry) {
-
-        if (!manualRedactionEntry.isApproved()) {
-            return false;
-        }
-
-        return (!manualRedactionEntry.isAddToDictionary() && !manualRedactionEntry.isAddToDossierDictionary()) || ((manualRedactionEntry.isAddToDictionary()
-                                                                                                                    || manualRedactionEntry.isAddToDossierDictionary())
-                                                                                                                   && manualRedactionEntry.getProcessedDate() == null);
-    }
-
-
-    private RedactionLogEntry createRedactionLogEntry(ManualRedactionEntry manualRedactionEntry, String id, int analysisNumber) {
-
-        var addToDictionary = manualRedactionEntry.isAddToDictionary() || manualRedactionEntry.isAddToDossierDictionary();
-
-        var change = ManualChange.from(manualRedactionEntry).withManualRedactionType(addToDictionary ? ManualRedactionType.ADD_TO_DICTIONARY : ManualRedactionType.ADD_LOCALLY);
-        List<ManualChange> changeList = new ArrayList<>();
-        changeList.add(change);
-
-        return RedactionLogEntry.builder()
-                .id(id)
-                .reason(manualRedactionEntry.getReason())
-                .isDictionaryEntry(manualRedactionEntry.isAddToDictionary())
-                .isDossierDictionaryEntry(manualRedactionEntry.isAddToDossierDictionary())
-                .legalBasis(manualRedactionEntry.getLegalBasis())
-                .value(manualRedactionEntry.getValue())
-                .sourceId(manualRedactionEntry.getSourceId())
-                .section(manualRedactionEntry.getSection())
-                .type(manualRedactionEntry.getType())
-                .redacted(true)
-                .isHint(false)
-                .sectionNumber(-1)
-                .rectangle(manualRedactionEntry.isRectangle())
-                .manualChanges(changeList)
-                .changes(List.of(new Change(analysisNumber + 1, ChangeType.ADDED, manualRedactionEntry.getRequestDate())))
-                .build();
-    }
-
-
-    @Data
-    @AllArgsConstructor
-    private static class ManualRedactionWrapper implements Comparable<ManualRedactionWrapper> {
-
-        private String id;
-        private OffsetDateTime date;
-        private Object item;
-        private boolean approved;
-
-
-        @Override
-        public int compareTo(ManualRedactionWrapper o) {
-
-            return this.date.compareTo(o.date);
-        }
-
-    }
-
-}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/migration/LegacyVersion0MigrationService.java

@@ -1,70 +0,0 @@
-package com.iqser.red.service.redaction.v1.server.migration;
-
-import java.lang.reflect.Field;
-import java.util.Comparator;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.stream.Collectors;
-
-import org.springframework.stereotype.Service;
-
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
-
-import lombok.SneakyThrows;
-
-@Service
-@SuppressWarnings("PMD")
-public class LegacyVersion0MigrationService {
-
-    public RedactionLog mergeDuplicateAnnotationIds(RedactionLog redactionLog) {
-
-        List<RedactionLogEntry> mergedEntries = new LinkedList<>();
-        Map<String, List<RedactionLogEntry>> entriesById = redactionLog.getRedactionLogEntry().stream().collect(Collectors.groupingBy(RedactionLogEntry::getId));
-        for (List<RedactionLogEntry> entries : entriesById.values()) {
-
-            if (entries.isEmpty()) {
-                continue;
-            }
-
-            if (entries.size() == 1) {
-                mergedEntries.add(entries.get(0));
-                continue;
-            }
-
-            List<RedactionLogEntry> sortedEntries = entries.stream().sorted(Comparator.comparing(entry -> entry.getChanges().get(0).getDateTime())).toList();
-
-            RedactionLogEntry initialEntry = sortedEntries.get(0);
-            for (RedactionLogEntry entry : sortedEntries.subList(1, sortedEntries.size())) {
-                copyNonNullFields(entry, initialEntry);
-            }
-            mergedEntries.add(initialEntry);
-        }
-
-        redactionLog.setRedactionLogEntry(mergedEntries);
-        return redactionLog;
-    }
-
-
-    @SneakyThrows
-    public static void copyNonNullFields(RedactionLogEntry source, RedactionLogEntry destination) {
-
-        if (source == null || destination == null) {
-            throw new IllegalArgumentException("Source and destination objects must not be null");
-        }
-
-        Class<?> sourceClass = source.getClass();
-
-        Field[] sourceFields = sourceClass.getDeclaredFields();
-
-        for (Field field : sourceFields) {
-            field.setAccessible(true);
-            Object value = field.get(source);
-            if (value != null) {
-                field.set(destination, value);
-            }
-        }
-    }
-
-}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/migration/MigrationMapper.java

@@ -1,93 +0,0 @@
-package com.iqser.red.service.redaction.v1.server.migration;
-
-import java.util.Collections;
-import java.util.List;
-import java.util.Set;
-import java.util.stream.Collectors;
-
-import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ChangeType;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualChange;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Change;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ManualRedactionType;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
-
-public class MigrationMapper {
-
-    public static com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Change toEntityLogChanges(Change change) {
-
-        return new com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Change(change.getAnalysisNumber(),
-                                                                                                              toEntityLogType(change.getType()),
-                                                                                                              change.getDateTime());
-    }
-
-
-    public static com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualChange toEntityLogManualChanges(com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ManualChange manualChange) {
-
-        return new com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualChange(toManualRedactionType(manualChange.getManualRedactionType()),
-                                                                                                                    manualChange.getProcessedDate(),
-                                                                                                                    manualChange.getRequestedDate(),
-                                                                                                                    manualChange.getUserId(),
-                                                                                                                    manualChange.getPropertyChanges());
-    }
-
-
-    public static ChangeType toEntityLogType(com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ChangeType type) {
-
-        return switch (type) {
-            case ADDED -> ChangeType.ADDED;
-            case REMOVED -> ChangeType.REMOVED;
-            case CHANGED -> ChangeType.CHANGED;
-        };
-    }
-
-
-    public static com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType toManualRedactionType(ManualRedactionType manualRedactionType) {
-
-        return switch (manualRedactionType) {
-            case ADD_LOCALLY -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.ADD;
-            case ADD_TO_DICTIONARY -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.ADD_TO_DICTIONARY;
-            case REMOVE_LOCALLY -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.REMOVE;
-            case REMOVE_FROM_DICTIONARY -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.REMOVE_FROM_DICTIONARY;
-            case FORCE_REDACT -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.FORCE;
-            case FORCE_HINT -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.FORCE;
-            case RECATEGORIZE -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.RECATEGORIZE;
-            case LEGAL_BASIS_CHANGE -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.LEGAL_BASIS_CHANGE;
-            case RESIZE -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.RESIZE;
-        };
-    }
-
-
-    public static com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine toEntityLogEngine(Engine engine) {
-
-        return switch (engine) {
-            case DICTIONARY -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine.DICTIONARY;
-            case NER -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine.NER;
-            case RULE -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine.RULE;
-        };
-    }
-
-
-    public static Set<com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine> getMigratedEngines(RedactionLogEntry entry) {
-
-        if (entry.getEngines() == null) {
-            return Collections.emptySet();
-        }
-        return entry.getEngines()
-                .stream()
-                .map(MigrationMapper::toEntityLogEngine)
-                .collect(Collectors.toSet());
-    }
-
-
-    public List<ManualChange> migrateManualChanges(List<com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ManualChange> manualChanges) {
-
-        if (manualChanges == null) {
-            return Collections.emptyList();
-        }
-        return manualChanges.stream()
-                .map(MigrationMapper::toEntityLogManualChanges)
-                .toList();
-    }
-
-}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/migration/MigrationMessageReceiver.java

@@ -1,93 +0,0 @@
-package com.iqser.red.service.redaction.v1.server.migration;
-
-import static com.iqser.red.service.redaction.v1.model.QueueNames.MIGRATION_QUEUE;
-import static com.iqser.red.service.redaction.v1.model.QueueNames.MIGRATION_RESPONSE_QUEUE;
-
-import org.springframework.amqp.core.Message;
-import org.springframework.amqp.rabbit.annotation.RabbitHandler;
-import org.springframework.amqp.rabbit.annotation.RabbitListener;
-import org.springframework.amqp.rabbit.core.RabbitTemplate;
-import org.springframework.stereotype.Service;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog;
-import com.iqser.red.service.redaction.v1.model.MigrationRequest;
-import com.iqser.red.service.redaction.v1.model.MigrationResponse;
-import com.iqser.red.service.redaction.v1.server.model.MigratedEntityLog;
-import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
-import com.iqser.red.service.redaction.v1.server.service.DictionaryService;
-import com.iqser.red.service.redaction.v1.server.service.document.DocumentGraphMapper;
-import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
-
-import lombok.AccessLevel;
-import lombok.RequiredArgsConstructor;
-import lombok.SneakyThrows;
-import lombok.experimental.FieldDefaults;
-import lombok.extern.slf4j.Slf4j;
-
-@Slf4j
-@Service
-@RequiredArgsConstructor
-@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
-public class MigrationMessageReceiver {
-
-    ObjectMapper objectMapper;
-    RedactionLogToEntityLogMigrationService redactionLogToEntityLogMigrationService;
-    RedactionStorageService redactionStorageService;
-    LegacyRedactionLogMergeService legacyRedactionLogMergeService;
-    LegacyVersion0MigrationService legacyVersion0MigrationService;
-    RabbitTemplate rabbitTemplate;
-    DictionaryService dictionaryService;
-
-
-    @SneakyThrows
-    @RabbitHandler
-    @RabbitListener(queues = MIGRATION_QUEUE)
-    public void receiveMigrationRequest(Message message) {
-
-        MigrationRequest migrationRequest = objectMapper.readValue(message.getBody(), MigrationRequest.class);
-        log.info("--------------------------------------------------------------------");
-        log.info("Starting redactionLog to entityLog migration for dossierId {} and fileId {}", migrationRequest.getDossierId(), migrationRequest.getFileId());
-
-        dictionaryService.updateDictionary(migrationRequest.getDossierTemplateId(), migrationRequest.getDossierId());
-
-        Document document = DocumentGraphMapper.toDocumentGraph(redactionStorageService.getDocumentData(migrationRequest.getDossierId(), migrationRequest.getFileId()));
-        RedactionLog redactionLog = redactionStorageService.getRedactionLog(migrationRequest.getDossierId(), migrationRequest.getFileId());
-
-        if (redactionLog.getAnalysisVersion() == 0) {
-            redactionLog = legacyVersion0MigrationService.mergeDuplicateAnnotationIds(redactionLog);
-        } else if (migrationRequest.getManualRedactions() != null) {
-            redactionLog = legacyRedactionLogMergeService.addManualAddEntriesAndRemoveSkippedImported(redactionLog,
-                                                                                                      migrationRequest.getManualRedactions(),
-                                                                                                      migrationRequest.getDossierTemplateId());
-        }
-
-        MigratedEntityLog migratedEntityLog = redactionLogToEntityLogMigrationService.migrate(redactionLog,
-                                                                                              document,
-                                                                                              migrationRequest.getDossierTemplateId(),
-                                                                                              migrationRequest.getManualRedactions(),
-                                                                                              migrationRequest.getFileId());
-
-        log.info("Storing migrated entityLog and ids to migrate in DB for file {}", migrationRequest.getFileId());
-        redactionStorageService.storeObject(migrationRequest.getDossierId(), migrationRequest.getFileId(), FileType.ENTITY_LOG, migratedEntityLog.getEntityLog());
-        redactionStorageService.storeObject(migrationRequest.getDossierId(), migrationRequest.getFileId(), FileType.MIGRATED_IDS, migratedEntityLog.getMigratedIds());
-
-        sendFinished(MigrationResponse.builder().dossierId(migrationRequest.getDossierId()).fileId(migrationRequest.getFileId()).build());
-        log.info("Migrated {} redactionLog entries, found {} annotation ids for migration in the db, {} new manual entries, for dossierId {} and fileId {}",
-                 migratedEntityLog.getEntityLog().getEntityLogEntry().size(),
-                 migratedEntityLog.getMigratedIds().getMappings().size(),
-                 migratedEntityLog.getMigratedIds().getManualRedactionEntriesToAdd().size(),
-                 migrationRequest.getDossierId(),
-                 migrationRequest.getFileId());
-        log.info("");
-    }
-
-
-    @SneakyThrows
-    public void sendFinished(MigrationResponse migrationResponse) {
-
-        rabbitTemplate.convertAndSend(MIGRATION_RESPONSE_QUEUE, migrationResponse);
-    }
-
-}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/migration/RedactionLogToEntityLogMigrationService.java

@@ -1,290 +0,0 @@
-package com.iqser.red.service.redaction.v1.server.migration;
-
-import java.awt.geom.Rectangle2D;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Optional;
-import java.util.function.Function;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
-
-import org.springframework.stereotype.Service;
-
-import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogLegalBasis;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.migration.MigratedIds;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.BaseAnnotation;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogLegalBasis;
-import com.iqser.red.service.redaction.v1.server.model.MigratedEntityLog;
-import com.iqser.red.service.redaction.v1.server.model.MigrationEntity;
-import com.iqser.red.service.redaction.v1.server.model.PrecursorEntity;
-import com.iqser.red.service.redaction.v1.server.model.RectangleWithPage;
-import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
-import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
-import com.iqser.red.service.redaction.v1.server.model.document.nodes.Image;
-import com.iqser.red.service.redaction.v1.server.model.document.nodes.ImageType;
-import com.iqser.red.service.redaction.v1.server.service.DictionaryService;
-import com.iqser.red.service.redaction.v1.server.service.ManualChangesApplicationService;
-import com.iqser.red.service.redaction.v1.server.service.document.EntityEnrichmentService;
-import com.iqser.red.service.redaction.v1.server.service.document.EntityFindingUtility;
-import com.iqser.red.service.redaction.v1.server.service.document.EntityFromPrecursorCreationService;
-import com.iqser.red.service.redaction.v1.server.utils.IdBuilder;
-import com.iqser.red.service.redaction.v1.server.utils.MigratedIdsCollector;
-
-import lombok.AccessLevel;
-import lombok.RequiredArgsConstructor;
-import lombok.experimental.FieldDefaults;
-import lombok.extern.slf4j.Slf4j;
-
-@Slf4j
-@Service
-@RequiredArgsConstructor
-@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
-//TODO: remove this, once the migration is done
-public class RedactionLogToEntityLogMigrationService {
-
-    private static final double MATCH_THRESHOLD = 10;
-    EntityFindingUtility entityFindingUtility;
-    EntityEnrichmentService entityEnrichmentService;
-    DictionaryService dictionaryService;
-    ManualChangesApplicationService manualChangesApplicationService;
-
-
-    public MigratedEntityLog migrate(RedactionLog redactionLog, Document document, String dossierTemplateId, ManualRedactions manualRedactions, String fileId) {
-
-        log.info("Migrating entities for file {}", fileId);
-        List<MigrationEntity> entitiesToMigrate = calculateMigrationEntitiesFromRedactionLog(redactionLog, document, dossierTemplateId, fileId);
-
-        MigratedIds migratedIds = entitiesToMigrate.stream()
-                .collect(new MigratedIdsCollector());
-
-        applyManualChanges(entitiesToMigrate, manualRedactions);
-        log.info("applying manual changes to migrated entities for file {}", fileId);
-
-        EntityLog entityLog = new EntityLog();
-        entityLog.setAnalysisNumber(redactionLog.getAnalysisNumber());
-        entityLog.setRulesVersion(redactionLog.getRulesVersion());
-        entityLog.setDictionaryVersion(redactionLog.getDictionaryVersion());
-        entityLog.setDossierDictionaryVersion(redactionLog.getDossierDictionaryVersion());
-        entityLog.setLegalBasisVersion(redactionLog.getLegalBasisVersion());
-        entityLog.setAnalysisVersion(redactionLog.getAnalysisVersion());
-        entityLog.setLegalBasis(redactionLog.getLegalBasis()
-                                        .stream()
-                                        .map(RedactionLogToEntityLogMigrationService::toEntityLogLegalBasis)
-                                        .toList());
-
-        Map<String, String> oldToNewIDMapping = migratedIds.buildOldToNewMapping();
-
-        log.info("Writing migrated entities to entityLog for file {}", fileId);
-        entityLog.setEntityLogEntry(entitiesToMigrate.stream()
-                                            .map(migrationEntity -> migrationEntity.toEntityLogEntry(oldToNewIDMapping))
-                                            .toList());
-
-        if (getNumberOfApprovedEntries(redactionLog) != entityLog.getEntityLogEntry().size()) {
-            String message = String.format("Not all entities have been found during the migration redactionLog has %d entries and new entityLog %d",
-                                           redactionLog.getRedactionLogEntry().size(),
-                                           entityLog.getEntityLogEntry().size());
-            log.error(message);
-            throw new AssertionError(message);
-        }
-
-        MigratedIds idsToMigrateInDb = entitiesToMigrate.stream()
-                .filter(MigrationEntity::hasManualChangesOrComments)
-                .filter(m -> !m.getOldId().equals(m.getNewId()))
-                .collect(new MigratedIdsCollector());
-
-        List<ManualRedactionEntry> manualRedactionEntriesToAdd = entitiesToMigrate.stream()
-                .filter(MigrationEntity::needsManualEntry)
-                .map(MigrationEntity::buildManualRedactionEntry)
-                .toList();
-
-        idsToMigrateInDb.setManualRedactionEntriesToAdd(manualRedactionEntriesToAdd);
-
-        return new MigratedEntityLog(idsToMigrateInDb, entityLog);
-    }
-
-
-    private void applyManualChanges(List<MigrationEntity> entitiesToMigrate, ManualRedactions manualRedactions) {
-
-        if (manualRedactions == null) {
-            return;
-        }
-
-        Map<String, List<BaseAnnotation>> manualChangesPerAnnotationId = Stream.of(manualRedactions.getIdsToRemove(),
-                                                                                   manualRedactions.getEntriesToAdd(),
-                                                                                   manualRedactions.getForceRedactions(),
-                                                                                   manualRedactions.getResizeRedactions(),
-                                                                                   manualRedactions.getLegalBasisChanges(),
-                                                                                   manualRedactions.getRecategorizations())
-                .flatMap(Collection::stream)
-                .collect(Collectors.groupingBy(BaseAnnotation::getAnnotationId));
-
-        entitiesToMigrate.forEach(migrationEntity -> migrationEntity.applyManualChanges(manualChangesPerAnnotationId.getOrDefault(migrationEntity.getOldId(),
-                                                                                                                                  Collections.emptyList()),
-                                                                                        manualChangesApplicationService));
-
-    }
-
-
-    private static long getNumberOfApprovedEntries(RedactionLog redactionLog) {
-
-        return redactionLog.getRedactionLogEntry().size();
-    }
-
-
-    private List<MigrationEntity> calculateMigrationEntitiesFromRedactionLog(RedactionLog redactionLog, Document document, String dossierTemplateId, String fileId) {
-
-        List<MigrationEntity> images = getImageBasedMigrationEntities(redactionLog, document, fileId);
-        List<MigrationEntity> textMigrationEntities = getTextBasedMigrationEntities(redactionLog, document, dossierTemplateId, fileId);
-        return Stream.of(textMigrationEntities.stream(), images.stream())
-                .flatMap(Function.identity())
-                .toList();
-    }
-
-
-    private static EntityLogLegalBasis toEntityLogLegalBasis(RedactionLogLegalBasis redactionLogLegalBasis) {
-
-        return new EntityLogLegalBasis(redactionLogLegalBasis.getName(), redactionLogLegalBasis.getDescription(), redactionLogLegalBasis.getReason());
-    }
-
-
-    private List<MigrationEntity> getImageBasedMigrationEntities(RedactionLog redactionLog, Document document, String fileId) {
-
-        List<Image> images = document.streamAllImages()
-                .collect(Collectors.toList());
-
-        List<RedactionLogEntry> redactionLogImages = redactionLog.getRedactionLogEntry()
-                .stream()
-                .filter(RedactionLogEntry::isImage)
-                .toList();
-
-        List<MigrationEntity> migrationEntities = new LinkedList<>();
-        for (RedactionLogEntry redactionLogImage : redactionLogImages) {
-            List<RectangleWithPage> imagePositions = redactionLogImage.getPositions()
-                    .stream()
-                    .map(RectangleWithPage::fromRedactionLogRectangle)
-                    .toList();
-            assert imagePositions.size() == 1;
-            Optional<Image> optionalClosestImage = images.stream()
-                    .filter(image -> image.onPage(redactionLogImage.getPositions()
-                                                          .get(0).getPage()))
-                    .min(Comparator.comparingDouble(image -> EntityFindingUtility.calculateDistance(image.getPosition(), imagePositions.get(0).rectangle2D())))
-                    .filter(image -> EntityFindingUtility.calculateDistance(image.getPosition(), imagePositions.get(0).rectangle2D()) <= MATCH_THRESHOLD);
-
-            Image closestImage;
-            if (optionalClosestImage.isEmpty()) { // if no fitting image can be found create a new one with the previous values!
-                closestImage = buildImageDirectly(document, redactionLogImage);
-            } else {
-                closestImage = optionalClosestImage.get();
-                images.remove(closestImage);
-            }
-
-            String ruleIdentifier;
-            String reason = Optional.ofNullable(redactionLogImage.getReason())
-                    .orElse("");
-            if (redactionLogImage.getMatchedRule().isBlank() || redactionLogImage.getMatchedRule() == null) {
-                ruleIdentifier = "OLDIMG.0.0";
-            } else {
-                ruleIdentifier = "OLDIMG." + redactionLogImage.getMatchedRule() + ".0";
-            }
-
-            if (redactionLogImage.lastChangeIsRemoved()) {
-                closestImage.remove(ruleIdentifier, reason);
-            } else if (redactionLogImage.isRedacted()) {
-                closestImage.apply(ruleIdentifier, reason, redactionLogImage.getLegalBasis());
-            } else {
-                closestImage.skip(ruleIdentifier, reason);
-            }
-            migrationEntities.add(MigrationEntity.fromRedactionLogImage(redactionLogImage, closestImage, fileId));
-        }
-        return migrationEntities;
-    }
-
-
-    private static Image buildImageDirectly(Document document, RedactionLogEntry redactionLogImage) {
-
-        Image image = Image.builder()
-                .documentTree(document.getDocumentTree())
-                .imageType(ImageType.fromString(redactionLogImage.getType()))
-                .transparent(redactionLogImage.isImageHasTransparency())
-                .page(document.getPages()
-                              .stream()
-                              .filter(p -> p.getNumber() == redactionLogImage.getPositions()
-                                      .get(0).getPage())
-                              .findFirst()
-                              .orElseThrow())
-                .position(toRectangle2D(redactionLogImage.getPositions()
-                                                .get(0)))
-                .build();
-
-        List<Integer> treeId = document.getDocumentTree().createNewMainEntryAndReturnId(image);
-        image.setTreeId(treeId);
-        image.setId(IdBuilder.buildId(image.getPages(),
-                                      image.getBBox().values()
-                                              .stream()
-                                              .toList(),
-                                      "",
-                                      ""));
-
-        return image;
-    }
-
-
-    private static Rectangle2D toRectangle2D(Rectangle rect) {
-
-        return new Rectangle2D.Double(rect.getTopLeft().getX(), rect.getTopLeft().getY(), rect.getWidth(), rect.getHeight());
-    }
-
-
-    private List<MigrationEntity> getTextBasedMigrationEntities(RedactionLog redactionLog, Document document, String dossierTemplateId, String fileId) {
-
-        List<MigrationEntity> entitiesToMigrate = redactionLog.getRedactionLogEntry()
-                .stream()
-                .filter(redactionLogEntry -> !redactionLogEntry.isImage())
-                .map(entry -> MigrationEntity.fromRedactionLogEntry(entry, dictionaryService.isHint(entry.getType(), dossierTemplateId), fileId))
-                .toList();
-
-        List<PrecursorEntity> precursorEntities = entitiesToMigrate.stream()
-                .map(MigrationEntity::getPrecursorEntity)
-                .toList();
-
-        log.info("Finding all possible entities");
-        Map<String, List<TextEntity>> tempEntitiesByValue = entityFindingUtility.findAllPossibleEntitiesAndGroupByValue(document, precursorEntities);
-
-        for (MigrationEntity migrationEntity : entitiesToMigrate) {
-            Optional<TextEntity> optionalTextEntity = entityFindingUtility.findClosestEntityAndReturnEmptyIfNotFound(migrationEntity.getPrecursorEntity(),
-                                                                                                                     tempEntitiesByValue,
-                                                                                                                     MATCH_THRESHOLD);
-
-            if (optionalTextEntity.isEmpty()) {
-                migrationEntity.setMigratedEntity(migrationEntity.getPrecursorEntity());
-                migrationEntity.setOldId(migrationEntity.getPrecursorEntity().getId());
-                migrationEntity.setNewId(migrationEntity.getPrecursorEntity().getId());
-                continue;
-            }
-
-            TextEntity migratedEntity = EntityFromPrecursorCreationService.createCorrectEntity(migrationEntity.getPrecursorEntity(), optionalTextEntity.get(), true);
-
-            migrationEntity.setMigratedEntity(migratedEntity);
-            migrationEntity.setOldId(migrationEntity.getPrecursorEntity().getId());
-            migrationEntity.setNewId(migratedEntity.getId());
-        }
-
-        tempEntitiesByValue.values()
-                .stream()
-                .flatMap(Collection::stream)
-                .forEach(TextEntity::removeFromGraph);
-
-        return entitiesToMigrate;
-    }
-
-}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/ClosestEntity.java

@@ -1,17 +0,0 @@
-package com.iqser.red.service.redaction.v1.server.model;
-
-import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
-
-import lombok.Builder;
-import lombok.Getter;
-import lombok.Setter;
-
-@Builder
-@Getter
-@Setter
-public class ClosestEntity {
-
-    private double distance;
-    private TextEntity textEntity;
-
-}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/ManualEntity.java

@@ -0,0 +1,106 @@
+package com.iqser.red.service.redaction.v1.server.model;
+
+import java.util.List;
+import java.util.PriorityQueue;
+
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
+import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
+import com.iqser.red.service.redaction.v1.server.model.document.entity.IEntity;
+import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
+import com.iqser.red.service.redaction.v1.server.model.document.entity.ManualChangeOverwrite;
+import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule;
+
+import lombok.AccessLevel;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Getter;
+import lombok.experimental.FieldDefaults;
+
+@Getter
+@Builder
+@AllArgsConstructor
+@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
+public class ManualEntity implements IEntity {
+
+    // The id must be mapped into a TextEntity as is for comments to work correctly
+    String id;
+    String value;
+    List<RectangleWithPage> entityPosition;
+    String ruleIdentifier;
+    String reason;
+    String legalBasis;
+    String type;
+    String section;
+    EntityType entityType;
+    boolean applied;
+    boolean isDictionaryEntry;
+    boolean isDossierDictionaryEntry;
+    boolean rectangle;
+
+    @Builder.Default
+    PriorityQueue<MatchedRule> matchedRuleList = new PriorityQueue<>();
+    @Builder.Default
+    ManualChangeOverwrite manualOverwrite = new ManualChangeOverwrite();
+
+
+    public static ManualEntity fromRedactionLogEntry(RedactionLogEntry redactionLogEntry) {
+
+        String ruleIdentifier = redactionLogEntry.getType() + "." + redactionLogEntry.getMatchedRule() + ".0";
+        List<RectangleWithPage> rectangleWithPages = redactionLogEntry.getPositions().stream().map(RectangleWithPage::fromRedactionLogRectangle).toList();
+        return ManualEntity.builder()
+                .id(redactionLogEntry.getId())
+                .value(redactionLogEntry.getValue())
+                .entityPosition(rectangleWithPages)
+                .ruleIdentifier(ruleIdentifier)
+                .reason(redactionLogEntry.getReason())
+                .legalBasis(redactionLogEntry.getLegalBasis())
+                .type(redactionLogEntry.getType())
+                .section(redactionLogEntry.getSection())
+                .entityType(redactionLogEntry.isRecommendation() ? EntityType.RECOMMENDATION : EntityType.ENTITY)
+                .applied(redactionLogEntry.isRedacted())
+                .isDictionaryEntry(redactionLogEntry.isDictionaryEntry())
+                .isDossierDictionaryEntry(redactionLogEntry.isDossierDictionaryEntry())
+                .rectangle(redactionLogEntry.isRectangle())
+                .build();
+    }
+
+
+    public static ManualEntity fromManualRedactionEntry(ManualRedactionEntry manualRedactionEntry) {
+
+        List<RectangleWithPage> rectangleWithPages = manualRedactionEntry.getPositions().stream().map(RectangleWithPage::fromAnnotationRectangle).toList();
+        ManualChangeOverwrite manualChangeOverwrite = new ManualChangeOverwrite();
+        manualChangeOverwrite.addChange(manualRedactionEntry);
+        return ManualEntity.builder()
+                .id(manualRedactionEntry.getAnnotationId())
+                .value(manualRedactionEntry.getValue())
+                .entityPosition(rectangleWithPages)
+                .ruleIdentifier("MAN.0.0")
+                .reason(manualRedactionEntry.getReason())
+                .legalBasis(manualRedactionEntry.getLegalBasis())
+                .type(manualRedactionEntry.getType())
+                .section(manualRedactionEntry.getSection())
+                .entityType(EntityType.ENTITY)
+                .applied(true)
+                .isDictionaryEntry(false)
+                .isDossierDictionaryEntry(false)
+                .rectangle(manualRedactionEntry.isRectangle())
+                .manualOverwrite(manualChangeOverwrite)
+                .build();
+    }
+
+
+    @Override
+    public TextRange getTextRange() {
+
+        return new TextRange(-1, -1);
+    }
+
+
+    @Override
+    public String type() {
+
+        return getManualOverwrite().getType().orElse(type);
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/MigratedEntityLog.java

@@ -1,22 +0,0 @@
-package com.iqser.red.service.redaction.v1.server.model;
-
-import java.util.List;
-
-import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.migration.MigratedIds;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
-
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.Data;
-import lombok.NoArgsConstructor;
-
-@Data
-@Builder
-@NoArgsConstructor
-@AllArgsConstructor
-public class MigratedEntityLog {
-
-    MigratedIds migratedIds;
-    EntityLog entityLog;
-}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/MigrationEntity.java

@@ -1,441 +0,0 @@
-package com.iqser.red.service.redaction.v1.server.model;
-
-import static com.iqser.red.service.redaction.v1.server.service.EntityLogCreatorService.buildEntryState;
-import static com.iqser.red.service.redaction.v1.server.service.EntityLogCreatorService.buildEntryType;
-
-import java.awt.geom.Rectangle2D;
-import java.util.Collections;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Optional;
-import java.util.Set;
-import java.util.stream.Collectors;
-
-import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryType;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.BaseAnnotation;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ManualRedactionType;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
-import com.iqser.red.service.redaction.v1.server.migration.MigrationMapper;
-import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
-import com.iqser.red.service.redaction.v1.server.model.document.entity.IEntity;
-import com.iqser.red.service.redaction.v1.server.model.document.entity.ManualChangeOverwrite;
-import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
-import com.iqser.red.service.redaction.v1.server.model.document.nodes.Image;
-import com.iqser.red.service.redaction.v1.server.service.ManualChangeFactory;
-import com.iqser.red.service.redaction.v1.server.service.ManualChangesApplicationService;
-
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.Data;
-import lombok.RequiredArgsConstructor;
-import lombok.extern.slf4j.Slf4j;
-
-@Slf4j
-@Data
-@Builder
-@AllArgsConstructor
-@RequiredArgsConstructor
-public final class MigrationEntity {
-
-    private final PrecursorEntity precursorEntity;
-    private final RedactionLogEntry redactionLogEntry;
-    private IEntity migratedEntity;
-    private String oldId;
-    private String newId;
-    private String fileId;
-
-    @Builder.Default
-    List<BaseAnnotation> manualChanges = new LinkedList<>();
-
-
-    public static MigrationEntity fromRedactionLogEntry(RedactionLogEntry redactionLogEntry, boolean hint, String fileId) {
-
-        PrecursorEntity precursorEntity = createPrecursorEntity(redactionLogEntry, hint);
-
-        if (precursorEntity.getEntityType().equals(EntityType.HINT) && !redactionLogEntry.isHint() && !redactionLogEntry.isRedacted()) {
-            precursorEntity.ignore(precursorEntity.getRuleIdentifier(), precursorEntity.getReason());
-        } else if (redactionLogEntry.lastChangeIsRemoved()) {
-            precursorEntity.remove(precursorEntity.getRuleIdentifier(), precursorEntity.getReason());
-        } else if (lastManualChangeIsRemove(redactionLogEntry)) {
-            precursorEntity.ignore(precursorEntity.getRuleIdentifier(), precursorEntity.getReason());
-        } else if (precursorEntity.isApplied() && redactionLogEntry.isRecommendation()) {
-            precursorEntity.skip(precursorEntity.getRuleIdentifier(), precursorEntity.getReason());
-        } else if (precursorEntity.isApplied()) {
-            precursorEntity.apply(precursorEntity.getRuleIdentifier(), precursorEntity.getReason(), precursorEntity.getLegalBasis());
-        } else {
-            precursorEntity.skip(precursorEntity.getRuleIdentifier(), precursorEntity.getReason());
-        }
-
-        return MigrationEntity.builder().precursorEntity(precursorEntity).redactionLogEntry(redactionLogEntry).oldId(redactionLogEntry.getId()).fileId(fileId).build();
-    }
-
-
-    public static MigrationEntity fromRedactionLogImage(RedactionLogEntry redactionLogImage, Image image, String fileId) {
-
-        return MigrationEntity.builder().redactionLogEntry(redactionLogImage).migratedEntity(image).oldId(redactionLogImage.getId()).newId(image.getId()).fileId(fileId).build();
-    }
-
-
-    private static boolean lastManualChangeIsRemove(RedactionLogEntry redactionLogEntry) {
-
-        if (redactionLogEntry.getManualChanges() == null) {
-            return false;
-        }
-
-        return redactionLogEntry.getManualChanges()
-                .stream()
-                .reduce((a, b) -> b)
-                .map(m -> m.getManualRedactionType().equals(ManualRedactionType.REMOVE_LOCALLY))
-                .orElse(false);
-    }
-
-
-    public static PrecursorEntity createPrecursorEntity(RedactionLogEntry redactionLogEntry, boolean hint) {
-
-        String ruleIdentifier = buildRuleIdentifier(redactionLogEntry);
-        List<RectangleWithPage> rectangleWithPages = redactionLogEntry.getPositions()
-                .stream()
-                .map(RectangleWithPage::fromRedactionLogRectangle)
-                .toList();
-        EntityType entityType = getEntityType(redactionLogEntry, hint);
-        return PrecursorEntity.builder()
-                .id(redactionLogEntry.getId())
-                .value(redactionLogEntry.getValue())
-                .entityPosition(rectangleWithPages)
-                .ruleIdentifier(ruleIdentifier)
-                .reason(Optional.ofNullable(redactionLogEntry.getReason())
-                                .orElse(""))
-                .legalBasis(redactionLogEntry.getLegalBasis())
-                .type(redactionLogEntry.getType())
-                .section(redactionLogEntry.getSection())
-                .engines(MigrationMapper.getMigratedEngines(redactionLogEntry))
-                .entityType(entityType)
-                .applied(redactionLogEntry.isRedacted())
-                .isDictionaryEntry(redactionLogEntry.isDictionaryEntry())
-                .isDossierDictionaryEntry(redactionLogEntry.isDossierDictionaryEntry())
-                .rectangle(redactionLogEntry.isRectangle())
-                .manualOverwrite(new ManualChangeOverwrite(entityType))
-                .build();
-    }
-
-
-    private static String buildRuleIdentifier(RedactionLogEntry redactionLogEntry) {
-
-        String ruleIdentifier;
-        if (redactionLogEntry.getMatchedRule() != null) {
-            ruleIdentifier = "OLD." + redactionLogEntry.getMatchedRule() + ".0";
-        } else {
-            ruleIdentifier = "MAN.5.0"; // pure ManualRedactions used to have no matched rule
-        }
-        return ruleIdentifier;
-    }
-
-
-    private static EntityType getEntityType(RedactionLogEntry redactionLogEntry, boolean hint) {
-
-        if (hint) {
-            return EntityType.HINT;
-        }
-        if (redactionLogEntry.isFalsePositive()) {
-            return EntityType.FALSE_POSITIVE;
-        }
-        if (redactionLogEntry.isHint()) {
-            return EntityType.HINT;
-        }
-        if (redactionLogEntry.isRecommendation()) {
-            return EntityType.RECOMMENDATION;
-        }
-        return EntityType.ENTITY;
-    }
-
-
-    private static EntryType getEntryType(EntityType entityType) {
-
-        return switch (entityType) {
-            case ENTITY -> EntryType.ENTITY;
-            case HINT -> EntryType.HINT;
-            case FALSE_POSITIVE -> EntryType.FALSE_POSITIVE;
-            case RECOMMENDATION -> EntryType.RECOMMENDATION;
-            case FALSE_RECOMMENDATION -> EntryType.FALSE_RECOMMENDATION;
-        };
-    }
-
-
-    public EntityLogEntry toEntityLogEntry(Map<String, String> oldToNewIdMapping) {
-
-        EntityLogEntry entityLogEntry;
-        if (migratedEntity instanceof Image image) {
-            entityLogEntry = createEntityLogEntry(image);
-        } else if (migratedEntity instanceof TextEntity textEntity) {
-            entityLogEntry = createEntityLogEntry(textEntity);
-        } else if (migratedEntity instanceof PrecursorEntity entity) {
-            entityLogEntry = createEntityLogEntry(entity);
-        } else {
-            throw new UnsupportedOperationException("Unknown subclass " + migratedEntity.getClass());
-        }
-
-        entityLogEntry.setManualChanges(ManualChangeFactory.toManualChangeList(migratedEntity.getManualOverwrite().getManualChangeLog(), redactionLogEntry.isHint()));
-        entityLogEntry.setColor(redactionLogEntry.getColor());
-        entityLogEntry.setChanges(redactionLogEntry.getChanges()
-                                          .stream()
-                                          .map(MigrationMapper::toEntityLogChanges)
-                                          .toList());
-        entityLogEntry.setReference(migrateSetOfIds(redactionLogEntry.getReference(), oldToNewIdMapping));
-        entityLogEntry.setImportedRedactionIntersections(migrateSetOfIds(redactionLogEntry.getImportedRedactionIntersections(), oldToNewIdMapping));
-        entityLogEntry.setEngines(MigrationMapper.getMigratedEngines(redactionLogEntry));
-        if (redactionLogEntry.getLegalBasis() != null) {
-            entityLogEntry.setLegalBasis(redactionLogEntry.getLegalBasis());
-        }
-
-        if (entityLogEntry.getEntryType().equals(EntryType.HINT) && lastManualChangeIsRemoveLocally(entityLogEntry)) {
-            entityLogEntry.setState(EntryState.IGNORED);
-        }
-
-        return entityLogEntry;
-    }
-
-
-    private static boolean lastManualChangeIsRemoveLocally(EntityLogEntry entityLogEntry) {
-
-        return entityLogEntry.getManualChanges()
-                .stream()
-                .reduce((a, b) -> b)
-                .filter(mc -> mc.getManualRedactionType().equals(com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.REMOVE))
-                .isPresent();
-    }
-
-
-    private Set<String> migrateSetOfIds(Set<String> ids, Map<String, String> oldToNewIdMapping) {
-
-        if (ids == null) {
-            return Collections.emptySet();
-        }
-        return ids.stream()
-                .map(oldToNewIdMapping::get)
-                .collect(Collectors.toSet());
-    }
-
-
-    public EntityLogEntry createEntityLogEntry(Image image) {
-
-        List<Position> positions = getPositionsFromOverride(image).orElse(List.of(new Position(image.getPosition(), image.getPage().getNumber())));
-        return EntityLogEntry.builder()
-                .id(image.getId())
-                .value(image.value())
-                .type(image.type())
-                .reason(image.buildReasonWithManualChangeDescriptions())
-                .legalBasis(image.legalBasis())
-                .matchedRule(image.getMatchedRule().getRuleIdentifier().toString())
-                .dictionaryEntry(false)
-                .positions(positions)
-                .containingNodeId(image.getTreeId())
-                .closestHeadline(image.getHeadline().getTextBlock().getSearchText())
-                .section(image.getManualOverwrite().getSection()
-                                 .orElse(redactionLogEntry.getSection()))
-                .textAfter(redactionLogEntry.getTextAfter())
-                .textBefore(redactionLogEntry.getTextBefore())
-                .imageHasTransparency(image.isTransparent())
-                .state(buildEntryState(image))
-                .entryType(redactionLogEntry.isHint() ? EntryType.IMAGE_HINT : EntryType.IMAGE)
-                .build();
-
-    }
-
-
-    public EntityLogEntry createEntityLogEntry(PrecursorEntity precursorEntity) {
-
-        return EntityLogEntry.builder()
-                .id(precursorEntity.getId())
-                .reason(precursorEntity.buildReasonWithManualChangeDescriptions())
-                .legalBasis(precursorEntity.legalBasis())
-                .value(precursorEntity.value())
-                .type(precursorEntity.type())
-                .state(buildEntryState(precursorEntity))
-                .entryType(buildEntryType(precursorEntity))
-                .section(precursorEntity.getManualOverwrite().getSection()
-                                 .orElse(redactionLogEntry.getSection()))
-                .textAfter(redactionLogEntry.getTextAfter())
-                .textBefore(redactionLogEntry.getTextBefore())
-                .containingNodeId(Collections.emptyList())
-                .closestHeadline("")
-                .matchedRule(precursorEntity.getMatchedRule().getRuleIdentifier().toString())
-                .dictionaryEntry(precursorEntity.isDictionaryEntry())
-                .dossierDictionaryEntry(precursorEntity.isDossierDictionaryEntry())
-                .startOffset(-1)
-                .endOffset(-1)
-                .positions(precursorEntity.getManualOverwrite().getPositions()
-                                   .orElse(precursorEntity.getEntityPosition())
-                                   .stream()
-                                   .map(entityPosition -> new Position(entityPosition.rectangle2D(), entityPosition.pageNumber()))
-                                   .toList())
-                .engines(Collections.emptySet())
-                .build();
-    }
-
-
-    public EntityLogEntry createEntityLogEntry(TextEntity entity) {
-
-        assert entity.getPositionsOnPagePerPage().size() == 1;
-        List<Position> rectanglesPerLine = getRectanglesPerLine(entity);
-        return EntityLogEntry.builder()
-                .id(entity.getId())
-                .positions(rectanglesPerLine)
-                .reason(entity.buildReasonWithManualChangeDescriptions())
-                .legalBasis(entity.legalBasis())
-                .value(entity.getManualOverwrite().getValue()
-                               .orElse(entity.getMatchedRule().isWriteValueWithLineBreaks() ? entity.getValueWithLineBreaks() : entity.getValue()))
-                .type(entity.type())
-                .section(entity.getManualOverwrite().getSection()
-                                 .orElse(redactionLogEntry.getSection()))
-                .textAfter(entity.getTextAfter())
-                .textBefore(entity.getTextBefore())
-                .containingNodeId(entity.getDeepestFullyContainingNode().getTreeId())
-                .closestHeadline(entity.getDeepestFullyContainingNode().getHeadline().getTextBlock().getSearchText())
-                .matchedRule(entity.getMatchedRule().getRuleIdentifier().toString())
-                .dictionaryEntry(entity.isDictionaryEntry())
-                .startOffset(entity.getTextRange().start())
-                .endOffset(entity.getTextRange().end())
-                .dossierDictionaryEntry(entity.isDossierDictionaryEntry())
-                .engines(entity.getEngines() != null ? entity.getEngines() : Collections.emptySet())
-                .state(buildEntryState(entity))
-                .entryType(buildEntryType(entity))
-                .build();
-    }
-
-
-    private static List<Position> getRectanglesPerLine(TextEntity entity) {
-
-        return getPositionsFromOverride(entity).orElse(entity.getPositionsOnPagePerPage()
-                                                               .get(0).getRectanglePerLine()
-                                                               .stream()
-                                                               .map(rectangle2D -> new Position(rectangle2D,
-                                                                                                entity.getPositionsOnPagePerPage()
-                                                                                                        .get(0).getPage().getNumber()))
-                                                               .toList());
-    }
-
-
-    private static Optional<List<Position>> getPositionsFromOverride(IEntity entity) {
-
-        return entity.getManualOverwrite().getPositions()
-                .map(rects -> rects.stream()
-                        .map(r -> new Position(r.rectangle2D(), r.pageNumber()))
-                        .toList());
-    }
-
-
-    public boolean hasManualChangesOrComments() {
-
-        return !(redactionLogEntry.getManualChanges() == null || redactionLogEntry.getManualChanges().isEmpty()) || //
-               !(redactionLogEntry.getComments() == null || redactionLogEntry.getComments().isEmpty()) //
-               || hasManualChanges();
-    }
-
-
-    public boolean hasManualChanges() {
-
-        return !manualChanges.isEmpty();
-    }
-
-
-    public void applyManualChanges(List<BaseAnnotation> manualChangesToApply, ManualChangesApplicationService manualChangesApplicationService) {
-
-        manualChanges.addAll(manualChangesToApply);
-        manualChangesToApply.forEach(manualChange -> {
-            if (manualChange instanceof ManualResizeRedaction manualResizeRedaction && migratedEntity instanceof TextEntity textEntity) {
-                // Due to the value in the old redaction log already being resized, there is no way to find the original entity ID and therefore to migrate the resize annotation correctly.
-                // Instead, we add an add_locally change to the db.
-                ManualResizeRedaction migratedManualResizeRedaction = ManualResizeRedaction.builder()
-                        .positions(manualResizeRedaction.getPositions())
-                        .annotationId(getNewId())
-                        .updateDictionary(manualResizeRedaction.getUpdateDictionary())
-                        .addToAllDossiers(manualResizeRedaction.isAddToAllDossiers())
-                        .textAfter(manualResizeRedaction.getTextAfter())
-                        .textBefore(manualResizeRedaction.getTextBefore())
-                        .build();
-                manualChangesApplicationService.resize(textEntity, migratedManualResizeRedaction);
-            } else {
-                migratedEntity.getManualOverwrite().addChange(manualChange);
-            }
-        });
-    }
-
-
-    public ManualRedactionEntry buildManualRedactionEntry() {
-
-        assert hasManualChanges();
-
-        // currently we need to insert a manual redaction entry, whenever an entity has been resized.
-        String user = manualChanges.stream()
-                .filter(mc -> mc instanceof ManualResizeRedaction)
-                .findFirst()
-                .orElse(manualChanges.get(0)).getUser();
-
-        return ManualRedactionEntry.builder()
-                .annotationId(newId)
-                .fileId(fileId)
-                .type(redactionLogEntry.getType())
-                .value(redactionLogEntry.getValue())
-                .reason(redactionLogEntry.getReason())
-                .legalBasis(redactionLogEntry.getLegalBasis())
-                .section(redactionLogEntry.getSection())
-                .addToDictionary(false)
-                .addToDossierDictionary(false)
-                .rectangle(false)
-                .positions(buildPositions(migratedEntity))
-                .user(user)
-                .build();
-    }
-
-
-    private List<Rectangle> buildPositions(IEntity entity) {
-
-        if (entity instanceof TextEntity textEntity) {
-
-            var positionsOnPage = textEntity.getPositionsOnPagePerPage()
-                    .get(0);
-            return positionsOnPage.getRectanglePerLine()
-                    .stream()
-                    .map(p -> new Rectangle((float) p.getX(), (float) p.getY(), (float) p.getWidth(), (float) p.getHeight(), positionsOnPage.getPage().getNumber()))
-                    .toList();
-        }
-        if (entity instanceof PrecursorEntity pEntity) {
-
-            return pEntity.getManualOverwrite().getPositions()
-                    .orElse(pEntity.getEntityPosition())
-                    .stream()
-                    .map(p -> new Rectangle((float) p.rectangle2D().getX(),
-                                            (float) p.rectangle2D().getY(),
-                                            (float) p.rectangle2D().getWidth(),
-                                            (float) p.rectangle2D().getHeight(),
-                                            p.pageNumber()))
-                    .toList();
-        }
-        if (entity instanceof Image image) {
-
-            Rectangle2D position = image.getManualOverwrite().getPositions()
-                    .map(p -> p.get(0).rectangle2D())
-                    .orElse(image.getPosition());
-
-            return List.of(new Rectangle((float) position.getX(), (float) position.getY(), (float) position.getWidth(), (float) position.getHeight(), image.getPage().getNumber()));
-
-        } else {
-            throw new UnsupportedOperationException();
-        }
-    }
-
-
-    public boolean needsManualEntry() {
-
-        return manualChanges.stream()
-                       .anyMatch(mc -> mc instanceof ManualResizeRedaction && !((ManualResizeRedaction) mc).getUpdateDictionary()) && !(migratedEntity instanceof Image);
-    }
-
-}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/PrecursorEntity.java

@@ -1,195 +0,0 @@
-package com.iqser.red.service.redaction.v1.server.model;
-
-import static com.iqser.red.service.redaction.v1.server.service.NotFoundImportedEntitiesService.IMPORTED_REDACTION_TYPE;
-
-import java.util.List;
-import java.util.Optional;
-import java.util.PriorityQueue;
-import java.util.Set;
-import java.util.UUID;
-
-import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryType;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedaction;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
-import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
-import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
-import com.iqser.red.service.redaction.v1.server.model.document.entity.IEntity;
-import com.iqser.red.service.redaction.v1.server.model.document.entity.ManualChangeOverwrite;
-import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule;
-
-import lombok.AccessLevel;
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.Getter;
-import lombok.experimental.FieldDefaults;
-
-@Getter
-@Builder
-@AllArgsConstructor
-@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
-@SuppressWarnings("PMD.AvoidFieldNameMatchingMethodName")
-public class PrecursorEntity implements IEntity {
-
-    // The id must be mapped into a TextEntity as is for comments to work correctly
-    String id;
-    String value;
-    List<RectangleWithPage> entityPosition;
-    String ruleIdentifier;
-    String reason;
-    String legalBasis;
-    String type;
-    String section;
-    EntityType entityType;
-    boolean applied;
-    boolean isDictionaryEntry;
-    boolean isDossierDictionaryEntry;
-    boolean rectangle;
-    Set<Engine> engines;
-
-    @Builder.Default
-    PriorityQueue<MatchedRule> matchedRuleList = new PriorityQueue<>();
-    ManualChangeOverwrite manualOverwrite;
-
-
-    public static PrecursorEntity fromManualRedactionEntry(ManualRedactionEntry manualRedactionEntry, boolean hint) {
-
-        List<RectangleWithPage> rectangleWithPages = manualRedactionEntry.getPositions()
-                .stream()
-                .map(RectangleWithPage::fromAnnotationRectangle)
-                .toList();
-
-        var entityType = hint ? EntityType.HINT : EntityType.ENTITY;
-        ManualChangeOverwrite manualChangeOverwrite = new ManualChangeOverwrite(entityType);
-        manualChangeOverwrite.addChange(manualRedactionEntry);
-        return PrecursorEntity.builder()
-                .id(manualRedactionEntry.getAnnotationId())
-                .value(manualRedactionEntry.getValue())
-                .entityPosition(rectangleWithPages)
-                .ruleIdentifier("MAN.0.0")
-                .reason(manualRedactionEntry.getReason())
-                .legalBasis(manualRedactionEntry.getLegalBasis())
-                .type(manualRedactionEntry.getType())
-                .section(manualRedactionEntry.getSection())
-                .entityType(entityType)
-                .applied(true)
-                .isDictionaryEntry(false)
-                .isDossierDictionaryEntry(false)
-                .rectangle(manualRedactionEntry.isRectangle())
-                .manualOverwrite(manualChangeOverwrite)
-                .engines(Set.of(Engine.MANUAL))
-                .build();
-    }
-
-
-    public static PrecursorEntity fromEntityLogEntry(EntityLogEntry entityLogEntry) {
-
-        List<RectangleWithPage> rectangleWithPages = entityLogEntry.getPositions()
-                .stream()
-                .map(RectangleWithPage::fromEntityLogPosition)
-                .toList();
-        EntityType entityType = getEntityType(entityLogEntry.getEntryType());
-        return PrecursorEntity.builder()
-                .id(entityLogEntry.getId())
-                .value(entityLogEntry.getValue())
-                .entityPosition(rectangleWithPages)
-                .ruleIdentifier(entityLogEntry.getMatchedRule())
-                .reason(entityLogEntry.getReason())
-                .legalBasis(entityLogEntry.getLegalBasis())
-                .type(entityLogEntry.getType())
-                .section(entityLogEntry.getSection())
-                .entityType(entityType)
-                .isDictionaryEntry(entityLogEntry.isDictionaryEntry())
-                .isDossierDictionaryEntry(entityLogEntry.isDossierDictionaryEntry())
-                .manualOverwrite(new ManualChangeOverwrite(entityType))
-                .engines(entityLogEntry.getEngines())
-                .build();
-    }
-
-
-    public static PrecursorEntity fromImportedEntry(ImportedRedaction importedRedaction) {
-
-        List<RectangleWithPage> rectangleWithPages = importedRedaction.getPositions()
-                .stream()
-                .map(RectangleWithPage::fromEntityLogPosition)
-                .toList();
-        EntryType entryType = Optional.ofNullable(importedRedaction.getEntryType())
-                .orElse(EntryType.ENTITY);
-        EntityType entityType = getEntityType(entryType);
-        String value = Optional.ofNullable(importedRedaction.getValue())
-                .orElse("");
-        return PrecursorEntity.builder()
-                .id(importedRedaction.getId())
-                .value(value)
-                .entityPosition(rectangleWithPages)
-                .reason(Optional.ofNullable(importedRedaction.getReason())
-                                .orElse(""))
-                .legalBasis(Optional.ofNullable(importedRedaction.getLegalBasis())
-                                    .orElse(""))
-                .type(Optional.ofNullable(importedRedaction.getType())
-                              .orElse(IMPORTED_REDACTION_TYPE))
-                .entityType(entityType)
-                .isDictionaryEntry(false)
-                .isDossierDictionaryEntry(false)
-                .rectangle(value.isBlank() || entryType.equals(EntryType.IMAGE) || entryType.equals(EntryType.IMAGE_HINT) || entryType.equals(EntryType.AREA))
-                .manualOverwrite(new ManualChangeOverwrite(entityType))
-                .engines(Set.of(Engine.IMPORTED))
-                .build();
-    }
-
-
-    public static PrecursorEntity fromManualResizeRedaction(ManualResizeRedaction manualResizeRedaction) {
-
-        List<RectangleWithPage> rectangleWithPages = manualResizeRedaction.getPositions()
-                .stream()
-                .map(RectangleWithPage::fromAnnotationRectangle)
-                .toList();
-        return PrecursorEntity.builder()
-                .id(UUID.randomUUID().toString())
-                .value(manualResizeRedaction.getValue())
-                .entityPosition(rectangleWithPages)
-                .entityType(EntityType.ENTITY)
-                .manualOverwrite(new ManualChangeOverwrite(EntityType.ENTITY))
-                .build();
-    }
-
-
-    @Override
-    public TextRange getTextRange() {
-
-        return new TextRange(-1, -1);
-    }
-
-
-    @Override
-    public String type() {
-
-        return getManualOverwrite().getType()
-                .orElse(type);
-    }
-
-
-    private static EntityType getEntityType(EntryType entryType) {
-
-        switch (entryType) {
-            case FALSE_RECOMMENDATION -> {
-                return EntityType.FALSE_RECOMMENDATION;
-            }
-            case FALSE_POSITIVE -> {
-                return EntityType.FALSE_POSITIVE;
-            }
-            case HINT -> {
-                return EntityType.HINT;
-            }
-            case RECOMMENDATION -> {
-                return EntityType.RECOMMENDATION;
-            }
-            default -> {
-                return EntityType.ENTITY;
-            }
-        }
-    }
-
-}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/RectangleWithPage.java

@@ -2,7 +2,6 @@ package com.iqser.red.service.redaction.v1.server.model;
 
 import java.awt.geom.Rectangle2D;
 
-import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
 
 public record RectangleWithPage(int pageNumber, Rectangle2D rectangle2D) {
@@ -13,12 +12,6 @@ public record RectangleWithPage(int pageNumber, Rectangle2D rectangle2D) {
     }
 
 
-    public static RectangleWithPage fromEntityLogPosition(Position position) {
-
-        return new RectangleWithPage(position.getPageNumber(), toRectangle2D(position));
-    }
-
-
     public static RectangleWithPage fromAnnotationRectangle(com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle rectangle) {
 
         return new RectangleWithPage(rectangle.getPage(), toRectangle2D(rectangle));
@@ -27,19 +20,13 @@ public record RectangleWithPage(int pageNumber, Rectangle2D rectangle2D) {
 
     private static Rectangle2D toRectangle2D(Rectangle rectangle) {
 
-        return new Rectangle2D.Float(rectangle.getTopLeft().getX(), rectangle.getTopLeft().getY(), rectangle.getWidth(), rectangle.getHeight());
-    }
-
-
-    private static Rectangle2D toRectangle2D(Position position) {
-
-        return new Rectangle2D.Float(position.x(), position.y(), position.w(), position.h());
+        return new Rectangle2D.Float(rectangle.getTopLeft().getX(), rectangle.getTopLeft().getY() + rectangle.getHeight(), rectangle.getWidth(), -rectangle.getHeight());
     }
 
 
     private static Rectangle2D toRectangle2D(com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle rectangle) {
 
-        return new Rectangle2D.Float(rectangle.getTopLeft().getX(), rectangle.getTopLeft().getY(), rectangle.getWidth(), rectangle.getHeight());
+        return new Rectangle2D.Float(rectangle.getTopLeft().getX(), rectangle.getTopLeft().getY() + rectangle.getHeight(), rectangle.getWidth(), -rectangle.getHeight());
     }
 
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/component/Component.java

@@ -20,9 +20,9 @@ import lombok.experimental.FieldDefaults;
 public class Component {
 
     RuleIdentifier matchedRule;
-    String name;
+    String category;
     String value;
-    String valueDescription;
+    String transformation;
 
     List<Entity> references;
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/component/Entity.java

@@ -1,7 +1,6 @@
 package com.iqser.red.service.redaction.v1.server.model.component;
 
 import java.util.List;
-import java.util.Optional;
 import java.util.Set;
 
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Change;
@@ -11,8 +10,6 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryType;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualChange;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position;
-import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
-import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
 
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
@@ -33,15 +30,16 @@ public class Entity {
     String value;
     String reason;
     String matchedRule;
+    boolean rectangle;
     String legalBasis;
 
     boolean imported;
 
-    SemanticNode containingNode;
     String section;
     float[] color;
 
     List<Position> positions;
+    int sectionNumber;
 
     String textBefore;
     String textAfter;
@@ -57,6 +55,8 @@ public class Entity {
 
     boolean excluded;
 
+    String sourceId;
+
     List<Change> changes;
 
     List<ManualChange> manualChanges;
@@ -68,7 +68,7 @@ public class Entity {
     Set<String> importedRedactionIntersections;
 
 
-    public static Entity fromEntityLogEntry(EntityLogEntry e, Document document) {
+    public static Entity fromEntityLogEntry(EntityLogEntry e) {
 
         return Entity.builder()
                 .id(e.getId())
@@ -78,21 +78,23 @@ public class Entity {
                 .value(e.getValue())
                 .reason(e.getReason())
                 .matchedRule(e.getMatchedRule())
+                .rectangle(e.isRectangle())
                 .legalBasis(e.getLegalBasis())
                 .imported(e.isImported())
                 .section(e.getSection())
                 .color(e.getColor())
                 .positions(e.getPositions())
-                .containingNode(document.getDocumentTree().getEntryById(e.getContainingNodeId()).getNode())
+                .sectionNumber(e.getSectionNumber())
                 .textBefore(e.getTextBefore())
                 .textAfter(e.getTextAfter())
                 .startOffset(e.getStartOffset())
                 .endOffset(e.getEndOffset())
-                .length(Optional.ofNullable(e.getValue()).orElse("").length())
+                .length(e.getValue().length())
                 .imageHasTransparency(e.isImageHasTransparency())
                 .isDictionaryEntry(e.isDictionaryEntry())
                 .isDossierDictionaryEntry(e.isDossierDictionaryEntry())
                 .excluded(e.isExcluded())
+                .sourceId(e.getSourceId())
                 .changes(e.getChanges())
                 .manualChanges(e.getManualChanges())
                 .engines(e.getEngines())

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/Dictionary.java

@@ -2,23 +2,23 @@ package com.iqser.red.service.redaction.v1.server.model.dictionary;
 
 import static java.lang.String.format;
 
-import java.util.Arrays;
 import java.util.Collection;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
-import java.util.Locale;
 import java.util.Map;
 import java.util.Set;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
 import org.apache.commons.lang3.StringUtils;
 
+import com.iqser.red.service.redaction.v1.server.utils.exception.NotFoundException;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
 import com.iqser.red.service.redaction.v1.server.utils.Patterns;
-import com.iqser.red.service.redaction.v1.server.utils.exception.NotFoundException;
 
 import lombok.Data;
 import lombok.Getter;
@@ -107,53 +107,39 @@ public class Dictionary {
         if (StringUtils.isEmpty(value)) {
             throw new IllegalArgumentException(format("%s is not a valid dictionary entry", value));
         }
-        boolean isCaseInsensitive = localAccessMap.get(type).isCaseInsensitive();
         Set<MatchedRule> matchedRulesSet = new HashSet<>(matchedRules);
-
-        String cleanedValue = value;
-        if (isCaseInsensitive) {
-            cleanedValue = cleanedValue.toLowerCase(Locale.US);
-        }
-        localAccessMap.get(type)
-                .getLocalEntriesWithMatchedRules()
-                .merge(cleanedValue.trim(), matchedRulesSet, (set1, set2) -> Stream.concat(set1.stream(), set2.stream()).collect(Collectors.toSet()));
+        localAccessMap.get(type).getLocalEntriesWithMatchedRules().merge(value.trim(), matchedRulesSet, (set1, set2) -> Stream.concat(set1.stream(), set2.stream()).collect(Collectors.toSet()));
         if (alsoAddLastname) {
-            String lastname = cleanedValue.split(" ")[0];
-            localAccessMap.get(type)
-                    .getLocalEntriesWithMatchedRules()
-                    .merge(lastname, matchedRulesSet, (set1, set2) -> Stream.concat(set1.stream(), set2.stream()).collect(Collectors.toSet()));
+            String lastname = value.split(" ")[0];
+            localAccessMap.get(type).getLocalEntriesWithMatchedRules().merge(lastname, matchedRulesSet, (set1, set2) -> Stream.concat(set1.stream(), set2.stream()).collect(Collectors.toSet()));
         }
     }
 
 
     public void recommendEverywhere(TextEntity textEntity) {
 
-        addLocalDictionaryEntry(textEntity.type(), textEntity.getValue(), textEntity.getMatchedRuleList(), false);
+        addLocalDictionaryEntry(textEntity.getType(), textEntity.getValue(), textEntity.getMatchedRuleList(), false);
     }
 
 
     public void recommendEverywhereWithLastNameSeparately(TextEntity textEntity) {
 
-        addLocalDictionaryEntry(textEntity.type(), textEntity.getValue(), textEntity.getMatchedRuleList(), true);
+        addLocalDictionaryEntry(textEntity.getType(), textEntity.getValue(), textEntity.getMatchedRuleList(), true);
     }
 
 
     public void addMultipleAuthorsAsRecommendation(TextEntity textEntity) {
 
-        splitIntoAuthorNames(textEntity).forEach(authorName -> addLocalDictionaryEntry(textEntity.type(), authorName, textEntity.getMatchedRuleList(), true));
+        String cleanedWord = textEntity.getValue().replaceAll(",", " ").replaceAll("  ", " ").trim() + " ";
+        Pattern pattern = Patterns.AUTHOR_TABLE_SPLITTER;
+        Matcher matcher = pattern.matcher(cleanedWord);
 
-    }
-
-
-    public static List<String> splitIntoAuthorNames(TextEntity textEntity) {
-
-        List<String> splitAuthorNames;
-        if (textEntity.getValue().contains(",")) {
-            splitAuthorNames = Arrays.asList(textEntity.getValue().split(","));
-        } else {
-            splitAuthorNames = Arrays.asList(textEntity.getValueWithLineBreaks().split("\n"));
+        while (matcher.find()) {
+            String match = matcher.group().trim();
+            if (match.length() >= 3) {
+                addLocalDictionaryEntry(textEntity.getType(), match, textEntity.getMatchedRuleList(), true);
+            }
         }
-        return splitAuthorNames.stream().map(String::trim).filter(authorName -> Patterns.AUTHOR_NAME_PATTERN.matcher(authorName).matches()).toList();
     }
 
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/DictionaryModel.java

@@ -2,8 +2,6 @@ package com.iqser.red.service.redaction.v1.server.model.dictionary;
 
 import java.io.Serializable;
 import java.util.HashMap;
-import java.util.List;
-import java.util.Locale;
 import java.util.Set;
 import java.util.stream.Collectors;
 
@@ -111,11 +109,4 @@ public class DictionaryModel implements Serializable {
         return falseRecommendationsSearch;
     }
 
-    public Set<MatchedRule> getMatchedRulesForLocalDictionaryEntry(String value) {
-        var cleanedValue = isCaseInsensitive() ? value.toLowerCase(Locale.US) : value;
-
-        return localEntriesWithMatchedRules.get(cleanedValue);
-
-    }
-
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/DocumentData.java

@@ -1,7 +1,5 @@
 package com.iqser.red.service.redaction.v1.server.model.document;
 
-import java.io.Serializable;
-
 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPage;
 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPositionData;
 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentStructure;
@@ -11,15 +9,13 @@ import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;
-import lombok.NoArgsConstructor;
 import lombok.experimental.FieldDefaults;
 
 @Data
 @Builder
 @AllArgsConstructor
-@NoArgsConstructor
-@FieldDefaults(level = AccessLevel.PRIVATE)
-public class DocumentData implements Serializable {
+@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
+public class DocumentData {
 
     DocumentPage[] documentPages;
     DocumentTextData[] documentTextData;

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/DocumentTree.java

@@ -5,15 +5,14 @@ import static java.lang.String.format;
 import java.util.Collections;
 import java.util.LinkedList;
 import java.util.List;
-import java.util.Optional;
 import java.util.stream.Stream;
 
-import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.GenericSemanticNode;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.NodeType;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
-import com.iqser.red.service.redaction.v1.server.model.document.nodes.Table;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.Table;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlockCollector;
 
@@ -136,48 +135,6 @@ public class DocumentTree {
     }
 
 
-    public Optional<SemanticNode> getNextSibling(List<Integer> treeId) {
-
-        var siblingTreeId = getNextSiblingId(treeId);
-        if (!entryExists(siblingTreeId)) {
-            return Optional.empty();
-        }
-        return Optional.of(getEntryById(siblingTreeId).getNode());
-    }
-
-
-    public List<Integer> getNextSiblingId(List<Integer> treeId) {
-
-        List<Integer> siblingTreeId = new LinkedList<>();
-        for (int i = 0; i < treeId.size() - 1; i++) {
-            siblingTreeId.add(treeId.get(i));
-        }
-        siblingTreeId.add(treeId.get(treeId.size() - 1) + 1);
-        return siblingTreeId;
-    }
-
-
-    public Optional<SemanticNode> getPreviousSibling(List<Integer> treeId) {
-
-        var siblingTreeId = getPreviousSiblingId(treeId);
-        if (!entryExists(siblingTreeId)) {
-            return Optional.empty();
-        }
-        return Optional.of(getEntryById(siblingTreeId).getNode());
-    }
-
-
-    public List<Integer> getPreviousSiblingId(List<Integer> treeId) {
-
-        List<Integer> siblingTreeId = new LinkedList<>();
-        for (int i = 0; i < treeId.size() - 1; i++) {
-            siblingTreeId.add(treeId.get(i));
-        }
-        siblingTreeId.add(treeId.get(treeId.size() - 1) - 1);
-        return siblingTreeId;
-    }
-
-
     public Entry getEntryById(List<Integer> treeId) {
 
         if (treeId.isEmpty()) {

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/TextRange.java

@@ -13,7 +13,6 @@ import lombok.Setter;
 
 @Setter
 @EqualsAndHashCode
-@SuppressWarnings("PMD.AvoidFieldNameMatchingMethodName")
 public class TextRange implements Comparable<TextRange> {
 
     private int start;

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/EntityType.java

@@ -2,7 +2,6 @@ package com.iqser.red.service.redaction.v1.server.model.document.entity;
 
 public enum EntityType {
     ENTITY,
-    HINT,
     RECOMMENDATION,
     FALSE_POSITIVE,
     FALSE_RECOMMENDATION

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/IEntity.java

@@ -35,7 +35,7 @@ public interface IEntity {
 
     default String value() {
 
-        return getManualOverwrite().getValue().orElse(getValue() == null ? "" : getValue());
+        return getManualOverwrite().getValue().orElse(getValue());
     }
 
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/ManualChangeOverwrite.java

@@ -10,29 +10,30 @@ import java.util.Optional;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.BaseAnnotation;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
-import com.iqser.red.service.redaction.v1.server.model.RectangleWithPage;
 
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
 import lombok.Builder;
+import lombok.NoArgsConstructor;
 import lombok.experimental.FieldDefaults;
 
 @Builder
+@NoArgsConstructor
 @AllArgsConstructor
 @FieldDefaults(level = AccessLevel.PRIVATE)
 public class ManualChangeOverwrite {
 
     private static final Map<Class<? extends BaseAnnotation>, String> MANUAL_CHANGE_DESCRIPTIONS = Map.of(//
-                                                                                                          ManualRedactionEntry.class, "created by manual change", //
-                                                                                                          ManualLegalBasisChange.class, "legal basis was manually changed", //
-                                                                                                          ManualResizeRedaction.class, "resized by manual override", //
-                                                                                                          ManualForceRedaction.class, "forced by manual override", //
-                                                                                                          IdRemoval.class, "removed by manual override", //
-                                                                                                          ManualRecategorization.class, "recategorized by manual override");
+            ManualRedactionEntry.class, "created by manual change", //
+            ManualLegalBasisChange.class, "legal basis was manually changed", //
+            ManualResizeRedaction.class, "resized by manual override", //
+            ManualForceRedaction.class, "forced by manual override", //
+            IdRemoval.class, "removed by manual override", //
+            ManualRecategorization.class, "recategorized by manual override");
 
     List<BaseAnnotation> manualChanges = new LinkedList<>();
     boolean changed;
@@ -41,25 +42,11 @@ public class ManualChangeOverwrite {
     String legalBasis;
     String section;
     String value;
-    List<RectangleWithPage> positions;
     Boolean applied;
     Boolean removed;
     Boolean ignored;
     Boolean resized;
     Boolean recategorized;
-    EntityType entityType;
-
-
-    public ManualChangeOverwrite() {
-
-        this(EntityType.ENTITY);
-    }
-
-
-    public ManualChangeOverwrite(EntityType entityType) {
-
-        this.entityType = entityType;
-    }
 
 
     public void calculateCurrentOverride() {
@@ -80,8 +67,7 @@ public class ManualChangeOverwrite {
         manualChanges.sort(Comparator.comparing(BaseAnnotation::getRequestDate));
         updateFields(manualChanges);
         // make list unmodifiable.
-        return manualChanges.stream()
-                .toList();
+        return manualChanges.stream().toList();
     }
 
 
@@ -94,20 +80,13 @@ public class ManualChangeOverwrite {
 
             if (manualChange instanceof IdRemoval) {
                 applied = false;
-                if (sortedManualChanges.get(0) instanceof ManualRedactionEntry) { // set to removed if it's a Manual Redaction
-                    removed = true;
-                } else {
-                    ignored = true;
-                }
-            }
-
-            if (manualChange instanceof ManualRedactionEntry manualRedactionEntry) {
-                setToApplied();
-                legalBasis = manualRedactionEntry.getLegalBasis();
+                ignored = true;
             }
 
             if (manualChange instanceof ManualForceRedaction manualForceRedaction) {
-                setToApplied();
+                removed = false;
+                ignored = false;
+                applied = true;
                 legalBasis = manualForceRedaction.getLegalBasis();
             }
 
@@ -117,23 +96,16 @@ public class ManualChangeOverwrite {
                 value = manualLegalBasisChange.getValue();
             }
 
-            if (manualChange instanceof ManualResizeRedaction manualResizeRedaction) {
-                // resizing logic for TextEntities and Images happens in ManualChangesApplicationService.
+            if (manualChange instanceof ManualResizeRedaction) {
+                // resizing logic happens in ManualChangesApplicationService.
                 resized = true;
-                // This is only for not found Manual Entities.
-                value = manualResizeRedaction.getValue();
-                positions = manualResizeRedaction.getPositions()
-                        .stream()
-                        .map(RectangleWithPage::fromAnnotationRectangle)
-                        .toList();
             }
 
             if (manualChange instanceof ManualRecategorization recategorization) {
+                // recategorization logic happens in ManualChangesApplicationService.
                 recategorized = true;
+                // this is only relevant for ManualEntities. Image and TextEntity is recategorized in the ManualChangesApplicationService.
                 type = recategorization.getType();
-                if (recategorization.getLegalBasis() != null && !recategorization.getLegalBasis().isEmpty()) {
-                    legalBasis = recategorization.getLegalBasis();
-                }
             }
 
             descriptions.add(MANUAL_CHANGE_DESCRIPTIONS.get(manualChange.getClass()));
@@ -142,16 +114,6 @@ public class ManualChangeOverwrite {
     }
 
 
-    private void setToApplied() {
-
-        removed = false;
-        ignored = false;
-        if (!isHint(entityType)) {
-            applied = true;
-        }
-    }
-
-
     public void addChange(BaseAnnotation manualChange) {
 
         changed = true;
@@ -235,17 +197,4 @@ public class ManualChangeOverwrite {
         return descriptions == null ? Collections.emptyList() : descriptions;
     }
 
-
-    public Optional<List<RectangleWithPage>> getPositions() {
-
-        calculateCurrentOverride();
-        return positions == null ? Optional.empty() : Optional.of(positions);
-    }
-
-
-    private boolean isHint(EntityType entityType) {
-
-        return entityType.equals(EntityType.HINT);
-    }
-
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/MatchedRule.java

@@ -24,19 +24,18 @@ public final class MatchedRule implements Comparable<MatchedRule> {
 
     public static final RuleType FINAL_TYPE = RuleType.fromString("FINAL");
     public static final RuleType ELIMINATION_RULE_TYPE = RuleType.fromString("X");
-    public static final RuleType IMPORTED_TYPE = RuleType.fromString("IMP");
-    private static final List<RuleType> RULE_TYPE_PRIORITIES = List.of(FINAL_TYPE, ELIMINATION_RULE_TYPE, IMPORTED_TYPE);
+    private static final List<RuleType> RULE_TYPE_PRIORITIES = List.of(FINAL_TYPE, ELIMINATION_RULE_TYPE);
 
     RuleIdentifier ruleIdentifier;
     @Builder.Default
     String reason = "";
     @Builder.Default
     String legalBasis = "";
-    boolean writeValueWithLineBreaks;
     boolean applied;
+    boolean writeValueWithLineBreaks;
     boolean removed;
     boolean ignored;
-
+    boolean resized;
     @Builder.Default
     Set<TextEntity> references = Collections.emptySet();
 
@@ -47,20 +46,6 @@ public final class MatchedRule implements Comparable<MatchedRule> {
     }
 
 
-    public MatchedRule asSkippedIfApplied() {
-
-        if (!this.isApplied()) {
-            return this;
-        }
-        return MatchedRule.builder().ruleIdentifier(getRuleIdentifier())
-                .writeValueWithLineBreaks(this.isWriteValueWithLineBreaks())
-                .legalBasis(this.getLegalBasis())
-                .reason(this.getReason())
-                .references(this.getReferences())
-                .build();
-    }
-
-
     @Override
     public int compareTo(MatchedRule matchedRule) {
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/TextEntity.java

@@ -1,7 +1,6 @@
 package com.iqser.red.service.redaction.v1.server.model.document.entity;
 
 import java.awt.geom.Rectangle2D;
-import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Comparator;
 import java.util.HashSet;
@@ -29,23 +28,21 @@ import lombok.experimental.FieldDefaults;
 @AllArgsConstructor
 @FieldDefaults(level = AccessLevel.PRIVATE)
 @EqualsAndHashCode(onlyExplicitlyIncluded = true)
-@SuppressWarnings("PMD.AvoidFieldNameMatchingMethodName")
 public class TextEntity implements IEntity {
 
     // primary key
     @EqualsAndHashCode.Include
-    final String id;
-    // primary key end
-
-    TextRange textRange;
-    @Builder.Default
-    List<TextRange> duplicateTextRanges = new ArrayList<>();
-    String type; // TODO: make final once ManualChangesApplicatioService recategorize is deleted
+    final TextRange textRange;
+    @EqualsAndHashCode.Include
+    final String type;
+    @EqualsAndHashCode.Include
     final EntityType entityType;
+    // primary key end
 
     @Builder.Default
     final PriorityQueue<MatchedRule> matchedRuleList = new PriorityQueue<>();
-    final ManualChangeOverwrite manualOverwrite;
+    @Builder.Default
+    final ManualChangeOverwrite manualOverwrite = new ManualChangeOverwrite();
 
     boolean dictionaryEntry;
     boolean dossierDictionaryEntry;
@@ -65,30 +62,12 @@ public class TextEntity implements IEntity {
     SemanticNode deepestFullyContainingNode;
 
 
-    public static TextEntity initialEntityNode(TextRange textRange, String type, EntityType entityType, SemanticNode node) {
+    public static TextEntity initialEntityNode(TextRange textRange, String type, EntityType entityType) {
 
-        return TextEntity.builder().id(buildId(node, textRange, type, entityType)).type(type).entityType(entityType).textRange(textRange).manualOverwrite(new ManualChangeOverwrite(entityType)).build();
+        return TextEntity.builder().type(type).entityType(entityType).textRange(textRange).build();
     }
 
 
-    public static TextEntity initialEntityNode(TextRange textRange, String type, EntityType entityType, String id) {
-
-        return TextEntity.builder().id(id).type(type).entityType(entityType).textRange(textRange).manualOverwrite(new ManualChangeOverwrite(entityType)).build();
-    }
-
-
-    private static String buildId(SemanticNode node, TextRange textRange, String type, EntityType entityType) {
-
-        Map<Page, List<Rectangle2D>> rectanglesPerLinePerPage = node.getPositionsPerPage(textRange);
-        return IdBuilder.buildId(rectanglesPerLinePerPage.keySet(), rectanglesPerLinePerPage.values().stream().flatMap(Collection::stream).toList(), type, entityType.name());
-    }
-
-
-    public void addTextRange(TextRange textRange) {
-
-        duplicateTextRanges.add(textRange);
-    }
-
     public boolean occursInNodeOfType(Class<? extends SemanticNode> clazz) {
 
         return intersectingNodes.stream().anyMatch(clazz::isInstance);
@@ -103,13 +82,13 @@ public class TextEntity implements IEntity {
 
     public boolean isType(String type) {
 
-        return type().equals(type);
+        return this.type.equals(type);
     }
 
 
     public boolean isAnyType(List<String> types) {
 
-        return types.contains(type());
+        return types.contains(type);
     }
 
 
@@ -146,6 +125,7 @@ public class TextEntity implements IEntity {
                     .min(Comparator.comparingInt(Page::getNumber))
                     .orElseThrow(() -> new RuntimeException("No Positions found on any page!"));
 
+            String id = IdBuilder.buildId(pages, rectanglesPerLinePerPage.values().stream().flatMap(Collection::stream).toList(), type, entityType.name());
             positionsOnPagePerPage = rectanglesPerLinePerPage.entrySet().stream().map(entry -> buildPositionOnPage(firstPage, id, entry)).toList();
         }
         return positionsOnPagePerPage;
@@ -213,7 +193,7 @@ public class TextEntity implements IEntity {
         });
         sb.delete(sb.length() - 2, sb.length());
         sb.append("], type = \"");
-        sb.append(type());
+        sb.append(type);
         sb.append("\", EntityType.");
         sb.append(entityType);
         sb.append("]");
@@ -227,11 +207,4 @@ public class TextEntity implements IEntity {
         return getManualOverwrite().getType().orElse(type);
     }
 
-
-    @Override
-    public String value() {
-
-        return getManualOverwrite().getValue().orElse(getMatchedRule().isWriteValueWithLineBreaks() ? getValueWithLineBreaks() : value);
-    }
-
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Document.java

@@ -14,13 +14,11 @@ import com.iqser.red.service.redaction.v1.server.model.document.DocumentTree;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlockCollector;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
 
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;
-import lombok.EqualsAndHashCode;
 import lombok.NoArgsConstructor;
 import lombok.experimental.FieldDefaults;
 
@@ -29,14 +27,8 @@ import lombok.experimental.FieldDefaults;
 @AllArgsConstructor
 @NoArgsConstructor
 @FieldDefaults(level = AccessLevel.PRIVATE)
-@EqualsAndHashCode(onlyExplicitlyIncluded = true)
 public class Document implements GenericSemanticNode {
 
-    @EqualsAndHashCode.Include
-    List<Integer> treeId = Collections.emptyList();
-    @Builder.Default
-    Set<LayoutEngine> engines = new HashSet<>(Set.of(LayoutEngine.ALGORITHM));
-
     Set<Page> pages;
     DocumentTree documentTree;
     Integer numberOfPages;
@@ -75,6 +67,13 @@ public class Document implements GenericSemanticNode {
     }
 
 
+    @Override
+    public List<Integer> getTreeId() {
+
+        return Collections.emptyList();
+    }
+
+
     @Override
     public void setTreeId(List<Integer> tocId) {
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Footer.java

@@ -1,15 +1,12 @@
 package com.iqser.red.service.redaction.v1.server.model.document.nodes;
 
-import java.awt.geom.Rectangle2D;
 import java.util.HashSet;
 import java.util.List;
-import java.util.Map;
 import java.util.Set;
 
 import com.iqser.red.service.redaction.v1.server.model.document.DocumentTree;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
 
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
@@ -24,25 +21,21 @@ import lombok.experimental.FieldDefaults;
 @AllArgsConstructor
 @NoArgsConstructor
 @FieldDefaults(level = AccessLevel.PRIVATE)
-@EqualsAndHashCode(onlyExplicitlyIncluded = true)
 public class Footer implements GenericSemanticNode {
 
-    final static SectionIdentifier sectionIdentifier = SectionIdentifier.empty();
     @Builder.Default
-    Set<LayoutEngine> engines = new HashSet<>(Set.of(LayoutEngine.ALGORITHM));
+    final static SectionIdentifier sectionIdentifier = SectionIdentifier.empty();
 
-    @EqualsAndHashCode.Include
     List<Integer> treeId;
-
     TextBlock leafTextBlock;
 
+    @EqualsAndHashCode.Exclude
     DocumentTree documentTree;
 
     @Builder.Default
+    @EqualsAndHashCode.Exclude
     Set<TextEntity> entities = new HashSet<>();
 
-    Map<Page, Rectangle2D> bBoxCache;
-
 
     @Override
     public NodeType getType() {
@@ -78,14 +71,4 @@ public class Footer implements GenericSemanticNode {
         return treeId + ": " + NodeType.FOOTER + ": " + leafTextBlock.buildSummary();
     }
 
-
-    @Override
-    public Map<Page, Rectangle2D> getBBox() {
-
-        if (bBoxCache == null) {
-            bBoxCache = GenericSemanticNode.super.getBBox();
-        }
-        return bBoxCache;
-    }
-
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Header.java

@@ -1,15 +1,12 @@
 package com.iqser.red.service.redaction.v1.server.model.document.nodes;
 
-import java.awt.geom.Rectangle2D;
 import java.util.HashSet;
 import java.util.List;
-import java.util.Map;
 import java.util.Set;
 
 import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.DocumentTree;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
 
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
@@ -24,25 +21,21 @@ import lombok.experimental.FieldDefaults;
 @AllArgsConstructor
 @NoArgsConstructor
 @FieldDefaults(level = AccessLevel.PRIVATE)
-@EqualsAndHashCode(onlyExplicitlyIncluded = true)
 public class Header implements GenericSemanticNode {
 
-    final static SectionIdentifier sectionIdentifier = SectionIdentifier.empty();
     @Builder.Default
-    Set<LayoutEngine> engines = new HashSet<>(Set.of(LayoutEngine.ALGORITHM));
+    final static SectionIdentifier sectionIdentifier = SectionIdentifier.empty();
 
-    @EqualsAndHashCode.Include
     List<Integer> treeId;
-
     TextBlock leafTextBlock;
 
+    @EqualsAndHashCode.Exclude
     DocumentTree documentTree;
 
     @Builder.Default
+    @EqualsAndHashCode.Exclude
     Set<TextEntity> entities = new HashSet<>();
 
-    Map<Page, Rectangle2D> bBoxCache;
-
 
     @Override
     public boolean isLeaf() {
@@ -78,14 +71,4 @@ public class Header implements GenericSemanticNode {
         return treeId + ": " + NodeType.HEADER + ": " + leafTextBlock.buildSummary();
     }
 
-
-    @Override
-    public Map<Page, Rectangle2D> getBBox() {
-
-        if (bBoxCache == null) {
-            bBoxCache = GenericSemanticNode.super.getBBox();
-        }
-        return bBoxCache;
-    }
-
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Headline.java

@@ -1,16 +1,13 @@
 package com.iqser.red.service.redaction.v1.server.model.document.nodes;
 
-import java.awt.geom.Rectangle2D;
 import java.util.HashSet;
 import java.util.List;
-import java.util.Map;
 import java.util.Set;
 
 import com.iqser.red.service.redaction.v1.server.model.document.DocumentTree;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.AtomicTextBlock;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
 
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
@@ -25,23 +22,19 @@ import lombok.experimental.FieldDefaults;
 @AllArgsConstructor
 @NoArgsConstructor
 @FieldDefaults(level = AccessLevel.PRIVATE)
-@EqualsAndHashCode(onlyExplicitlyIncluded = true)
 public class Headline implements GenericSemanticNode {
 
-    @Builder.Default
-    Set<LayoutEngine> engines = new HashSet<>(Set.of(LayoutEngine.ALGORITHM));
-    @EqualsAndHashCode.Include
     List<Integer> treeId;
     TextBlock leafTextBlock;
     SectionIdentifier sectionIdentifier;
 
+    @EqualsAndHashCode.Exclude
     DocumentTree documentTree;
 
     @Builder.Default
+    @EqualsAndHashCode.Exclude
     Set<TextEntity> entities = new HashSet<>();
 
-    Map<Page, Rectangle2D> bBoxCache;
-
 
     @Override
     public NodeType getType() {
@@ -78,16 +71,6 @@ public class Headline implements GenericSemanticNode {
     }
 
 
-    @Override
-    public Map<Page, Rectangle2D> getBBox() {
-
-        if (bBoxCache == null) {
-            bBoxCache = GenericSemanticNode.super.getBBox();
-        }
-        return bBoxCache;
-    }
-
-
     @Override
     public SectionIdentifier getSectionIdentifier() {
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Image.java

@@ -18,7 +18,6 @@ import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRu
 import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlockCollector;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
 
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
@@ -33,13 +32,8 @@ import lombok.experimental.FieldDefaults;
 @AllArgsConstructor
 @NoArgsConstructor
 @FieldDefaults(level = AccessLevel.PRIVATE)
-@EqualsAndHashCode(onlyExplicitlyIncluded = true)
 public class Image implements GenericSemanticNode, IEntity {
 
-    @Builder.Default
-    Set<LayoutEngine> engines = new HashSet<>(Set.of(LayoutEngine.ALGORITHM));
-
-    @EqualsAndHashCode.Include
     List<Integer> treeId;
     String id;
 
@@ -74,9 +68,7 @@ public class Image implements GenericSemanticNode, IEntity {
     @Override
     public TextBlock getTextBlock() {
 
-        return streamAllSubNodes().filter(SemanticNode::isLeaf)
-                .map(SemanticNode::getLeafTextBlock)
-                .collect(new TextBlockCollector());
+        return streamAllSubNodes().filter(SemanticNode::isLeaf).map(SemanticNode::getLeafTextBlock).collect(new TextBlockCollector());
     }
 
 
@@ -97,8 +89,7 @@ public class Image implements GenericSemanticNode, IEntity {
     @Override
     public String type() {
 
-        return getManualOverwrite().getType()
-                .orElse(imageType.toString());
+        return getManualOverwrite().getType().orElse(imageType.toString());
     }
 
 
@@ -131,32 +122,6 @@ public class Image implements GenericSemanticNode, IEntity {
     }
 
 
-    public boolean mostlyContainedBy(Image image, double containmentThreshold) {
-
-        Map<Page, Rectangle2D> bboxImage = image.getBBox();
-        Map<Page, Rectangle2D> bbox = this.getBBox();
-        //image needs to be on the same page
-        if(bboxImage.get(this.page) != null) {
-            Rectangle2D intersection = bboxImage.get(this.page).createIntersection(bbox.get(this.page));
-            double calculatedIntersection = intersection.getWidth() * intersection.getHeight();
-            double area = bbox.get(this.page).getWidth() * bbox.get(this.page).getHeight();
-            return (calculatedIntersection / area) > containmentThreshold;
-        }
-        return false;
-    }
-
-
-    public boolean mostlyContains(Image image, double containmentThreshold) {
-
-        Map<Page, Rectangle2D> bboxImage = image.getBBox();
-        Map<Page, Rectangle2D> bbox = this.getBBox();
-        Rectangle2D intersection = bbox.get(this.page).createIntersection(bboxImage.get(this.page));
-        double calculatedIntersection = intersection.getWidth() * intersection.getHeight();
-        double area = bbox.get(this.page).getWidth() * bbox.get(this.page).getHeight();
-        return (area / calculatedIntersection) > containmentThreshold;
-    }
-
-
     public int length() {
 
         return 0;

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/ImageType.java

@@ -6,22 +6,10 @@ public enum ImageType {
     LOGO,
     FORMULA,
     SIGNATURE,
-    OTHER {
-        @Override
-        public String toString() {
-
-            return "image";
-        }
-    },
+    OTHER,
     OCR;
 
 
-    public String toString() {
-
-        return name().toLowerCase(Locale.ENGLISH);
-    }
-
-
     public static ImageType fromString(String imageType) {
 
         return switch (imageType.toLowerCase(Locale.ROOT)) {

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Page.java

@@ -23,23 +23,26 @@ import lombok.experimental.FieldDefaults;
 @NoArgsConstructor
 @AllArgsConstructor
 @FieldDefaults(level = AccessLevel.PRIVATE)
-@EqualsAndHashCode(onlyExplicitlyIncluded = true)
 public class Page {
 
-    @EqualsAndHashCode.Include
     Integer number;
     Integer height;
     Integer width;
     Integer rotation;
 
+    @EqualsAndHashCode.Exclude
     List<SemanticNode> mainBody;
+    @EqualsAndHashCode.Exclude
     Header header;
+    @EqualsAndHashCode.Exclude
     Footer footer;
 
     @Builder.Default
+    @EqualsAndHashCode.Exclude
     Set<TextEntity> entities = new HashSet<>();
 
     @Builder.Default
+    @EqualsAndHashCode.Exclude
     Set<Image> images = new HashSet<>();
 
 
@@ -54,4 +57,19 @@ public class Page {
 
         return String.valueOf(number);
     }
+
+
+    @Override
+    public int hashCode() {
+
+        return number;
+    }
+
+
+    @Override
+    public boolean equals(Object o) {
+
+        return o instanceof Page && o.hashCode() == this.hashCode();
+    }
+
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Paragraph.java

@@ -1,15 +1,12 @@
 package com.iqser.red.service.redaction.v1.server.model.document.nodes;
 
-import java.awt.geom.Rectangle2D;
 import java.util.HashSet;
 import java.util.List;
-import java.util.Map;
 import java.util.Set;
 
 import com.iqser.red.service.redaction.v1.server.model.document.DocumentTree;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
 
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
@@ -22,23 +19,18 @@ import lombok.experimental.FieldDefaults;
 @Builder
 @AllArgsConstructor
 @FieldDefaults(level = AccessLevel.PRIVATE)
-@EqualsAndHashCode(onlyExplicitlyIncluded = true)
 public class Paragraph implements GenericSemanticNode {
 
-    @Builder.Default
-    Set<LayoutEngine> engines = new HashSet<>(Set.of(LayoutEngine.ALGORITHM));
-
-    @EqualsAndHashCode.Include
     List<Integer> treeId;
     TextBlock leafTextBlock;
 
+    @EqualsAndHashCode.Exclude
     DocumentTree documentTree;
 
     @Builder.Default
+    @EqualsAndHashCode.Exclude
     Set<TextEntity> entities = new HashSet<>();
 
-    Map<Page, Rectangle2D> bBoxCache;
-
 
     @Override
     public NodeType getType() {
@@ -67,14 +59,4 @@ public class Paragraph implements GenericSemanticNode {
         return treeId + ": " + NodeType.PARAGRAPH + ": " + leafTextBlock.buildSummary();
     }
 
-
-    @Override
-    public Map<Page, Rectangle2D> getBBox() {
-
-        if (bBoxCache == null) {
-            bBoxCache = GenericSemanticNode.super.getBBox();
-        }
-        return bBoxCache;
-    }
-
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Section.java

@@ -1,23 +1,19 @@
 package com.iqser.red.service.redaction.v1.server.model.document.nodes;
 
-import java.awt.geom.Rectangle2D;
 import java.util.HashSet;
 import java.util.List;
-import java.util.Map;
 import java.util.Set;
 
 import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.DocumentTree;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlockCollector;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
 
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;
 import lombok.EqualsAndHashCode;
-import lombok.RequiredArgsConstructor;
 import lombok.experimental.FieldDefaults;
 import lombok.extern.slf4j.Slf4j;
 
@@ -26,23 +22,18 @@ import lombok.extern.slf4j.Slf4j;
 @Builder
 @AllArgsConstructor
 @FieldDefaults(level = AccessLevel.PRIVATE)
-@EqualsAndHashCode(onlyExplicitlyIncluded = true)
 public class Section implements GenericSemanticNode {
 
-    @EqualsAndHashCode.Include
     List<Integer> treeId;
 
     TextBlock textBlock;
+    @EqualsAndHashCode.Exclude
     DocumentTree documentTree;
 
     @Builder.Default
+    @EqualsAndHashCode.Exclude
     Set<TextEntity> entities = new HashSet<>();
 
-    Map<Page, Rectangle2D> bBoxCache;
-
-    @Builder.Default
-    Set<LayoutEngine> engines = new HashSet<>(Set.of(LayoutEngine.ALGORITHM));
-
 
     @Override
     public NodeType getType() {
@@ -101,14 +92,4 @@ public class Section implements GenericSemanticNode {
         return streamAllSubNodesOfType(NodeType.HEADLINE).anyMatch(h -> h.containsStringIgnoreCase(value));
     }
 
-
-    @Override
-    public Map<Page, Rectangle2D> getBBox() {
-
-        if (bBoxCache == null) {
-            bBoxCache = GenericSemanticNode.super.getBBox();
-        }
-        return bBoxCache;
-    }
-
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/SemanticNode.java

@@ -9,20 +9,17 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
-import java.util.Optional;
 import java.util.Set;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
-import com.iqser.red.service.redaction.v1.server.model.document.DocumentTree;
 import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
+import com.iqser.red.service.redaction.v1.server.model.document.DocumentTree;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.AtomicTextBlock;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
-import com.iqser.red.service.redaction.v1.server.service.document.NodeVisitor;
 import com.iqser.red.service.redaction.v1.server.utils.RectangleTransformations;
 import com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
 
 public interface SemanticNode {
 
@@ -63,7 +60,6 @@ public interface SemanticNode {
         return getTextBlock().getPages();
     }
 
-
     /**
      * Finds the first page associated with this Node.
      *
@@ -82,19 +78,13 @@ public interface SemanticNode {
      */
     default Set<Page> getPages(TextRange textRange) {
 
-        if (!getTextRange().intersects(textRange)) {
-            throw new IllegalArgumentException(format("%s which was used to query for pages is not intersected in the %s of this node!", textRange, getTextRange()));
+        if (!getTextRange().contains(textRange)) {
+            throw new IllegalArgumentException(format("%s which was used to query for pages is not contained in the %s of this node!", textRange, getTextRange()));
         }
         return getTextBlock().getPages(textRange);
     }
 
 
-    /**
-     * Checks if the given page number exists in the list of pages.
-     *
-     * @param pageNumber the page number to be checked
-     * @return true if the page number exists, otherwise false
-     */
     default boolean onPage(int pageNumber) {
 
         return getPages().stream().anyMatch(page -> page.getNumber() == pageNumber);
@@ -180,30 +170,6 @@ public interface SemanticNode {
     }
 
 
-    /**
-     * Returns the next sibling node of this SemanticNode in the document tree, if any.
-     * If there is no next sibling node, an empty Optional is returned.
-     *
-     * @return Optional containing the next sibling node, or empty if there is none
-     */
-    default Optional<SemanticNode> getNextSibling() {
-
-        return getDocumentTree().getNextSibling(getTreeId());
-    }
-
-
-    /**
-     * Returns the previous sibling node of this SemanticNode in the document tree, if any.
-     * If there is no previous sibling node, an empty Optional is returned.
-     *
-     * @return Optional containing the previous sibling node, or empty if there is none
-     */
-    default Optional<SemanticNode> getPreviousSibling() {
-
-        return getDocumentTree().getPreviousSibling(getTreeId());
-    }
-
-
     /**
      * Leaf means a SemanticNode has direct access to a TextBlock, by default this is false and must be overridden.
      * Currently only Sections, Images, and Tables are not leaves.
@@ -249,7 +215,7 @@ public interface SemanticNode {
      */
     default boolean hasEntitiesOfType(String type) {
 
-        return getEntities().stream().filter(TextEntity::active).anyMatch(redactionEntity -> redactionEntity.type().equals(type));
+        return getEntities().stream().filter(TextEntity::active).anyMatch(redactionEntity -> redactionEntity.getType().equals(type));
     }
 
 
@@ -258,11 +224,11 @@ public interface SemanticNode {
      * Ignores Entity with ignored == true or removed == true.
      *
      * @param types an array of strings representing the types of entities to check for
-     * @return true, if this SemanticNode has at least one Entity of the provided types
+     * @return true, if this SemanticNode has at least one Entity of any of the provided types
      */
     default boolean hasEntitiesOfAnyType(String... types) {
 
-        return getEntities().stream().filter(TextEntity::active).anyMatch(redactionEntity -> Arrays.stream(types).anyMatch(type -> redactionEntity.type().equals(type)));
+        return getEntities().stream().filter(TextEntity::active).anyMatch(redactionEntity -> Arrays.stream(types).anyMatch(type -> redactionEntity.getType().equals(type)));
     }
 
 
@@ -275,7 +241,11 @@ public interface SemanticNode {
      */
     default boolean hasEntitiesOfAllTypes(String... types) {
 
-        return getEntities().stream().filter(TextEntity::active).map(TextEntity::type).collect(Collectors.toUnmodifiableSet()).containsAll(Arrays.stream(types).toList());
+        return getEntities().stream()
+                .filter(TextEntity::active)
+                .map(TextEntity::getType)
+                .collect(Collectors.toUnmodifiableSet())
+                .containsAll(Arrays.stream(types).toList());
     }
 
 
@@ -288,7 +258,7 @@ public interface SemanticNode {
      */
     default List<TextEntity> getEntitiesOfType(String type) {
 
-        return getEntities().stream().filter(TextEntity::active).filter(redactionEntity -> redactionEntity.type().equals(type)).toList();
+        return getEntities().stream().filter(TextEntity::active).filter(redactionEntity -> redactionEntity.getType().equals(type)).toList();
     }
 
 
@@ -357,13 +327,6 @@ public interface SemanticNode {
         return getTextBlock().getSearchText().contains(string);
     }
 
-    Set<LayoutEngine> getEngines();
-
-    default void addEngine(LayoutEngine engine) {
-        getEngines().add(engine);
-    }
-
-
 
     /**
      * Checks whether this SemanticNode contains all the provided Strings.
@@ -437,78 +400,6 @@ public interface SemanticNode {
     }
 
 
-    /**
-     * Checks whether this SemanticNode contains exactly the provided String as a word.
-     *
-     * @param word - String which the TextBlock might contain
-     * @return true, if this node's TextBlock contains string
-     */
-    default boolean containsWord(String word) {
-
-        return getTextBlock().getWords().stream().anyMatch(s -> s.equals(word));
-    }
-
-
-    /**
-     * Checks whether this SemanticNode contains exactly the provided String as a word ignoring case.
-     *
-     * @param word - String which the TextBlock might contain
-     * @return true, if this node's TextBlock contains string
-     */
-    default boolean containsWordIgnoreCase(String word) {
-
-        return getTextBlock().getWords().stream().map(String::toLowerCase).anyMatch(s -> s.equals(word.toLowerCase(Locale.ENGLISH)));
-    }
-
-
-    /**
-     * Checks whether this SemanticNode contains any of the provided Strings as a word.
-     *
-     * @param words - A List of Strings which the TextBlock might contain
-     * @return true, if this node's TextBlock contains any of the provided strings
-     */
-    default boolean containsAnyWord(String... words) {
-
-        return Arrays.stream(words).anyMatch(word -> getTextBlock().getWords().stream().anyMatch(word::equals));
-    }
-
-
-    /**
-     * Checks whether this SemanticNode contains any of the provided Strings as a word ignoring case.
-     *
-     * @param words - A List of Strings which the TextBlock might contain
-     * @return true, if this node's TextBlock contains any of the provided strings
-     */
-    default boolean containsAnyWordIgnoreCase(String... words) {
-
-        return Arrays.stream(words).map(String::toLowerCase).anyMatch(word -> getTextBlock().getWords().stream().map(String::toLowerCase).anyMatch(word::equals));
-    }
-
-
-    /**
-     * Checks whether this SemanticNode contains all the provided Strings as word.
-     *
-     * @param words - A List of Strings which the TextBlock might contain
-     * @return true, if this node's TextBlock contains all the provided strings
-     */
-    default boolean containsAllWords(String... words) {
-
-        return Arrays.stream(words).allMatch(word -> getTextBlock().getWords().stream().anyMatch(word::equals));
-    }
-
-
-    /**
-     * Checks whether this SemanticNode contains all the provided Strings as word ignoring case.
-     *
-     * @param words - A List of Strings which the TextBlock might contain
-     * @return true, if this node's TextBlock contains all the provided strings
-     */
-    default boolean containsAllWordsIgnoreCase(String... words) {
-
-        return Arrays.stream(words).map(String::toLowerCase).allMatch(word -> getTextBlock().getWords().stream().map(String::toLowerCase).anyMatch(word::equals));
-    }
-
-
     /**
      * Checks whether this SemanticNode matches the provided regex pattern.
      *
@@ -535,12 +426,11 @@ public interface SemanticNode {
 
     /**
      * Checks whether this SemanticNode intersects the provided rectangle.
-     *
-     * @param x          the lower left corner X value
-     * @param y          the lower left corner Y value
-     * @param w          width
-     * @param h          height
-     * @param pageNumber the pageNumber of the rectangle
+     * @param x the lower left corner X value
+     * @param y the lower left corner Y value
+     * @param w width
+     * @param h height
+     * @param pageNumber the pagenumber of the rectangle
      * @return true if intersects, false otherwise
      */
     default boolean intersectsRectangle(int x, int y, int w, int h, int pageNumber) {
@@ -562,6 +452,7 @@ public interface SemanticNode {
             if (textBlock.containsTextRange(textEntity.getTextRange())) {
                 textEntity.setDeepestFullyContainingNode(this);
             }
+
             textEntity.addIntersectingNode(this);
             streamChildren().filter(semanticNode -> semanticNode.getTextRange().intersects(textEntity.getTextRange()))
                     .forEach(node -> node.addThisToEntityIfIntersects(textEntity));
@@ -635,27 +526,6 @@ public interface SemanticNode {
     }
 
 
-    /**
-     * For a given TextRange this function returns a List of rectangle around the text in the range.
-     * These Rectangles are split either by a new line or by a large gap in the current line.
-     * This is mainly used to find the positions of TextEntities
-     *
-     * @param textRange A TextRange to calculate the positions for.
-     * @return A Map, where the keys are the pages and the values are a list of rectangles describing the position of words
-     */
-    default Map<Page, List<Rectangle2D>> getPositionsPerPage(TextRange textRange) {
-
-        if (isLeaf()) {
-            return getTextBlock().getPositionsPerPage(textRange);
-        }
-        Optional<SemanticNode> containingChildNode = streamChildren().filter(child -> child.getTextRange().contains(textRange)).findFirst();
-        if (containingChildNode.isEmpty()) {
-            return getTextBlock().getPositionsPerPage(textRange);
-        }
-        return containingChildNode.get().getPositionsPerPage(textRange);
-    }
-
-
     /**
      * If this Node is a Leaf it will calculate the boundingBox of its LeafTextBlock, otherwise it will calculate the Union of the BoundingBoxes of all its Children.
      * If called on the Document, it will return the cropbox of each page
@@ -664,11 +534,12 @@ public interface SemanticNode {
      */
     default Map<Page, Rectangle2D> getBBox() {
 
+        Map<Page, Rectangle2D> bBoxPerPage = new HashMap<>();
         if (isLeaf()) {
-            return getBBoxFromLeafTextBlock();
+            return getBBoxFromLeafTextBlock(bBoxPerPage);
         }
 
-        return getBBoxFromChildren();
+        return getBBoxFromChildren(bBoxPerPage);
     }
 
 
@@ -691,49 +562,28 @@ public interface SemanticNode {
 
     /**
      * TODO: this produces unwanted results for sections spanning multiple columns.
-     * Computes the Union of the bounding boxes of all children recursively.
      *
+     * @param bBoxPerPage initial empty BoundingBox
      * @return The union of the BoundingBoxes of all children
      */
-    private Map<Page, Rectangle2D> getBBoxFromChildren() {
+    private Map<Page, Rectangle2D> getBBoxFromChildren(Map<Page, Rectangle2D> bBoxPerPage) {
 
-        Map<Page, Rectangle2D> bBoxPerPage = new HashMap<>();
-        List<Map<Page, Rectangle2D>> childrenBBoxes = streamChildren().map(SemanticNode::getBBox).toList();
-        Set<Page> pages = childrenBBoxes.stream().flatMap(map -> map.keySet().stream()).collect(Collectors.toSet());
-        for (Page page : pages) {
-            Rectangle2D bBoxOnPage = childrenBBoxes.stream()
-                    .filter(childBboxPerPage -> childBboxPerPage.containsKey(page))
-                    .map(childBboxPerPage -> childBboxPerPage.get(page))
-                    .collect(RectangleTransformations.collectBBox());
-            bBoxPerPage.put(page, bBoxOnPage);
-        }
-        return bBoxPerPage;
+        return streamChildren().map(SemanticNode::getBBox).reduce((map1, map2) -> {
+            map1.forEach((page, rectangle) -> map2.merge(page, rectangle, (rect1, rect2) -> rect1.createUnion(rect2).getBounds2D()));
+            return map2;
+        }).orElse(bBoxPerPage);
     }
 
 
     /**
+     * @param bBoxPerPage initial empty BoundingBox
      * @return The union of all BoundingBoxes of the TextBlock of this node
      */
-    private Map<Page, Rectangle2D> getBBoxFromLeafTextBlock() {
+    private Map<Page, Rectangle2D> getBBoxFromLeafTextBlock(Map<Page, Rectangle2D> bBoxPerPage) {
 
-        Map<Page, Rectangle2D> bBoxPerPage = new HashMap<>();
         Map<Page, List<AtomicTextBlock>> atomicTextBlockPerPage = getTextBlock().getAtomicTextBlocks().stream().collect(Collectors.groupingBy(AtomicTextBlock::getPage));
-        atomicTextBlockPerPage.forEach((page, atomicTextBlocks) -> bBoxPerPage.put(page, RectangleTransformations.atomicTextBlockBBox(atomicTextBlocks)));
+        atomicTextBlockPerPage.forEach((page, atbs) -> bBoxPerPage.put(page, RectangleTransformations.atomicTextBlockBBox(atbs)));
         return bBoxPerPage;
     }
 
-
-    /**
-     * Accepts a {@link NodeVisitor} and initiates a depth-first traversal of the semantic tree rooted at this node.
-     * The visitor's {@link NodeVisitor#visit(SemanticNode)} method is invoked for each node encountered during the traversal.
-     *
-     * @param visitor The {@link NodeVisitor} to accept and apply during the traversal.
-     * @see NodeVisitor
-     */
-    default void accept(NodeVisitor visitor) {
-
-        visitor.visit(this);
-        streamChildren().forEach(childNode -> childNode.accept(visitor));
-    }
-
-}
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Table.java

@@ -2,14 +2,11 @@ package com.iqser.red.service.redaction.v1.server.model.document.nodes;
 
 import static java.lang.String.format;
 
-import java.awt.geom.Rectangle2D;
 import java.util.Collection;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Locale;
-import java.util.Map;
 import java.util.Set;
-import java.util.stream.Collectors;
 import java.util.stream.IntStream;
 import java.util.stream.Stream;
 
@@ -17,7 +14,6 @@ import com.iqser.red.service.redaction.v1.server.model.document.DocumentTree;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlockCollector;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
 
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
@@ -30,12 +26,8 @@ import lombok.experimental.FieldDefaults;
 @Builder
 @AllArgsConstructor
 @FieldDefaults(level = AccessLevel.PRIVATE)
-@EqualsAndHashCode(onlyExplicitlyIncluded = true)
 public class Table implements SemanticNode {
 
-    @Builder.Default
-    Set<LayoutEngine> engines = new HashSet<>(Set.of(LayoutEngine.ALGORITHM));
-    @EqualsAndHashCode.Include
     List<Integer> treeId;
     DocumentTree documentTree;
 
@@ -45,20 +37,9 @@ public class Table implements SemanticNode {
     TextBlock textBlock;
 
     @Builder.Default
+    @EqualsAndHashCode.Exclude
     Set<TextEntity> entities = new HashSet<>();
 
-    Map<Page, Rectangle2D> bBoxCache;
-
-
-    @Override
-    public Map<Page, Rectangle2D> getBBox() {
-
-        if (bBoxCache == null) {
-            bBoxCache = SemanticNode.super.getBBox();
-        }
-        return bBoxCache;
-    }
-
 
     /**
      * Streams all entities in this table, that appear in a row, which contains any of the provided strings.
@@ -68,7 +49,8 @@ public class Table implements SemanticNode {
      */
     public Stream<TextEntity> streamEntitiesWhereRowContainsStringsIgnoreCase(List<String> strings) {
 
-        return IntStream.range(0, numberOfRows).boxed()
+        return IntStream.range(0, numberOfRows)
+                .boxed()
                 .filter(row -> rowContainsStringsIgnoreCase(row, strings))
                 .flatMap(this::streamRow)
                 .map(TableCell::getEntities)
@@ -85,11 +67,8 @@ public class Table implements SemanticNode {
      */
     public boolean rowContainsStringsIgnoreCase(Integer row, List<String> strings) {
 
-        String rowText = streamRow(row).map(TableCell::getTextBlock)
-                .collect(new TextBlockCollector()).getSearchText().toLowerCase(Locale.ROOT);
-        return strings.stream()
-                .map(String::toLowerCase)
-                .allMatch(rowText::contains);
+        String rowText = streamRow(row).map(TableCell::getTextBlock).collect(new TextBlockCollector()).getSearchText().toLowerCase(Locale.ROOT);
+        return strings.stream().map(String::toLowerCase).allMatch(rowText::contains);
     }
 
 
@@ -102,13 +81,9 @@ public class Table implements SemanticNode {
      */
     public Stream<TextEntity> streamEntitiesWhereRowHasHeaderAndValue(String header, String value) {
 
-        List<Integer> vertebrateStudyCols = streamHeaders().filter(headerNode -> headerNode.containsString(header))
-                .map(TableCell::getCol)
-                .toList();
+        List<Integer> vertebrateStudyCols = streamHeaders().filter(headerNode -> headerNode.containsString(header)).map(TableCell::getCol).toList();
         return streamTableCells().filter(tableCellNode -> vertebrateStudyCols.stream()
-                        .anyMatch(vertebrateStudyCol -> getCell(tableCellNode.getRow(), vertebrateStudyCol).containsString(value)))
-                .map(TableCell::getEntities)
-                .flatMap(Collection::stream);
+                .anyMatch(vertebrateStudyCol -> getCell(tableCellNode.getRow(), vertebrateStudyCol).containsString(value))).map(TableCell::getEntities).flatMap(Collection::stream);
     }
 
 
@@ -121,13 +96,9 @@ public class Table implements SemanticNode {
      */
     public Stream<TextEntity> streamEntitiesWhereRowHasHeaderAndAnyValue(String header, List<String> values) {
 
-        List<Integer> colsWithHeader = streamHeaders().filter(headerNode -> headerNode.containsString(header))
-                .map(TableCell::getCol)
-                .toList();
+        List<Integer> colsWithHeader = streamHeaders().filter(headerNode -> headerNode.containsString(header)).map(TableCell::getCol).toList();
         return streamTableCells().filter(tableCellNode -> colsWithHeader.stream()
-                        .anyMatch(colWithHeader -> getCell(tableCellNode.getRow(), colWithHeader).containsAnyString(values)))
-                .map(TableCell::getEntities)
-                .flatMap(Collection::stream);
+                .anyMatch(colWithHeader -> getCell(tableCellNode.getRow(), colWithHeader).containsAnyString(values))).map(TableCell::getEntities).flatMap(Collection::stream);
     }
 
 
@@ -140,33 +111,16 @@ public class Table implements SemanticNode {
      */
     public Stream<TextEntity> streamEntitiesWhereRowContainsEntitiesOfType(List<String> types) {
 
-        return IntStream.range(0, numberOfRows).boxed()
-                .filter(rowNumber -> streamTextEntitiesInRow(rowNumber).map(TextEntity::type)
-                        .anyMatch(types::contains))
-                .flatMap(this::streamRow)
-                .map(TableCell::getEntities)
-                .flatMap(Collection::stream);
-    }
+        List<Integer> rowsWithEntityOfType = getEntities().stream()
+                .filter(TextEntity::active)
+                .filter(redactionEntity -> types.stream().anyMatch(type -> type.equals(redactionEntity.getType())))
+                .map(TextEntity::getIntersectingNodes)
+                .filter(node -> node instanceof TableCell)
+                .map(node -> (TableCell) node)
+                .map(TableCell::getRow)
+                .toList();
 
-
-    /**
-     * Streams all entities in this table, that appear in a row, which contains at least one entity of each of the provided types.
-     * Ignores Entity with ignored == true or removed == true.
-     *
-     * @param types type strings to check whether a row contains an entity like them
-     * @return Stream of all entities in this table, that appear in a row, which contains at least one entity of each of the provided types.
-     */
-    public Stream<TextEntity> streamEntitiesWhereRowContainsEntitiesOfEachType(List<String> types) {
-
-        return IntStream.range(0, numberOfRows).boxed()
-                .filter(rowNumber -> {
-                    Set<String> entityTypes = streamTextEntitiesInRow(rowNumber).map(TextEntity::type)
-                            .collect(Collectors.toSet());
-                    return entityTypes.containsAll(types);
-                })
-                .flatMap(this::streamRow)
-                .map(TableCell::getEntities)
-                .flatMap(Collection::stream);
+        return rowsWithEntityOfType.stream().flatMap(this::streamRow).map(TableCell::getEntities).flatMap(Collection::stream);
     }
 
 
@@ -179,43 +133,18 @@ public class Table implements SemanticNode {
      */
     public Stream<TextEntity> streamEntitiesWhereRowContainsNoEntitiesOfType(List<String> types) {
 
-        return IntStream.range(0, numberOfRows).boxed()
-                .filter(rowNumber -> streamTextEntitiesInRow(rowNumber).map(TextEntity::type)
-                        .noneMatch(types::contains))
+        return IntStream.range(0, numberOfRows)
+                .boxed()
+                .filter(rowNumber -> streamRow(rowNumber).map(TableCell::getEntities)
+                        .flatMap(Collection::stream)
+                        .filter(TextEntity::active)
+                        .noneMatch(entity -> types.contains(entity.getType())))
                 .flatMap(this::streamRow)
                 .map(TableCell::getEntities)
                 .flatMap(Collection::stream);
     }
 
 
-    /**
-     * Streams all Entities in the given row.
-     *
-     * @param rowNumber the row number to look for
-     * @return stream of TextEntities occurring in row
-     */
-    public Stream<TextEntity> streamTextEntitiesInRow(int rowNumber) {
-
-        return streamRow(rowNumber).map(TableCell::getEntities)
-                .flatMap(Collection::stream)
-                .filter(TextEntity::active);
-    }
-
-
-    /**
-     * Streams all Entities in the given col.
-     *
-     * @param colNumber the column number to look for
-     * @return stream of TextEntities occurring in row
-     */
-    public Stream<TextEntity> streamTextEntitiesInCol(int colNumber) {
-
-        return streamCol(colNumber).map(TableCell::getEntities)
-                .flatMap(Collection::stream)
-                .filter(TextEntity::active);
-    }
-
-
     /**
      * Returns a TableCell at the provided row and column location.
      *
@@ -229,8 +158,7 @@ public class Table implements SemanticNode {
             throw new IllegalArgumentException(format("row %d, col %d is out of bounds for number of rows of %d and number of cols %d", row, col, numberOfRows, numberOfCols));
         }
         int idx = row * numberOfCols + col;
-        return (TableCell) documentTree.getEntryById(treeId).getChildren()
-                .get(idx).getNode();
+        return (TableCell) documentTree.getEntryById(treeId).getChildren().get(idx).getNode();
     }
 
 
@@ -245,18 +173,6 @@ public class Table implements SemanticNode {
     }
 
 
-    /**
-     * Streams all TableCells that contain at least one entity of a given type in this Table row-wise.
-     *
-     * @param type The type of the entity
-     * @return Stream of filtered TableCells
-     */
-    public Stream<TableCell> streamTableCellsWhichContainType(String type) {
-
-        return streamTableCells().filter(tableCell -> tableCell.hasEntitiesOfType(type));
-    }
-
-
     /**
      * Streams all TableCells in this Table which have the provided header row-wise.
      *
@@ -279,8 +195,7 @@ public class Table implements SemanticNode {
      */
     public Stream<TableCell> streamCol(int col) {
 
-        return IntStream.range(0, numberOfRows).boxed()
-                .map(row -> getCell(row, col));
+        return IntStream.range(0, numberOfRows).boxed().map(row -> getCell(row, col));
     }
 
 
@@ -292,8 +207,7 @@ public class Table implements SemanticNode {
      */
     public Stream<TableCell> streamRow(int row) {
 
-        return IntStream.range(0, numberOfCols).boxed()
-                .map(col -> getCell(row, col));
+        return IntStream.range(0, numberOfCols).boxed().map(col -> getCell(row, col));
     }
 
 
@@ -317,8 +231,7 @@ public class Table implements SemanticNode {
      */
     public Stream<TableCell> streamHeadersForCell(int row, int col) {
 
-        return Stream.concat(streamRow(row), streamCol(col))
-                .filter(TableCell::isHeader);
+        return Stream.concat(streamRow(row), streamCol(col)).filter(TableCell::isHeader);
     }
 
 
@@ -342,11 +255,7 @@ public class Table implements SemanticNode {
      */
     public boolean hasHeaderIgnoreCase(String header) {
 
-        return streamHeaders().anyMatch(tableCellNode -> tableCellNode.getTextBlock()
-                .getSearchText()
-                .strip()
-                .toLowerCase(Locale.ENGLISH)
-                .equals(header.toLowerCase(Locale.ENGLISH)));
+        return streamHeaders().anyMatch(tableCellNode -> tableCellNode.getTextBlock().getSearchText().strip().toLowerCase(Locale.ENGLISH).equals(header.toLowerCase(Locale.ENGLISH)));
     }
 
 
@@ -380,7 +289,7 @@ public class Table implements SemanticNode {
      * Finds all entities of the provided type, which appear in the same row that the provided entity appears in.
      * Ignores Entity with ignored == true or removed == true.
      *
-     * @param type       the type of entities to search for
+     * @param type            the type of entities to search for
      * @param textEntity the entity, which appears in the row to search
      * @return List of all entities of the provided type, which appear in the same row that the provided entity appears in.
      */
@@ -408,9 +317,7 @@ public class Table implements SemanticNode {
     public TextBlock getTextBlock() {
 
         if (textBlock == null) {
-            textBlock = streamAllSubNodes().filter(SemanticNode::isLeaf)
-                    .map(SemanticNode::getLeafTextBlock)
-                    .collect(new TextBlockCollector());
+            textBlock = streamAllSubNodes().filter(SemanticNode::isLeaf).map(SemanticNode::getLeafTextBlock).collect(new TextBlockCollector());
         }
         return textBlock;
     }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/TableCell.java

@@ -11,7 +11,6 @@ import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntit
 import com.iqser.red.service.redaction.v1.server.model.document.DocumentTree;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlockCollector;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
 
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
@@ -24,13 +23,8 @@ import lombok.experimental.FieldDefaults;
 @Builder
 @AllArgsConstructor
 @FieldDefaults(level = AccessLevel.PRIVATE)
-@EqualsAndHashCode(onlyExplicitlyIncluded = true)
 public class TableCell implements GenericSemanticNode {
 
-    @Builder.Default
-    Set<LayoutEngine> engines = new HashSet<>(Set.of(LayoutEngine.ALGORITHM));
-
-    @EqualsAndHashCode.Include
     List<Integer> treeId;
     int row;
     int col;
@@ -42,9 +36,11 @@ public class TableCell implements GenericSemanticNode {
 
     TextBlock textBlock;
 
+    @EqualsAndHashCode.Exclude
     DocumentTree documentTree;
 
     @Builder.Default
+    @EqualsAndHashCode.Exclude
     Set<TextEntity> entities = new HashSet<>();
 
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/textblock/AtomicTextBlock.java

@@ -3,14 +3,11 @@ package com.iqser.red.service.redaction.v1.server.model.document.textblock;
 import static java.lang.String.format;
 
 import java.awt.geom.Rectangle2D;
-import java.text.BreakIterator;
-import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
-import java.util.Locale;
 import java.util.Map;
 import java.util.Set;
 import java.util.stream.Collectors;
@@ -43,7 +40,6 @@ public class AtomicTextBlock implements TextBlock {
     //string coordinates
     TextRange textRange;
     String searchText;
-    List<String> words;
     List<Integer> lineBreaks;
 
     //position coordinates
@@ -118,20 +114,6 @@ public class AtomicTextBlock implements TextBlock {
         return new TextRange(lineBreaks.get(lineNumber - 1) + textRange.start(), lineBreaks.get(lineNumber) + textRange.start());
     }
 
-    public List<String> getWords() {
-
-        if (words == null) {
-            words = new ArrayList<>();
-            BreakIterator iterator = BreakIterator.getWordInstance(Locale.ENGLISH);
-            iterator.setText(searchText);
-            int start = iterator.first();
-            for (int end = iterator.next(); end != BreakIterator.DONE; start = end, end = iterator.next()) {
-                words.add(searchText.substring(start, end));
-            }
-        }
-        return words;
-    }
-
 
     @Override
     public List<AtomicTextBlock> getAtomicTextBlocks() {

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/textblock/ConcatenatedTextBlock.java

@@ -7,7 +7,6 @@ import java.util.Collections;
 import java.util.HashMap;
 import java.util.LinkedList;
 import java.util.List;
-import java.util.Collection;
 import java.util.Map;
 import java.util.stream.Stream;
 
@@ -71,16 +70,7 @@ public class ConcatenatedTextBlock implements TextBlock {
 
     private List<AtomicTextBlock> getAllAtomicTextBlocksPartiallyInStringBoundary(TextRange textRange) {
 
-        List<AtomicTextBlock> intersectingAtomicTextBlocks = new LinkedList<>();
-        for (AtomicTextBlock atomicTextBlock : atomicTextBlocks) {
-            if (atomicTextBlock.getTextRange().start() > textRange.end()) {
-                break; // early stop, following TextBlocks will never intersect
-            }
-            if (atomicTextBlock.getTextRange().intersects(textRange)) {
-                intersectingAtomicTextBlocks.add(atomicTextBlock);
-            }
-        }
-        return intersectingAtomicTextBlocks;
+        return atomicTextBlocks.stream().filter(tb -> tb.getTextRange().intersects(textRange)).toList();
     }
 
 
@@ -96,13 +86,6 @@ public class ConcatenatedTextBlock implements TextBlock {
     }
 
 
-    @Override
-    public List<String> getWords() {
-
-        return atomicTextBlocks.stream().map(AtomicTextBlock::getWords).flatMap(Collection::stream).toList();
-    }
-
-
     @Override
     public int numberOfLines() {
 
@@ -131,13 +114,13 @@ public class ConcatenatedTextBlock implements TextBlock {
     }
 
 
+
     @Override
     public Rectangle2D getPosition(int stringIdx) {
 
         return getAtomicTextBlockByStringIndex(stringIdx).getPosition(stringIdx);
     }
 
-
     public TextRange getLineTextRange(int lineNumber) {
 
         if (atomicTextBlocks.size() == 1) {
@@ -153,10 +136,10 @@ public class ConcatenatedTextBlock implements TextBlock {
         return new TextRange(textRange.start(), textRange.start());
     }
 
-
     @Override
     public List<Rectangle2D> getPositions(TextRange stringTextRange) {
 
+
         List<AtomicTextBlock> textBlocks = getAllAtomicTextBlocksPartiallyInStringBoundary(stringTextRange);
 
         if (textBlocks.isEmpty()) {

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/textblock/TextBlock.java

@@ -18,7 +18,6 @@ public interface TextBlock extends CharSequence {
 
     String getSearchText();
 
-    List<String> getWords();
 
     List<AtomicTextBlock> getAtomicTextBlocks();
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/drools/BasicQuery.java

@@ -1,15 +0,0 @@
-package com.iqser.red.service.redaction.v1.server.model.drools;
-
-import lombok.AccessLevel;
-import lombok.Data;
-import lombok.experimental.FieldDefaults;
-
-@Data
-@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
-public class BasicQuery {
-
-    String name;
-    int line;
-    String code;
-
-}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/drools/BasicRule.java

@@ -1,6 +1,5 @@
 package com.iqser.red.service.redaction.v1.server.model.drools;
 
-import org.drools.drl.ast.descr.AttributeDescr;
 import org.drools.drl.ast.descr.RuleDescr;
 
 import lombok.AccessLevel;
@@ -19,7 +18,6 @@ public class BasicRule {
     RuleIdentifier identifier;
     String name;
     String code;
-    String agendaGroup;
     int line;
 
 
@@ -28,8 +26,7 @@ public class BasicRule {
         RuleIdentifier identifier = RuleIdentifier.fromName(rule.getName());
         String nameWithoutIdentifier = rule.getName().replace(identifier + ":", "");
         String code = rulesString.substring(rule.getStartCharacter(), rule.getEndCharacter());
-        String agendaGroup = rule.getAttributes().getOrDefault("agenda-group", new AttributeDescr("agenda-group", "DEFAULT")).getValue();
-        return new BasicRule(identifier, nameWithoutIdentifier, code, agendaGroup, rule.getLine());
+        return new BasicRule(identifier, nameWithoutIdentifier, code, rule.getLine());
     }
 
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/drools/RuleClass.java

@@ -2,14 +2,21 @@ package com.iqser.red.service.redaction.v1.server.model.drools;
 
 import java.util.List;
 import java.util.Objects;
-import java.util.Optional;
 
 public record RuleClass(RuleType ruleType, List<RuleUnit> ruleUnits) {
 
-    public Optional<RuleUnit> findRuleUnitByInteger(Integer unit) {
+    public RuleUnit findRuleUnitByInteger(Integer unit) {
 
         return ruleUnits.stream()
-                .filter(ruleUnit -> Objects.equals(ruleUnit.unit(), unit)).findFirst();
+                .filter(ruleUnit -> Objects.equals(ruleUnit.unit(), unit))
+                .findFirst()
+                .orElseThrow(() -> new IllegalArgumentException(String.format("RuleUnit %d does not exist in class %s", unit, this)));
+    }
+
+
+    public boolean ruleUnitExists(Integer unit) {
+
+        return ruleUnits.stream().anyMatch(ruleUnit -> Objects.equals(ruleUnit.unit(), unit));
     }
 
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/drools/RuleFileBluePrint.java

@@ -1,16 +1,13 @@
 package com.iqser.red.service.redaction.v1.server.model.drools;
 
-import java.util.Arrays;
 import java.util.Collection;
-import java.util.Collections;
 import java.util.List;
 import java.util.Objects;
-import java.util.Optional;
-import java.util.Set;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
 import com.iqser.red.service.redaction.v1.model.DroolsSyntaxValidation;
+import com.iqser.red.service.redaction.v1.server.utils.exception.NotFoundException;
 
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
@@ -30,39 +27,36 @@ public final class RuleFileBluePrint {
     int importLine;
     String globals;
     int globalsLine;
-    List<BasicQuery> queries;
+    String queries;
+    int queriesLine;
     List<RuleClass> ruleClasses;
     DroolsSyntaxValidation droolsSyntaxValidation;
 
 
-    public Optional<RuleClass> findRuleClassByType(RuleType ruleType) {
+    public RuleClass findRuleClassByType(RuleType ruleType) {
 
-        return ruleClasses.stream().filter(ruleClass -> Objects.equals(ruleClass.ruleType(), ruleType)).findFirst();
+        return ruleClasses.stream()
+                .filter(ruleClass -> Objects.equals(ruleClass.ruleType(), ruleType))
+                .findFirst()
+                .orElseThrow(() -> new NotFoundException(String.format("RuleType %s does not exist in this BluePrint %s", ruleType.name(), this)));
     }
 
 
-    public Set<String> getImportSplitByKeyword() {
-
-        return Arrays.stream(imports.replaceAll("\n", "").split("import")).map(String::trim).collect(Collectors.toSet());
-    }
-
 
     public List<BasicRule> findRulesByIdentifier(RuleIdentifier ruleIdentifier) {
 
         if (Objects.isNull(ruleIdentifier.unit())) {
-            return findRuleClassByType(ruleIdentifier.type()).map(RuleClass::ruleUnits)
-                    .orElse(Collections.emptyList())
-                    .stream()
-                    .flatMap(ruleUnit -> ruleUnit.rules().stream().filter(rule -> rule.getIdentifier().matches(ruleIdentifier)))
-                    .toList();
+            return findRuleClassByType(ruleIdentifier.type()).ruleUnits().stream().map(RuleUnit::rules).flatMap(Collection::stream).toList();
         }
-        return findRuleClassByType(ruleIdentifier.type()).flatMap(ruleClass -> ruleClass.findRuleUnitByInteger(ruleIdentifier.unit()))
-                .map(ruleUnit -> ruleUnit.rules().stream().filter(rule -> rule.getIdentifier().matches(ruleIdentifier)))
-                .orElse(Stream.empty())
+        return findRuleClassByType(ruleIdentifier.type()).findRuleUnitByInteger(ruleIdentifier.unit())
+                .rules()
+                .stream()
+                .filter(rule -> rule.getIdentifier().matches(ruleIdentifier))
                 .toList();
     }
 
 
+
     public List<RuleIdentifier> getAllRuleIdentifiers() {
 
         return streamAllRules().map(BasicRule::getIdentifier).collect(Collectors.toList());

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/drools/RuleIdentifier.java

@@ -1,6 +1,9 @@
 package com.iqser.red.service.redaction.v1.server.model.drools;
 
+import java.util.Arrays;
 import java.util.Objects;
+import java.util.Set;
+import java.util.stream.Collectors;
 
 import lombok.NonNull;
 
@@ -32,6 +35,18 @@ public record RuleIdentifier(@NonNull RuleType type, Integer unit, Integer id) {
     }
 
 
+    public static Set<RuleIdentifier> fromListOfIdentifiersString(String input) {
+
+        return Arrays.stream(input.split(",")).map(String::trim).map(RuleIdentifier::fromString).collect(Collectors.toSet());
+    }
+
+
+    private static Integer parseIntOrStar(String value) {
+
+        return !value.equals("*") ? Integer.parseInt(value) : null;
+    }
+
+
     /**
      * This is used to filter rules, if the field Integer unit or Integer id is null, the field will match any other RuleIdentifier.
      * Therefore, to compare RuleIdentifiers one should always use the function matches.
@@ -51,10 +66,6 @@ public record RuleIdentifier(@NonNull RuleType type, Integer unit, Integer id) {
     @Override
     public String toString() {
 
-        if (type.name().isBlank() || type.name().isEmpty()) {
-            return "";
-        }
-
         StringBuilder sb = new StringBuilder();
         sb.append(type().name());
         if (Objects.nonNull(unit())) {

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/drools/RuleType.java

@@ -2,6 +2,7 @@ package com.iqser.red.service.redaction.v1.server.model.drools;
 
 import java.util.regex.Pattern;
 
+
 public record RuleType(String name) {
 
     static Pattern alphaNumeric = Pattern.compile("^[a-zA-Z0-9]+$");

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/queue/MessageReceiver.java

@@ -1,11 +1,13 @@
 package com.iqser.red.service.redaction.v1.server.queue;
 
-import static com.iqser.red.service.redaction.v1.model.QueueNames.REDACTION_QUEUE;
+import static com.iqser.red.service.redaction.v1.server.queue.MessagingConfiguration.REDACTION_QUEUE;
 
 import org.springframework.amqp.core.Message;
 import org.springframework.amqp.rabbit.annotation.RabbitHandler;
 import org.springframework.amqp.rabbit.annotation.RabbitListener;
 
+import com.fasterxml.jackson.core.JsonProcessingException;
+
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/queue/MessagingConfiguration.java

@@ -1,11 +1,6 @@
 package com.iqser.red.service.redaction.v1.server.queue;
 
-import static com.iqser.red.service.redaction.v1.model.QueueNames.MIGRATION_DLQ;
-import static com.iqser.red.service.redaction.v1.model.QueueNames.MIGRATION_QUEUE;
-import static com.iqser.red.service.redaction.v1.model.QueueNames.REDACTION_ANALYSIS_RESPONSE_QUEUE;
-import static com.iqser.red.service.redaction.v1.model.QueueNames.REDACTION_DQL;
-import static com.iqser.red.service.redaction.v1.model.QueueNames.REDACTION_PRIORITY_QUEUE;
-import static com.iqser.red.service.redaction.v1.model.QueueNames.REDACTION_QUEUE;
+import lombok.RequiredArgsConstructor;
 
 import org.springframework.amqp.core.Queue;
 import org.springframework.amqp.core.QueueBuilder;
@@ -13,12 +8,15 @@ import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;
 
-import lombok.RequiredArgsConstructor;
-
 @Configuration
 @RequiredArgsConstructor
 public class MessagingConfiguration {
 
+    public static final String REDACTION_QUEUE = "redactionQueue";
+    public static final String REDACTION_DQL = "redactionDQL";
+
+    public static final String REDACTION_PRIORITY_QUEUE = "redactionPriorityQueue";
+
     public static final String X_ERROR_INFO_HEADER = "x-error-message";
     public static final String X_ERROR_INFO_TIMESTAMP_HEADER = "x-error-message-timestamp";
 
@@ -39,20 +37,6 @@ public class MessagingConfiguration {
     }
 
 
-    @Bean
-    public Queue migrationQueue() {
-
-        return QueueBuilder.durable(MIGRATION_QUEUE).withArgument("x-dead-letter-exchange", "").withArgument("x-dead-letter-routing-key", MIGRATION_DLQ).maxPriority(2).build();
-    }
-
-
-    @Bean
-    public Queue migrationDLQ() {
-
-        return QueueBuilder.durable(MIGRATION_DLQ).build();
-    }
-
-
     @Bean
     public Queue redactionQueue() {
 
@@ -70,16 +54,6 @@ public class MessagingConfiguration {
                 .build();
     }
 
-    @Bean
-    public Queue redactionAnalysisResponseQueue() {
-
-        return QueueBuilder.durable(REDACTION_ANALYSIS_RESPONSE_QUEUE)
-                .withArgument("x-dead-letter-exchange", "")
-                .withArgument("x-dead-letter-routing-key", REDACTION_DQL)
-                .maxPriority(2)
-                .build();
-    }
-
 
     @Bean
     public Queue redactionDeadLetterQueue() {

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/queue/PriorityMessageReceiver.java

@@ -1,11 +1,13 @@
 package com.iqser.red.service.redaction.v1.server.queue;
 
-import static com.iqser.red.service.redaction.v1.model.QueueNames.REDACTION_PRIORITY_QUEUE;
+import static com.iqser.red.service.redaction.v1.server.queue.MessagingConfiguration.REDACTION_PRIORITY_QUEUE;
 
 import org.springframework.amqp.core.Message;
 import org.springframework.amqp.rabbit.annotation.RabbitHandler;
 import org.springframework.amqp.rabbit.annotation.RabbitListener;
 
+import com.fasterxml.jackson.core.JsonProcessingException;
+
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/queue/RedactionMessageReceiver.java

@@ -1,8 +1,8 @@
 package com.iqser.red.service.redaction.v1.server.queue;
 
-import static com.iqser.red.service.redaction.v1.model.QueueNames.REDACTION_DQL;
-import static com.iqser.red.service.redaction.v1.model.QueueNames.REDACTION_PRIORITY_QUEUE;
-import static com.iqser.red.service.redaction.v1.model.QueueNames.REDACTION_QUEUE;
+import static com.iqser.red.service.redaction.v1.server.queue.MessagingConfiguration.REDACTION_DQL;
+import static com.iqser.red.service.redaction.v1.server.queue.MessagingConfiguration.REDACTION_PRIORITY_QUEUE;
+import static com.iqser.red.service.redaction.v1.server.queue.MessagingConfiguration.REDACTION_QUEUE;
 import static com.iqser.red.service.redaction.v1.server.queue.MessagingConfiguration.X_ERROR_INFO_HEADER;
 import static com.iqser.red.service.redaction.v1.server.queue.MessagingConfiguration.X_ERROR_INFO_TIMESTAMP_HEADER;
 import static java.lang.String.format;
@@ -23,9 +23,8 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeResu
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileErrorInfo;
 import com.iqser.red.service.redaction.v1.server.client.FileStatusProcessingUpdateClient;
 import com.iqser.red.service.redaction.v1.server.client.RulesClient;
-import com.iqser.red.service.redaction.v1.server.service.AnalyzeService;
-import com.iqser.red.service.redaction.v1.server.service.UnprocessedChangesService;
 import com.iqser.red.service.redaction.v1.server.utils.exception.DroolsTimeoutException;
+import com.iqser.red.service.redaction.v1.server.service.AnalyzeService;
 
 import lombok.RequiredArgsConstructor;
 import lombok.SneakyThrows;
@@ -40,7 +39,6 @@ public class RedactionMessageReceiver {
     private final AnalyzeService analyzeService;
     private final FileStatusProcessingUpdateClient fileStatusProcessingUpdateClient;
     private final RulesClient rulesClient;
-    private final UnprocessedChangesService unprocessedChangesService;
 
 
     @SneakyThrows
@@ -72,8 +70,7 @@ public class RedactionMessageReceiver {
         log.info("");
         log.info("Processing AnalyzeRequest for file: {},  priority: {} ", analyzeRequest.getFileId(), priority);
         log.info("");
-        AnalyzeResult result = new AnalyzeResult();
-        boolean shouldRespond = true;
+        AnalyzeResult result;
 
         try {
             switch (analyzeRequest.getMessageType()) {
@@ -101,26 +98,13 @@ public class RedactionMessageReceiver {
                             format("%.2f", result.getDuration() / 1000.0));
                     log.info("----------------------------------------------------------------------------------");
                     break;
-                case SURROUNDING_TEXT_ANALYSIS:
-                    log.info("------------------------------Surrounding Text Analysis------------------------------------------");
-                    log.info("Starting Surrounding Text Analysis for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
-                    log.debug(analyzeRequest.getManualRedactions().toString());
-                    unprocessedChangesService.analyseSurroundingText(analyzeRequest);
-                    log.info("Successful Surrounding Text Analysis dossier {} file {} ",
-                            analyzeRequest.getDossierId(),
-                            analyzeRequest.getFileId());
-                    log.info("-------------------------------------------------------------------------------------------------");
-                    shouldRespond = false;
-                    break;
 
                 default:
                     throw new IllegalArgumentException("Unknown MessageType: " + analyzeRequest.getMessageType());
             }
             log.info("");
-            if (shouldRespond) {
-                result.setMessageType(analyzeRequest.getMessageType());
-                fileStatusProcessingUpdateClient.analysisSuccessful(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), result);
-            }
+            result.setMessageType(analyzeRequest.getMessageType());
+            fileStatusProcessingUpdateClient.analysisSuccessful(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), result);
         } catch (DroolsTimeoutException droolsTimeoutException) {
             if (!droolsTimeoutException.isReported()) {
                 rulesClient.setRulesTimeoutDetected(analyzeRequest.getDossierTemplateId(), droolsTimeoutException.getRuleFileType());
@@ -144,7 +128,7 @@ public class RedactionMessageReceiver {
 
     @RabbitHandler
     @RabbitListener(queues = REDACTION_DQL)
-    public void receiveAnalyzeRequestDLQ(Message in) throws IOException {
+    public void receiveAnalyzeRequestDQL(Message in) throws IOException {
 
         var analyzeRequest = objectMapper.readValue(in.getBody(), AnalyzeRequest.class);
         log.info("Failed to process analyze request: {}", analyzeRequest);

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalyzeService.java

@@ -1,13 +1,11 @@
 package com.iqser.red.service.redaction.v1.server.service;
 
-import java.util.Collection;
 import java.util.Collections;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.stream.Collectors;
-import java.util.stream.Stream;
 
 import org.springframework.stereotype.Service;
 import org.springframework.web.bind.annotation.RequestBody;
@@ -20,7 +18,9 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileTyp
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentLog;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogChanges;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedactions;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogLegalBasis;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Comment;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.legalbasis.LegalBasis;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog;
@@ -31,27 +31,24 @@ import com.iqser.red.service.redaction.v1.server.RedactionServiceSettings;
 import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient;
 import com.iqser.red.service.redaction.v1.server.client.model.NerEntitiesModel;
 import com.iqser.red.service.redaction.v1.server.model.KieWrapper;
-import com.iqser.red.service.redaction.v1.server.model.PrecursorEntity;
+import com.iqser.red.service.redaction.v1.server.model.ManualEntity;
 import com.iqser.red.service.redaction.v1.server.model.NerEntities;
-import com.iqser.red.service.redaction.v1.server.model.component.Component;
 import com.iqser.red.service.redaction.v1.server.model.dictionary.Dictionary;
 import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryIncrement;
 import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryVersion;
+import com.iqser.red.service.redaction.v1.server.model.component.Component;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
 import com.iqser.red.service.redaction.v1.server.service.document.DocumentGraphMapper;
-import com.iqser.red.service.redaction.v1.server.service.document.ImportedRedactionEntryService;
 import com.iqser.red.service.redaction.v1.server.service.document.ManualRedactionEntryService;
 import com.iqser.red.service.redaction.v1.server.service.document.NerEntitiesAdapter;
 import com.iqser.red.service.redaction.v1.server.service.document.SectionFinderService;
 import com.iqser.red.service.redaction.v1.server.service.drools.ComponentDroolsExecutionService;
 import com.iqser.red.service.redaction.v1.server.service.drools.EntityDroolsExecutionService;
 import com.iqser.red.service.redaction.v1.server.service.drools.KieContainerCreationService;
-import com.iqser.red.service.redaction.v1.server.storage.ObservedStorageService;
 import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
 
 import io.micrometer.core.annotation.Timed;
-import io.micrometer.observation.annotation.Observed;
 import lombok.AccessLevel;
 import lombok.RequiredArgsConstructor;
 import lombok.SneakyThrows;
@@ -64,123 +61,29 @@ import lombok.extern.slf4j.Slf4j;
 @RequiredArgsConstructor
 public class AnalyzeService {
 
+    private static final String REDACTMANAGER_ANALYZE_PAGEWISE_METRIC_NAME = "redactmanager_analyze.pagewise";
+
     DictionaryService dictionaryService;
     EntityDroolsExecutionService entityDroolsExecutionService;
     ComponentDroolsExecutionService componentDroolsExecutionService;
     KieContainerCreationService kieContainerCreationService;
     DictionarySearchService dictionarySearchService;
+    RedactionLogCreatorService redactionLogCreatorService;
     EntityLogCreatorService entityLogCreatorService;
     ComponentLogCreatorService componentLogCreatorService;
     RedactionStorageService redactionStorageService;
+    RedactionChangeLogService redactionChangeLogService;
+    EntityChangeLogService entityChangeLogService;
+    LegalBasisClient legalBasisClient;
     RedactionServiceSettings redactionServiceSettings;
-    NotFoundImportedEntitiesService notFoundImportedEntitiesService;
+    ImportedRedactionService importedRedactionService;
     SectionFinderService sectionFinderService;
     ManualRedactionEntryService manualRedactionEntryService;
-    ImportedRedactionEntryService importedRedactionEntryService;
-    ObservedStorageService observedStorageService;
+
     FunctionTimerValues redactmanagerAnalyzePagewiseValues;
 
 
-    @Timed("redactmanager_reanalyze")
-    @SneakyThrows
-    @Observed(name = "AnalyzeService", contextualName = "reanalyze")
-    public AnalyzeResult reanalyze(@RequestBody AnalyzeRequest analyzeRequest) {
-
-        long startTime = System.currentTimeMillis();
-        EntityLog previousEntityLog = redactionStorageService.getEntityLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
-        log.info("Loaded previous entity log for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
-
-        Document document = DocumentGraphMapper.toDocumentGraph(observedStorageService.getDocumentData(analyzeRequest.getDossierId(), analyzeRequest.getFileId()));
-        log.info("Loaded Document Graph for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
-
-        ImportedRedactions importedRedactions = redactionStorageService.getImportedRedactions(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
-        log.info("Loaded Imported Redactions for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
-
-        // not yet ready for reanalysis
-        if (previousEntityLog == null || document == null || document.getNumberOfPages() == 0) {
-            return analyze(analyzeRequest);
-        }
-
-        DictionaryIncrement dictionaryIncrement = dictionaryService.getDictionaryIncrements(analyzeRequest.getDossierTemplateId(),
-                                                                                            new DictionaryVersion(previousEntityLog.getDictionaryVersion(),
-                                                                                                                  previousEntityLog.getDossierDictionaryVersion()),
-                                                                                            analyzeRequest.getDossierId());
-
-        Set<Integer> sectionsToReanalyseIds = getSectionsToReanalyseIds(analyzeRequest, previousEntityLog, document, dictionaryIncrement, importedRedactions);
-        List<SemanticNode> sectionsToReAnalyse = getSectionsToReAnalyse(document, sectionsToReanalyseIds);
-        log.info("{} Sections to reanalyze found for file {} in dossier {}", sectionsToReanalyseIds.size(), analyzeRequest.getFileId(), analyzeRequest.getDossierId());
-
-        if (sectionsToReAnalyse.isEmpty()) {
-
-            EntityLogChanges entityLogChanges = entityLogCreatorService.updateVersionsAndReturnChanges(previousEntityLog,
-                                                                                                       dictionaryIncrement.getDictionaryVersion(),
-                                                                                                       analyzeRequest,
-                                                                                                       false);
-
-            return finalizeAnalysis(analyzeRequest,
-                                    startTime,
-                                    kieContainerCreationService.getLatestKieContainer(analyzeRequest.getDossierTemplateId(), RuleFileType.COMPONENT),
-                                    entityLogChanges,
-                                    document,
-                                    document.getNumberOfPages(),
-                                    dictionaryIncrement.getDictionaryVersion(),
-                                    true,
-                                    Collections.emptySet());
-        }
-
-        KieWrapper kieWrapperEntityRules = kieContainerCreationService.getLatestKieContainer(analyzeRequest.getDossierTemplateId(), RuleFileType.ENTITY);
-        log.info("Updated entity rules to version {} for file {} in dossier {}", kieWrapperEntityRules.rulesVersion(), analyzeRequest.getFileId(), analyzeRequest.getDossierId());
-
-        NerEntities nerEntities = getEntityRecognitionEntitiesFilteredBySectionIds(analyzeRequest, document, sectionsToReanalyseIds);
-        log.info("Loaded Ner Entities for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
-
-        var notFoundManualRedactionEntries = manualRedactionEntryService.addManualRedactionEntriesAndReturnNotFoundEntries(analyzeRequest,
-                                                                                                                           document,
-                                                                                                                           analyzeRequest.getDossierTemplateId());
-        var notFoundImportedEntries = importedRedactionEntryService.addImportedEntriesAndReturnNotFoundEntries(analyzeRequest, importedRedactions, document);
-        var notFoundManualOrImportedEntries = Stream.of(notFoundManualRedactionEntries, notFoundImportedEntries)
-                .flatMap(Collection::stream)
-                .collect(Collectors.toList());
-
-        Dictionary dictionary = dictionaryService.getDeepCopyDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId());
-        log.info("Updated Dictionaries for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
-
-        dictionarySearchService.addDictionaryEntities(dictionary, sectionsToReAnalyse);
-        log.info("Finished Dictionary Search for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
-
-        // we could add the imported redactions similar to the manual redactions here as well for additional processing
-        List<FileAttribute> allFileAttributes = entityDroolsExecutionService.executeRules(kieWrapperEntityRules.container(),
-                                                                                          document,
-                                                                                          sectionsToReAnalyse,
-                                                                                          dictionary,
-                                                                                          analyzeRequest.getFileAttributes(),
-                                                                                          analyzeRequest.getManualRedactions(),
-                                                                                          nerEntities);
-        log.info("Finished entity rule execution for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
-
-        EntityLogChanges entityLogChanges = entityLogCreatorService.updatePreviousEntityLog(analyzeRequest,
-                                                                                            document,
-                                                                                            notFoundManualOrImportedEntries,
-                                                                                            previousEntityLog,
-                                                                                            sectionsToReanalyseIds,
-                                                                                            dictionary.getVersion());
-
-        notFoundImportedEntitiesService.processEntityLog(entityLogChanges.getEntityLog(), analyzeRequest, notFoundImportedEntries);
-
-        return finalizeAnalysis(analyzeRequest,
-                                startTime,
-                                kieContainerCreationService.getLatestKieContainer(analyzeRequest.getDossierTemplateId(), RuleFileType.COMPONENT),
-                                entityLogChanges,
-                                document,
-                                document.getNumberOfPages(),
-                                dictionaryIncrement.getDictionaryVersion(),
-                                true,
-                                new HashSet<>(allFileAttributes));
-    }
-
-
     @Timed("redactmanager_analyze")
-    @Observed(name = "AnalyzeService", contextualName = "analyze")
     public AnalyzeResult analyze(AnalyzeRequest analyzeRequest) {
 
         long startTime = System.currentTimeMillis();
@@ -191,12 +94,9 @@ public class AnalyzeService {
         var kieWrapperComponentRules = kieContainerCreationService.getLatestKieContainer(analyzeRequest.getDossierTemplateId(), RuleFileType.COMPONENT);
         log.info("Updated Rules to Version {} for file {} in dossier {}", kieWrapperEntityRules.rulesVersion(), analyzeRequest.getFileId(), analyzeRequest.getDossierId());
 
-        Document document = DocumentGraphMapper.toDocumentGraph(observedStorageService.getDocumentData(analyzeRequest.getDossierId(), analyzeRequest.getFileId()));
+        Document document = DocumentGraphMapper.toDocumentGraph(redactionStorageService.getDocumentData(analyzeRequest.getDossierId(), analyzeRequest.getFileId()));
         log.info("Loaded Document Graph for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
 
-        ImportedRedactions importedRedactions = redactionStorageService.getImportedRedactions(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
-        log.info("Loaded Imported Redactions for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
-
         NerEntities nerEntities = getEntityRecognitionEntities(analyzeRequest, document);
         log.info("Loaded Ner Entities for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
 
@@ -204,62 +104,195 @@ public class AnalyzeService {
         Dictionary dictionary = dictionaryService.getDeepCopyDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId());
         log.info("Updated Dictionaries for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
 
-        var notFoundManualRedactionEntries = manualRedactionEntryService.addManualRedactionEntriesAndReturnNotFoundEntries(analyzeRequest,
-                                                                                                                           document,
-                                                                                                                           analyzeRequest.getDossierTemplateId());
-        var notFoundImportedEntries = importedRedactionEntryService.addImportedEntriesAndReturnNotFoundEntries(analyzeRequest, importedRedactions, document);
-        var notFoundManualOrImportedEntries = Stream.of(notFoundManualRedactionEntries, notFoundImportedEntries)
-                .flatMap(Collection::stream)
-                .collect(Collectors.toList());
+        List<ManualEntity> notFoundManualRedactionEntries = manualRedactionEntryService.addManualRedactionEntriesAndReturnNotFoundEntries(analyzeRequest, document);
 
         dictionarySearchService.addDictionaryEntities(dictionary, document);
         log.info("Finished Dictionary Search for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
 
-        // we could add the imported redactions similar to the manual redactions here as well for additional processing
         List<FileAttribute> allFileAttributes = entityDroolsExecutionService.executeRules(kieWrapperEntityRules.container(),
-                                                                                          document,
-                                                                                          dictionary,
-                                                                                          analyzeRequest.getFileAttributes(),
-                                                                                          analyzeRequest.getManualRedactions(),
-                                                                                          nerEntities);
+                document,
+                dictionary,
+                analyzeRequest.getFileAttributes(),
+                analyzeRequest.getManualRedactions(),
+                nerEntities);
         log.info("Finished entity rule execution for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
 
-        EntityLog entityLog = entityLogCreatorService.createInitialEntityLog(analyzeRequest,
-                                                                             document,
-                                                                             notFoundManualOrImportedEntries,
-                                                                             dictionary.getVersion(),
-                                                                             kieWrapperEntityRules.rulesVersion());
-
-        notFoundImportedEntitiesService.processEntityLog(entityLog, analyzeRequest, notFoundImportedEntries);
+        RedactionLog redactionLog = createRedactionLog(analyzeRequest, document, notFoundManualRedactionEntries, dictionary, kieWrapperEntityRules);
+        EntityLog entityLog = createEntityLog(analyzeRequest, document, notFoundManualRedactionEntries, dictionary, kieWrapperEntityRules);
 
         return finalizeAnalysis(analyzeRequest,
-                                startTime,
-                                kieWrapperComponentRules,
-                                new EntityLogChanges(entityLog, false),
-                                document,
-                                document.getNumberOfPages(),
-                                dictionary.getVersion(),
-                                false,
-                                new HashSet<>(allFileAttributes));
+                startTime, kieWrapperComponentRules, entityLog,
+                redactionLog,
+                document.getNumberOfPages(),
+                dictionary.getVersion(), false, new HashSet<>(allFileAttributes));
+    }
+
+
+    private EntityLog createEntityLog(AnalyzeRequest analyzeRequest,
+                                      Document document,
+                                      List<ManualEntity> notFoundManualRedactionEntries,
+                                      Dictionary dictionary,
+                                      KieWrapper wrapper) {
+
+        List<EntityLogEntry> entityLogEntries = entityLogCreatorService.createEntityLogEntries(document, analyzeRequest.getDossierTemplateId(), notFoundManualRedactionEntries);
+
+        List<LegalBasis> legalBasis = legalBasisClient.getLegalBasisMapping(analyzeRequest.getDossierTemplateId());
+
+        EntityLog entityLog = new EntityLog(redactionServiceSettings.getAnalysisVersion(),
+                analyzeRequest.getAnalysisNumber(),
+                entityLogEntries,
+                toEntityLogLegalBasis(legalBasis),
+                dictionary.getVersion().getDossierTemplateVersion(),
+                dictionary.getVersion().getDossierVersion(),
+                wrapper.rulesVersion(),
+                legalBasisClient.getVersion(analyzeRequest.getDossierTemplateId()));
+
+        List<EntityLogEntry> importedRedactionFilteredEntries = importedRedactionService.processImportedEntities(analyzeRequest.getDossierTemplateId(),
+                analyzeRequest.getDossierId(),
+                analyzeRequest.getFileId(),
+                entityLog.getEntityLogEntry(),
+                true);
+
+        entityLog.setEntityLogEntry(importedRedactionFilteredEntries);
+        return entityLog;
+    }
+
+
+    private List<EntityLogLegalBasis> toEntityLogLegalBasis(List<LegalBasis> legalBasis) {
+
+        return legalBasis.stream().map(l -> new EntityLogLegalBasis(l.getName(), l.getDescription(), l.getReason())).collect(Collectors.toList());
+    }
+
+
+    @Timed("redactmanager_reanalyze")
+    @SneakyThrows
+    public AnalyzeResult reanalyze(@RequestBody AnalyzeRequest analyzeRequest) {
+
+        long startTime = System.currentTimeMillis();
+        RedactionLog previousRedactionLog = redactionStorageService.getRedactionLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
+        EntityLog previousEntityLog = redactionStorageService.getEntityLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
+        log.info("Loaded previous entity log for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
+        ComponentLog previousComponentLog = new ComponentLog();
+
+        Document document = DocumentGraphMapper.toDocumentGraph(redactionStorageService.getDocumentData(analyzeRequest.getDossierId(), analyzeRequest.getFileId()));
+        log.info("Loaded Document Graph for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
+
+        // not yet ready for reanalysis
+        if (previousEntityLog == null || document == null || document.getNumberOfPages() == 0) {
+            return analyze(analyzeRequest);
+        }
+
+        DictionaryIncrement dictionaryIncrement = dictionaryService.getDictionaryIncrements(analyzeRequest.getDossierTemplateId(),
+                new DictionaryVersion(previousEntityLog.getDictionaryVersion(), previousEntityLog.getDossierDictionaryVersion()),
+                analyzeRequest.getDossierId());
+
+        Set<Integer> sectionsToReanalyseIds = getSectionsToReanalyseIds(analyzeRequest, previousEntityLog, document, dictionaryIncrement);
+        List<SemanticNode> sectionsToReAnalyse = getSectionsToReAnalyse(document, sectionsToReanalyseIds);
+        log.info("{} Sections to reanalyze found for file {} in dossier {}", sectionsToReanalyseIds.size(), analyzeRequest.getFileId(), analyzeRequest.getDossierId());
+
+        if (sectionsToReAnalyse.isEmpty()) {
+            return finalizeAnalysis(analyzeRequest,
+                    startTime, kieContainerCreationService.getLatestKieContainer(analyzeRequest.getDossierTemplateId(), RuleFileType.COMPONENT), previousEntityLog,
+                    previousRedactionLog,
+                    document.getNumberOfPages(),
+                    dictionaryIncrement.getDictionaryVersion(),
+                    true,
+                    Collections.emptySet());
+        }
+        KieWrapper kieWrapperEntityRules = kieContainerCreationService.getLatestKieContainer(analyzeRequest.getDossierTemplateId(), RuleFileType.ENTITY);
+        log.info("Updated entity rules to version {} for file {} in dossier {}", kieWrapperEntityRules.rulesVersion(), analyzeRequest.getFileId(), analyzeRequest.getDossierId());
+
+        NerEntities nerEntities = getEntityRecognitionEntitiesFilteredBySectionIds(analyzeRequest, document, sectionsToReanalyseIds);
+        log.info("Loaded Ner Entities for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
+
+        List<ManualEntity> notFoundManualRedactionEntries = manualRedactionEntryService.addManualRedactionEntriesAndReturnNotFoundEntries(analyzeRequest, document);
+
+        Dictionary dictionary = dictionaryService.getDeepCopyDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId());
+        log.info("Updated Dictionaries for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
+
+        sectionsToReAnalyse.forEach(node -> dictionarySearchService.addDictionaryEntities(dictionary, node));
+        log.info("Finished Dictionary Search for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
+
+        List<FileAttribute> allFileAttributes = entityDroolsExecutionService.executeRules(kieWrapperEntityRules.container(),
+                document,
+                sectionsToReAnalyse,
+                dictionary,
+                analyzeRequest.getFileAttributes(),
+                analyzeRequest.getManualRedactions(),
+                nerEntities);
+        log.info("Finished entity rule execution for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
+
+        RedactionLog redactionLog = updatePreviousRedactionLog(analyzeRequest, document, notFoundManualRedactionEntries, previousRedactionLog, sectionsToReanalyseIds);
+        EntityLog entityLog = updatePreviousEntityLog(analyzeRequest, document, notFoundManualRedactionEntries, previousEntityLog, sectionsToReanalyseIds);
+
+        return finalizeAnalysis(analyzeRequest,
+                startTime, kieContainerCreationService.getLatestKieContainer(analyzeRequest.getDossierTemplateId(), RuleFileType.COMPONENT), entityLog,
+                redactionLog,
+                document.getNumberOfPages(),
+                dictionaryIncrement.getDictionaryVersion(),
+                true,
+                new HashSet<>(allFileAttributes));
+    }
+
+
+    @Deprecated(forRemoval = true)
+    private RedactionLog updatePreviousRedactionLog(AnalyzeRequest analyzeRequest,
+                                                    Document document,
+                                                    List<ManualEntity> notFoundManualRedactionEntries,
+                                                    RedactionLog previousRedactionLog,
+                                                    Set<Integer> sectionsToReanalyseIds) {
+
+        List<RedactionLogEntry> newRedactionLogEntries = redactionLogCreatorService.createRedactionLog(document,
+                analyzeRequest.getDossierTemplateId(),
+                notFoundManualRedactionEntries,
+                getComments(analyzeRequest));
+
+        var importedRedactionFilteredEntries = importedRedactionService.processImportedRedactions(analyzeRequest.getDossierTemplateId(),
+                analyzeRequest.getDossierId(),
+                analyzeRequest.getFileId(),
+                newRedactionLogEntries,
+                false);
+
+        previousRedactionLog.getRedactionLogEntry()
+                .removeIf(entry -> sectionsToReanalyseIds.contains(entry.getSectionNumber()) && !entry.getType().equals(ImportedRedactionService.IMPORTED_REDACTION_TYPE));
+        previousRedactionLog.getRedactionLogEntry().addAll(importedRedactionFilteredEntries);
+        return previousRedactionLog;
+    }
+
+
+    private EntityLog updatePreviousEntityLog(AnalyzeRequest analyzeRequest,
+                                              Document document,
+                                              List<ManualEntity> notFoundManualRedactionEntries,
+                                              EntityLog previousEntityLog,
+                                              Set<Integer> sectionsToReanalyseIds) {
+
+        List<EntityLogEntry> newEntityLogEntries = entityLogCreatorService.createEntityLogEntries(document, analyzeRequest.getDossierTemplateId(), notFoundManualRedactionEntries);
+
+        var importedRedactionFilteredEntries = importedRedactionService.processImportedEntities(analyzeRequest.getDossierTemplateId(),
+                analyzeRequest.getDossierId(),
+                analyzeRequest.getFileId(),
+                newEntityLogEntries,
+                false);
+
+        previousEntityLog.getEntityLogEntry()
+                .removeIf(entry -> sectionsToReanalyseIds.contains(entry.getSectionNumber()) && !entry.getType().equals(ImportedRedactionService.IMPORTED_REDACTION_TYPE));
+        previousEntityLog.getEntityLogEntry().addAll(importedRedactionFilteredEntries);
+        return previousEntityLog;
     }
 
 
     private AnalyzeResult finalizeAnalysis(AnalyzeRequest analyzeRequest,
-                                           long startTime,
-                                           KieWrapper kieWrapperComponentRules,
-                                           EntityLogChanges entityLogChanges,
-                                           Document document,
+                                           long startTime, KieWrapper kieWrapperComponentRules, EntityLog entityLog,
+                                           RedactionLog redactionLog,
                                            int numberOfPages,
                                            DictionaryVersion dictionaryVersion,
                                            boolean isReanalysis,
                                            Set<FileAttribute> addedFileAttributes) {
 
-        EntityLog entityLog = entityLogChanges.getEntityLog();
-        redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.ENTITY_LOG, entityLogChanges.getEntityLog());
+        EntityLogChanges entityLogChanges = finalizeEntityLog(analyzeRequest, entityLog, redactionLog, dictionaryVersion);
 
-        log.info("Created entity log for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
-        if (entityLogChanges.isHasChanges() || !isReanalysis) {
-            computeComponentsWhenRulesArePresent(analyzeRequest, kieWrapperComponentRules, document, addedFileAttributes, entityLogChanges, dictionaryVersion);
+        if (entityLogChanges.isHasChanges()) {
+            computeComponentsWhenRulesArePresent(analyzeRequest, kieWrapperComponentRules, addedFileAttributes, entityLogChanges);
         }
 
         log.info("Stored analysis logs for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
@@ -277,7 +310,6 @@ public class AnalyzeService {
                 .analysisVersion(redactionServiceSettings.getAnalysisVersion())
                 .analysisNumber(analyzeRequest.getAnalysisNumber())
                 .rulesVersion(entityLog.getRulesVersion())
-                .componentRulesVersion(kieWrapperComponentRules.rulesVersion())
                 .dictionaryVersion(entityLog.getDictionaryVersion())
                 .legalBasisVersion(entityLog.getLegalBasisVersion())
                 .dossierDictionaryVersion(entityLog.getDossierDictionaryVersion())
@@ -290,45 +322,57 @@ public class AnalyzeService {
 
     private void computeComponentsWhenRulesArePresent(AnalyzeRequest analyzeRequest,
                                                       KieWrapper kieWrapperComponentRules,
-                                                      Document document,
                                                       Set<FileAttribute> addedFileAttributes,
-                                                      EntityLogChanges entityLogChanges,
-                                                      DictionaryVersion dictionaryVersion) {
+                                                      EntityLogChanges entityLogChanges) {
 
         if (!kieWrapperComponentRules.isPresent()) {
             return;
         }
 
         List<Component> components = componentDroolsExecutionService.executeRules(kieWrapperComponentRules.container(),
-                                                                                  entityLogChanges.getEntityLog(),
-                                                                                  document,
-                                                                                  addedFileAttributes.stream()
-                                                                                          .toList());
+                entityLogChanges.getEntityLog(),
+                addedFileAttributes.stream().toList());
         log.info("Finished component rule execution for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
 
-        ComponentLog componentLog = componentLogCreatorService.buildComponentLog(analyzeRequest.getAnalysisNumber(), components, kieWrapperComponentRules.rulesVersion());
+        ComponentLog componentLog = componentLogCreatorService.buildComponentLog(analyzeRequest.getAnalysisNumber(), components);
 
         redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.COMPONENT_LOG, componentLog);
 
     }
 
 
-    private static List<SemanticNode> getSectionsToReAnalyse(Document document, Set<Integer> sectionsToReanalyseIds) {
+    private EntityLogChanges finalizeEntityLog(AnalyzeRequest analyzeRequest, EntityLog entityLog, RedactionLog redactionLog, DictionaryVersion dictionaryVersion) {
 
-        return document.streamChildren()
-                .filter(section -> sectionsToReanalyseIds.contains(section.getTreeId()
-                                                                           .get(0)))
-                .collect(Collectors.toList());
+        redactionLog.setDictionaryVersion(dictionaryVersion.getDossierTemplateVersion());
+        redactionLog.setDossierDictionaryVersion(dictionaryVersion.getDossierVersion());
+        excludeExcludedPages(redactionLog, analyzeRequest.getExcludedPages());
+        EntityLogChanges entityLogChanges = entityChangeLogService.computeChanges(analyzeRequest.getDossierId(),
+                analyzeRequest.getFileId(),
+                entityLog,
+                analyzeRequest.getAnalysisNumber());
+        RedactionLogChanges redactionLogChange = redactionChangeLogService.computeChanges(analyzeRequest.getDossierId(),
+                analyzeRequest.getFileId(),
+                redactionLog,
+                analyzeRequest.getAnalysisNumber());
+        log.info("Created entity log for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
+
+        redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.REDACTION_LOG, redactionLogChange.getRedactionLog());
+        redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.ENTITY_LOG, entityLogChanges.getEntityLog());
+        return entityLogChanges;
     }
 
 
-    private Set<Integer> getSectionsToReanalyseIds(AnalyzeRequest analyzeRequest,
-                                                   EntityLog entityLog,
-                                                   Document document,
-                                                   DictionaryIncrement dictionaryIncrement,
-                                                   ImportedRedactions importedRedactions) {
+    private static List<SemanticNode> getSectionsToReAnalyse(Document document, Set<Integer> sectionsToReanalyseIds) {
 
-        return sectionFinderService.findSectionsToReanalyse(dictionaryIncrement, entityLog, document, analyzeRequest, importedRedactions);
+        return document.streamChildren().filter(section -> sectionsToReanalyseIds.contains(section.getTreeId().get(0))).collect(Collectors.toList());
+    }
+
+
+    private Set<Integer> getSectionsToReanalyseIds(AnalyzeRequest analyzeRequest, EntityLog entityLog, Document document, DictionaryIncrement dictionaryIncrement) {
+
+        return analyzeRequest.getSectionsToReanalyse().isEmpty() //
+                ? sectionFinderService.findSectionsToReanalyse(dictionaryIncrement, entityLog, document, analyzeRequest) //
+                : analyzeRequest.getSectionsToReanalyse();
     }
 
 
@@ -348,10 +392,9 @@ public class AnalyzeService {
 
     private static NerEntitiesModel filterNerEntitiesModelBySectionIds(Set<Integer> sectionsToReanalyseIds, NerEntitiesModel nerEntitiesModel) {
 
-        return new NerEntitiesModel(nerEntitiesModel.getData().entrySet()
-                                            .stream() //
-                                            .filter(entry -> sectionsToReanalyseIds.contains(entry.getKey())) //
-                                            .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)));
+        return new NerEntitiesModel(nerEntitiesModel.getData().entrySet().stream() //
+                .filter(entry -> sectionsToReanalyseIds.contains(entry.getKey())) //
+                .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)));
     }
 
 
@@ -366,4 +409,66 @@ public class AnalyzeService {
         return nerEntities;
     }
 
+
+    @Deprecated(forRemoval = true)
+    private RedactionLog createRedactionLog(AnalyzeRequest analyzeRequest,
+                                            Document document,
+                                            List<ManualEntity> notFoundManualRedactionEntries,
+                                            Dictionary dictionary,
+                                            KieWrapper wrapper) {
+
+        List<RedactionLogEntry> redactionLogEntries = redactionLogCreatorService.createRedactionLog(document,
+                analyzeRequest.getDossierTemplateId(),
+                notFoundManualRedactionEntries,
+                getComments(analyzeRequest));
+
+        List<LegalBasis> legalBasis = legalBasisClient.getLegalBasisMapping(analyzeRequest.getDossierTemplateId());
+        RedactionLog redactionLog = new RedactionLog(redactionServiceSettings.getAnalysisVersion(),
+                analyzeRequest.getAnalysisNumber(),
+                redactionLogEntries,
+                toRedactionLogLegalBasis(legalBasis),
+                dictionary.getVersion().getDossierTemplateVersion(),
+                dictionary.getVersion().getDossierVersion(),
+                wrapper.rulesVersion(),
+                legalBasisClient.getVersion(analyzeRequest.getDossierTemplateId()));
+
+        List<RedactionLogEntry> importedRedactionFilteredEntries = importedRedactionService.processImportedRedactions(analyzeRequest.getDossierTemplateId(),
+                analyzeRequest.getDossierId(),
+                analyzeRequest.getFileId(),
+                redactionLog.getRedactionLogEntry(),
+                true);
+        redactionLog.setRedactionLogEntry(importedRedactionFilteredEntries);
+        return redactionLog;
+    }
+
+
+    private static Map<String, List<Comment>> getComments(AnalyzeRequest analyzeRequest) {
+
+        if (analyzeRequest.getManualRedactions() == null) {
+            return Collections.emptyMap();
+        }
+        if (analyzeRequest.getManualRedactions().getComments() == null) {
+            return Collections.emptyMap();
+        }
+        return analyzeRequest.getManualRedactions().getComments();
+    }
+
+
+    public List<RedactionLogLegalBasis> toRedactionLogLegalBasis(List<LegalBasis> legalBasis) {
+
+        return legalBasis.stream().map(l -> new RedactionLogLegalBasis(l.getName(), l.getDescription(), l.getReason())).collect(Collectors.toList());
+    }
+
+
+    private void excludeExcludedPages(RedactionLog redactionLog, Set<Integer> excludedPages) {
+
+        if (excludedPages != null && !excludedPages.isEmpty()) {
+            redactionLog.getRedactionLogEntry().forEach(entry -> entry.getPositions().forEach(pos -> {
+                if (excludedPages.contains(pos.getPage())) {
+                    entry.setExcluded(true);
+                }
+            }));
+        }
+    }
+
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/ComponentLogCreatorService.java

@@ -1,61 +1,57 @@
 package com.iqser.red.service.redaction.v1.server.service;
 
-import java.util.ArrayList;
-import java.util.HashMap;
 import java.util.List;
-import java.util.Map;
+import java.util.stream.Collectors;
 
 import org.springframework.stereotype.Service;
 
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentEntityReference;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentLog;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentLogEntityReference;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentLogCategory;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentLogEntry;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentLogEntryValue;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position;
 import com.iqser.red.service.redaction.v1.server.model.component.Component;
 import com.iqser.red.service.redaction.v1.server.model.component.Entity;
-import com.iqser.red.service.redaction.v1.server.service.document.ComponentComparator;
 import com.iqser.red.service.redaction.v1.server.service.document.EntityComparators;
 
 @Service
 public class ComponentLogCreatorService {
 
-    public ComponentLog buildComponentLog(int analysisNumber, List<Component> components, long componentRulesVersion) {
+    public ComponentLog buildComponentLog(int analysisNumber, List<Component> components) {
 
-        Map<String, List<ComponentLogEntryValue>> map = new HashMap<>();
-        components.stream().sorted(ComponentComparator.first()).forEach(component -> {
-            ComponentLogEntryValue componentLogEntryValue = buildComponentLogEntry(component);
-            map.computeIfAbsent(component.getName(), k -> new ArrayList<>()).add(componentLogEntryValue);
-        });
-        List<ComponentLogEntry> componentLogComponents = map
+        List<ComponentLogCategory> componentLogCategories = components.stream()
+                .collect(Collectors.groupingBy(Component::getCategory, Collectors.mapping(this::buildComponentLogEntry, Collectors.toList())))
                 .entrySet()
-                .stream().map(entry -> new ComponentLogEntry(entry.getKey(), entry.getValue()))
+                .stream()
+                .map(entry -> new ComponentLogCategory(entry.getKey(), entry.getValue()))
                 .toList();
-        return new ComponentLog(analysisNumber, componentRulesVersion, componentLogComponents);
+        return new ComponentLog(analysisNumber, componentLogCategories, -1, -1, -1);
     }
 
 
-    private ComponentLogEntryValue buildComponentLogEntry(Component component) {
+    private ComponentLogEntry buildComponentLogEntry(Component component) {
 
-        return ComponentLogEntryValue.builder()
-                .value(component.getValue()).originalValue(component.getValue())
-                .componentRuleId(component.getMatchedRule().toString())
-                .valueDescription(component.getValueDescription())
-                .componentLogEntityReferences(toComponentEntityReferences(component.getReferences().stream().sorted(EntityComparators.first()).toList()))
+        return ComponentLogEntry.builder()
+                .value(component.getValue())
+                .transformation(component.getTransformation()).originalValue(component.getReferences().stream().sorted(EntityComparators.start()).map(Entity::getValue).toList())
+                .componentEntityReferences(toComponentEntityReferences(component.getReferences()))
                 .build();
     }
 
 
-    private List<ComponentLogEntityReference> toComponentEntityReferences(List<Entity> references) {
+    private List<ComponentEntityReference> toComponentEntityReferences(List<Entity> references) {
 
         return references.stream().map(this::toComponentEntityReference).toList();
     }
 
 
-    private ComponentLogEntityReference toComponentEntityReference(Entity entity) {
+    private ComponentEntityReference toComponentEntityReference(Entity entity) {
 
-        return ComponentLogEntityReference.builder().id(entity.getId())
-                .page(entity.getPositions().stream().findFirst().map(Position::getPageNumber).orElse(0)).entityRuleId(entity.getMatchedRule())
+        return ComponentEntityReference.builder()
+                .id(entity.getId())
+                .page(entity.getPositions().stream().findFirst().map(Position::getPageNumber).orElse(0))
+                .reason(entity.getReason())
+                .ruleIdentifier(entity.getMatchedRule())
                 .type(entity.getType())
                 .build();
     }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/DictionarySearchService.java

@@ -1,8 +1,5 @@
 package com.iqser.red.service.redaction.v1.server.service;
 
-import java.util.List;
-import java.util.Set;
-
 import org.springframework.stereotype.Service;
 
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine;
@@ -13,7 +10,6 @@ import com.iqser.red.service.redaction.v1.server.model.dictionary.Dictionary;
 import com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService;
 import com.iqser.red.service.redaction.v1.server.service.document.EntityEnrichmentService;
 
-import io.micrometer.observation.annotation.Observed;
 import lombok.AccessLevel;
 import lombok.RequiredArgsConstructor;
 import lombok.experimental.FieldDefaults;
@@ -28,18 +24,10 @@ public class DictionarySearchService {
     EntityEnrichmentService entityEnrichmentService;
 
 
-    @Observed(name = "DictionarySearchService", contextualName = "add-dictionary-entries")
-    public void addDictionaryEntities(Dictionary dictionary, List<SemanticNode> semanticNodes) {
-
-        semanticNodes.forEach(node -> addDictionaryEntities(dictionary, node));
-    }
-
-
-    @Observed(name = "DictionarySearchService", contextualName = "add-dictionary-entries")
     public void addDictionaryEntities(Dictionary dictionary, SemanticNode node) {
 
         for (var model : dictionary.getDictionaryModels()) {
-            bySearchImplementationAsDictionary(model.getEntriesSearch(), model.getType(), model.isHint() ? EntityType.HINT : EntityType.ENTITY, node, model.isDossierDictionary());
+            bySearchImplementationAsDictionary(model.getEntriesSearch(), model.getType(), EntityType.ENTITY, node, model.isDossierDictionary());
             bySearchImplementationAsDictionary(model.getFalsePositiveSearch(), model.getType(), EntityType.FALSE_POSITIVE, node, model.isDossierDictionary());
             bySearchImplementationAsDictionary(model.getFalseRecommendationsSearch(), model.getType(), EntityType.FALSE_RECOMMENDATION, node, model.isDossierDictionary());
         }
@@ -55,11 +43,11 @@ public class DictionarySearchService {
         EntityCreationService entityCreationService = new EntityCreationService(entityEnrichmentService);
         searchImplementation.getBoundaries(node.getTextBlock(), node.getTextRange())
                 .stream()
-                .filter(boundary -> entityCreationService.isValidEntityTextRange(node.getTextBlock(), boundary))
-                .forEach(bounds -> entityCreationService.byTextRangeWithEngine(bounds, type, entityType, node, Set.of(Engine.DICTIONARY)).ifPresent(entity -> {
-                    entity.setDictionaryEntry(true);
-                    entity.setDossierDictionaryEntry(isDossierDictionaryEntry);
-                }));
+                .filter(boundary -> entityCreationService.isValidEntityBoundary(node.getTextBlock(), boundary))
+                .map(bounds -> entityCreationService.forceByBoundary(bounds, type, entityType, node))
+                .peek(entity -> entity.setDictionaryEntry(true))
+                .peek(entity -> entity.setDossierDictionaryEntry(isDossierDictionaryEntry))
+                .forEach(entity -> entity.addEngine(Engine.DICTIONARY));
     }
 
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/DictionaryService.java

@@ -23,21 +23,20 @@ import com.iqser.red.service.dictionarymerge.commons.DictionaryEntry;
 import com.iqser.red.service.dictionarymerge.commons.DictionaryEntryModel;
 import com.iqser.red.service.dictionarymerge.commons.DictionaryMergeService;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.configuration.Colors;
-import com.iqser.red.service.redaction.v1.server.RedactionServiceSettings;
 import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
 import com.iqser.red.service.redaction.v1.server.model.dictionary.Dictionary;
-import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryEntries;
 import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryIncrement;
-import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryIncrementValue;
 import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryModel;
+import com.iqser.red.service.redaction.v1.server.model.dictionary.TenantDictionary;
+import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryEntries;
+import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryIncrementValue;
 import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryRepresentation;
 import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryVersion;
-import com.iqser.red.service.redaction.v1.server.model.dictionary.TenantDictionary;
+import com.iqser.red.service.redaction.v1.server.RedactionServiceSettings;
 import com.knecon.fforesight.tenantcommons.TenantContext;
 
 import feign.FeignException;
 import io.micrometer.core.annotation.Timed;
-import io.micrometer.observation.annotation.Observed;
 import jakarta.annotation.PostConstruct;
 import lombok.RequiredArgsConstructor;
 import lombok.SneakyThrows;
@@ -48,7 +47,7 @@ import lombok.extern.slf4j.Slf4j;
 @RequiredArgsConstructor
 public class DictionaryService {
 
-    public static final String DEFAULT_COLOR = "#cccccc";
+    private static final String DEFAULT_COLOR = "#cccccc";
     private final DictionaryClient dictionaryClient;
 
     private final RedactionServiceSettings settings;
@@ -74,7 +73,6 @@ public class DictionaryService {
 
 
     @SneakyThrows
-    @Observed(name = "DictionaryService", contextualName = "update-dictionary")
     @Timed("redactmanager_updateDictionary")
     public DictionaryVersion updateDictionary(String dossierTemplateId, String dossierId) {
 
@@ -96,7 +94,6 @@ public class DictionaryService {
 
 
     @SneakyThrows
-    @Observed(name = "DictionaryService", contextualName = "update-dictionary-increments")
     @Timed("redactmanager_getDictionaryIncrements")
     public DictionaryIncrement getDictionaryIncrements(String dossierTemplateId, DictionaryVersion fromVersion, String dossierId) {
 
@@ -104,7 +101,6 @@ public class DictionaryService {
 
         Set<DictionaryIncrementValue> newValues = new HashSet<>();
         List<DictionaryModel> dictionaryModels = getDossierTemplateDictionary(dossierTemplateId).getDictionary();
-
         dictionaryModels.forEach(dictionaryModel -> {
             dictionaryModel.getEntries().forEach(dictionaryEntry -> {
                 if (dictionaryEntry.getVersion() > fromVersion.getDossierTemplateVersion()) {
@@ -154,25 +150,19 @@ public class DictionaryService {
         try {
             DictionaryRepresentation dictionaryRepresentation = new DictionaryRepresentation();
 
-            var typeResponse = dossierId == null ? dictionaryClient.getAllTypesForDossierTemplate(dossierTemplateId, true) : dictionaryClient.getAllTypesForDossier(dossierId,
-                    true);
+            var typeResponse = dossierId == null ? dictionaryClient.getAllTypesForDossierTemplate(dossierTemplateId, false) : dictionaryClient.getAllTypesForDossier(dossierId,
+                    false);
             if (CollectionUtils.isNotEmpty(typeResponse)) {
 
                 List<DictionaryModel> dictionary = typeResponse.stream().map(t -> {
 
-                    Optional<DictionaryModel> optionalOldModel;
+                    Optional<DictionaryModel> oldModel;
                     if (dossierId == null) {
                         var representation = getDossierTemplateDictionary(dossierTemplateId);
-                        optionalOldModel = representation != null ? representation.getDictionary()
-                                .stream()
-                                .filter(f -> f.getType().equals(t.getType()))
-                                .findAny() : Optional.empty();
+                        oldModel = representation != null ? representation.getDictionary().stream().filter(f -> f.getType().equals(t.getType())).findAny() : Optional.empty();
                     } else {
                         var representation = getDossierDictionary(dossierId);
-                        optionalOldModel = representation != null ? representation.getDictionary()
-                                .stream()
-                                .filter(f -> f.getType().equals(t.getType()))
-                                .findAny() : Optional.empty();
+                        oldModel = representation != null ? representation.getDictionary().stream().filter(f -> f.getType().equals(t.getType())).findAny() : Optional.empty();
                     }
 
                     Set<DictionaryEntryModel> entries = new HashSet<>();
@@ -185,38 +175,19 @@ public class DictionaryService {
                     var newFalsePositivesValues = newEntries.getFalsePositives().stream().map(DictionaryEntry::getValue).collect(Collectors.toSet());
                     var newFalseRecommendationsValues = newEntries.getFalseRecommendations().stream().map(DictionaryEntry::getValue).collect(Collectors.toSet());
 
-                    optionalOldModel.ifPresent(oldDictionaryModel -> {
+                    oldModel.ifPresent(oldDictionaryModel -> {
 
                     });
-                    if (optionalOldModel.isPresent()) {
-                        var oldModel = optionalOldModel.get();
-                        if (oldModel.isCaseInsensitive() && !t.isCaseInsensitive()) {
-                            // add old entries from existing DictionaryModel but exclude lower case representation
-                            entries.addAll(oldModel.getEntries().stream().filter(f -> !newValues.stream().map(s -> s.toLowerCase(Locale.ROOT)).toList().contains(f.getValue())).toList());
-                            falsePositives.addAll(oldModel.getFalsePositives()
-                                    .stream()
-                                    .filter(f -> !newFalsePositivesValues.stream().map(s -> s.toLowerCase(Locale.ROOT)).toList().contains(f.getValue()))
-                                    .toList());
-                            falseRecommendations.addAll(oldModel.getFalseRecommendations()
-                                    .stream()
-                                    .filter(f -> !newFalseRecommendationsValues.stream().map(s -> s.toLowerCase(Locale.ROOT)).toList().contains(f.getValue()))
-                                    .toList());
-                        } else if (!oldModel.isCaseInsensitive() && t.isCaseInsensitive()) {
-                            // add old entries from existing DictionaryModel but exclude upper case representation
-                            entries.addAll(oldModel.getEntries().stream().filter(f -> !newValues.contains(f.getValue().toLowerCase(Locale.ROOT))).toList());
-                            falsePositives.addAll(oldModel.getFalsePositives().stream().filter(f -> !newFalsePositivesValues.contains(f.getValue().toLowerCase(Locale.ROOT))).toList());
-                            falseRecommendations.addAll(oldModel.getFalseRecommendations()
-                                    .stream()
-                                    .filter(f -> !newFalseRecommendationsValues.contains(f.getValue().toLowerCase(Locale.ROOT)))
-                                    .toList());
-
-                        } else {
-                            // add old entries from existing DictionaryModel
-                            entries.addAll(oldModel.getEntries().stream().filter(f -> !newValues.contains(f.getValue())).toList());
-                            falsePositives.addAll(oldModel.getFalsePositives().stream().filter(f -> !newFalsePositivesValues.contains(f.getValue())).toList());
-                            falseRecommendations.addAll(oldModel.getFalseRecommendations().stream().filter(f -> !newFalseRecommendationsValues.contains(f.getValue())).toList());
-                        }
-                    }
+                    // add old entries from existing DictionaryModel
+                    oldModel.ifPresent(dictionaryModel -> entries.addAll(dictionaryModel.getEntries().stream().filter(f -> !newValues.contains(f.getValue())).toList()));
+                    oldModel.ifPresent(dictionaryModel -> falsePositives.addAll(dictionaryModel.getFalsePositives()
+                            .stream()
+                            .filter(f -> !newFalsePositivesValues.contains(f.getValue()))
+                            .toList()));
+                    oldModel.ifPresent(dictionaryModel -> falseRecommendations.addAll(dictionaryModel.getFalseRecommendations()
+                            .stream()
+                            .filter(f -> !newFalseRecommendationsValues.contains(f.getValue()))
+                            .toList()));
 
                     // Add Increments
                     entries.addAll(newEntries.getEntries());
@@ -321,7 +292,6 @@ public class DictionaryService {
 
     @SneakyThrows
     @Timed("redactmanager_getDeepCopyDictionary")
-    @Observed(name = "DictionaryService", contextualName = "deep-copy-dictionary")
     public Dictionary getDeepCopyDictionary(String dossierTemplateId, String dossierId) {
 
         List<DictionaryModel> mergedDictionaries;

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityChangeLogService.java

@@ -1,9 +1,11 @@
 package com.iqser.red.service.redaction.v1.server.service;
 
 import java.time.OffsetDateTime;
-import java.util.Comparator;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
-import java.util.Optional;
+import java.util.Map;
 import java.util.Set;
 import java.util.stream.Collectors;
 
@@ -11,115 +13,104 @@ import org.springframework.stereotype.Service;
 
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Change;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ChangeType;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogChanges;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualChange;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;
+import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
 
 import io.micrometer.core.annotation.Timed;
-import lombok.AccessLevel;
 import lombok.RequiredArgsConstructor;
-import lombok.experimental.FieldDefaults;
 import lombok.extern.slf4j.Slf4j;
 
 @Slf4j
 @Service
 @RequiredArgsConstructor
-@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
 public class EntityChangeLogService {
 
+    private final RedactionStorageService redactionStorageService;
+
+
     @Timed("redactmanager_computeChanges")
-    public boolean computeChanges(List<EntityLogEntry> previousEntityLogEntries, List<EntityLogEntry> newEntityLogEntries, ManualRedactions manualRedactions, int analysisNumber) {
+    public EntityLogChanges computeChanges(String dossierId, String fileId, EntityLog currentEntityLog, int analysisNumber) {
 
-        var now = OffsetDateTime.now();
-        if (previousEntityLogEntries.isEmpty()) {
-            newEntityLogEntries.forEach(entry -> entry.getChanges().add(new Change(analysisNumber, ChangeType.ADDED, now)));
-            return true;
+        long start = System.currentTimeMillis();
+
+        EntityLog previousEntityLog = redactionStorageService.getEntityLog(dossierId, fileId);
+
+        if (previousEntityLog == null) {
+            currentEntityLog.getEntityLogEntry().forEach(entry -> {
+                entry.getChanges().add(new Change(analysisNumber, ChangeType.ADDED, OffsetDateTime.now()));
+            });
+            return new EntityLogChanges(currentEntityLog, false);
         }
 
-        boolean hasChanges = false;
+        List<EntityLogEntry> previouslyExistingEntries = previousEntityLog.getEntityLogEntry().stream().filter(entry -> !entry.lastChangeIsRemoved()).toList();
 
-        for (EntityLogEntry entityLogEntry : newEntityLogEntries) {
-            Optional<EntityLogEntry> optionalPreviousEntity = previousEntityLogEntries.stream()
-                    .filter(entry -> entry.getId().equals(entityLogEntry.getId()))
-                    .findAny();
-            if (optionalPreviousEntity.isEmpty()) {
-                hasChanges = true;
-                entityLogEntry.getChanges().add(new Change(analysisNumber, ChangeType.ADDED, now));
-                continue;
-            }
+        Map<String, EntityLogEntry> addedEntryIds = getEntriesThatExistInCurrentButNotInPreviousEntityLog(currentEntityLog, previouslyExistingEntries);
+        Set<String> removedIds = getEntryIdsThatExistInPreviousButNotInCurrentEntityLog(currentEntityLog, previouslyExistingEntries);
 
-            EntityLogEntry previousEntity = optionalPreviousEntity.get();
-            entityLogEntry.getChanges().addAll(previousEntity.getChanges());
-            if (!previousEntity.getState().equals(entityLogEntry.getState())) {
-                hasChanges = true;
-                ChangeType changeType = calculateChangeType(entityLogEntry.getState(), previousEntity.getState());
-                entityLogEntry.getChanges().add(new Change(analysisNumber, changeType, now));
+        List<EntityLogEntry> newEntityLogEntries = previousEntityLog.getEntityLogEntry();
+
+        List<EntityLogEntry> toRemove = new ArrayList<>();
+        newEntityLogEntries.forEach(entry -> {
+            if (removedIds.contains(entry.getId()) && addedEntryIds.containsKey(entry.getId())) {
+                List<Change> changes = entry.getChanges();
+                changes.add(new Change(analysisNumber, ChangeType.CHANGED, OffsetDateTime.now()));
+                var newEntry = addedEntryIds.get(entry.getId());
+                newEntry.setChanges(changes);
+                addedEntryIds.put(entry.getId(), newEntry);
+                toRemove.add(entry);
+            } else if (removedIds.contains(entry.getId())) {
+                entry.getChanges().add(new Change(analysisNumber, ChangeType.REMOVED, OffsetDateTime.now()));
+            } else if (addedEntryIds.containsKey(entry.getId())) {
+                List<Change> changes = entry.getChanges();
+                changes.add(new Change(analysisNumber, ChangeType.ADDED, OffsetDateTime.now()));
+                var newEntry = addedEntryIds.get(entry.getId());
+                newEntry.setChanges(changes);
+                addedEntryIds.put(entry.getId(), newEntry);
+                toRemove.add(entry);
             }
-        }
-        addRemovedEntriesAsRemoved(previousEntityLogEntries, newEntityLogEntries, manualRedactions, analysisNumber, now);
-        return hasChanges;
+        });
+
+        newEntityLogEntries.removeAll(toRemove);
+
+        addedEntryIds.forEach((k, v) -> {
+            if (v.getChanges().isEmpty()) {
+                v.getChanges().add(new Change(analysisNumber, ChangeType.ADDED, OffsetDateTime.now()));
+            }
+            newEntityLogEntries.add(v);
+        });
+
+        currentEntityLog.setEntityLogEntry(newEntityLogEntries);
+
+        log.debug("Change computation took: {}", System.currentTimeMillis() - start);
+        return new EntityLogChanges(currentEntityLog, !addedEntryIds.isEmpty() || !removedIds.isEmpty());
     }
 
 
-    private void addRemovedEntriesAsRemoved(List<EntityLogEntry> previousEntityLogEntries,
-                                            List<EntityLogEntry> newEntityLogEntries,
-                                            ManualRedactions manualRedactions,
-                                            int analysisNumber,
-                                            OffsetDateTime now) {
+    private static Set<String> getEntryIdsThatExistInPreviousButNotInCurrentEntityLog(EntityLog currentEntityLog, List<EntityLogEntry> previouslyExistingEntries) {
 
-        Set<String> existingIds = newEntityLogEntries.stream()
-                .map(EntityLogEntry::getId)
-                .collect(Collectors.toSet());
-        List<EntityLogEntry> removedEntries = previousEntityLogEntries.stream()
-                .filter(entry -> !existingIds.contains(entry.getId()))
-                .collect(Collectors.toList());
-        List<EntityLogEntry> removedDossierRedaction = removedEntries.stream()
-                .filter(e -> e.getState() == EntryState.REMOVED && e.getType().equals("dossier_redaction"))
-                .toList();
-        previousEntityLogEntries.removeAll(removedDossierRedaction);
-        removedEntries.removeAll(removedDossierRedaction);
-        removedEntries.forEach(entry -> entry.getChanges().add(new Change(analysisNumber, ChangeType.REMOVED, now)));
-        removedEntries.forEach(entry -> entry.setState(EntryState.REMOVED));
-        removedEntries.forEach(entry -> addManualChangeForDictionaryRemovals(entry, manualRedactions));
-        newEntityLogEntries.addAll(removedEntries);
+        Set<EntityLogEntry> removed = new HashSet<>(previouslyExistingEntries);
+        currentEntityLog.getEntityLogEntry().forEach(removed::remove);
+
+        Set<String> removedIds = removed.stream().map(EntityLogEntry::getId).collect(Collectors.toSet());
+        return removedIds;
     }
 
 
-    private void addManualChangeForDictionaryRemovals(EntityLogEntry entry, ManualRedactions manualRedactions) {
+    private static Map<String, EntityLogEntry> getEntriesThatExistInCurrentButNotInPreviousEntityLog(EntityLog currentEntityLog, List<EntityLogEntry> previouslyExistingEntries) {
 
-        if (manualRedactions == null || manualRedactions.getIdsToRemove().isEmpty()) {
-            return;
-        }
-
-        manualRedactions.getIdsToRemove()
+        Set<EntityLogEntry> currentExistingEntries = currentEntityLog.getEntityLogEntry()
                 .stream()
-                .filter(IdRemoval::isRemoveFromDictionary)//
-                .filter(removed -> removed.getAnnotationId().equals(entry.getId()))//
-                .findFirst()//
-                .ifPresent(idRemove -> entry.getManualChanges().add(ManualChangeFactory.toManualChange(idRemove, false)));
-    }
+                .filter(entry -> entry.getChanges().isEmpty() || !entry.lastChangeIsRemoved())
+                .collect(Collectors.toSet());
 
-
-    private ChangeType calculateChangeType(EntryState state, EntryState previousState) {
-
-        if (state.equals(previousState)) {
-            throw new IllegalArgumentException("States are equal, can't calculate ChangeType.");
-        }
-        if (!isRemoved(previousState) && isRemoved(state)) {
-            return ChangeType.REMOVED;
-        }
-        if (isRemoved(previousState) && !isRemoved(state)) {
-            return ChangeType.ADDED;
-        }
-        return ChangeType.CHANGED;
-    }
-
-
-    private static boolean isRemoved(EntryState state) {
-
-        return (state.equals(EntryState.REMOVED) || state.equals(EntryState.IGNORED));
+        previouslyExistingEntries.forEach(currentExistingEntries::remove);
+        Map<String, EntityLogEntry> addedIds = new HashMap<>();
+        currentExistingEntries.forEach(entry -> {
+            addedIds.put(entry.getId(), entry);
+        });
+        return addedIds;
     }
 
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java

@@ -1,5 +1,6 @@
 package com.iqser.red.service.redaction.v1.server.service;
 
+import java.awt.geom.Rectangle2D;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashSet;
@@ -10,307 +11,190 @@ import java.util.stream.Collectors;
 
 import org.springframework.stereotype.Service;
 
-import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogChanges;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogLegalBasis;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryType;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.legalbasis.LegalBasis;
-import com.iqser.red.service.redaction.v1.server.RedactionServiceSettings;
-import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient;
-import com.iqser.red.service.redaction.v1.server.model.PrecursorEntity;
-import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryVersion;
-import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
+import com.iqser.red.service.redaction.v1.server.model.ManualEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.IEntity;
-import com.iqser.red.service.redaction.v1.server.model.document.entity.ManualChangeOverwrite;
+import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.PositionOnPage;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Image;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.ImageType;
-import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
 
-import lombok.AccessLevel;
 import lombok.RequiredArgsConstructor;
-import lombok.experimental.FieldDefaults;
 import lombok.extern.slf4j.Slf4j;
 
 @Service
 @Slf4j
 @RequiredArgsConstructor
-@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
 public class EntityLogCreatorService {
 
-    DictionaryService dictionaryService;
-    RedactionServiceSettings redactionServiceSettings;
-    LegalBasisClient legalBasisClient;
-    EntityChangeLogService entityChangeLogService;
-    RedactionStorageService redactionStorageService;
+    private final DictionaryService dictionaryService;
+    private final ManualChangeFactory manualChangeFactory;
 
 
-    private static boolean notFalsePositiveOrFalseRecommendation(TextEntity textEntity) {
-
-        return !(textEntity.getEntityType().equals(EntityType.FALSE_POSITIVE) || textEntity.getEntityType().equals(EntityType.FALSE_RECOMMENDATION));
-    }
-
-
-    public EntityLog createInitialEntityLog(AnalyzeRequest analyzeRequest,
-                                            Document document,
-                                            List<PrecursorEntity> notFoundEntities,
-                                            DictionaryVersion dictionaryVersion,
-                                            long rulesVersion) {
-
-        List<EntityLogEntry> entityLogEntries = createEntityLogEntries(document, analyzeRequest, notFoundEntities);
-
-        List<LegalBasis> legalBasis = legalBasisClient.getLegalBasisMapping(analyzeRequest.getDossierTemplateId());
-
-        List<EntityLogEntry> previousExistingEntityLogEntries = getPreviousEntityLogEntries(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
-
-        entityChangeLogService.computeChanges(previousExistingEntityLogEntries, entityLogEntries, analyzeRequest.getManualRedactions(), analyzeRequest.getAnalysisNumber());
-
-        return new EntityLog(redactionServiceSettings.getAnalysisVersion(),
-                             analyzeRequest.getAnalysisNumber(),
-                             entityLogEntries,
-                             toEntityLogLegalBasis(legalBasis),
-                             dictionaryVersion.getDossierTemplateVersion(),
-                             dictionaryVersion.getDossierVersion(),
-                             rulesVersion,
-                             legalBasisClient.getVersion(analyzeRequest.getDossierTemplateId()));
-    }
-
-
-    private List<EntityLogEntry> getPreviousEntityLogEntries(String dossierId, String fileId) {
-
-        if (redactionStorageService.entityLogExists(dossierId, fileId)) {
-            return redactionStorageService.getEntityLog(dossierId, fileId).getEntityLogEntry();
-        } else {
-            return Collections.emptyList();
-        }
-    }
-
-
-    public EntityLogChanges updateVersionsAndReturnChanges(EntityLog entityLog, DictionaryVersion dictionaryVersion, AnalyzeRequest analyzeRequest, boolean hasChanges) {
-
-        List<LegalBasis> legalBasis = legalBasisClient.getLegalBasisMapping(analyzeRequest.getDossierTemplateId());
-        entityLog.setLegalBasisVersion(legalBasisClient.getVersion(analyzeRequest.getDossierTemplateId()));
-        entityLog.setLegalBasis(toEntityLogLegalBasis(legalBasis));
-        entityLog.setDictionaryVersion(dictionaryVersion.getDossierTemplateVersion());
-        entityLog.setDossierDictionaryVersion(dictionaryVersion.getDossierVersion());
-        entityLog.setAnalysisNumber(analyzeRequest.getAnalysisNumber());
-
-        return new EntityLogChanges(entityLog, hasChanges);
-    }
-
-
-    public EntityLogChanges updatePreviousEntityLog(AnalyzeRequest analyzeRequest,
-                                                    Document document,
-                                                    List<PrecursorEntity> notFoundEntries,
-                                                    EntityLog previousEntityLog,
-                                                    Set<Integer> sectionsToReanalyseIds,
-                                                    DictionaryVersion dictionaryVersion) {
-
-        List<EntityLogEntry> newEntityLogEntries = createEntityLogEntries(document, analyzeRequest, notFoundEntries).stream()
-                .filter(entry -> entry.getContainingNodeId().isEmpty() || sectionsToReanalyseIds.contains(entry.getContainingNodeId()
-                                                                                                                  .get(0)))
-                .collect(Collectors.toList());
-        Set<String> newEntityIds = newEntityLogEntries.stream()
-                .map(EntityLogEntry::getId)
-                .collect(Collectors.toSet());
-
-        List<EntityLogEntry> previousEntriesFromReAnalyzedSections = previousEntityLog.getEntityLogEntry()
-                .stream()
-                .filter(entry -> (newEntityIds.contains(entry.getId()) || entry.getContainingNodeId().isEmpty() || sectionsToReanalyseIds.contains(entry.getContainingNodeId()
-                                                                                                                                                           .get(0))))
-                .collect(Collectors.toList());
-        previousEntityLog.getEntityLogEntry().removeAll(previousEntriesFromReAnalyzedSections);
-
-        boolean hasChanges = entityChangeLogService.computeChanges(previousEntriesFromReAnalyzedSections,
-                                                                   newEntityLogEntries,
-                                                                   analyzeRequest.getManualRedactions(),
-                                                                   analyzeRequest.getAnalysisNumber());
-        previousEntityLog.getEntityLogEntry().addAll(newEntityLogEntries);
-
-        return updateVersionsAndReturnChanges(previousEntityLog, dictionaryVersion, analyzeRequest, hasChanges);
-    }
-
-
-    private List<EntityLogEntry> createEntityLogEntries(Document document, AnalyzeRequest analyzeRequest, List<PrecursorEntity> notFoundPrecursorEntries) {
-
-        String dossierTemplateId = analyzeRequest.getDossierTemplateId();
+    public List<EntityLogEntry> createEntityLogEntries(Document document, String dossierTemplateId, List<ManualEntity> notFoundManualRedactionEntries) {
 
         List<EntityLogEntry> entries = new ArrayList<>();
-
         document.getEntities()
                 .stream()
-                .filter(entity -> !entity.getValue().isEmpty())
-                .filter(EntityLogCreatorService::notFalsePositiveOrFalseRecommendation)
+                .filter(EntityLogCreatorService::isEntityOrRecommendationType)
                 .filter(entity -> !entity.removed())
-                .forEach(entityNode -> entries.addAll(toEntityLogEntries(entityNode)));
-        document.streamAllImages()
-                .filter(entity -> !entity.removed())
-                .forEach(imageNode -> entries.add(createEntityLogEntry(imageNode, dossierTemplateId)));
-        notFoundPrecursorEntries.stream()
-                .filter(entity -> !entity.removed())
-                .forEach(precursorEntity -> entries.add(createEntityLogEntry(precursorEntity, dossierTemplateId)));
+                .forEach(entityNode -> entries.addAll(toEntityLogEntries(entityNode, dossierTemplateId)));
+        document.streamAllImages().filter(image -> !image.removed()).forEach(imageNode -> entries.add(createEntityLogEntry(imageNode, dossierTemplateId)));
+        notFoundManualRedactionEntries.forEach(entityIdentifier -> entries.add(createEntityLogEntry(entityIdentifier, dossierTemplateId)));
         return entries;
     }
 
 
-    private List<EntityLogEntry> toEntityLogEntries(TextEntity textEntity) {
+    private static boolean isEntityOrRecommendationType(TextEntity textEntity) {
 
-        List<EntityLogEntry> entityLogEntries = new ArrayList<>();
+        return textEntity.getEntityType() == EntityType.ENTITY || textEntity.getEntityType() == EntityType.RECOMMENDATION;
+    }
+
+
+    private List<EntityLogEntry> toEntityLogEntries(TextEntity textEntity, String dossierTemplateId) {
+
+        Set<String> processedIds = new HashSet<>();
+        List<EntityLogEntry> redactionLogEntities = new ArrayList<>();
 
-        // split entity into multiple entries if it occurs on multiple pages, since FE can't handle multi page entities
         for (PositionOnPage positionOnPage : textEntity.getPositionsOnPagePerPage()) {
 
-            EntityLogEntry entityLogEntry = createEntityLogEntry(textEntity);
+            // Duplicates should be removed. They might exist due to table extraction duplicating cells spanning multiple columns/rows.
+            if (processedIds.contains(positionOnPage.getId())) {
+                continue;
+            }
+            processedIds.add(positionOnPage.getId());
+
+            EntityLogEntry redactionLogEntry = createEntityLogEntry(textEntity, dossierTemplateId);
+            redactionLogEntry.setId(positionOnPage.getId());
 
             List<Position> rectanglesPerLine = positionOnPage.getRectanglePerLine()
                     .stream()
-                    .map(rectangle2D -> new Position(rectangle2D, positionOnPage.getPage().getNumber()))
+                    .map(rectangle2D -> toRedactionLogRectangle(rectangle2D, positionOnPage.getPage().getNumber()))
                     .toList();
 
-            // set the ID from the positions, since it might contain a "-" with the page number if the entity is split across multiple pages
-            entityLogEntry.setId(positionOnPage.getId());
-            entityLogEntry.setPositions(rectanglesPerLine);
-            entityLogEntries.add(entityLogEntry);
+            redactionLogEntry.setPositions(rectanglesPerLine);
+            redactionLogEntities.add(redactionLogEntry);
         }
 
-        return entityLogEntries;
+        return redactionLogEntities;
     }
 
 
-    private EntityLogEntry createEntityLogEntry(Image image, String dossierTemplateId) {
-
-        boolean isHint = dictionaryService.isHint(image.type(), dossierTemplateId);
-        return EntityLogEntry.builder()
-                .id(image.getId())
-                .value(image.value())
-                .type(image.type())
-                .reason(image.buildReasonWithManualChangeDescriptions())
-                .legalBasis(image.legalBasis())
-                .matchedRule(image.getMatchedRule().getRuleIdentifier().toString())
-                .dictionaryEntry(false)
-                .positions(List.of(new Position(image.getPosition(), image.getPage().getNumber())))
-                .containingNodeId(image.getTreeId())
-                .closestHeadline(image.getHeadline().getTextBlock().getSearchText())
-                .section(image.getManualOverwrite().getSection()
-                                 .orElse(image.getParent().toString()))
-                .imageHasTransparency(image.isTransparent())
-                .manualChanges(ManualChangeFactory.toManualChangeList(image.getManualOverwrite().getManualChangeLog(), isHint))
-                .state(buildEntryState(image))
-                .entryType(isHint ? EntryType.IMAGE_HINT : EntryType.IMAGE)
-                .engines(getEngines(null, image.getManualOverwrite()))
-                .build();
-
-    }
-
-
-    private EntityLogEntry createEntityLogEntry(PrecursorEntity precursorEntity, String dossierTemplateId) {
-
-        String type = precursorEntity.getManualOverwrite().getType()
-                .orElse(precursorEntity.getType());
-        boolean isHint = isHint(precursorEntity.getEntityType());
-        return EntityLogEntry.builder()
-                .id(precursorEntity.getId())
-                .reason(precursorEntity.buildReasonWithManualChangeDescriptions())
-                .legalBasis(precursorEntity.legalBasis())
-                .value(precursorEntity.value())
-                .type(type)
-                .state(buildEntryState(precursorEntity))
-                .entryType(buildEntryType(precursorEntity))
-                .section(precursorEntity.getManualOverwrite().getSection()
-                                 .orElse(precursorEntity.getSection()))
-                .containingNodeId(Collections.emptyList())
-                .closestHeadline("")
-                .matchedRule(precursorEntity.getMatchedRule().getRuleIdentifier().toString())
-                .dictionaryEntry(precursorEntity.isDictionaryEntry())
-                .dossierDictionaryEntry(precursorEntity.isDossierDictionaryEntry())
-                .textAfter("")
-                .textBefore("")
-                .startOffset(-1)
-                .endOffset(-1)
-                .positions(precursorEntity.getManualOverwrite().getPositions()
-                                   .orElse(precursorEntity.getEntityPosition())
-                                   .stream()
-                                   .map(entityPosition -> new Position(entityPosition.rectangle2D(), entityPosition.pageNumber()))
-                                   .toList())
-                .engines(getEngines(precursorEntity.getEngines(), precursorEntity.getManualOverwrite()))
-                //imported is no longer used, frontend should check engines
-                //(was .imported(precursorEntity.getEngines() != null && precursorEntity.getEngines().contains(Engine.IMPORTED)))
-                .imported(false)
-                .reference(Collections.emptySet())
-                .manualChanges(ManualChangeFactory.toManualChangeList(precursorEntity.getManualOverwrite().getManualChangeLog(), isHint))
-                .build();
-    }
-
-
-    private EntityLogEntry createEntityLogEntry(TextEntity entity) {
+    private EntityLogEntry createEntityLogEntry(TextEntity entity, String dossierTemplateId) {
 
         Set<String> referenceIds = new HashSet<>();
-        entity.references()
-                .stream()
-                .filter(TextEntity::active)
-                .forEach(ref -> ref.getPositionsOnPagePerPage()
-                        .forEach(pos -> referenceIds.add(pos.getId())));
-        boolean isHint = isHint(entity.getEntityType());
+        entity.references().stream().filter(TextEntity::active).forEach(ref -> ref.getPositionsOnPagePerPage().forEach(pos -> referenceIds.add(pos.getId())));
+        int sectionNumber = entity.getDeepestFullyContainingNode().getTreeId().isEmpty() ? 0 : entity.getDeepestFullyContainingNode().getTreeId().get(0);
+        boolean isHint = isHint(entity.getType(), dossierTemplateId);
         return EntityLogEntry.builder()
+                .color(getColor(entity.getType(), dossierTemplateId, entity.applied()))
                 .reason(entity.buildReasonWithManualChangeDescriptions())
                 .legalBasis(entity.legalBasis())
-                .value(entity.getManualOverwrite().getValue()
-                               .orElse(entity.getMatchedRule().isWriteValueWithLineBreaks() ? entity.getValueWithLineBreaks() : entity.getValue()))
-                .type(entity.type())
-                .section(entity.getManualOverwrite().getSection()
-                                 .orElse(entity.getDeepestFullyContainingNode().toString()))
-                .containingNodeId(entity.getDeepestFullyContainingNode().getTreeId())
-                .closestHeadline(entity.getDeepestFullyContainingNode().getHeadline().getTextBlock().getSearchText())
+                .value(entity.getManualOverwrite().getValue().orElse(entity.getMatchedRule().isWriteValueWithLineBreaks() ? entity.getValueWithLineBreaks() : entity.getValue()))
+                .type(entity.getType())
+                .section(entity.getManualOverwrite().getSection().orElse(entity.getDeepestFullyContainingNode().toString()))
+                .sectionNumber(sectionNumber)
                 .matchedRule(entity.getMatchedRule().getRuleIdentifier().toString())
-                .dictionaryEntry(entity.isDictionaryEntry())
+                .isDictionaryEntry(entity.isDictionaryEntry())
                 .textAfter(entity.getTextAfter())
                 .textBefore(entity.getTextBefore())
                 .startOffset(entity.getTextRange().start())
                 .endOffset(entity.getTextRange().end())
-                .dossierDictionaryEntry(entity.isDossierDictionaryEntry())
-                .engines(getEngines(entity.getEngines(), entity.getManualOverwrite()))
-                //imported is no longer used, frontend should check engines
-                //(was .imported(entity.getEngines() != null && entity.getEngines().contains(Engine.IMPORTED)))
-                .imported(false)
+                .isDossierDictionaryEntry(entity.isDossierDictionaryEntry())
+                .engines(entity.getEngines() != null ? entity.getEngines() : Collections.emptySet())
                 .reference(referenceIds)
-                .manualChanges(ManualChangeFactory.toManualChangeList(entity.getManualOverwrite().getManualChangeLog(), isHint))
+                .manualChanges(manualChangeFactory.toManualChangeList(entity.getManualOverwrite().getManualChangeLog(), isHint))
                 .state(buildEntryState(entity))
-                .entryType(buildEntryType(entity))
+                .entryType(buildEntryType(entity, isHint))
                 .build();
     }
 
 
-    private Set<Engine> getEngines(Set<Engine> currentEngines, ManualChangeOverwrite manualChangeOverwrite) {
+    public EntityLogEntry createEntityLogEntry(ManualEntity manualEntity, String dossierTemplateId) {
 
-        Set<Engine> engines = currentEngines != null ? new HashSet<>(currentEngines) : new HashSet<>();
+        String type = manualEntity.getManualOverwrite().getType().orElse(manualEntity.getType());
+        boolean isHint = isHint(type, dossierTemplateId);
+        return EntityLogEntry.builder()
+                .id(manualEntity.getId())
+                .color(getColor(type, dossierTemplateId, manualEntity.applied()))
+                .reason(manualEntity.buildReasonWithManualChangeDescriptions())
+                .legalBasis(manualEntity.legalBasis())
+                .value(manualEntity.getManualOverwrite().getValue().orElse(manualEntity.getValue()))
+                .type(type)
+                .state(buildEntryState(manualEntity))
+                .entryType(buildEntryType(manualEntity, isHint))
+                .section(manualEntity.getManualOverwrite().getSection().orElse(manualEntity.getSection()))
+                .sectionNumber(0)
+                .matchedRule("ManualRedaction")
+                .rectangle(manualEntity.isRectangle())
+                .isDictionaryEntry(manualEntity.isDictionaryEntry())
+                .isDossierDictionaryEntry(manualEntity.isDossierDictionaryEntry())
+                .textAfter("")
+                .textBefore("")
+                .startOffset(-1)
+                .endOffset(-1)
+                .positions(manualEntity.getEntityPosition()
+                        .stream()
+                        .map(entityPosition -> toRedactionLogRectangle(entityPosition.rectangle2D(), entityPosition.pageNumber()))
+                        .collect(Collectors.toList()))
+                .engines(Collections.emptySet())
+                .reference(Collections.emptySet())
+                .manualChanges(manualChangeFactory.toManualChangeList(manualEntity.getManualOverwrite().getManualChangeLog(), isHint))
+                .build();
+    }
 
-        if (manualChangeOverwrite != null && !manualChangeOverwrite.getManualChangeLog().isEmpty()) {
-            engines.add(Engine.MANUAL);
+
+    public EntityLogEntry createEntityLogEntry(Image image, String dossierTemplateId) {
+
+        String imageType = image.getImageType().equals(ImageType.OTHER) ? "image" : image.getImageType().toString().toLowerCase(Locale.ENGLISH);
+        boolean isHint = dictionaryService.isHint(imageType, dossierTemplateId);
+        return EntityLogEntry.builder()
+                .id(image.getId())
+                .color(getColor(imageType, dossierTemplateId, image.applied()))
+                .type(imageType)
+                .reason(image.buildReasonWithManualChangeDescriptions())
+                .legalBasis(image.legalBasis())
+                .matchedRule(image.getMatchedRule().getRuleIdentifier().toString())
+                .isDictionaryEntry(false)
+                .positions(List.of(toRedactionLogRectangle(image.getPosition(), image.getPage().getNumber())))
+                .sectionNumber(image.getTreeId().get(0))
+                .section(image.getManualOverwrite().getSection().orElse(image.getParent().toString()))
+                .imageHasTransparency(image.isTransparent())
+                .manualChanges(manualChangeFactory.toManualChangeList(image.getManualOverwrite().getManualChangeLog(), isHint))
+                .state(buildEntryState(image))
+                .entryType(buildEntryType(image, isHint))
+                .build();
+
+    }
+
+
+    private float[] getColor(String type, String dossierTemplateId, boolean isRedaction) {
+
+        if (!isRedaction && !isHint(type, dossierTemplateId)) {
+            return dictionaryService.getNotRedactedColor(dossierTemplateId);
         }
-        return engines;
+        return dictionaryService.getColor(type, dossierTemplateId);
     }
 
 
-    private boolean isHint(EntityType entityType) {
+    private boolean isHint(String type, String dossierTemplateId) {
 
-        return entityType.equals(EntityType.HINT);
+        return dictionaryService.isHint(type, dossierTemplateId);
     }
 
 
-    public static EntryState buildEntryState(IEntity entity) {
+    private EntryState buildEntryState(IEntity IEntity) {
 
-        if (entity.applied() && entity.active()) {
+        if (IEntity.applied()) {
             return EntryState.APPLIED;
-        } else if (entity.skipped() && entity.active()) {
+        } else if (IEntity.skipped()) {
             return EntryState.SKIPPED;
-        } else if (entity.ignored()) {
+        } else if (IEntity.ignored()) {
             return EntryState.IGNORED;
         } else {
             return EntryState.REMOVED;
@@ -318,27 +202,23 @@ public class EntityLogCreatorService {
     }
 
 
-    public static EntryType buildEntryType(IEntity entity) {
+    private EntryType buildEntryType(IEntity IEntity, boolean isHint) {
 
-        if (entity instanceof TextEntity textEntity) {
-            return getEntryType(textEntity.getEntityType());
-        } else if (entity instanceof PrecursorEntity precursorEntity) {
-            if (precursorEntity.isRectangle()) {
-                return EntryType.AREA;
-            }
-            return getEntryType(precursorEntity.getEntityType());
-        } else if (entity instanceof Image) {
+        if (IEntity instanceof TextEntity textEntity) {
+            return getEntryType(isHint, textEntity.getEntityType());
+        } else if (IEntity instanceof ManualEntity manualEntity) {
+            return getEntryType(isHint, manualEntity.getEntityType());
+        } else if (IEntity instanceof Image) {
             return EntryType.IMAGE;
         }
-        throw new UnsupportedOperationException(String.format("Entity subclass %s is not implemented!", entity.getClass()));
+        throw new UnsupportedOperationException("Entity subclass %s not implemented!");
     }
 
 
-    private static EntryType getEntryType(EntityType entityType) {
+    private static EntryType getEntryType(boolean isHint, EntityType entityType) {
 
         return switch (entityType) {
-            case ENTITY -> EntryType.ENTITY;
-            case HINT -> EntryType.HINT;
+            case ENTITY -> isHint ? EntryType.HINT : EntryType.ENTITY;
             case FALSE_POSITIVE -> EntryType.FALSE_POSITIVE;
             case RECOMMENDATION -> EntryType.RECOMMENDATION;
             case FALSE_RECOMMENDATION -> EntryType.FALSE_RECOMMENDATION;
@@ -346,11 +226,10 @@ public class EntityLogCreatorService {
     }
 
 
-    private List<EntityLogLegalBasis> toEntityLogLegalBasis(List<LegalBasis> legalBasis) {
+    private Position toRedactionLogRectangle(Rectangle2D rectangle2D, int pageNumber) {
 
-        return legalBasis.stream()
-                .map(l -> new EntityLogLegalBasis(l.getName(), l.getDescription(), l.getReason()))
-                .collect(Collectors.toList());
+        return new Position(new float[]{(float) rectangle2D.getMinX(), (float) (rectangle2D.getMinY() + rectangle2D.getHeight()), (float) rectangle2D.getWidth(), -(float) rectangle2D.getHeight()},
+                pageNumber);
     }
 
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/ImportedRedactionService.java

@@ -0,0 +1,263 @@
+package com.iqser.red.service.redaction.v1.server.service;
+
+import java.util.HashSet;
+import java.util.List;
+
+import org.springframework.stereotype.Service;
+
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Point;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.imported.ImportedRedaction;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.imported.ImportedRedactions;
+import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
+
+import io.micrometer.core.annotation.Timed;
+import lombok.RequiredArgsConstructor;
+
+@Service
+@RequiredArgsConstructor
+public class ImportedRedactionService {
+
+    public static final String IMPORTED_REDACTION_TYPE = "imported_redaction";
+
+    private final DictionaryService dictionaryService;
+    private final RedactionStorageService redactionStorageService;
+
+
+    @Deprecated(forRemoval = true)
+    @Timed("redactmanager_processImportedRedactions")
+    public List<RedactionLogEntry> processImportedRedactions(String dossierTemplateId,
+                                                             String dossierId,
+                                                             String fileId,
+                                                             List<RedactionLogEntry> redactionLogEntries,
+                                                             boolean addImportedRedactions) {
+
+        var importedRedactions = redactionStorageService.getImportedRedactions(dossierId, fileId);
+        if (importedRedactions == null) {
+            return redactionLogEntries;
+        }
+
+        redactionLogEntries.forEach(redactionLogEntry -> addIntersections(redactionLogEntry, importedRedactions));
+
+        if (addImportedRedactions) {
+            return addImportedRedactionsRedactionLogEntries(dossierTemplateId, redactionLogEntries, importedRedactions);
+        }
+
+        return redactionLogEntries;
+    }
+
+
+    @Timed("redactmanager_processImportedRedactions")
+    public List<EntityLogEntry> processImportedEntities(String dossierTemplateId,
+                                                        String dossierId,
+                                                        String fileId,
+                                                        List<EntityLogEntry> redactionLogEntries,
+                                                        boolean addImportedRedactions) {
+
+        var importedRedactions = redactionStorageService.getImportedRedactions(dossierId, fileId);
+        if (importedRedactions == null) {
+            return redactionLogEntries;
+        }
+
+        redactionLogEntries.forEach(redactionLogEntry -> addIntersections(redactionLogEntry, importedRedactions));
+
+        if (addImportedRedactions) {
+            return addImportedRedactionsEntityLogEntries(dossierTemplateId, redactionLogEntries, importedRedactions);
+        }
+
+        return redactionLogEntries;
+    }
+
+
+    @Deprecated(forRemoval = true)
+    private List<RedactionLogEntry> addImportedRedactionsRedactionLogEntries(String dossierTemplateId,
+                                                                             List<RedactionLogEntry> redactionLogEntries,
+                                                                             ImportedRedactions importedRedactions) {
+
+        for (var importedRedactionsValues : importedRedactions.getImportedRedactions().values()) {
+            for (var importedRedaction : importedRedactionsValues) {
+                RedactionLogEntry redactionLogEntry = RedactionLogEntry.builder()
+                        .id(importedRedaction.getId())
+                        .type(IMPORTED_REDACTION_TYPE)
+                        .imported(true)
+                        .redacted(true)
+                        .positions(importedRedaction.getPositions())
+                        .color(getColor(IMPORTED_REDACTION_TYPE, dossierTemplateId))
+                        .build();
+
+                redactionLogEntries.add(redactionLogEntry);
+            }
+        }
+
+        return redactionLogEntries;
+    }
+
+
+    private List<EntityLogEntry> addImportedRedactionsEntityLogEntries(String dossierTemplateId, List<EntityLogEntry> entityLogEntries, ImportedRedactions importedRedactions) {
+
+        for (var importedRedactionsValues : importedRedactions.getImportedRedactions().values()) {
+            for (var importedRedaction : importedRedactionsValues) {
+                EntityLogEntry redactionLogEntry = EntityLogEntry.builder()
+                        .id(importedRedaction.getId())
+                        .type(IMPORTED_REDACTION_TYPE)
+                        .imported(true)
+                        .state(EntryState.APPLIED)
+                        .positions(toPositions(importedRedaction.getPositions()))
+                        .color(getColor(IMPORTED_REDACTION_TYPE, dossierTemplateId))
+                        .build();
+
+                entityLogEntries.add(redactionLogEntry);
+            }
+        }
+
+        return entityLogEntries;
+    }
+
+
+    private List<Position> toPositions(List<Rectangle> positions) {
+
+        return positions.stream().map(this::toPosition).toList();
+    }
+
+
+    private Position toPosition(Rectangle rectangle) {
+
+        return new Position(new float[]{rectangle.getTopLeft().getX(), rectangle.getTopLeft().getY(), rectangle.getWidth(), rectangle.getHeight()}, rectangle.getPage());
+    }
+
+
+    @Deprecated(forRemoval = true)
+    private void addIntersections(RedactionLogEntry redactionLogEntry, ImportedRedactions importedRedactions) {
+
+        for (Rectangle rectangle : redactionLogEntry.getPositions()) {
+            var normalizedRectangle = normalize(rectangle);
+            if (importedRedactions.getImportedRedactions().containsKey(rectangle.getPage())) {
+                var importedRedactionsOnPage = importedRedactions.getImportedRedactions().get(rectangle.getPage());
+                for (ImportedRedaction importedRedaction : importedRedactionsOnPage) {
+                    for (Rectangle importedRedactionPosition : importedRedaction.getPositions()) {
+                        if (rectOverlap(normalizedRectangle, normalize(importedRedactionPosition))) {
+                            if (redactionLogEntry.getImportedRedactionIntersections() == null) {
+                                redactionLogEntry.setImportedRedactionIntersections(new HashSet<>());
+                            }
+                            redactionLogEntry.getImportedRedactionIntersections().add(importedRedaction.getId());
+                        }
+                    }
+                }
+            }
+        }
+    }
+
+
+    private void addIntersections(EntityLogEntry redactionLogEntry, ImportedRedactions importedRedactions) {
+
+        for (Position rectangle : redactionLogEntry.getPositions()) {
+            var normalizedRectangle = normalize(rectangle);
+            if (importedRedactions.getImportedRedactions().containsKey(rectangle.getPageNumber())) {
+                var importedRedactionsOnPage = importedRedactions.getImportedRedactions().get(rectangle.getPageNumber());
+                for (ImportedRedaction importedRedaction : importedRedactionsOnPage) {
+                    for (Rectangle importedRedactionPosition : importedRedaction.getPositions()) {
+                        if (rectOverlap(normalizedRectangle, normalize(importedRedactionPosition))) {
+                            if (redactionLogEntry.getImportedRedactionIntersections() == null) {
+                                redactionLogEntry.setImportedRedactionIntersections(new HashSet<>());
+                            }
+                            redactionLogEntry.getImportedRedactionIntersections().add(importedRedaction.getId());
+                        }
+                    }
+                }
+            }
+        }
+    }
+
+
+    boolean valueInRange(float value, float min, float max) {
+
+        return round(value) >= round(min) && round(value) <= round(max);
+    }
+
+
+    boolean rectOverlap(Rectangle a, Rectangle b) {
+
+        boolean xOverlap = valueInRange(a.getTopLeft().getX(), b.getTopLeft().getX(), b.getTopLeft().getX() + b.getWidth()) || valueInRange(b.getTopLeft().getX(),
+                a.getTopLeft().getX(),
+                a.getTopLeft().getX() + a.getWidth());
+
+        boolean yOverlap = valueInRange(a.getTopLeft().getY(), b.getTopLeft().getY(), b.getTopLeft().getY() + b.getHeight()) || valueInRange(b.getTopLeft().getY(),
+                a.getTopLeft().getY(),
+                a.getTopLeft().getY() + a.getHeight());
+
+        return xOverlap && yOverlap;
+    }
+
+
+    @Deprecated(forRemoval = true)
+    private Rectangle normalize(Rectangle rectangle) {
+
+        Rectangle r = new Rectangle();
+        Point p = new Point();
+
+        if (rectangle.getWidth() < 0) {
+            p.setX(rectangle.getTopLeft().getX() + rectangle.getWidth());
+            r.setWidth(Math.abs(rectangle.getWidth()));
+        } else {
+            p.setX(rectangle.getTopLeft().getX());
+            r.setWidth(rectangle.getWidth());
+        }
+
+        if (rectangle.getHeight() < 0) {
+            p.setY(rectangle.getTopLeft().getY() + rectangle.getHeight());
+            r.setHeight(Math.abs(rectangle.getHeight()));
+        } else {
+            p.setY(rectangle.getTopLeft().getY());
+            r.setHeight(rectangle.getHeight());
+        }
+
+        r.setTopLeft(p);
+        r.setPage(rectangle.getPage());
+        return r;
+    }
+
+
+    private Rectangle normalize(Position position) {
+
+        Rectangle r = new Rectangle();
+        Point p = new Point();
+
+        if (position.getRectangle()[2] < 0) {
+            p.setX(position.getRectangle()[0] + position.getRectangle()[2]);
+            r.setWidth(Math.abs(position.getRectangle()[2]));
+        } else {
+            p.setX(position.getRectangle()[0]);
+            r.setWidth(position.getRectangle()[2]);
+        }
+
+        if (position.getRectangle()[3] < 0) {
+            p.setY(position.getRectangle()[1] + position.getRectangle()[3]);
+            r.setHeight(Math.abs(position.getRectangle()[3]));
+        } else {
+            p.setY(position.getRectangle()[1]);
+            r.setHeight(position.getRectangle()[3]);
+        }
+
+        r.setTopLeft(p);
+        r.setPage(position.getPageNumber());
+        return r;
+    }
+
+
+    private float round(float value) {
+
+        double d = Math.pow(10, 0);
+        return (float) (Math.round(value * d) / d);
+    }
+
+
+    private float[] getColor(String type, String dossierTemplateId) {
+
+        return dictionaryService.getColor(type, dossierTemplateId);
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/ManualChangeFactory.java

@@ -2,10 +2,9 @@ package com.iqser.red.service.redaction.v1.server.service;
 
 import java.time.OffsetDateTime;
 import java.util.List;
-import java.util.stream.Collectors;
 
-import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualChange;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType;
+import org.springframework.stereotype.Service;
+
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.BaseAnnotation;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction;
@@ -13,34 +12,31 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualChange;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType;
 
-import lombok.experimental.UtilityClass;
-
-@UtilityClass
+@Service
 public class ManualChangeFactory {
 
     public List<ManualChange> toManualChangeList(List<BaseAnnotation> manualChanges, boolean isHint) {
 
-        return manualChanges.stream()
-                .map(baseAnnotation -> toManualChange(baseAnnotation, isHint))
-                .collect(Collectors.toList());
+        return manualChanges.stream().map(baseAnnotation -> toManualChange(baseAnnotation, isHint)).toList();
     }
 
 
-    public ManualChange toManualChange(BaseAnnotation baseAnnotation, boolean isHint) {
+    private ManualChange toManualChange(BaseAnnotation baseAnnotation, boolean isHint) {
 
         ManualChange manualChange = ManualChange.from(baseAnnotation);
         if (baseAnnotation instanceof ManualRecategorization imageRecategorization) {
             manualChange.withManualRedactionType(ManualRedactionType.RECATEGORIZE).withChange("type", imageRecategorization.getType());
         } else if (baseAnnotation instanceof IdRemoval manualRemoval) {
-            manualChange.withManualRedactionType(manualRemoval.isRemoveFromDictionary() ? ManualRedactionType.REMOVE_FROM_DICTIONARY : ManualRedactionType.REMOVE);
-        } else if (baseAnnotation instanceof ManualForceRedaction manualForceRedaction) {
-            manualChange.withManualRedactionType(ManualRedactionType.FORCE).withChange("legalBasis", manualForceRedaction.getLegalBasis());
+            manualChange.withManualRedactionType(manualRemoval.isRemoveFromDictionary() ? ManualRedactionType.REMOVE_FROM_DICTIONARY : ManualRedactionType.REMOVE_LOCALLY);
+        } else if (baseAnnotation instanceof ManualForceRedaction) {
+            manualChange.withManualRedactionType(isHint ? ManualRedactionType.FORCE_HINT : ManualRedactionType.FORCE_REDACT);
         } else if (baseAnnotation instanceof ManualResizeRedaction manualResizeRedact) {
-            manualChange.withManualRedactionType(manualResizeRedact.getUpdateDictionary() ? ManualRedactionType.RESIZE_IN_DICTIONARY : ManualRedactionType.RESIZE)
-                    .withChange("value", manualResizeRedact.getValue());
+            manualChange.withManualRedactionType(ManualRedactionType.RESIZE).withChange("value", manualResizeRedact.getValue());
         } else if (baseAnnotation instanceof ManualRedactionEntry manualRedactionEntry) {
-            manualChange.withManualRedactionType(manualRedactionEntry.isAddToDictionary() ? ManualRedactionType.ADD_TO_DICTIONARY : ManualRedactionType.ADD)
+            manualChange.withManualRedactionType(manualRedactionEntry.isAddToDictionary() ? ManualRedactionType.ADD_TO_DICTIONARY : ManualRedactionType.ADD_LOCALLY)
                     .withChange("value", manualRedactionEntry.getValue());
         } else if (baseAnnotation instanceof ManualLegalBasisChange manualLegalBasisChange) {
             manualChange.withManualRedactionType(ManualRedactionType.LEGAL_BASIS_CHANGE)

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/ManualChangesApplicationService.java

@@ -1,70 +1,45 @@
 package com.iqser.red.service.redaction.v1.server.service;
 
 import java.awt.geom.Rectangle2D;
-import java.util.ArrayList;
-import java.util.Collection;
 import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
+import java.util.LinkedList;
 import java.util.NoSuchElementException;
-import java.util.Optional;
-import java.util.Set;
-import java.util.stream.Collectors;
 
-import org.springframework.stereotype.Service;
-
-import com.google.common.collect.Sets;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
-import com.iqser.red.service.redaction.v1.server.model.PrecursorEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.IEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.PositionOnPage;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Image;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.ImageType;
-import com.iqser.red.service.redaction.v1.server.model.document.nodes.Page;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
-import com.iqser.red.service.redaction.v1.server.service.document.EntityFindingUtility;
+import com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService;
 import com.iqser.red.service.redaction.v1.server.utils.RectangleTransformations;
 
-import lombok.AccessLevel;
 import lombok.RequiredArgsConstructor;
-import lombok.experimental.FieldDefaults;
 
-@Service
 @RequiredArgsConstructor
-@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
 public class ManualChangesApplicationService {
 
-    static double MATCH_THRESHOLD = 10;
-
-    EntityFindingUtility entityFindingUtility;
+    private final EntityCreationService entityCreationService;
 
 
-    @Deprecated
-    public void recategorize(IEntity entityToBeReCategorized, ManualRecategorization manualRecategorization) {
+    public void recategorize(IEntity IEntityToBeReCategorized, ManualRecategorization manualRecategorization) {
 
-        entityToBeReCategorized.getMatchedRuleList().clear();
-        entityToBeReCategorized.getManualOverwrite().addChange(manualRecategorization);
-
-        if (entityToBeReCategorized instanceof Image image) {
+        if (IEntityToBeReCategorized instanceof Image image) {
             image.setImageType(ImageType.fromString(manualRecategorization.getType()));
             return;
         }
-
-        if (entityToBeReCategorized instanceof TextEntity textEntity) {
-            textEntity.setType(manualRecategorization.getType());
+        // need to create a new entity and copy over all values, since type is part of the primary key for entities and should never be changed!
+        if (IEntityToBeReCategorized instanceof TextEntity textEntity) {
+            TextEntity recategorizedEntity = entityCreationService.copyEntity(textEntity, manualRecategorization.getType(), textEntity.getEntityType(), textEntity.getDeepestFullyContainingNode());
+            recategorizedEntity.setPositionsOnPagePerPage(textEntity.getPositionsOnPagePerPage());
+            recategorizedEntity.getManualOverwrite().addChange(manualRecategorization);
+            textEntity.removeFromGraph();
         }
     }
 
 
-    public void resize(TextEntity entityToBeResized, ManualResizeRedaction manualResizeRedaction) {
-
-        resizeEntityAndReinsert(entityToBeResized, manualResizeRedaction);
-    }
-
-
-    @Deprecated
     public void resizeEntityAndReinsert(TextEntity entityToBeResized, ManualResizeRedaction manualResizeRedaction) {
 
         PositionOnPage positionOnPageToBeResized = entityToBeResized.getPositionsOnPagePerPage()
@@ -73,60 +48,32 @@ public class ManualChangesApplicationService {
                 .findFirst()
                 .orElseThrow(() -> new NoSuchElementException("No redaction position with matching annotation id found!"));
 
-        positionOnPageToBeResized.setRectanglePerLine(manualResizeRedaction.getPositions()
-                .stream()
-                .map(ManualChangesApplicationService::toRectangle2D)
-                .collect(Collectors.toList()));
+        positionOnPageToBeResized.setRectanglePerLine(manualResizeRedaction.getPositions().stream().map(ManualChangesApplicationService::toRectangle2D).toList());
 
-        entityToBeResized.getManualOverwrite().addChange(manualResizeRedaction);
-
-        SemanticNode node = entityToBeResized.getDeepestFullyContainingNode();
-        PrecursorEntity searchEntity = PrecursorEntity.fromManualResizeRedaction(manualResizeRedaction);
-        // Loop through nodes starting from the deepest fully containing node all the way to the document node
-        while (node != null) {
-
-            Map<String, List<TextEntity>> possibleEntities = entityFindingUtility.findAllPossibleEntitiesAndGroupByValue(node, List.of(searchEntity));
-            Optional<TextEntity> closestEntity = entityFindingUtility.findClosestEntityAndReturnEmptyIfNotFound(searchEntity, possibleEntities, MATCH_THRESHOLD);
-
-            if (closestEntity.isPresent()) {
-                copyValuesFromClosestEntity(entityToBeResized, manualResizeRedaction, closestEntity.get());
-                possibleEntities.values().stream().flatMap(Collection::stream).forEach(TextEntity::removeFromGraph);
-                return;
-            }
-
-            possibleEntities.values().stream().flatMap(Collection::stream).forEach(TextEntity::removeFromGraph);
-
-            if (node.hasParent()) {
-                node = node.getParent();
-            } else {
-                break;
-            }
+        int newStartOffset;
+        if (manualResizeRedaction.getValue().length() > entityToBeResized.getValue().length()) {
+            newStartOffset = entityToBeResized.getTextRange().start() - manualResizeRedaction.getValue().indexOf(entityToBeResized.getValue());
+        } else {
+            newStartOffset = entityToBeResized.getTextRange().start() + entityToBeResized.getValue().indexOf(manualResizeRedaction.getValue());
         }
+
+        // need to reinsert the entity, due to the boundary having changed.
+        removeAndUpdateAndReInsertEntity(entityToBeResized, manualResizeRedaction, newStartOffset);
+        entityToBeResized.getManualOverwrite().addChange(manualResizeRedaction);
     }
 
 
-    private static void copyValuesFromClosestEntity(TextEntity entityToBeResized, ManualResizeRedaction manualResizeRedaction, TextEntity closestEntity) {
+    private void removeAndUpdateAndReInsertEntity(TextEntity entityToBeResized, ManualResizeRedaction manualResizeRedaction, int newStartOffset) {
 
-        Set<SemanticNode> currentIntersectingNodes = new HashSet<>(entityToBeResized.getIntersectingNodes());
-        Set<SemanticNode> newIntersectingNodes = new HashSet<>(closestEntity.getIntersectingNodes());
-
-        Sets.difference(currentIntersectingNodes, newIntersectingNodes).forEach(removedNode -> removedNode.getEntities().remove(entityToBeResized));
-        Sets.difference(newIntersectingNodes, currentIntersectingNodes).forEach(addedNode -> addedNode.getEntities().add(entityToBeResized));
-
-        Set<Page> currentIntersectingPages = new HashSet<>(entityToBeResized.getPages());
-        Set<Page> newIntersectingPages = new HashSet<>(closestEntity.getPages());
-
-        Sets.difference(currentIntersectingPages, newIntersectingPages).forEach(removedPage -> removedPage.getEntities().remove(entityToBeResized));
-        Sets.difference(newIntersectingPages, currentIntersectingPages).forEach(addedPage -> addedPage.getEntities().add(entityToBeResized));
-
-        entityToBeResized.setDeepestFullyContainingNode(closestEntity.getDeepestFullyContainingNode());
-        entityToBeResized.setIntersectingNodes(new ArrayList<>(newIntersectingNodes));
-        entityToBeResized.setTextRange(closestEntity.getTextRange());
-        entityToBeResized.setTextAfter(closestEntity.getTextAfter());
-        entityToBeResized.setTextBefore(closestEntity.getTextBefore());
-        entityToBeResized.setDuplicateTextRanges(new ArrayList<>(closestEntity.getDuplicateTextRanges()));
-        entityToBeResized.setValue(closestEntity.getValue());
-        entityToBeResized.setPages(newIntersectingPages);
+        SemanticNode nodeToInsertInto = entityToBeResized.getDeepestFullyContainingNode().getDocumentTree().getRoot().getNode();
+        entityToBeResized.getIntersectingNodes().forEach(node -> node.getEntities().remove(this));
+        entityToBeResized.getPages().forEach(page -> page.getEntities().remove(this));
+        entityToBeResized.setIntersectingNodes(new LinkedList<>());
+        entityToBeResized.setDeepestFullyContainingNode(null);
+        entityToBeResized.setPages(new HashSet<>());
+        entityToBeResized.getTextRange().setStart(newStartOffset);
+        entityToBeResized.getTextRange().setEnd(newStartOffset + manualResizeRedaction.getValue().length());
+        entityCreationService.addEntityToGraph(entityToBeResized, nodeToInsertInto);
     }
 
 
@@ -141,9 +88,9 @@ public class ManualChangesApplicationService {
     }
 
 
-    public static Rectangle2D toRectangle2D(com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle rect) {
+    private static Rectangle2D toRectangle2D(com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle rect) {
 
-        return new Rectangle2D.Double(rect.getTopLeftX(), rect.getTopLeftY(), rect.getWidth(), rect.getHeight());
+        return new Rectangle2D.Double(rect.getTopLeftX() - rect.getWidth(), rect.getTopLeftY() - rect.getHeight(), rect.getWidth(), rect.getHeight());
     }
 
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/NotFoundImportedEntitiesService.java

@@ -1,102 +0,0 @@
-package com.iqser.red.service.redaction.v1.server.service;
-
-import java.time.OffsetDateTime;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.stream.Collectors;
-
-import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Change;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ChangeType;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position;
-import com.iqser.red.service.redaction.v1.server.model.PrecursorEntity;
-import com.iqser.red.service.redaction.v1.server.model.RectangleWithPage;
-import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
-
-import org.springframework.stereotype.Service;
-
-import io.micrometer.core.annotation.Timed;
-import lombok.RequiredArgsConstructor;
-import lombok.extern.slf4j.Slf4j;
-
-@Service
-@Slf4j
-@RequiredArgsConstructor
-public class NotFoundImportedEntitiesService {
-
-    public static final String IMPORTED_REDACTION_TYPE = "imported_redaction";
-
-
-    @Timed("redactmanager_processEntityLog")
-    public void processEntityLog(EntityLog entityLog, AnalyzeRequest analyzeRequest, List<PrecursorEntity> notFoundEntities) {
-        // recreate imported redactions with manual changes
-        if (notFoundEntities == null || notFoundEntities.isEmpty()) {
-            log.debug("checkForImportedRedactions - no not found imported entities");
-            return;
-        }
-
-        // remove all  intersections and update imported entities
-        calculateIntersectionsForNotFound(notFoundEntities, entityLog, analyzeRequest.getAnalysisNumber());
-    }
-
-
-    private void calculateIntersectionsForNotFound(List<PrecursorEntity> notFoundEntities, EntityLog entityLog, int analysisNumber) {
-
-        if (!notFoundEntities.isEmpty()) {
-            // imported redactions present, intersections must be added with merged imported redactions
-            Map<Integer, List<PrecursorEntity>> importedRedactionsMap = mapImportedRedactionsOnPage(notFoundEntities);
-            entityLog.getEntityLogEntry().stream().filter(entry -> !entry.getEngines().contains(Engine.IMPORTED)).forEach(redactionLogEntry -> {
-                redactionLogEntry.setImportedRedactionIntersections(new HashSet<>());
-                addIntersections(redactionLogEntry, importedRedactionsMap, analysisNumber);
-            });
-        }
-    }
-
-
-    public Map<Integer, List<PrecursorEntity>> mapImportedRedactionsOnPage(List<PrecursorEntity> importedEntities) {
-
-        Map<Integer, List<PrecursorEntity>> importedRedactionsMap = new HashMap<>();
-        Set<Integer> pageNumbers = importedEntities.stream()
-                .map(PrecursorEntity::getEntityPosition)
-                .flatMap(Collection::stream)
-                .map(RectangleWithPage::pageNumber)
-                .collect(Collectors.toSet());
-        pageNumbers.forEach(pageNumber -> importedRedactionsMap.put(pageNumber,
-                importedEntities.stream().filter(i -> pageNumber == i.getEntityPosition().get(0).pageNumber()).collect(Collectors.toList())));
-        return importedRedactionsMap;
-    }
-
-
-    public void addIntersections(EntityLogEntry entityLogEntry, Map<Integer, List<PrecursorEntity>> importedEntitiesMap, int analysisNumber) {
-
-        for (Position rectangle : entityLogEntry.getPositions()) {
-            if (importedEntitiesMap.containsKey(rectangle.getPageNumber())) {
-                var importedEntitiesOnPage = importedEntitiesMap.get(rectangle.getPageNumber());
-                for (PrecursorEntity precursorEntity : importedEntitiesOnPage) {
-                    for (RectangleWithPage rectangleWithPage : precursorEntity.getEntityPosition()) {
-                        if (rectangle.toRectangle2D().intersects(rectangleWithPage.rectangle2D())) {
-                            if (entityLogEntry.getImportedRedactionIntersections() == null) {
-                                entityLogEntry.setImportedRedactionIntersections(new HashSet<>());
-                            }
-                            entityLogEntry.getImportedRedactionIntersections().add(precursorEntity.getId());
-
-                            if (entityLogEntry.getState() != EntryState.REMOVED) {
-                                entityLogEntry.setState(EntryState.REMOVED);
-                                entityLogEntry.getChanges().add(new Change(analysisNumber, ChangeType.REMOVED, OffsetDateTime.now()));
-                            }
-                        }
-                    }
-                }
-            }
-        }
-    }
-
-}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RedactionChangeLogService.java

@@ -0,0 +1,117 @@
+package com.iqser.red.service.redaction.v1.server.service;
+
+import java.time.OffsetDateTime;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import org.springframework.stereotype.Service;
+
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Change;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ChangeType;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogChanges;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
+import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
+
+import io.micrometer.core.annotation.Timed;
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+@Service
+@RequiredArgsConstructor
+public class RedactionChangeLogService {
+
+    private final RedactionStorageService redactionStorageService;
+
+
+    @Timed("redactmanager_computeChanges")
+    public RedactionLogChanges computeChanges(String dossierId, String fileId, RedactionLog currentRedactionLog, int analysisNumber) {
+
+        long start = System.currentTimeMillis();
+
+        RedactionLog previousRedactionLog = redactionStorageService.getRedactionLog(dossierId, fileId);
+
+        if (previousRedactionLog == null) {
+            currentRedactionLog.getRedactionLogEntry().forEach(entry -> {
+                entry.getChanges().add(new Change(analysisNumber, ChangeType.ADDED, OffsetDateTime.now()));
+            });
+            return new RedactionLogChanges(currentRedactionLog, false);
+        }
+
+        List<RedactionLogEntry> previouslyExistingEntries = previousRedactionLog.getRedactionLogEntry().stream().filter(entry -> !entry.lastChangeIsRemoved()).toList();
+
+        Map<String, RedactionLogEntry> addedEntryIds = getEntriesThatExistInCurrentButNotInPreviousRedactionLog(currentRedactionLog, previouslyExistingEntries);
+        Set<String> removedIds = getEntryIdsThatExistInPreviousButNotInCurrentRedactionLog(currentRedactionLog, previouslyExistingEntries);
+
+        List<RedactionLogEntry> newRedactionLogEntries = previousRedactionLog.getRedactionLogEntry();
+
+        List<RedactionLogEntry> toRemove = new ArrayList<>();
+        newRedactionLogEntries.forEach(entry -> {
+            if (removedIds.contains(entry.getId()) && addedEntryIds.containsKey(entry.getId())) {
+                List<Change> changes = entry.getChanges();
+                changes.add(new Change(analysisNumber, ChangeType.CHANGED, OffsetDateTime.now()));
+                var newEntry = addedEntryIds.get(entry.getId());
+                newEntry.setChanges(changes);
+                addedEntryIds.put(entry.getId(), newEntry);
+                toRemove.add(entry);
+            } else if (removedIds.contains(entry.getId())) {
+                entry.getChanges().add(new Change(analysisNumber, ChangeType.REMOVED, OffsetDateTime.now()));
+            } else if (addedEntryIds.containsKey(entry.getId())) {
+                List<Change> changes = entry.getChanges();
+                changes.add(new Change(analysisNumber, ChangeType.ADDED, OffsetDateTime.now()));
+                var newEntry = addedEntryIds.get(entry.getId());
+                newEntry.setChanges(changes);
+                addedEntryIds.put(entry.getId(), newEntry);
+                toRemove.add(entry);
+            }
+        });
+
+        newRedactionLogEntries.removeAll(toRemove);
+
+        addedEntryIds.forEach((k, v) -> {
+            if (v.getChanges().isEmpty()) {
+                v.getChanges().add(new Change(analysisNumber, ChangeType.ADDED, OffsetDateTime.now()));
+            }
+            newRedactionLogEntries.add(v);
+        });
+
+        currentRedactionLog.setRedactionLogEntry(newRedactionLogEntries);
+
+        log.debug("Change computation took: {}", System.currentTimeMillis() - start);
+        return new RedactionLogChanges(currentRedactionLog, !addedEntryIds.isEmpty() || !removedIds.isEmpty());
+    }
+
+
+    private static Set<String> getEntryIdsThatExistInPreviousButNotInCurrentRedactionLog(RedactionLog currentRedactionLog, List<RedactionLogEntry> previouslyExistingEntries) {
+
+        Set<RedactionLogEntry> removed = new HashSet<>(previouslyExistingEntries);
+        currentRedactionLog.getRedactionLogEntry().forEach(removed::remove);
+
+        Set<String> removedIds = removed.stream().map(RedactionLogEntry::getId).collect(Collectors.toSet());
+        return removedIds;
+    }
+
+
+    private static Map<String, RedactionLogEntry> getEntriesThatExistInCurrentButNotInPreviousRedactionLog(RedactionLog currentRedactionLog,
+                                                                                                           List<RedactionLogEntry> previouslyExistingEntries) {
+
+        Set<RedactionLogEntry> currentExistingEntries = currentRedactionLog.getRedactionLogEntry()
+                .stream()
+                .filter(entry -> entry.getChanges().isEmpty() || !entry.lastChangeIsRemoved())
+                .collect(Collectors.toSet());
+
+        previouslyExistingEntries.forEach(currentExistingEntries::remove);
+        Map<String, RedactionLogEntry> addedIds = new HashMap<>();
+        currentExistingEntries.forEach(entry -> {
+            addedIds.put(entry.getId(), entry);
+        });
+        return addedIds;
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/RedactionLogCreatorService.java

@@ -0,0 +1,279 @@
+package com.iqser.red.service.redaction.v1.server.service;
+
+import java.awt.geom.Rectangle2D;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import org.springframework.beans.BeanUtils;
+import org.springframework.stereotype.Service;
+
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ManualChange;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Comment;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Point;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogComment;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
+import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
+import com.iqser.red.service.redaction.v1.server.model.document.entity.ManualChangeOverwrite;
+import com.iqser.red.service.redaction.v1.server.model.document.entity.PositionOnPage;
+import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.Image;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.ImageType;
+import com.iqser.red.service.redaction.v1.server.model.ManualEntity;
+
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+
+@Service
+@Slf4j
+@RequiredArgsConstructor
+public class RedactionLogCreatorService {
+
+    private final DictionaryService dictionaryService;
+    private final ManualChangeFactory manualChangeFactory;
+
+
+    public List<RedactionLogEntry> createRedactionLog(Document document,
+                                                      String dossierTemplateId,
+                                                      List<ManualEntity> notFoundManualRedactionEntries,
+                                                      Map<String, List<Comment>> comments) {
+
+        List<RedactionLogEntry> entries = new ArrayList<>();
+        document.getEntities()
+                .stream()
+                .filter(RedactionLogCreatorService::isEntityOrRecommendationType)
+                .filter(entity -> !entity.removed())
+                .forEach(entityNode -> entries.addAll(toRedactionLogEntries(entityNode, dossierTemplateId, comments)));
+        document.streamAllImages().filter(image -> !image.removed()).forEach(imageNode -> entries.add(createRedactionLogEntry(imageNode, dossierTemplateId, comments)));
+        notFoundManualRedactionEntries.forEach(entityIdentifier -> entries.add(createRedactionLogEntry(entityIdentifier, dossierTemplateId, comments)));
+        return entries;
+    }
+
+
+    private static boolean isEntityOrRecommendationType(TextEntity textEntity) {
+
+        return textEntity.getEntityType() == EntityType.ENTITY || textEntity.getEntityType() == EntityType.RECOMMENDATION;
+    }
+
+
+    private List<RedactionLogEntry> toRedactionLogEntries(TextEntity textEntity, String dossierTemplateId, Map<String, List<Comment>> comments) {
+
+        Set<String> processedIds = new HashSet<>();
+        List<RedactionLogEntry> redactionLogEntities = new ArrayList<>();
+
+        for (PositionOnPage positionOnPage : textEntity.getPositionsOnPagePerPage()) {
+
+            // Duplicates should be removed. They might exist due to table extraction duplicating cells spanning multiple columns/rows.
+            if (processedIds.contains(positionOnPage.getId())) {
+                continue;
+            }
+            processedIds.add(positionOnPage.getId());
+
+            RedactionLogEntry redactionLogEntry = createRedactionLogEntry(textEntity, dossierTemplateId);
+            redactionLogEntry.setId(positionOnPage.getId());
+            redactionLogEntry.setComments(buildRedactionLogComments(comments, positionOnPage.getId()));
+
+            List<Rectangle> rectanglesPerLine = positionOnPage.getRectanglePerLine()
+                    .stream()
+                    .map(rectangle2D -> toRedactionLogRectangle(rectangle2D, positionOnPage.getPage().getNumber()))
+                    .toList();
+
+            redactionLogEntry.setPositions(rectanglesPerLine);
+            redactionLogEntities.add(redactionLogEntry);
+        }
+
+        return redactionLogEntities;
+    }
+
+
+    private List<RedactionLogComment> buildRedactionLogComments(Map<String, List<Comment>> commentsPerId, String id) {
+
+        if (!commentsPerId.containsKey(id)) {
+            return Collections.emptyList();
+        }
+        List<Comment> comments = commentsPerId.get(id);
+        if (comments == null || comments.isEmpty()) {
+            return Collections.emptyList();
+        }
+        return toRedactionLogComments(comments);
+    }
+
+
+    private List<RedactionLogComment> toRedactionLogComments(List<Comment> comments) {
+
+        return comments.stream().map(this::toRedactionLogComment).toList();
+    }
+
+
+    private RedactionLogComment toRedactionLogComment(Comment comment) {
+
+        return new RedactionLogComment(comment.getId(),
+                comment.getUser(),
+                comment.getText(),
+                comment.getAnnotationId(),
+                comment.getFileId(),
+                comment.getDate(),
+                comment.getSoftDeletedTime());
+    }
+
+
+    private RedactionLogEntry createRedactionLogEntry(TextEntity entity, String dossierTemplateId) {
+
+        Set<String> referenceIds = new HashSet<>();
+        entity.references().stream().filter(TextEntity::active).forEach(ref -> ref.getPositionsOnPagePerPage().forEach(pos -> referenceIds.add(pos.getId())));
+        int sectionNumber = entity.getDeepestFullyContainingNode().getTreeId().isEmpty() ? 0 : entity.getDeepestFullyContainingNode().getTreeId().get(0);
+        boolean isHint = isHint(entity.getType(), dossierTemplateId);
+        return RedactionLogEntry.builder()
+                .color(getColor(entity.getType(), dossierTemplateId, entity.applied()))
+                .reason(entity.buildReasonWithManualChangeDescriptions())
+                .legalBasis(entity.legalBasis())
+                .value(entity.getManualOverwrite().getValue().orElse(entity.getMatchedRule().isWriteValueWithLineBreaks() ? entity.getValueWithLineBreaks() : entity.getValue()))
+                .type(entity.getType())
+                .redacted(entity.applied())
+                .isHint(isHint)
+                .isRecommendation(entity.getEntityType().equals(EntityType.RECOMMENDATION))
+                .isFalsePositive(entity.getEntityType().equals(EntityType.FALSE_POSITIVE) || entity.getEntityType().equals(EntityType.FALSE_RECOMMENDATION))
+                .section(entity.getManualOverwrite().getSection().orElse(entity.getDeepestFullyContainingNode().toString()))
+                .sectionNumber(sectionNumber)
+                .matchedRule(entity.getMatchedRule().getRuleIdentifier().toString())
+                .isDictionaryEntry(entity.isDictionaryEntry())
+                .textAfter(entity.getTextAfter())
+                .textBefore(entity.getTextBefore())
+                .startOffset(entity.getTextRange().start())
+                .endOffset(entity.getTextRange().end())
+                .isDossierDictionaryEntry(entity.isDossierDictionaryEntry())
+                .engines(getEngines(entity))
+                .reference(referenceIds)
+                .manualChanges(mapManualChanges(entity.getManualOverwrite(), isHint))
+                .build();
+    }
+
+
+    private Set<Engine> getEngines(TextEntity entity) {
+
+        return entity.getEngines() != null ? mapToEngines(entity.getEngines()) : Collections.emptySet();
+    }
+
+
+    private Set<Engine> mapToEngines(Set<com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine> engines) {
+
+        return engines.stream().map(engine -> switch (engine) {
+            case NER -> Engine.NER;
+            case RULE -> Engine.RULE;
+            case DICTIONARY -> Engine.DICTIONARY;
+            default -> null;
+        }).filter(Objects::nonNull).collect(Collectors.toSet());
+    }
+
+
+    public RedactionLogEntry createRedactionLogEntry(ManualEntity manualEntity, String dossierTemplateId, Map<String, List<Comment>> comments) {
+
+        String type = manualEntity.getManualOverwrite().getType().orElse(manualEntity.getType());
+        boolean isHint = isHint(type, dossierTemplateId);
+        return RedactionLogEntry.builder()
+                .id(manualEntity.getId())
+                .color(getColor(type, dossierTemplateId, manualEntity.applied()))
+                .reason(manualEntity.buildReasonWithManualChangeDescriptions())
+                .legalBasis(manualEntity.legalBasis())
+                .value(manualEntity.getManualOverwrite().getValue().orElse(manualEntity.getValue()))
+                .type(type)
+                .redacted(manualEntity.applied())
+                .isHint(isHint)
+                .isRecommendation(manualEntity.getEntityType().equals(EntityType.RECOMMENDATION))
+                .isFalsePositive(manualEntity.getEntityType().equals(EntityType.FALSE_POSITIVE) || manualEntity.getEntityType().equals(EntityType.FALSE_RECOMMENDATION))
+                .section(manualEntity.getManualOverwrite().getSection().orElse(manualEntity.getSection()))
+                .sectionNumber(0)
+                .matchedRule("ManualRedaction")
+                .rectangle(manualEntity.isRectangle())
+                .isDictionaryEntry(manualEntity.isDictionaryEntry())
+                .isDossierDictionaryEntry(manualEntity.isDossierDictionaryEntry())
+                .textAfter("")
+                .textBefore("")
+                .startOffset(-1)
+                .endOffset(-1)
+                .positions(manualEntity.getEntityPosition()
+                        .stream()
+                        .map(entityPosition -> toRedactionLogRectangle(entityPosition.rectangle2D(), entityPosition.pageNumber()))
+                        .collect(Collectors.toList()))
+                .engines(Collections.emptySet())
+                .reference(Collections.emptySet())
+                .manualChanges(mapManualChanges(manualEntity.getManualOverwrite(), isHint))
+                .comments(buildRedactionLogComments(comments, manualEntity.getId()))
+                .build();
+    }
+
+
+    private List<ManualChange> mapManualChanges(ManualChangeOverwrite manualEntity, boolean isHint) {
+
+        return manualChangeFactory.toManualChangeList(manualEntity.getManualChangeLog(), isHint).stream().map(this::mapManualChange).toList();
+    }
+
+
+    private ManualChange mapManualChange(com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualChange manualChange) {
+
+        ManualChange manualChange1 = new ManualChange();
+        BeanUtils.copyProperties(manualChange, manualChange1);
+        return manualChange1;
+    }
+
+
+    public RedactionLogEntry createRedactionLogEntry(Image image, String dossierTemplateId, Map<String, List<Comment>> comments) {
+
+        String imageType = image.getImageType().equals(ImageType.OTHER) ? "image" : image.getImageType().toString().toLowerCase(Locale.ENGLISH);
+        boolean isHint = dictionaryService.isHint(imageType, dossierTemplateId);
+        return RedactionLogEntry.builder()
+                .id(image.getId())
+                .color(getColor(imageType, dossierTemplateId, image.applied()))
+                .isImage(true)
+                .type(imageType)
+                .redacted(image.applied())
+                .reason(image.buildReasonWithManualChangeDescriptions())
+                .legalBasis(image.legalBasis())
+                .matchedRule(image.getMatchedRule().getRuleIdentifier().toString())
+                .isHint(isHint)
+                .isDictionaryEntry(false)
+                .isRecommendation(false)
+                .positions(List.of(toRedactionLogRectangle(image.getPosition(), image.getPage().getNumber())))
+                .sectionNumber(image.getTreeId().get(0))
+                .section(image.getManualOverwrite().getSection().orElse(image.getParent().toString()))
+                .imageHasTransparency(image.isTransparent())
+                .manualChanges(mapManualChanges(image.getManualOverwrite(), isHint))
+                .comments(buildRedactionLogComments(comments, image.getId()))
+                .build();
+
+    }
+
+
+    private float[] getColor(String type, String dossierTemplateId, boolean isRedaction) {
+
+        if (!isRedaction && !isHint(type, dossierTemplateId)) {
+            return dictionaryService.getNotRedactedColor(dossierTemplateId);
+        }
+        return dictionaryService.getColor(type, dossierTemplateId);
+    }
+
+
+    private boolean isHint(String type, String dossierTemplateId) {
+
+        return dictionaryService.isHint(type, dossierTemplateId);
+    }
+
+
+    private Rectangle toRedactionLogRectangle(Rectangle2D rectangle2D, int pageNumber) {
+
+        return new Rectangle(new Point((float) rectangle2D.getMinX(), (float) (rectangle2D.getMinY() + rectangle2D.getHeight())),
+                (float) rectangle2D.getWidth(),
+                -(float) rectangle2D.getHeight(),
+                pageNumber);
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/UnprocessedChangesService.java

@@ -1,198 +0,0 @@
-package com.iqser.red.service.redaction.v1.server.service;
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Optional;
-import java.util.Set;
-import java.util.stream.Collectors;
-
-import org.springframework.amqp.rabbit.core.RabbitTemplate;
-import org.springframework.stereotype.Service;
-
-import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.BaseAnnotation;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
-import com.iqser.red.service.redaction.v1.model.AnalyzeResponse;
-import com.iqser.red.service.redaction.v1.model.QueueNames;
-import com.iqser.red.service.redaction.v1.model.UnprocessedManualEntity;
-import com.iqser.red.service.redaction.v1.server.model.PrecursorEntity;
-import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
-import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
-import com.iqser.red.service.redaction.v1.server.service.document.DocumentGraphMapper;
-import com.iqser.red.service.redaction.v1.server.service.document.EntityFindingUtility;
-import com.iqser.red.service.redaction.v1.server.service.document.EntityFromPrecursorCreationService;
-import com.iqser.red.service.redaction.v1.server.storage.ObservedStorageService;
-import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
-
-import io.micrometer.observation.annotation.Observed;
-import lombok.AccessLevel;
-import lombok.RequiredArgsConstructor;
-import lombok.experimental.FieldDefaults;
-import lombok.extern.slf4j.Slf4j;
-
-@Slf4j
-@Service
-@RequiredArgsConstructor
-@FieldDefaults(level = AccessLevel.PRIVATE)
-public class UnprocessedChangesService {
-
-    private static final double THRESHOLD = 10;
-
-    final RabbitTemplate rabbitTemplate;
-    final ObservedStorageService observedStorageService;
-    final EntityFindingUtility entityFindingUtility;
-    final RedactionStorageService redactionStorageService;
-    final EntityFromPrecursorCreationService entityFromPrecursorCreationService;
-    final DictionaryService dictionaryService;
-    final ManualChangesApplicationService manualChangesApplicationService;
-
-
-    @Observed(name = "UnprocessedChangesService", contextualName = "analyse-surrounding-text")
-    public void analyseSurroundingText(AnalyzeRequest analyzeRequest) {
-
-        List<UnprocessedManualEntity> unprocessedManualEntities = new ArrayList<>();
-
-        dictionaryService.updateDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId());
-
-        EntityLog previousEntityLog = redactionStorageService.getEntityLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
-        Document document = DocumentGraphMapper.toDocumentGraph(observedStorageService.getDocumentData(analyzeRequest.getDossierId(), analyzeRequest.getFileId()));
-
-        Set<String> allAnnotationIds = analyzeRequest.getManualRedactions().getEntriesToAdd()
-                .stream()
-                .map(ManualRedactionEntry::getAnnotationId)
-                .collect(Collectors.toSet());
-        Set<String> resizeIds = analyzeRequest.getManualRedactions().getResizeRedactions()
-                .stream()
-                .map(ManualResizeRedaction::getAnnotationId)
-                .collect(Collectors.toSet());
-        allAnnotationIds.addAll(resizeIds);
-
-        List<ManualResizeRedaction> manualResizeRedactions = analyzeRequest.getManualRedactions().getResizeRedactions()
-                .stream()
-                .toList();
-        List<PrecursorEntity> manualEntitiesToBeResized = previousEntityLog.getEntityLogEntry()
-                .stream()
-                .filter(entityLogEntry -> resizeIds.contains(entityLogEntry.getId()))
-                .toList()
-                .stream()
-                .map(PrecursorEntity::fromEntityLogEntry)
-                .toList();
-
-        if (!manualResizeRedactions.isEmpty()) {
-            processManualResizeRedactions(document, manualEntitiesToBeResized, unprocessedManualEntities, manualResizeRedactions);
-        }
-
-        List<PrecursorEntity> notFoundManualEntities = new ArrayList<>();
-        List<PrecursorEntity> manualEntities = manualEntitiesConverter(analyzeRequest.getManualRedactions(), analyzeRequest.getDossierTemplateId());
-        if (!manualEntities.isEmpty()) {
-            notFoundManualEntities = entityFromPrecursorCreationService.toTextEntity(manualEntities, document);
-        }
-
-        document.getEntities()
-                .forEach(textEntity -> {
-                    Set<String> processedIds = new HashSet<>();
-                    for (var positionsOnPerPage : textEntity.getPositionsOnPagePerPage()) {
-                        if (processedIds.contains(positionsOnPerPage.getId())) {
-                            continue;
-                        }
-                        processedIds.add(positionsOnPerPage.getId());
-                        List<Position> positions = positionsOnPerPage.getRectanglePerLine()
-                                .stream()
-                                .map(rectangle2D -> new Position(rectangle2D, positionsOnPerPage.getPage().getNumber()))
-                                .collect(Collectors.toList());
-                        unprocessedManualEntities.add(UnprocessedManualEntity.builder()
-                                                              .annotationId(allAnnotationIds.stream()
-                                                                                    .filter(textEntity::matchesAnnotationId)
-                                                                                    .findFirst()
-                                                                                    .orElse(""))
-                                                              .textBefore(textEntity.getTextBefore())
-                                                              .textAfter(textEntity.getTextAfter())
-                                                              .section(textEntity.getManualOverwrite().getSection()
-                                                                               .orElse(textEntity.getDeepestFullyContainingNode().toString()))
-                                                              .positions(positions)
-                                                              .build());
-                    }
-                });
-
-        notFoundManualEntities.forEach(manualEntity -> unprocessedManualEntities.add(builDefaultUnprocessedManualEntity(manualEntity)));
-
-        rabbitTemplate.convertAndSend(QueueNames.REDACTION_ANALYSIS_RESPONSE_QUEUE,
-                                      AnalyzeResponse.builder().fileId(analyzeRequest.getFileId()).unprocessedManualEntities(unprocessedManualEntities).build());
-    }
-
-
-    private void processManualResizeRedactions(Document document,
-                                               List<PrecursorEntity> manualEntities,
-                                               List<UnprocessedManualEntity> unprocessedManualEntities,
-                                               List<ManualResizeRedaction> manualResizeRedactions) {
-
-        Map<String, List<TextEntity>> tempEntities = entityFindingUtility.findAllPossibleEntitiesAndGroupByValue(document, manualEntities);
-
-        for (PrecursorEntity precursorEntity : manualEntities) {
-
-            Optional<TextEntity> optionalTextEntity = entityFindingUtility.findClosestEntityAndReturnEmptyIfNotFound(precursorEntity, tempEntities, THRESHOLD);
-
-            if (optionalTextEntity.isEmpty()) {
-                unprocessedManualEntities.add(builDefaultUnprocessedManualEntity(precursorEntity));
-                continue;
-            }
-
-            TextEntity correctEntity = EntityFromPrecursorCreationService.createCorrectEntity(precursorEntity, optionalTextEntity.get());
-            Optional<ManualResizeRedaction> optionalManualResizeRedaction = manualResizeRedactions.stream()
-                    .filter(manualResizeRedaction -> manualResizeRedaction.getAnnotationId().equals(precursorEntity.getId()))
-                    .findFirst();
-            if (optionalManualResizeRedaction.isPresent()) {
-                ManualResizeRedaction manualResizeRedaction = optionalManualResizeRedaction.get();
-                manualChangesApplicationService.resize(correctEntity, manualResizeRedaction);
-
-                // If the entity's value is not the same as the manual resize request's value it means we didn't find it anywhere and we want to remove it
-                // from the graph, so it does not get processed and sent back to persistence-service to update its value.
-                if (!correctEntity.getValue().equals(manualResizeRedaction.getValue())) {
-                    correctEntity.removeFromGraph();
-                }
-            }
-        }
-
-        // remove all temp entities from the graph
-        tempEntities.values()
-                .stream()
-                .flatMap(Collection::stream)
-                .forEach(TextEntity::removeFromGraph);
-    }
-
-
-    private UnprocessedManualEntity builDefaultUnprocessedManualEntity(PrecursorEntity precursorEntity) {
-
-        return UnprocessedManualEntity.builder()
-                .annotationId(precursorEntity.getId())
-                .textAfter("")
-                .textBefore("")
-                .section("")
-                .positions(precursorEntity.getManualOverwrite().getPositions()
-                                   .orElse(precursorEntity.getEntityPosition())
-                                   .stream()
-                                   .map(entityPosition -> new Position(entityPosition.rectangle2D(), entityPosition.pageNumber()))
-                                   .toList())
-                .build();
-    }
-
-
-    private List<PrecursorEntity> manualEntitiesConverter(ManualRedactions manualRedactions, String dossierTemplateId) {
-
-        return manualRedactions.getEntriesToAdd()
-                .stream()
-                .filter(manualRedactionEntry -> manualRedactionEntry.getPositions() != null && !manualRedactionEntry.getPositions().isEmpty())
-                .filter(BaseAnnotation::isLocal)
-                .map(manualRedactionEntry -> PrecursorEntity.fromManualRedactionEntry(manualRedactionEntry,
-                                                                                      dictionaryService.isHint(manualRedactionEntry.getType(), dossierTemplateId)))
-                .toList();
-    }
-
-}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/ComponentComparator.java

@@ -1,33 +0,0 @@
-package com.iqser.red.service.redaction.v1.server.service.document;
-
-import java.util.Comparator;
-
-import com.iqser.red.service.redaction.v1.server.model.component.Component;
-
-public class ComponentComparator implements Comparator<Component> {
-
-    public static ComponentComparator first() {
-
-        return new ComponentComparator();
-    }
-
-
-    @Override
-    public int compare(Component component1, Component component2) {
-
-        var firstEntity1 = component1.getReferences().stream().min(EntityComparators.first());
-        var firstEntity2 = component2.getReferences().stream().min(EntityComparators.first());
-        if (firstEntity1.isEmpty() && firstEntity2.isEmpty()) {
-            return 0;
-        }
-        if (firstEntity1.isEmpty() && firstEntity2.isPresent()) {
-            return -1;
-        }
-        if (firstEntity1.isPresent() && firstEntity2.isEmpty()) {
-            return 1;
-        }
-
-        return new EntityComparators.FirstEntity().compare(firstEntity1.get(), firstEntity2.get());
-    }
-
-}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/ComponentCreationService.java

@@ -1,28 +1,17 @@
 package com.iqser.red.service.redaction.v1.server.service.document;
 
-import java.text.BreakIterator;
 import java.util.Collection;
 import java.util.Collections;
-import java.util.Comparator;
-import java.util.Date;
 import java.util.HashSet;
 import java.util.LinkedList;
 import java.util.List;
-import java.util.Locale;
-import java.util.Map;
-import java.util.Optional;
 import java.util.Set;
 import java.util.stream.Collectors;
-import java.util.stream.Stream;
 
 import org.kie.api.runtime.KieSession;
 
 import com.iqser.red.service.redaction.v1.server.model.component.Component;
 import com.iqser.red.service.redaction.v1.server.model.component.Entity;
-import com.iqser.red.service.redaction.v1.server.model.document.nodes.Paragraph;
-import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
-import com.iqser.red.service.redaction.v1.server.model.document.nodes.Table;
-import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell;
 import com.iqser.red.service.redaction.v1.server.model.drools.RuleIdentifier;
 import com.iqser.red.service.redaction.v1.server.utils.DateConverter;
 
@@ -38,493 +27,103 @@ public class ComponentCreationService {
     Set<Entity> referencedEntities = new HashSet<>();
 
 
-    private static List<Entity> findEntitiesFromLongestSection(Collection<Entity> entities) {
+    public void firstOrElse(String ruleIdentifier, String category, Collection<Entity> entities, String fallback) {
 
-        var entitiesBySection = entities.stream().collect(Collectors.groupingBy(entity -> entity.getContainingNode().getHighestParent()));
-        Optional<SemanticNode> longestSection = entitiesBySection.entrySet()
-                .stream()
-                .sorted(Comparator.comparingInt(ComponentCreationService::getTotalLengthOfEntities).reversed())
-                .map(Map.Entry::getKey)
-                .findFirst();
-
-        if (longestSection.isEmpty()) {
-            return Collections.emptyList();
-        }
-
-        return entitiesBySection.get(longestSection.get());
+        String transformation = String.format("First found value or else '%s'", fallback);
+        String value = entities.stream().min(EntityComparators.start()).map(Entity::getValue).orElse(fallback);
+        create(ruleIdentifier, category, value, transformation, entities);
     }
 
 
-    /**
-     * Joins entity values, and creates a component from the result.
-     *
-     * @param ruleIdentifier The identifier of the rule.
-     * @param name           The name of the entity.
-     * @param entities       The collection of entities to process.
-     */
-    public void joining(String ruleIdentifier, String name, Collection<Entity> entities) {
+    public void joining(String ruleIdentifier, String category, Collection<Entity> entities) {
 
-        joining(ruleIdentifier, name, entities, ", ");
+        joining(ruleIdentifier, category, entities, ", ");
     }
 
 
-    /**
-     * Finds the first value from the collection of entities and creates a component from it. If no value is found, the fallback value is used instead.
-     *
-     * @param ruleIdentifier the identifier for the rule
-     * @param name           the name of the operation
-     * @param entities       the collection of entities to search for the first value
-     * @param fallback       the value to be returned if no value is found in the collection
-     */
-    public void firstOrElse(String ruleIdentifier, String name, Collection<Entity> entities, String fallback) {
+    public void joining(String ruleIdentifier, String category, Collection<Entity> entities, String delimiter) {
 
-        String valueDescription = String.format("First found value of type %s or else '%s'", joinTypes(entities), fallback);
-        String value = entities.stream().min(EntityComparators.first()).map(Entity::getValue).orElse(fallback);
-        create(ruleIdentifier, name, value, valueDescription, entities);
+        String transformation = String.format("Joining all values with '%s'", delimiter);
+        String value = entities.stream().sorted(EntityComparators.start()).map(Entity::getValue).collect(Collectors.joining(delimiter));
+        create(ruleIdentifier, category, value, transformation, entities);
     }
 
 
-    private static String joinTypes(Collection<Entity> entities) {
+    public void joiningUnique(String ruleIdentifier, String category, Collection<Entity> entities) {
 
-        return entities.stream().map(Entity::getType).distinct().collect(Collectors.joining(", "));
+        joining(ruleIdentifier, category, entities, ", ");
     }
 
 
-    /**
-     * Creates a new component with the given parameters and inserts it into the kieSession.
-     *
-     * @param ruleIdentifier   The rule identifier for the component.
-     * @param name             The name of the component.
-     * @param value            The value of the component.
-     * @param valueDescription The description of the value.
-     * @param references       A collection of Entity objects that the component references.
-     */
-    public void create(String ruleIdentifier, String name, String value, String valueDescription, Collection<Entity> references) {
+    public void joiningUnique(String ruleIdentifier, String category, Collection<Entity> entities, String delimiter) {
+
+        String transformation = String.format("Joining all values with '%s'", delimiter);
+        String value = entities.stream().sorted(EntityComparators.start()).map(Entity::getValue).distinct().collect(Collectors.joining(delimiter));
+        create(ruleIdentifier, category, value, transformation, entities);
+    }
+
+
+    public void convertDates(String ruleIdentifier, String category, Collection<Entity> entities) {
+
+        convertDates(ruleIdentifier, category, entities, "dd/MM/yyyy");
+    }
+
+
+    public void convertDates(String ruleIdentifier, String category, Collection<Entity> entities, String resultFormat) {
+
+        String transformation = "Convert values of type to dd/MM/yyyy joined with ', '";
+        String date = entities.stream().map(Entity::getValue).map(value -> DateConverter.convertDate(value, resultFormat)).collect(Collectors.joining(", "));
+        create(ruleIdentifier, category, date, transformation, entities);
+    }
+
+
+    public void createComponentsForUnMappedEntities(String ruleIdentifier, Collection<Entity> entities) {
+
+        entities.forEach(entity -> create(ruleIdentifier, entity.getType(), entity.getValue(), "Unmapped Entity", List.of(entity)));
+    }
+
+
+    public void create(String ruleIdentifier, String category, String value, String transformation, Collection<Entity> references) {
 
         referencedEntities.addAll(references);
 
         kieSession.insert(Component.builder()
                 .matchedRule(RuleIdentifier.fromString(ruleIdentifier))
-                .name(name)
+                .category(category)
                 .value(value)
-                .valueDescription(valueDescription)
-                .references(new LinkedList<>(references))
-                .build());
+                .transformation(transformation)
+                .references(new LinkedList<>(references)));
     }
 
 
-    /**
-     * Joins entity values from the first section entities appear in, and creates a component from the result.
-     *
-     * @param ruleIdentifier The identifier of the rule.
-     * @param name           The name of the entity.
-     * @param entities       The collection of entities to process.
-     */
-    public void joiningFromFirstSectionOnly(String ruleIdentifier, String name, Collection<Entity> entities) {
-
-        joiningFromFirstSectionOnly(ruleIdentifier, name, entities, ", ");
-    }
-
-
-    /**
-     * Joins unique entity values from the first section entities appear in, and creates a component from the result.
-     *
-     * @param ruleIdentifier The identifier of the rule.
-     * @param name           The name of the entity.
-     * @param entities       The collection of entities to process.
-     */
-    public void joiningFromFirstSectionOnly(String ruleIdentifier, String name, Collection<Entity> entities, String delimiter) {
-
-        List<Entity> entitiesFromFirstSection = findEntitiesFromFirstSection(entities);
-        joining(ruleIdentifier, name, entitiesFromFirstSection, delimiter);
-    }
-
-
-    private static List<Entity> findEntitiesFromFirstSection(Collection<Entity> entities) {
-
-        var entitiesBySection = entities.stream().collect(Collectors.groupingBy(entity -> entity.getContainingNode().getHighestParent()));
-        Optional<SemanticNode> firstSection = entitiesBySection.keySet().stream().min(SemanticNodeComparators.first());
-        if (firstSection.isEmpty()) {
-            return Collections.emptyList();
-        }
-        return entitiesBySection.get(firstSection.get());
-    }
-
-
-    /**
-     * Joins unique entity values from the first section entities appear in, and creates a component from the result.
-     *
-     * @param ruleIdentifier The identifier of the rule.
-     * @param name           The name of the entity.
-     * @param entities       The collection of entities to process.
-     */
-    public void joiningUniqueFromFirstSectionOnly(String ruleIdentifier, String name, Collection<Entity> entities) {
-
-        joiningUniqueFromFirstSectionOnly(ruleIdentifier, name, entities, ", ");
-    }
-
-
-    /**
-     * Joins entity values from the first section entities appear in, and creates a component from the result.
-     *
-     * @param ruleIdentifier The identifier of the rule.
-     * @param name           The name of the entity.
-     * @param entities       The collection of entities to process.
-     */
-    public void joiningUniqueFromFirstSectionOnly(String ruleIdentifier, String name, Collection<Entity> entities, String delimiter) {
-
-        List<Entity> entitiesFromFirstSection = findEntitiesFromFirstSection(entities);
-        joiningUnique(ruleIdentifier, name, entitiesFromFirstSection, delimiter);
-    }
-
-
-    /**
-     * Joins entity values, and creates a component from the result.
-     *
-     * @param ruleIdentifier The identifier of the rule.
-     * @param name           The name of the entity.
-     * @param entities       The collection of entities to process.
-     */
-    public void joining(String ruleIdentifier, String name, Collection<Entity> entities, String delimiter) {
-
-        String valueDescription = String.format("Joining all values of type %s with '%s'", joinTypes(entities), delimiter);
-        String value = entities.stream().sorted(EntityComparators.first()).map(Entity::getValue).collect(Collectors.joining(delimiter));
-        create(ruleIdentifier, name, value, valueDescription, entities);
-    }
-
-
-    /**
-     * Joins entity values with delimiter ', ' from the section with the longest combined entity values only, and creates a component from the result.
-     *
-     * @param ruleIdentifier The identifier of the rule.
-     * @param name           The name of the entity.
-     * @param entities       The collection of entities to process.
-     */
-    public void joiningFromLongestSectionOnly(String ruleIdentifier, String name, Collection<Entity> entities) {
-
-        joiningFromLongestSectionOnly(ruleIdentifier, name, entities, ", ");
-    }
-
-
-    /**
-     * Joins entity values from the section with the longest combined entity values only, and creates a component from the result.
-     *
-     * @param ruleIdentifier The identifier of the rule.
-     * @param name           The name of the entity.
-     * @param entities       The collection of entities to process.
-     */
-    public void joiningFromLongestSectionOnly(String ruleIdentifier, String name, Collection<Entity> entities, String delimiter) {
-
-        List<Entity> entitiesFromLongestSection = findEntitiesFromLongestSection(entities);
-        joining(ruleIdentifier, name, entitiesFromLongestSection, delimiter);
-    }
-
-
-    /**
-     * Joins all unique values from a collection of entities into a single string using a specified delimiter and creates a component from the result.
-     *
-     * @param ruleIdentifier the identifier of the rule
-     * @param name           the name of the joining operation
-     * @param entities       the collection of entities
-     * @param delimiter      the delimiter to use for joining the values
-     */
-    public void joiningUnique(String ruleIdentifier, String name, Collection<Entity> entities, String delimiter) {
-
-        String valueDescription = String.format("Joining all unique values of type %s with '%s'", joinTypes(entities), delimiter);
-        String value = entities.stream().sorted(EntityComparators.first()).map(Entity::getValue).distinct().collect(Collectors.joining(delimiter));
-        create(ruleIdentifier, name, value, valueDescription, entities);
-    }
-
-
-    private static int getTotalLengthOfEntities(Map.Entry<SemanticNode, List<Entity>> entry) {
-
-        return entry.getValue().stream().mapToInt(Entity::getLength).sum();
-    }
-
-
-    /**
-     * Joins unique entity values with delimiter ', ' from the section with the longest combined entity values only, and creates a component from the result.
-     *
-     * @param ruleIdentifier The identifier of the rule.
-     * @param name           The name of the entity.
-     * @param entities       The collection of entities to process.
-     */
-    public void joiningUniqueFromLongestSectionOnly(String ruleIdentifier, String name, Collection<Entity> entities) {
-
-        joiningUniqueFromLongestSectionOnly(ruleIdentifier, name, entities, ", ");
-    }
-
-
-    /**
-     * Joins unique entity values from the section with the longest combined entity values only, and creates a component from the result.
-     *
-     * @param ruleIdentifier The identifier of the rule.
-     * @param name           The name of the entity.
-     * @param entities       The collection of entities to process.
-     * @param delimiter      The string delimiter to join the entities.
-     */
-    public void joiningUniqueFromLongestSectionOnly(String ruleIdentifier, String name, Collection<Entity> entities, String delimiter) {
-
-        List<Entity> entitiesFromLongestSection = findEntitiesFromLongestSection(entities);
-        joiningUnique(ruleIdentifier, name, entitiesFromLongestSection, delimiter);
-    }
-
-
-    /**
-     * Joins the names of the entities in the provided collection, separated by a comma,
-     * and creates a component from the result.
-     *
-     * @param ruleIdentifier The unique identifier of the rule being applied.
-     * @param name           The name to which the joined result will be set.
-     * @param entities       The collection of entities whose names will be joined.
-     */
-    public void joiningUnique(String ruleIdentifier, String name, Collection<Entity> entities) {
-
-        joiningUnique(ruleIdentifier, name, entities, ", ");
-    }
-
-
-    /**
-     * Computes the number of unique values in the collection of entities and creates a component with the result.
-     *
-     * @param ruleIdentifier the identifier of the rule
-     * @param name           the name of the record
-     * @param entities       the collection of entities to compute unique values from
-     */
-    public void uniqueValueCount(String ruleIdentifier, String name, Collection<Entity> entities) {
-
-        long count = entities.stream().map(Entity::getValue).distinct().count();
-        create(ruleIdentifier, name, String.valueOf(count), "Number of unique values in the entity references", entities);
-    }
-
-
-    /**
-     * Counts the distinct rows where any of the provided entities occur and creates a component for each distinct table with its respective row count.
-     *
-     * @param ruleIdentifier the identifier of the rule
-     * @param name           the name of the record
-     * @param entities       the collection of entities to count row values from
-     */
-    public void rowValueCount(String ruleIdentifier, String name, Collection<Entity> entities) {
-
-        entities.stream().collect(Collectors.groupingBy(this::getFirstTable)).forEach((optionalTable, groupedEntities) -> {
-
-            if (optionalTable.isEmpty()) {
-                return;
-            }
-
-            long count = groupedEntities.stream()
-                    .collect(Collectors.groupingBy(entity -> getFirstTableCell(entity).map(TableCell::getRow).orElse(-1)))
-                    .size();
-
-            create(ruleIdentifier, name, String.valueOf(count), "Count rows with values in the entity references in same table", entities);
-        });
-    }
-
-
-    /**
-     * Creates a component for each sentence in the collection of entities.
-     *
-     * @param ruleIdentifier The identifier of the rule.
-     * @param name           The name of the entity.
-     * @param entities       A collection of Entity objects.
-     */
-    public void asSentences(String ruleIdentifier, String name, Collection<Entity> entities) {
-
-        if (entities.isEmpty()) {
-            return;
-        }
-        entities.stream().sorted(EntityComparators.first()).forEach(entity -> {
-            BreakIterator iterator = BreakIterator.getSentenceInstance(Locale.ENGLISH);
-            iterator.setText(entity.getValue());
-            int start = iterator.first();
-            for (int end = iterator.next(); end != BreakIterator.DONE; start = end, end = iterator.next()) {
-                create(ruleIdentifier,
-                        name,
-                        entity.getValue().substring(start, end).replaceAll("\\n", "").trim(),
-                        String.format("Values of type '%s' as sentences", entity.getType()),
-                        entity);
-            }
-        });
-    }
-
-
-    /**
-     * Creates a new component with the given rule identifier, name, value, value description, and reference.
-     * The component is built using the provided parameters and inserted into the knowledge session.
-     * The reference is added to the referencedEntities list.
-     *
-     * @param ruleIdentifier   The identifier of the rule for the component.
-     * @param name             The name of the component.
-     * @param value            The value of the component.
-     * @param valueDescription The description of the value.
-     * @param reference        The reference entity for the component.
-     */
-    public void create(String ruleIdentifier, String name, String value, String valueDescription, Entity reference) {
+    public void create(String ruleIdentifier, String category, String value, String transformation, Entity reference) {
 
         referencedEntities.add(reference);
         List<Entity> referenceList = new LinkedList<>();
         referenceList.add(reference);
         kieSession.insert(Component.builder()
                 .matchedRule(RuleIdentifier.fromString(ruleIdentifier))
-                .name(name)
+                .category(category)
                 .value(value)
-                .valueDescription(valueDescription)
+                .transformation(transformation)
                 .references(referenceList)
                 .build());
     }
 
 
-    /**
-     * Creates components for unmapped entities.
-     *
-     * @param ruleIdentifier The identifier of the rule being applied.
-     * @param entities       The collection of entities to create components for.
-     */
-    public void createComponentsForUnMappedEntities(String ruleIdentifier, Collection<Entity> entities) {
+    public void create(String ruleIdentifier, String category, String value, String transformation) {
 
-        entities.stream()
-                .filter(entity -> !referencedEntities.contains(entity))
-                .sorted(EntityComparators.first())
-                .forEach(entity -> create(ruleIdentifier, entity.getType(), entity.getValue(), "Unmapped Entity", List.of(entity)));
+        create(ruleIdentifier, category, value, transformation, Collections.emptyList());
     }
 
 
-    /**
-     * Converts entity values to the 'dd/MM/yyyy' format and joins them with ', '. If the value could not be parsed as a date, it will be created as is.
-     *
-     * @param ruleIdentifier the identifier of the rule
-     * @param name           the name of the entity
-     * @param entities       the collection of entities
-     */
-    public void convertDates(String ruleIdentifier, String name, Collection<Entity> entities) {
-
-        convertDates(ruleIdentifier, name, entities, "dd/MM/yyyy");
-    }
-
-
-    /**
-     * Converts entity values to provided format and joins them with ', '. If the value could not be parsed as a date, it will be created as is.
-     *
-     * @param ruleIdentifier the identifier of the rule
-     * @param name           the name of the entity
-     * @param entities       the collection of entities
-     * @param resultFormat   the desired format for the converted dates
-     */
-    public void convertDates(String ruleIdentifier, String name, Collection<Entity> entities, String resultFormat) {
-
-        String valueDescription = String.format("Convert values of type '%s' to %s joined with ', '", joinTypes(entities), resultFormat);
-        List<String> unparsedDates = new LinkedList<>();
-        List<Date> dates = new LinkedList<>();
-
-        for (Entity entity : entities) {
-            String value = entity.getValue();
-            Optional<Date> optionalDate = DateConverter.parseDate(value);
-            if (optionalDate.isPresent()) {
-                dates.add(optionalDate.get());
-            } else {
-                unparsedDates.add(value);
-            }
-        }
-
-        String formattedDateStrings = Stream.concat(//
-                        dates.stream().sorted().map(date -> DateConverter.convertDate(date, resultFormat)), //
-                        unparsedDates.stream())//
-                .collect(Collectors.joining(", "));
-
-        create(ruleIdentifier, name, formattedDateStrings, valueDescription, entities);
-    }
-
-
-    /**
-     * Joins values from entities that are in the same table row. They are sorted from left to right. If entities are not in a table cell they are added as a single component.
-     *
-     * @param ruleIdentifier the identifier of the rule
-     * @param name           the name of the entity
-     * @param entities       the collection of entities
-     */
-    public void joiningFromSameTableRow(String ruleIdentifier, String name, Collection<Entity> entities) {
-
-        String types = entities.stream().map(Entity::getType).sorted(Comparator.reverseOrder()).distinct().collect(Collectors.joining(", "));
-        String valueDescription = String.format("Combine values of %s that are in same table row", types);
-        entities.stream().collect(Collectors.groupingBy(this::getFirstTable)).forEach((optionalTable, groupedEntities) -> {
-            if (optionalTable.isEmpty()) {
-                groupedEntities.forEach(entity -> create(ruleIdentifier, name, entity.getValue(), valueDescription, entity));
-            }
-
-            groupedEntities.stream()
-                    .filter(entity -> entity.getContainingNode() instanceof TableCell)
-                    .collect(Collectors.groupingBy(entity -> ((TableCell) entity.getContainingNode()).getRow()))
-                    .entrySet()
-                    .stream()
-                    .sorted(Comparator.comparingInt(Map.Entry::getKey))
-                    .map(Map.Entry::getValue)
-                    .forEach(entitiesInSameRow -> create(ruleIdentifier,
-                            name,
-                            entitiesInSameRow.stream().sorted(EntityComparators.first()).map(Entity::getValue).collect(Collectors.joining(", ")),
-                            valueDescription,
-                            entitiesInSameRow));
-        });
-    }
-
-
-    private Optional<Table> getFirstTable(Entity entity) {
-
-        SemanticNode node = entity.getContainingNode();
-        while (!(node instanceof Table)) {
-            if (!node.hasParent()) {
-                return Optional.empty();
-            }
-            node = node.getParent();
-        }
-
-        return Optional.of((Table) node);
-    }
-
-
-    private Optional<TableCell> getFirstTableCell(Entity entity) {
-
-        SemanticNode node = entity.getContainingNode();
-        while (!(node instanceof TableCell)) {
-            if (!node.hasParent()) {
-                return Optional.empty();
-            }
-            node = node.getParent();
-        }
-
-        return Optional.of((TableCell) node);
-    }
-
-
-    /**
-     * Creates a new component with the given rule identifier, name, value, and value description.
-     * If the component is part of a table, it also takes a list of entities that belong to the same table row.
-     *
-     * @param ruleIdentifier   the identifier of the rule
-     * @param name             the name of the entity
-     * @param value            the value of the component
-     * @param valueDescription the description of the value
-     */
-    public void create(String ruleIdentifier, String name, String value, String valueDescription) {
-
-        create(ruleIdentifier, name, value, valueDescription, Collections.emptyList());
-    }
-
-
-    /**
-     * Creates a new component with the given rule identifier, name, and value.
-     *
-     * @param ruleIdentifier the identifier of the rule
-     * @param name           the name of the entity
-     * @param value          the value of the component
-     */
-    public void create(String ruleIdentifier, String name, String value) {
+    public void create(String ruleIdentifier, String category, String value) {
 
         kieSession.insert(Component.builder()
                 .matchedRule(RuleIdentifier.fromString(ruleIdentifier))
-                .name(name)
+                .category(category)
                 .value(value)
-                .valueDescription("")
+                .transformation("")
                 .references(Collections.emptyList())
                 .build());
     }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/DocumentGraphMapper.java

@@ -6,7 +6,6 @@ import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
 import java.util.NoSuchElementException;
-import java.util.Set;
 
 import com.iqser.red.service.redaction.v1.server.model.document.DocumentTree;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Footer;
@@ -77,7 +76,6 @@ public class DocumentGraphMapper {
                 node.setLeafTextBlock(textBlock);
             }
             List<Integer> treeId = Arrays.stream(entryData.getTreeId()).boxed().toList();
-            entryData.getEngines().forEach(engine -> node.addEngine(engine));
             node.setTreeId(treeId);
 
             switch (entryData.getType()) {

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityComparators.java

@@ -6,12 +6,7 @@ import com.iqser.red.service.redaction.v1.server.model.component.Entity;
 
 public abstract class EntityComparators implements Comparator<Entity> {
 
-    public static Comparator<Entity> first() {
-
-        return new FirstEntity();
-    }
-
-    public static class LongestEntity implements Comparator<Entity> {
+    private static class LongestEntity implements Comparator<Entity> {
 
         @Override
         public int compare(Entity Entity, Entity otherEntity) {
@@ -21,20 +16,26 @@ public abstract class EntityComparators implements Comparator<Entity> {
 
     }
 
+    private static class FirstEntity implements Comparator<Entity> {
+
+        @Override
+        public int compare(Entity Entity, Entity otherEntity) {
+
+            return Integer.compare(Entity.getStartOffset(), otherEntity.getStartOffset());
+        }
+
+    }
+
 
     public static Comparator<Entity> length() {
 
         return new LongestEntity();
     }
 
-    public static class FirstEntity implements Comparator<Entity> {
 
-        @Override
-        public int compare(Entity Entity, Entity otherEntity) {
-
-            return Integer.compare(Entity.getStartOffset(), otherEntity.getStartOffset());
-        }
+    public static Comparator<Entity> start() {
 
+        return new FirstEntity();
     }
 
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityCreationService.java

@@ -1,6 +1,5 @@
 package com.iqser.red.service.redaction.v1.server.service.document;
 
-import static com.iqser.red.service.redaction.v1.server.service.document.EntityCreationUtility.*;
 import static com.iqser.red.service.redaction.v1.server.utils.SeparatorUtils.boundaryIsSurroundedBySeparators;
 
 import java.util.Collection;
@@ -19,22 +18,24 @@ import org.kie.api.runtime.KieSession;
 
 import com.google.common.base.Functions;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine;
-import com.iqser.red.service.redaction.v1.server.model.NerEntities;
-import com.iqser.red.service.redaction.v1.server.model.dictionary.SearchImplementation;
+import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
 import com.iqser.red.service.redaction.v1.server.model.document.ConsecutiveBoundaryCollector;
 import com.iqser.red.service.redaction.v1.server.model.document.DocumentTree;
-import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.ManualChangeOverwrite;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
+import com.iqser.red.service.redaction.v1.server.model.document.entity.PositionOnPage;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.NodeType;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.Page;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Table;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
 import com.iqser.red.service.redaction.v1.server.utils.RectangleTransformations;
 import com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility;
-import com.iqser.red.service.redaction.v1.server.utils.exception.NotFoundException;
+import com.iqser.red.service.redaction.v1.server.model.NerEntities;
+import com.iqser.red.service.redaction.v1.server.model.dictionary.SearchImplementation;
+import com.iqser.red.service.redaction.v1.server.utils.IdBuilder;
 
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
@@ -56,197 +57,144 @@ public class EntityCreationService {
 
     public Stream<TextEntity> betweenStrings(String start, String stop, String type, EntityType entityType, SemanticNode node) {
 
-        checkIfBothStartAndEndAreEmpty(start, stop);
+        List<TextRange> startBoundaries = RedactionSearchUtility.findBoundariesByString(start, node.getTextBlock());
+        List<TextRange> stopBoundaries = RedactionSearchUtility.findBoundariesByString(stop, node.getTextBlock());
 
-        List<TextRange> startTextRanges = RedactionSearchUtility.findTextRangesByString(start, node.getTextBlock());
-        List<TextRange> stopTextRanges = RedactionSearchUtility.findTextRangesByString(stop, node.getTextBlock());
-
-        return betweenTextRanges(startTextRanges, stopTextRanges, type, entityType, node);
+        return betweenBoundaries(startBoundaries, stopBoundaries, type, entityType, node);
     }
 
 
     public Stream<TextEntity> betweenStringsIgnoreCase(String start, String stop, String type, EntityType entityType, SemanticNode node) {
 
-        checkIfBothStartAndEndAreEmpty(start, stop);
+        List<TextRange> startBoundaries = RedactionSearchUtility.findBoundariesByStringIgnoreCase(start, node.getTextBlock());
+        List<TextRange> stopBoundaries = RedactionSearchUtility.findBoundariesByStringIgnoreCase(stop, node.getTextBlock());
 
-        List<TextRange> startBoundaries = RedactionSearchUtility.findTextRangesByStringIgnoreCase(start, node.getTextBlock());
-        List<TextRange> stopBoundaries = RedactionSearchUtility.findTextRangesByStringIgnoreCase(stop, node.getTextBlock());
-
-        return betweenTextRanges(startBoundaries, stopBoundaries, type, entityType, node);
+        return betweenBoundaries(startBoundaries, stopBoundaries, type, entityType, node);
     }
 
 
     public Stream<TextEntity> betweenStringsIncludeStart(String start, String stop, String type, EntityType entityType, SemanticNode node) {
 
-        checkIfBothStartAndEndAreEmpty(start, stop);
+        List<TextRange> startBoundaries = RedactionSearchUtility.findBoundariesByString(start, node.getTextBlock());
+        List<TextRange> stopBoundaries = RedactionSearchUtility.findBoundariesByString(stop, node.getTextBlock());
 
-        List<TextRange> startBoundaries = RedactionSearchUtility.findTextRangesByString(start, node.getTextBlock());
-        List<TextRange> stopBoundaries = RedactionSearchUtility.findTextRangesByString(stop, node.getTextBlock());
-
-        startBoundaries.forEach(textRange -> {
-            textRange.setStart(textRange.start() - start.length());
-            textRange.setEnd(textRange.end() - start.length());
+        startBoundaries.forEach(boundary -> {
+            boundary.setStart(boundary.start() - start.length());
+            boundary.setEnd(boundary.end() - start.length());
         });
 
-        return betweenTextRanges(startBoundaries, stopBoundaries, type, entityType, node);
+        return betweenBoundaries(startBoundaries, stopBoundaries, type, entityType, node);
     }
 
 
     public Stream<TextEntity> betweenStringsIncludeStartIgnoreCase(String start, String stop, String type, EntityType entityType, SemanticNode node) {
 
-        checkIfBothStartAndEndAreEmpty(start, stop);
+        List<TextRange> startBoundaries = RedactionSearchUtility.findBoundariesByStringIgnoreCase(start, node.getTextBlock());
+        List<TextRange> stopBoundaries = RedactionSearchUtility.findBoundariesByStringIgnoreCase(stop, node.getTextBlock());
 
-        List<TextRange> startBoundaries = RedactionSearchUtility.findTextRangesByStringIgnoreCase(start, node.getTextBlock());
-        List<TextRange> stopBoundaries = RedactionSearchUtility.findTextRangesByStringIgnoreCase(stop, node.getTextBlock());
-
-        startBoundaries.forEach(textRange -> {
-            textRange.setStart(textRange.start() - start.length());
-            textRange.setEnd(textRange.end() - start.length());
+        startBoundaries.forEach(boundary -> {
+            boundary.setStart(boundary.start() - start.length());
+            boundary.setEnd(boundary.end() - start.length());
         });
 
-        return betweenTextRanges(startBoundaries, stopBoundaries, type, entityType, node);
+        return betweenBoundaries(startBoundaries, stopBoundaries, type, entityType, node);
     }
 
 
     public Stream<TextEntity> betweenStringsIncludeEnd(String start, String stop, String type, EntityType entityType, SemanticNode node) {
 
-        checkIfBothStartAndEndAreEmpty(start, stop);
+        List<TextRange> startBoundaries = RedactionSearchUtility.findBoundariesByString(start, node.getTextBlock());
+        List<TextRange> stopBoundaries = RedactionSearchUtility.findBoundariesByString(stop, node.getTextBlock());
 
-        List<TextRange> startBoundaries = RedactionSearchUtility.findTextRangesByString(start, node.getTextBlock());
-        List<TextRange> stopBoundaries = RedactionSearchUtility.findTextRangesByString(stop, node.getTextBlock());
-
-        stopBoundaries.forEach(textRange -> {
-            textRange.setStart(textRange.start() + stop.length());
-            textRange.setEnd(textRange.end() + stop.length());
+        stopBoundaries.forEach(boundary -> {
+            boundary.setStart(boundary.start() + stop.length());
+            boundary.setEnd(boundary.end() + stop.length());
         });
 
-        return betweenTextRanges(startBoundaries, stopBoundaries, type, entityType, node);
+        return betweenBoundaries(startBoundaries, stopBoundaries, type, entityType, node);
     }
 
 
     public Stream<TextEntity> betweenStringsIncludeEndIgnoreCase(String start, String stop, String type, EntityType entityType, SemanticNode node) {
 
-        checkIfBothStartAndEndAreEmpty(start, stop);
+        List<TextRange> startBoundaries = RedactionSearchUtility.findBoundariesByStringIgnoreCase(start, node.getTextBlock());
+        List<TextRange> stopBoundaries = RedactionSearchUtility.findBoundariesByStringIgnoreCase(stop, node.getTextBlock());
 
-        List<TextRange> startBoundaries = RedactionSearchUtility.findTextRangesByStringIgnoreCase(start, node.getTextBlock());
-        List<TextRange> stopBoundaries = RedactionSearchUtility.findTextRangesByStringIgnoreCase(stop, node.getTextBlock());
-
-        stopBoundaries.forEach(textRange -> {
-            textRange.setStart(textRange.start() + stop.length());
-            textRange.setEnd(textRange.end() + stop.length());
+        stopBoundaries.forEach(boundary -> {
+            boundary.setStart(boundary.start() + stop.length());
+            boundary.setEnd(boundary.end() + stop.length());
         });
 
-        return betweenTextRanges(startBoundaries, stopBoundaries, type, entityType, node);
+        return betweenBoundaries(startBoundaries, stopBoundaries, type, entityType, node);
     }
 
 
     public Stream<TextEntity> betweenStringsIncludeStartAndEnd(String start, String stop, String type, EntityType entityType, SemanticNode node) {
 
-        checkIfBothStartAndEndAreEmpty(start, stop);
+        List<TextRange> startBoundaries = RedactionSearchUtility.findBoundariesByString(start, node.getTextBlock());
+        List<TextRange> stopBoundaries = RedactionSearchUtility.findBoundariesByString(stop, node.getTextBlock());
 
-        List<TextRange> startBoundaries = RedactionSearchUtility.findTextRangesByString(start, node.getTextBlock());
-        List<TextRange> stopBoundaries = RedactionSearchUtility.findTextRangesByString(stop, node.getTextBlock());
-
-        startBoundaries.forEach(textRange -> {
-            textRange.setStart(textRange.start() - start.length());
-            textRange.setEnd(textRange.end() - start.length());
+        startBoundaries.forEach(boundary -> {
+            boundary.setStart(boundary.start() - start.length());
+            boundary.setEnd(boundary.end() - start.length());
         });
-        stopBoundaries.forEach(textRange -> {
-            textRange.setStart(textRange.start() + stop.length());
-            textRange.setEnd(textRange.end() + stop.length());
+        stopBoundaries.forEach(boundary -> {
+            boundary.setStart(boundary.start() + stop.length());
+            boundary.setEnd(boundary.end() + stop.length());
         });
 
-        return betweenTextRanges(startBoundaries, stopBoundaries, type, entityType, node);
+        return betweenBoundaries(startBoundaries, stopBoundaries, type, entityType, node);
     }
 
 
     public Stream<TextEntity> betweenStringsIncludeStartAndEndIgnoreCase(String start, String stop, String type, EntityType entityType, SemanticNode node) {
 
-        checkIfBothStartAndEndAreEmpty(start, stop);
+        List<TextRange> startBoundaries = RedactionSearchUtility.findBoundariesByStringIgnoreCase(start, node.getTextBlock());
+        List<TextRange> stopBoundaries = RedactionSearchUtility.findBoundariesByStringIgnoreCase(stop, node.getTextBlock());
 
-        List<TextRange> startBoundaries = RedactionSearchUtility.findTextRangesByStringIgnoreCase(start, node.getTextBlock());
-        List<TextRange> stopBoundaries = RedactionSearchUtility.findTextRangesByStringIgnoreCase(stop, node.getTextBlock());
-
-        startBoundaries.forEach(textRange -> {
-            textRange.setStart(textRange.start() - start.length());
-            textRange.setEnd(textRange.end() - start.length());
+        startBoundaries.forEach(boundary -> {
+            boundary.setStart(boundary.start() - start.length());
+            boundary.setEnd(boundary.end() - start.length());
         });
-        stopBoundaries.forEach(textRange -> {
-            textRange.setStart(textRange.start() + stop.length());
-            textRange.setEnd(textRange.end() + stop.length());
+        stopBoundaries.forEach(boundary -> {
+            boundary.setStart(boundary.start() + stop.length());
+            boundary.setEnd(boundary.end() + stop.length());
         });
 
-        return betweenTextRanges(startBoundaries, stopBoundaries, type, entityType, node);
-    }
-
-
-    public Stream<TextEntity> shortestBetweenAnyString(List<String> starts, List<String> stops, String type, EntityType entityType, SemanticNode node) {
-
-        checkIfBothStartAndEndAreEmpty(starts, stops);
-
-        List<TextRange> startTextRanges = RedactionSearchUtility.findTextRangesByList(starts, node.getTextBlock());
-        List<TextRange> stopTextRanges = RedactionSearchUtility.findTextRangesByList(stops, node.getTextBlock());
-
-        return betweenTextRanges(startTextRanges, stopTextRanges, type, entityType, node);
-    }
-
-
-    public Stream<TextEntity> shortestBetweenAnyStringIgnoreCase(List<String> starts, List<String> stops, String type, EntityType entityType, SemanticNode node) {
-
-        checkIfBothStartAndEndAreEmpty(starts, stops);
-
-        List<TextRange> startTextRanges = RedactionSearchUtility.findTextRangesByListIgnoreCase(starts, node.getTextBlock());
-        List<TextRange> stopTextRanges = RedactionSearchUtility.findTextRangesByListIgnoreCase(stops, node.getTextBlock());
-
-        return betweenTextRanges(startTextRanges, stopTextRanges, type, entityType, node);
-    }
-
-
-    public Stream<TextEntity> shortestBetweenAnyStringIgnoreCase(List<String> starts, List<String> stops, String type, EntityType entityType, SemanticNode node, int limit) {
-
-        checkIfBothStartAndEndAreEmpty(starts, stops);
-
-        List<TextRange> startTextRanges = RedactionSearchUtility.findTextRangesByListIgnoreCase(starts, node.getTextBlock());
-        List<TextRange> stopTextRanges = RedactionSearchUtility.findTextRangesByListIgnoreCase(stops, node.getTextBlock());
-
-        return betweenTextRanges(startTextRanges, stopTextRanges, type, entityType, node, limit);
+        return betweenBoundaries(startBoundaries, stopBoundaries, type, entityType, node);
     }
 
 
     public Stream<TextEntity> betweenRegexes(String regexStart, String regexStop, String type, EntityType entityType, SemanticNode node) {
 
         TextBlock textBlock = node.getTextBlock();
-        List<TextRange> startBoundaries = RedactionSearchUtility.findTextRangesByRegex(regexStart, textBlock);
-        List<TextRange> stopBoundaries = RedactionSearchUtility.findTextRangesByRegex(regexStop, textBlock);
+        List<TextRange> startBoundaries = RedactionSearchUtility.findBoundariesByRegex(regexStart, textBlock);
+        List<TextRange> stopBoundaries = RedactionSearchUtility.findBoundariesByRegex(regexStop, textBlock);
 
-        return betweenTextRanges(startBoundaries, stopBoundaries, type, entityType, node);
+        return betweenBoundaries(startBoundaries, stopBoundaries, type, entityType, node);
     }
 
 
     public Stream<TextEntity> betweenRegexesIgnoreCase(String regexStart, String regexStop, String type, EntityType entityType, SemanticNode node) {
 
         TextBlock textBlock = node.getTextBlock();
-        List<TextRange> startBoundaries = RedactionSearchUtility.findTextRangesByRegexIgnoreCase(regexStart, 0, textBlock);
-        List<TextRange> stopBoundaries = RedactionSearchUtility.findTextRangesByRegexIgnoreCase(regexStop, 0, textBlock);
+        List<TextRange> startBoundaries = RedactionSearchUtility.findBoundariesByRegexIgnoreCase(regexStart, 0, textBlock);
+        List<TextRange> stopBoundaries = RedactionSearchUtility.findBoundariesByRegexIgnoreCase(regexStop, 0, textBlock);
 
-        return betweenTextRanges(startBoundaries, stopBoundaries, type, entityType, node);
+        return betweenBoundaries(startBoundaries, stopBoundaries, type, entityType, node);
     }
 
 
-    public Stream<TextEntity> betweenTextRanges(List<TextRange> startBoundaries, List<TextRange> stopBoundaries, String type, EntityType entityType, SemanticNode node) {
-
-        return betweenTextRanges(startBoundaries, stopBoundaries, type, entityType, node, 0);
-    }
-
-
-    public Stream<TextEntity> betweenTextRanges(List<TextRange> startBoundaries, List<TextRange> stopBoundaries, String type, EntityType entityType, SemanticNode node, int limit) {
+    public Stream<TextEntity> betweenBoundaries(List<TextRange> startBoundaries, List<TextRange> stopBoundaries, String type, EntityType entityType, SemanticNode node) {
 
+        if (startBoundaries.isEmpty() || stopBoundaries.isEmpty()) {
+            return Stream.empty();
+        }
         List<TextRange> entityBoundaries = findNonOverlappingBoundariesBetweenBoundariesWithMinimalDistances(startBoundaries, stopBoundaries);
         return entityBoundaries.stream()
-                .filter(range -> (limit == 0 || range.length() <= limit))
-                .map(textRange -> textRange.trim(node.getTextBlock()))
-                .filter(textRange -> isValidEntityTextRange(node.getTextBlock(), textRange))
-                .map(textRange -> byTextRange(textRange, type, entityType, node))
+                .map(boundary -> boundary.trim(node.getTextBlock()))
+                .filter(boundary -> isValidEntityBoundary(node.getTextBlock(), boundary))
+                .map(boundary -> byTextRange(boundary, type, entityType, node))
                 .filter(Optional::isPresent)
                 .map(Optional::get);
     }
@@ -256,27 +204,27 @@ public class EntityCreationService {
 
         List<TextRange> entityBoundaries = new LinkedList<>();
         for (TextRange startTextRange : startBoundaries) {
-            Optional<TextRange> optionalStopTextRangeWithMinimalDistance = stopBoundaries.stream()
+            Optional<TextRange> optionalStopBoundaryWithMinimalDistance = stopBoundaries.stream()
                     .filter(stopBoundary -> stopBoundary.start() > startTextRange.end())
                     .min(Comparator.comparingInt(TextRange::start));
-            if (optionalStopTextRangeWithMinimalDistance.isEmpty()) {
-                continue;
+            if (optionalStopBoundaryWithMinimalDistance.isEmpty()) {
+                break;
             }
-            entityBoundaries.add(new TextRange(startTextRange.end(), optionalStopTextRangeWithMinimalDistance.get().start()));
+            entityBoundaries.add(new TextRange(startTextRange.end(), optionalStopBoundaryWithMinimalDistance.get().start()));
         }
         return removeOuterOverlappingBoundaries(entityBoundaries);
     }
 
 
-    private static List<TextRange> removeOuterOverlappingBoundaries(List<TextRange> entityTextRanges) {
+    private static List<TextRange> removeOuterOverlappingBoundaries(List<TextRange> entityBoundaries) {
         /*
         In some cases we get boundaries, where one contains the other. This happens for Example when we have two start boundaries and one stop boundary after the two start boundaries.
         Then we get two boundaries where one is entirely contained in the other. So we want to remove the outer boundary.
         For Example consider the text: "a this is some text. a here is more text b". If "a" is the start string and "b" is the stop string, there are two possibilities.
         "this is some text. a here is more text" and "here is more text". We only want to keep the latter.
          */
-        return entityTextRanges.stream()
-                .filter(boundary -> entityTextRanges.stream().noneMatch(innerBoundary -> !innerBoundary.equals(boundary) && innerBoundary.containedBy(boundary)))
+        return entityBoundaries.stream()
+                .filter(boundary -> entityBoundaries.stream().noneMatch(innerBoundary -> innerBoundary != boundary && innerBoundary.containedBy(boundary)))
                 .toList();
 
     }
@@ -286,7 +234,7 @@ public class EntityCreationService {
 
         return searchImplementation.getBoundaries(node.getTextBlock(), node.getTextRange())
                 .stream()
-                .filter(boundary -> isValidEntityTextRange(node.getTextBlock(), boundary))
+                .filter(boundary -> isValidEntityBoundary(node.getTextBlock(), boundary))
                 .map(bounds -> byTextRange(bounds, type, entityType, node))
                 .filter(Optional::isPresent)
                 .map(Optional::get);
@@ -299,8 +247,8 @@ public class EntityCreationService {
         SearchImplementation searchImplementation = new SearchImplementation(strings, false);
         return searchImplementation.getBoundaries(textBlock, node.getTextRange())
                 .stream()
-                .map(boundary -> toLineAfterTextRange(textBlock, boundary))
-                .filter(boundary -> isValidEntityTextRange(textBlock, boundary))
+                .map(boundary -> toLineAfterBoundary(textBlock, boundary))
+                .filter(boundary -> isValidEntityBoundary(textBlock, boundary))
                 .map(boundary -> byTextRange(boundary, type, entityType, node))
                 .filter(Optional::isPresent)
                 .map(Optional::get);
@@ -313,8 +261,8 @@ public class EntityCreationService {
         SearchImplementation searchImplementation = new SearchImplementation(strings, true);
         return searchImplementation.getBoundaries(textBlock, node.getTextRange())
                 .stream()
-                .map(boundary -> toLineAfterTextRange(textBlock, boundary))
-                .filter(boundary -> isValidEntityTextRange(textBlock, boundary))
+                .map(boundary -> toLineAfterBoundary(textBlock, boundary))
+                .filter(boundary -> isValidEntityBoundary(textBlock, boundary))
                 .map(boundary -> byTextRange(boundary, type, entityType, node))
                 .filter(Optional::isPresent)
                 .map(Optional::get);
@@ -324,10 +272,10 @@ public class EntityCreationService {
     public Stream<TextEntity> lineAfterString(String string, String type, EntityType entityType, SemanticNode node) {
 
         TextBlock textBlock = node.getTextBlock();
-        return RedactionSearchUtility.findTextRangesByString(string, textBlock)
+        return RedactionSearchUtility.findBoundariesByString(string, textBlock)
                 .stream()
-                .map(boundary -> toLineAfterTextRange(textBlock, boundary))
-                .filter(boundary -> isValidEntityTextRange(textBlock, boundary))
+                .map(boundary -> toLineAfterBoundary(textBlock, boundary))
+                .filter(boundary -> isValidEntityBoundary(textBlock, boundary))
                 .map(boundary -> byTextRange(boundary, type, entityType, node))
                 .filter(Optional::isPresent)
                 .map(Optional::get);
@@ -337,10 +285,10 @@ public class EntityCreationService {
     public Stream<TextEntity> lineAfterStringIgnoreCase(String string, String type, EntityType entityType, SemanticNode node) {
 
         TextBlock textBlock = node.getTextBlock();
-        return RedactionSearchUtility.findTextRangesByStringIgnoreCase(string, textBlock)
+        return RedactionSearchUtility.findBoundariesByStringIgnoreCase(string, textBlock)
                 .stream()
-                .map(boundary -> toLineAfterTextRange(textBlock, boundary))
-                .filter(boundary -> isValidEntityTextRange(textBlock, boundary))
+                .map(boundary -> toLineAfterBoundary(textBlock, boundary))
+                .filter(boundary -> isValidEntityBoundary(textBlock, boundary))
                 .map(boundary -> byTextRange(boundary, type, entityType, node))
                 .filter(Optional::isPresent)
                 .map(Optional::get);
@@ -350,7 +298,7 @@ public class EntityCreationService {
     public Stream<TextEntity> lineAfterStringAcrossColumns(String string, String type, EntityType entityType, Table tableNode) {
 
         return tableNode.streamTableCells()
-                .flatMap(tableCell -> lineAfterBoundariesAcrossColumns(RedactionSearchUtility.findTextRangesByString(string, tableCell.getTextBlock()),
+                .flatMap(tableCell -> lineAfterBoundariesAcrossColumns(RedactionSearchUtility.findBoundariesByString(string, tableCell.getTextBlock()),
                         tableCell,
                         type,
                         entityType,
@@ -361,7 +309,7 @@ public class EntityCreationService {
     public Stream<TextEntity> lineAfterStringAcrossColumnsIgnoreCase(String string, String type, EntityType entityType, Table tableNode) {
 
         return tableNode.streamTableCells()
-                .flatMap(tableCell -> lineAfterBoundariesAcrossColumns(RedactionSearchUtility.findTextRangesByStringIgnoreCase(string, tableCell.getTextBlock()),
+                .flatMap(tableCell -> lineAfterBoundariesAcrossColumns(RedactionSearchUtility.findBoundariesByStringIgnoreCase(string, tableCell.getTextBlock()),
                         tableCell,
                         type,
                         entityType,
@@ -372,23 +320,23 @@ public class EntityCreationService {
     /**
      * Looks across the remaining table row to the right of the provided TableCell if any line intersects the y coordinates of the found text.
      *
-     * @param TextRanges a list of textRanges
+     * @param boundaries a list of boundaries
      * @param tableCell  the table cell
      * @param type       the type
      * @param entityType the entity type
      * @param tableNode  the table node
      * @return a stream of RedactionEntities
      */
-    private Stream<TextEntity> lineAfterBoundariesAcrossColumns(List<TextRange> TextRanges, TableCell tableCell, String type, EntityType entityType, Table tableNode) {
+    private Stream<TextEntity> lineAfterBoundariesAcrossColumns(List<TextRange> boundaries, TableCell tableCell, String type, EntityType entityType, Table tableNode) {
 
-        return TextRanges.stream()
+        return boundaries.stream()
                 .map(boundary -> RectangleTransformations.rectangle2DBBox(tableCell.getTextBlock().getPositions(boundary)))
                 .map(bBox -> Pair.of(bBox.getMaxY(), bBox.getMinY()))
                 .map(maxMinPair -> tableNode.streamRow(tableCell.getRow())
                         .filter(nextTableCell -> nextTableCell.getCol() > tableCell.getCol())
-                        .map(nextTableCell -> RedactionSearchUtility.findTextRangesOfAllLinesInYRange(maxMinPair.getLeft(), maxMinPair.getRight(), nextTableCell.getTextBlock()))
+                        .map(nextTableCell -> RedactionSearchUtility.findBoundaryOfAllLinesInYRange(maxMinPair.getLeft(), maxMinPair.getRight(), nextTableCell.getTextBlock()))
                         .map(b -> b.trim(tableNode.getTextBlock()))
-                        .filter(boundary -> isValidEntityTextRange(tableNode.getTextBlock(), boundary))
+                        .filter(boundary -> isValidEntityBoundary(tableNode.getTextBlock(), boundary))
                         .map(boundary -> byTextRange(boundary, type, entityType, tableNode))
                         .filter(Optional::isPresent)
                         .map(Optional::get))
@@ -399,15 +347,12 @@ public class EntityCreationService {
     public Optional<TextEntity> semanticNodeAfterString(SemanticNode semanticNode, String string, String type, EntityType entityType) {
 
         var textBlock = semanticNode.getTextBlock();
-        int startIndex = textBlock.indexOf(string);
-        if (startIndex == -1) {
-            return Optional.empty();
-        }
+        int startIndex = Math.min(textBlock.indexOf(string), 0);
         var boundary = new TextRange(startIndex, semanticNode.getTextRange().end());
         if (boundary.length() > 0) {
             boundary = new TextRange(boundary.start(), boundary.end() - 1);
         }
-        if (!isValidEntityTextRange(textBlock, boundary)) {
+        if (!isValidEntityBoundary(textBlock, boundary)) {
             return Optional.empty();
         }
         return byTextRange(boundary, type, entityType, semanticNode);
@@ -440,7 +385,7 @@ public class EntityCreationService {
 
     public Stream<TextEntity> byRegexWithLineBreaks(String regexPattern, String type, EntityType entityType, int group, SemanticNode node) {
 
-        return RedactionSearchUtility.findTextRangesByRegexWithLineBreaks(regexPattern, group, node.getTextBlock())
+        return RedactionSearchUtility.findBoundariesByRegexWithLineBreaks(regexPattern, group, node.getTextBlock())
                 .stream()
                 .map(boundary -> byTextRange(boundary, type, entityType, node))
                 .filter(Optional::isPresent)
@@ -450,7 +395,7 @@ public class EntityCreationService {
 
     public Stream<TextEntity> byRegexWithLineBreaksIgnoreCase(String regexPattern, String type, EntityType entityType, int group, SemanticNode node) {
 
-        return RedactionSearchUtility.findTextRangesByRegexWithLineBreaksIgnoreCase(regexPattern, group, node.getTextBlock())
+        return RedactionSearchUtility.findBoundariesByRegexWithLineBreaksIgnoreCase(regexPattern, group, node.getTextBlock())
                 .stream()
                 .map(boundary -> byTextRange(boundary, type, entityType, node))
                 .filter(Optional::isPresent)
@@ -460,7 +405,7 @@ public class EntityCreationService {
 
     public Stream<TextEntity> byRegex(String regexPattern, String type, EntityType entityType, int group, SemanticNode node) {
 
-        return RedactionSearchUtility.findTextRangesByRegex(regexPattern, group, node.getTextBlock())
+        return RedactionSearchUtility.findBoundariesByRegex(regexPattern, group, node.getTextBlock())
                 .stream()
                 .map(boundary -> byTextRange(boundary, type, entityType, node))
                 .filter(Optional::isPresent)
@@ -470,7 +415,7 @@ public class EntityCreationService {
 
     public Stream<TextEntity> byRegexIgnoreCase(String regexPattern, String type, EntityType entityType, int group, SemanticNode node) {
 
-        return RedactionSearchUtility.findTextRangesByRegexIgnoreCase(regexPattern, group, node.getTextBlock())
+        return RedactionSearchUtility.findBoundariesByRegexIgnoreCase(regexPattern, group, node.getTextBlock())
                 .stream()
                 .map(boundary -> byTextRange(boundary, type, entityType, node))
                 .filter(Optional::isPresent)
@@ -480,7 +425,7 @@ public class EntityCreationService {
 
     public Stream<TextEntity> byString(String keyword, String type, EntityType entityType, SemanticNode node) {
 
-        return RedactionSearchUtility.findTextRangesByString(keyword, node.getTextBlock())
+        return RedactionSearchUtility.findBoundariesByString(keyword, node.getTextBlock())
                 .stream()
                 .map(boundary -> byTextRange(boundary, type, entityType, node))
                 .filter(Optional::isPresent)
@@ -490,7 +435,7 @@ public class EntityCreationService {
 
     public Stream<TextEntity> byStringIgnoreCase(String keyword, String type, EntityType entityType, SemanticNode node) {
 
-        return RedactionSearchUtility.findTextRangesByStringIgnoreCase(keyword, node.getTextBlock())
+        return RedactionSearchUtility.findBoundariesByStringIgnoreCase(keyword, node.getTextBlock())
                 .stream()
                 .map(boundary -> byTextRange(boundary, type, entityType, node))
                 .filter(Optional::isPresent)
@@ -533,7 +478,7 @@ public class EntityCreationService {
         if (textRange.length() > 0) {
             textRange = new TextRange(textRange.start(), textRange.end() - 1);
         }
-        if (!isValidEntityTextRange(node.getTextBlock(), textRange)) {
+        if (!isValidEntityBoundary(node.getTextBlock(), textRange)) {
             return Optional.empty();
         }
         return byTextRange(textRange, type, entityType, node);
@@ -543,7 +488,7 @@ public class EntityCreationService {
     public Optional<TextEntity> byPrefixExpansionRegex(TextEntity entity, String regexPattern) {
 
         int expandedStart = RedactionSearchUtility.getExpandedStartByRegex(entity, regexPattern);
-        return byTextRange(new TextRange(expandedStart, entity.getTextRange().end()), entity.type(), entity.getEntityType(), entity.getDeepestFullyContainingNode());
+        return byTextRange(new TextRange(expandedStart, entity.getTextRange().end()), entity.getType(), entity.getEntityType(), entity.getDeepestFullyContainingNode());
     }
 
 
@@ -551,7 +496,16 @@ public class EntityCreationService {
 
         int expandedEnd = RedactionSearchUtility.getExpandedEndByRegex(entity, regexPattern);
         expandedEnd = truncateEndIfLineBreakIsBetween(entity.getTextRange().end(), expandedEnd, entity.getDeepestFullyContainingNode().getTextBlock());
-        return byTextRange(new TextRange(entity.getTextRange().start(), expandedEnd), entity.type(), entity.getEntityType(), entity.getDeepestFullyContainingNode());
+        return byTextRange(new TextRange(entity.getTextRange().start(), expandedEnd), entity.getType(), entity.getEntityType(), entity.getDeepestFullyContainingNode());
+    }
+
+
+    private int truncateEndIfLineBreakIsBetween(int end, int expandedEnd, TextBlock textBlock) {
+
+        if (textBlock.getNextLinebreak(end) < expandedEnd) {
+            return end;
+        }
+        return expandedEnd;
     }
 
 
@@ -568,68 +522,25 @@ public class EntityCreationService {
      */
     public Optional<TextEntity> byTextRange(TextRange textRange, String type, EntityType entityType, SemanticNode node) {
 
-        return byTextRangeWithEngine(textRange, type, entityType, node, Set.of(Engine.RULE));
-    }
-
-
-    /**
-     * Creates a redaction entity based on the given boundary, type, entity type, and semantic node.
-     * If the document already contains an equal redaction entity, then the original Entity is returned.
-     * Also inserts the Entity into the kieSession.
-     *
-     * @param textRange  The boundary of the redaction entity.
-     * @param type       The type of the redaction entity.
-     * @param entityType The entity type of the redaction entity.
-     * @param node       The semantic node to associate with the redaction entity.
-     * @param engines    Set of engines to be used.
-     * @return An Optional containing the redaction entity, or the previous entity if the entity already exists.
-     */
-    public Optional<TextEntity> byTextRangeWithEngine(TextRange textRange, String type, EntityType entityType, SemanticNode node, Set<Engine> engines) {
-
         if (!node.getTextRange().contains(textRange)) {
             throw new IllegalArgumentException(String.format("%s is not in the %s of the provided semantic node %s", textRange, node.getTextRange(), node));
         }
         TextRange trimmedTextRange = textRange.trim(node.getTextBlock());
-        TextEntity entity = TextEntity.initialEntityNode(trimmedTextRange, type, entityType, node);
+        TextEntity entity = TextEntity.initialEntityNode(trimmedTextRange, type, entityType);
         if (node.getEntities().contains(entity)) {
-            Optional<TextEntity> optionalTextEntity = node.getEntities().stream().filter(e -> e.equals(entity) && e.type().equals(type)).peek(e -> e.addEngines(engines)).findAny();
-            if (optionalTextEntity.isEmpty()) {
-                return optionalTextEntity; // Entity has been recategorized and should not be created at all.
-            }
-            TextEntity existingEntity = optionalTextEntity.get();
-            if (existingEntity.getTextRange().equals(textRange)) {
-                return optionalTextEntity; // exactly the same entity, return directly
-            }
-            if (!existingEntity.resized()) {
-                addDuplicateEntityToGraph(existingEntity, textRange, node);
-                return optionalTextEntity; // If Entity has not been resized, insert as duplicate at appropriate position
-            }
-            return Optional.empty(); // Entity has been resized, if there are duplicates they should be treated there
+            return node.getEntities().stream().filter(entity::equals).peek(e -> e.addEngine(Engine.RULE)).findAny();
         }
         addEntityToGraph(entity, node);
-        entity.addEngines(engines);
+        entity.addEngine(Engine.RULE);
         insertToKieSession(entity);
         return Optional.of(entity);
     }
 
 
-    // Do not use anymore. This might not work correctly due to duplicate textranges not being taken into account here.
-    @Deprecated(forRemoval = true)
-    public TextEntity forceByTextRange(TextRange textRange, String type, EntityType entityType, SemanticNode node) {
+    public TextEntity forceByBoundary(TextRange textRange, String type, EntityType entityType, SemanticNode node) {
 
         TextRange trimmedTextRange = textRange.trim(node.getTextBlock());
-        TextEntity entity = TextEntity.initialEntityNode(trimmedTextRange, type, entityType, node);
-        addEntityToGraph(entity, node);
-        return entity;
-    }
-
-
-    // Do not use anymore. This might not work correctly due to duplicate textranges not being taken into account here.
-    @Deprecated(forRemoval = true)
-    public TextEntity forceByTextRange(TextRange textRange, String type, EntityType entityType, SemanticNode node, String id) {
-
-        TextRange trimmedTextRange = textRange.trim(node.getTextBlock());
-        TextEntity entity = TextEntity.initialEntityNode(trimmedTextRange, type, entityType, id);
+        TextEntity entity = TextEntity.initialEntityNode(trimmedTextRange, type, entityType);
         addEntityToGraph(entity, node);
         return entity;
     }
@@ -647,7 +558,7 @@ public class EntityCreationService {
             return entitiesToMerge.get(0);
         }
 
-        TextEntity mergedEntity = TextEntity.initialEntityNode(TextRange.merge(entitiesToMerge.stream().map(TextEntity::getTextRange).toList()), type, entityType, node);
+        TextEntity mergedEntity = TextEntity.initialEntityNode(TextRange.merge(entitiesToMerge.stream().map(TextEntity::getTextRange).toList()), type, entityType);
         mergedEntity.addEngines(entitiesToMerge.stream().flatMap(entityNode -> entityNode.getEngines().stream()).collect(Collectors.toSet()));
         entitiesToMerge.stream().map(TextEntity::getMatchedRuleList).flatMap(Collection::stream).forEach(matchedRule -> mergedEntity.getMatchedRuleList().add(matchedRule));
         entitiesToMerge.stream()
@@ -673,19 +584,14 @@ public class EntityCreationService {
 
     public TextEntity copyEntity(TextEntity entity, String type, EntityType entityType, SemanticNode node) {
 
-        var newEntity = copyEntityWithoutRules(entity, type, entityType, node);
+        TextEntity newEntity = TextEntity.initialEntityNode(entity.getTextRange(), type, entityType);
+        newEntity.addEngines(entity.getEngines());
         newEntity.addMatchedRules(entity.getMatchedRuleList());
-        return newEntity;
-    }
-
-
-    public TextEntity copyEntityWithoutRules(TextEntity entity, String type, EntityType entityType, SemanticNode node) {
-
-        TextEntity newEntity = byTextRangeWithEngine(entity.getTextRange(), type, entityType, node, entity.getEngines()).orElseThrow(() -> new NotFoundException(
-                "No entity present!"));
         newEntity.getManualOverwrite().addChanges(entity.getManualOverwrite().getManualChangeLog());
         newEntity.setDictionaryEntry(entity.isDictionaryEntry());
         newEntity.setDossierDictionaryEntry(entity.isDossierDictionaryEntry());
+        addEntityToGraph(newEntity, node);
+        insertToKieSession(newEntity);
         return newEntity;
     }
 
@@ -700,39 +606,47 @@ public class EntityCreationService {
 
     public TextEntity byNerEntity(NerEntities.NerEntity nerEntity, EntityType entityType, SemanticNode semanticNode) {
 
-        return byTextRangeWithEngine(nerEntity.textRange(), nerEntity.type(), entityType, semanticNode, Set.of(Engine.NER)).orElseThrow(() -> new NotFoundException(
-                "No entity present!"));
+        var entity = forceByBoundary(nerEntity.textRange(), nerEntity.type(), entityType, semanticNode);
+        entity.addEngine(Engine.NER);
+        insertToKieSession(entity);
+        return entity;
     }
 
 
     public TextEntity byNerEntity(NerEntities.NerEntity nerEntity, String type, EntityType entityType, SemanticNode semanticNode) {
 
-        return byTextRangeWithEngine(nerEntity.textRange(), type, entityType, semanticNode, Set.of(Engine.NER)).orElseThrow(() -> new NotFoundException("No entity present!"));
-    }
-
-
-    public Optional<TextEntity> optionalByNerEntity(NerEntities.NerEntity nerEntity, EntityType entityType, SemanticNode semanticNode) {
-
-        return byTextRangeWithEngine(nerEntity.textRange(), nerEntity.type(), entityType, semanticNode, Set.of(Engine.NER));
-    }
-
-
-    public Optional<TextEntity> optionalByNerEntity(NerEntities.NerEntity nerEntity, String type, EntityType entityType, SemanticNode semanticNode) {
-
-        return byTextRangeWithEngine(nerEntity.textRange(), type, entityType, semanticNode, Set.of(Engine.NER));
+        var entity = forceByBoundary(nerEntity.textRange(), type, entityType, semanticNode);
+        entity.addEngine(Engine.NER);
+        insertToKieSession(entity);
+        return entity;
     }
 
 
     public Stream<TextEntity> combineNerEntitiesToCbiAddressDefaults(NerEntities nerEntities, String type, EntityType entityType, SemanticNode semanticNode) {
 
         return NerEntitiesAdapter.combineNerEntitiesToCbiAddressDefaults(nerEntities)
-                .map(boundary -> byTextRangeWithEngine(boundary, type, entityType, semanticNode, Set.of(Engine.NER)))
-                .filter(Optional::isPresent)
-                .map(Optional::get);
+                .map(boundary -> forceByBoundary(boundary, type, entityType, semanticNode))
+                .peek(entity -> entity.addEngine(Engine.NER))
+                .peek(this::insertToKieSession);
     }
 
 
-    public boolean isValidEntityTextRange(TextBlock textBlock, TextRange textRange) {
+    public TextEntity byTableCellAsHighlight(TableCell tableCell, String type, EntityType entityType) {
+
+        TextEntity highlightEntity = TextEntity.initialEntityNode(new TextRange(tableCell.getTextRange().start(), tableCell.getTextRange().start()), type, entityType);
+
+        String positionId = IdBuilder.buildId(tableCell.getBBox().keySet(), tableCell.getBBox().values().stream().toList(), type, entityType.name());
+        highlightEntity.setPositionsOnPagePerPage(tableCell.getBBox()
+                .entrySet()
+                .stream()
+                .map(entry -> new PositionOnPage(positionId, entry.getKey(), List.of(entry.getValue())))
+                .toList());
+        addEntityToGraph(highlightEntity, tableCell);
+        return highlightEntity;
+    }
+
+
+    public boolean isValidEntityBoundary(TextBlock textBlock, TextRange textRange) {
 
         return textRange.length() > 0 && boundaryIsSurroundedBySeparators(textBlock, textRange);
     }
@@ -742,17 +656,9 @@ public class EntityCreationService {
 
         DocumentTree documentTree = node.getDocumentTree();
         try {
-            if (node.getEntities().contains(entity)) {
-                // If entity already exists and it has a different text range, we add the text range to the list of duplicated text ranges
-                node.getEntities().stream()//
-                        .filter(e -> e.equals(entity))//
-                        .filter(e -> !e.getTextRange().equals(entity.getTextRange()))//
-                        .findAny()//
-                        .ifPresent(entityToDuplicate -> addDuplicateEntityToGraph(entityToDuplicate, entity.getTextRange(), node));
-            } else {
-                entity.addIntersectingNode(documentTree.getRoot().getNode());
-                addEntityToGraph(entity, documentTree);
-            }
+            addEntityToGraph(entity, documentTree);
+            entity.addIntersectingNode(documentTree.getRoot().getNode());
+            documentTree.getRoot().getNode().getEntities().add(entity);
         } catch (NoSuchElementException e) {
             entity.setDeepestFullyContainingNode(documentTree.getRoot().getNode());
             entityEnrichmentService.enrichEntity(entity, entity.getDeepestFullyContainingNode().getTextBlock());
@@ -763,33 +669,6 @@ public class EntityCreationService {
     }
 
 
-    private void addDuplicateEntityToGraph(TextEntity entityToDuplicate, TextRange newTextRange, SemanticNode node) {
-
-        entityToDuplicate.addTextRange(newTextRange);
-
-        SemanticNode deepestSharedNode = entityToDuplicate.getIntersectingNodes()
-                .stream()
-                .sorted(Comparator.comparingInt(n -> -n.getTreeId().size()))
-                .filter(intersectingNode -> entityToDuplicate.getDuplicateTextRanges().stream().allMatch(tr -> intersectingNode.getTextRange().contains(tr)) && //
-                        intersectingNode.getTextRange().contains(entityToDuplicate.getTextRange()))
-                .findFirst()
-                .orElse(node.getDocumentTree().getRoot().getNode());
-
-        entityToDuplicate.setDeepestFullyContainingNode(deepestSharedNode);
-
-        Set<SemanticNode> additionalIntersectingNodes = findIntersectingSubNodes(deepestSharedNode, newTextRange);
-
-        additionalIntersectingNodes.forEach(additionalIntersectingNode -> {
-            if (entityToDuplicate.getIntersectingNodes().contains(additionalIntersectingNode)) {
-                return;
-            }
-            additionalIntersectingNode.getEntities().add(entityToDuplicate);
-            additionalIntersectingNode.getPages(newTextRange).forEach(page -> page.getEntities().add(entityToDuplicate));
-            entityToDuplicate.addIntersectingNode(additionalIntersectingNode);
-        });
-    }
-
-
     private void addEntityToGraph(TextEntity entity, DocumentTree documentTree) {
 
         SemanticNode containingNode = documentTree.childNodes(Collections.emptyList())
@@ -807,4 +686,41 @@ public class EntityCreationService {
     }
 
 
+    private static void addToPages(TextEntity entity) {
+
+        Set<Page> pages = entity.getDeepestFullyContainingNode().getPages(entity.getTextRange());
+        entity.getPages().addAll(pages);
+        pages.forEach(page -> page.getEntities().add(entity));
+    }
+
+
+    private static void addEntityToNodeEntitySets(TextEntity entity) {
+
+        entity.getIntersectingNodes().forEach(node -> node.getEntities().add(entity));
+    }
+
+
+    private static boolean allEntitiesIntersectAndHaveSameTypes(List<TextEntity> entitiesToMerge) {
+
+        if (entitiesToMerge.isEmpty()) {
+            return true;
+        }
+        TextEntity previousEntity = entitiesToMerge.get(0);
+        for (TextEntity textEntity : entitiesToMerge.subList(1, entitiesToMerge.size())) {
+            boolean typeMatches = textEntity.getType().equals(previousEntity.getType());
+            boolean entityTypeMatches = textEntity.getEntityType().equals(previousEntity.getEntityType());
+            boolean intersects = textEntity.intersects(previousEntity);
+            if (!typeMatches || !entityTypeMatches || !intersects) {
+                return false;
+            }
+        }
+        return true;
+    }
+
+
+    private static TextRange toLineAfterBoundary(TextBlock textBlock, TextRange textRange) {
+
+        return new TextRange(textRange.end(), textBlock.getNextLinebreak(textRange.end())).trim(textBlock);
+    }
+
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityCreationUtility.java

@@ -1,91 +0,0 @@
-package com.iqser.red.service.redaction.v1.server.service.document;
-
-import java.util.List;
-import java.util.Set;
-
-import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
-import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
-import com.iqser.red.service.redaction.v1.server.model.document.nodes.Page;
-import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
-import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
-
-public class EntityCreationUtility {
-
-
-    public static void checkIfBothStartAndEndAreEmpty(String start, String end) {
-
-        checkIfBothStartAndEndAreEmpty(List.of(start), List.of(end));
-    }
-
-
-    public static <T> void checkIfBothStartAndEndAreEmpty(List<T> start, List<T> end) {
-
-        if ((start == null || start.isEmpty()) && (end == null || end.isEmpty())) {
-            throw new IllegalArgumentException("Start and end values are empty!");
-        }
-    }
-
-
-    public static int truncateEndIfLineBreakIsBetween(int end, int expandedEnd, TextBlock textBlock) {
-
-        if (textBlock.getNextLinebreak(end) < expandedEnd) {
-            return end;
-        }
-        return expandedEnd;
-    }
-
-
-    public static Set<SemanticNode> findIntersectingSubNodes(SemanticNode initialIntersectingNode, TextRange textRange) {
-
-        IntersectingNodeVisitor visitor = new IntersectingNodeVisitor(textRange);
-
-        if (initialIntersectingNode.getTextRange().intersects(textRange)) {
-            initialIntersectingNode.accept(visitor);
-        }
-
-        return visitor.getIntersectingNodes();
-    }
-
-
-    public static void addToPages(TextEntity entity) {
-
-        Set<Page> pages = entity.getDeepestFullyContainingNode().getPages(entity.getTextRange());
-        entity.getPages().addAll(pages);
-        pages.forEach(page -> page.getEntities().add(entity));
-    }
-
-
-    public static void addEntityToNodeEntitySets(TextEntity entity) {
-
-        entity.getIntersectingNodes().forEach(node -> node.getEntities().add(entity));
-    }
-
-
-    public static boolean allEntitiesIntersectAndHaveSameTypes(List<TextEntity> entitiesToMerge) {
-
-        if (entitiesToMerge.isEmpty()) {
-            return true;
-        }
-        TextEntity previousEntity = entitiesToMerge.get(0);
-        for (TextEntity textEntity : entitiesToMerge.subList(1, entitiesToMerge.size())) {
-            boolean typeMatches = textEntity.type().equals(previousEntity.type());
-            boolean entityTypeMatches = textEntity.getEntityType().equals(previousEntity.getEntityType());
-            boolean intersects = textEntity.intersects(previousEntity);
-            if (!typeMatches || !entityTypeMatches || !intersects) {
-                return false;
-            }
-        }
-        return true;
-    }
-
-
-    public static TextRange toLineAfterTextRange(TextBlock textBlock, TextRange textRange) {
-
-        if (textBlock.getTextRange().end() == textRange.end()) {
-            return new TextRange(textRange.end(), textRange.end());
-        }
-
-        return new TextRange(textRange.end(), textBlock.getNextLinebreak(textRange.end())).trim(textBlock);
-    }
-
-}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityFindingUtility.java

@@ -1,175 +0,0 @@
-package com.iqser.red.service.redaction.v1.server.service.document;
-
-import static java.lang.String.format;
-import static java.util.stream.Collectors.groupingBy;
-
-import java.awt.geom.Rectangle2D;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.List;
-import java.util.Locale;
-import java.util.Map;
-import java.util.Objects;
-import java.util.Optional;
-import java.util.Set;
-import java.util.stream.Collectors;
-
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.stereotype.Service;
-
-import com.iqser.red.service.redaction.v1.server.model.ClosestEntity;
-import com.iqser.red.service.redaction.v1.server.model.PrecursorEntity;
-import com.iqser.red.service.redaction.v1.server.model.RectangleWithPage;
-import com.iqser.red.service.redaction.v1.server.model.dictionary.SearchImplementation;
-import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
-import com.iqser.red.service.redaction.v1.server.model.document.entity.PositionOnPage;
-import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
-import com.iqser.red.service.redaction.v1.server.model.document.nodes.Page;
-import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
-import com.iqser.red.service.redaction.v1.server.service.DictionaryService;
-
-import lombok.extern.slf4j.Slf4j;
-
-@Slf4j
-@Service
-public class EntityFindingUtility {
-
-    EntityCreationService entityCreationService;
-    DictionaryService dictionaryService;
-
-
-    @Autowired
-    public EntityFindingUtility(EntityEnrichmentService entityEnrichmentService, DictionaryService dictionaryService) {
-
-        entityCreationService = new EntityCreationService(entityEnrichmentService);
-        this.dictionaryService = dictionaryService;
-    }
-
-
-    public Optional<TextEntity> findClosestEntityAndReturnEmptyIfNotFound(PrecursorEntity precursorEntity, Map<String, List<TextEntity>> entitiesWithSameValue, double matchThreshold) {
-
-        if (precursorEntity.getValue() == null) {
-            return Optional.empty();
-        }
-
-        List<TextEntity> possibleEntities = entitiesWithSameValue.get(precursorEntity.getValue().toLowerCase(Locale.ENGLISH));
-
-        if (entityIdentifierValueNotFound(possibleEntities)) {
-            log.warn("Entity could not be created with precursorEntity: {}, due to the value {} not being found anywhere.", precursorEntity, precursorEntity.getValue());
-            return Optional.empty();
-        }
-
-        Optional<ClosestEntity> optionalClosestEntity = possibleEntities.stream()
-                .filter(entity -> pagesMatch(entity, precursorEntity.getEntityPosition()))
-                .map(entity -> ClosestEntity.builder().distance(calculateMinDistance(precursorEntity.getEntityPosition(), entity)).textEntity(entity).build())
-                .min(Comparator.comparingDouble(ClosestEntity::getDistance));
-
-        if (optionalClosestEntity.isEmpty()) {
-            log.warn("No Entity with value {} found on page {}", precursorEntity.getValue(), precursorEntity.getEntityPosition());
-            return Optional.empty();
-        }
-
-        ClosestEntity closestEntity = optionalClosestEntity.get();
-        if (closestEntity.getDistance() > matchThreshold) {
-            log.warn("For entity {} on page {} with positions {} distance to closest found entity is {} and therefore higher than the threshold of {}",
-                    precursorEntity.getValue(),
-                    precursorEntity.getEntityPosition().get(0).pageNumber(),
-                    precursorEntity.getEntityPosition().stream().map(RectangleWithPage::rectangle2D).toList(),
-                    closestEntity.getDistance(),
-                    matchThreshold);
-            return Optional.empty();
-        }
-
-        return Optional.of(closestEntity.getTextEntity());
-    }
-
-
-    private static boolean entityIdentifierValueNotFound(List<TextEntity> possibleEntities) {
-
-        return possibleEntities == null || possibleEntities.isEmpty();
-    }
-
-
-    private static boolean pagesMatch(TextEntity entity, List<RectangleWithPage> originalPositions) {
-
-        Set<Integer> entityPageNumbers = entity.getPositionsOnPagePerPage().stream().map(PositionOnPage::getPage).map(Page::getNumber).collect(Collectors.toSet());
-        Set<Integer> originalPageNumbers = originalPositions.stream().map(RectangleWithPage::pageNumber).collect(Collectors.toSet());
-        return entityPageNumbers.containsAll(originalPageNumbers);
-    }
-
-
-    public static double calculateMinDistance(List<RectangleWithPage> originalPositions, TextEntity entity) {
-
-        if (originalPositions.size() != countRectangles(entity)) {
-            return Double.MAX_VALUE;
-        }
-        return originalPositions.stream()
-                .mapToDouble(rectangleWithPage -> calculateMinDistancePerRectangle(entity, rectangleWithPage.pageNumber(), rectangleWithPage.rectangle2D()))
-                .average()
-                .orElse(Double.MAX_VALUE);
-    }
-
-
-    private static long countRectangles(TextEntity entity) {
-
-        return entity.getPositionsOnPagePerPage().stream().mapToLong(redactionPosition -> redactionPosition.getRectanglePerLine().size()).sum();
-    }
-
-
-    private static double calculateMinDistancePerRectangle(TextEntity entity, int pageNumber, Rectangle2D originalRectangle) {
-
-        return entity.getPositionsOnPagePerPage()
-                .stream()
-                .filter(redactionPosition -> redactionPosition.getPage().getNumber() == pageNumber)
-                .map(PositionOnPage::getRectanglePerLine)
-                .flatMap(Collection::stream)
-                .mapToDouble(rectangle -> calculateDistance(rectangle, originalRectangle))
-                .min()
-                .orElse(Double.MAX_VALUE);
-    }
-
-
-    public static double calculateDistance(Rectangle2D rectangle1, Rectangle2D rectangle2) {
-
-        // mirrored coordinates safe comparison
-        double minX1 = Math.min(rectangle1.getMinX(), rectangle1.getMaxX());
-        double minY1 = Math.min(rectangle1.getMinY(), rectangle1.getMaxY());
-        double maxX1 = Math.max(rectangle1.getMinX(), rectangle1.getMaxX());
-        double maxY1 = Math.max(rectangle1.getMinY(), rectangle1.getMaxY());
-
-        double minX2 = Math.min(rectangle2.getMinX(), rectangle2.getMaxX());
-        double minY2 = Math.min(rectangle2.getMinY(), rectangle2.getMaxY());
-        double maxX2 = Math.max(rectangle2.getMinX(), rectangle2.getMaxX());
-        double maxY2 = Math.max(rectangle2.getMinY(), rectangle2.getMaxY());
-
-        return Math.abs(minX1 - minX2) //
-                + Math.abs(minY1 - minY2) //
-                + Math.abs(maxX1 - maxX2) //
-                + Math.abs(maxY1 - maxY2);
-    }
-
-
-    public Map<String, List<TextEntity>> findAllPossibleEntitiesAndGroupByValue(SemanticNode node, List<PrecursorEntity> manualEntities) {
-
-        Set<Integer> pageNumbers = manualEntities.stream().flatMap(entry -> entry.getEntityPosition().stream().map(RectangleWithPage::pageNumber)).collect(Collectors.toSet());
-        Set<String> entryValues = manualEntities.stream().map(PrecursorEntity::getValue).filter(Objects::nonNull).map(String::toLowerCase).collect(Collectors.toSet());
-
-        if (!pageNumbers.stream().allMatch(node::onPage)) {
-            throw new IllegalArgumentException(format("SemanticNode \"%s\" does not contain these pages %s, it has pages: %s",
-                    node,
-                    pageNumbers.stream().filter(pageNumber -> !node.onPage(pageNumber)).toList(),
-                    node.getPages()));
-        }
-        SearchImplementation searchImplementation = new SearchImplementation(entryValues, true);
-
-        return searchImplementation.getBoundaries(node.getTextBlock(), node.getTextRange())
-                .stream()
-                .map(boundary -> entityCreationService.byTextRangeWithEngine(boundary, "temp", EntityType.ENTITY, node, Collections.emptySet()))
-                .filter(Optional::isPresent)
-                .map(Optional::get)
-                .distinct()
-                .collect(groupingBy(entity -> entity.getValue().toLowerCase(Locale.ROOT)));
-    }
-
-}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityFromPrecursorCreationService.java

@@ -1,161 +0,0 @@
-package com.iqser.red.service.redaction.v1.server.service.document;
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Optional;
-import java.util.Set;
-import java.util.stream.Collectors;
-
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.stereotype.Service;
-
-import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedactions;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.BaseAnnotation;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;
-import com.iqser.red.service.redaction.v1.server.model.PrecursorEntity;
-import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
-import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
-import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
-import com.iqser.red.service.redaction.v1.server.service.DictionaryService;
-
-import lombok.AccessLevel;
-import lombok.experimental.FieldDefaults;
-import lombok.extern.slf4j.Slf4j;
-
-@Slf4j
-@Service
-@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
-public class EntityFromPrecursorCreationService {
-
-    static double MATCH_THRESHOLD = 10; // Is compared to the average sum of distances in pdf coordinates for each corner of the bounding box of the entities
-    EntityFindingUtility entityFindingUtility;
-    EntityCreationService entityCreationService;
-    DictionaryService dictionaryService;
-
-
-    @Autowired
-    public EntityFromPrecursorCreationService(EntityEnrichmentService entityEnrichmentService, DictionaryService dictionaryService, EntityFindingUtility entityFindingUtility) {
-
-        this.entityFindingUtility = entityFindingUtility;
-        entityCreationService = new EntityCreationService(entityEnrichmentService);
-        this.dictionaryService = dictionaryService;
-    }
-
-
-    public List<PrecursorEntity> createEntitiesIfFoundAndReturnNotFoundEntries(ManualRedactions manualRedactions, SemanticNode node, String dossierTemplateId) {
-
-        Set<IdRemoval> idRemovals = manualRedactions.getIdsToRemove();
-        List<PrecursorEntity> manualEntities = manualRedactions.getEntriesToAdd()
-                .stream()
-                .filter(BaseAnnotation::isLocal)
-                .filter(manualRedactionEntry -> idRemovals.stream()
-                        .filter(idRemoval -> idRemoval.getAnnotationId().equals(manualRedactionEntry.getAnnotationId()))
-                        .filter(idRemoval -> idRemoval.getRequestDate().isBefore(manualRedactionEntry.getRequestDate()))
-                        .findAny()//
-                        .isEmpty())
-                .map(manualRedactionEntry -> //
-                             PrecursorEntity.fromManualRedactionEntry(manualRedactionEntry, dictionaryService.isHint(manualRedactionEntry.getType(), dossierTemplateId)))
-                .peek(manualEntity -> {
-                    if (manualEntity.getEntityType().equals(EntityType.HINT)) {
-                        manualEntity.skip("MAN.5.1", "manual hint is skipped by default");
-                    } else {
-                        manualEntity.apply("MAN.5.0", "manual entries are applied by default", manualEntity.getLegalBasis());
-                    }
-                })
-                .toList();
-        return toTextEntity(manualEntities, node);
-    }
-
-
-    public List<PrecursorEntity> createEntitiesIfFoundAndReturnNotFoundEntries(ImportedRedactions importedRedactions, SemanticNode node) {
-
-        List<PrecursorEntity> importedEntities = importedRedactions.getImportedRedactions()
-                .stream()
-                .map(PrecursorEntity::fromImportedEntry)
-                .peek(importedEntity -> importedEntity.apply("IMP.0.0", importedEntity.getReason(), importedEntity.getLegalBasis()))
-                .toList();
-        //we could add image handling here as well and create not only text entities
-        return toTextEntity(importedEntities, node);
-    }
-
-
-    public List<PrecursorEntity> toTextEntity(List<PrecursorEntity> precursorEntities, SemanticNode node) {
-
-        var notFoundEntities = precursorEntities.stream()
-                .filter(PrecursorEntity::isRectangle)
-                .collect(Collectors.toList());
-
-        var findableEntities = precursorEntities.stream()
-                .filter(precursorEntity -> !precursorEntity.isRectangle())
-                .toList();
-
-        Map<String, List<TextEntity>> tempEntitiesByValue = entityFindingUtility.findAllPossibleEntitiesAndGroupByValue(node, findableEntities);
-
-        for (PrecursorEntity precursorEntity : findableEntities) {
-            Optional<TextEntity> optionalClosestEntity = entityFindingUtility.findClosestEntityAndReturnEmptyIfNotFound(precursorEntity, tempEntitiesByValue, MATCH_THRESHOLD);
-            if (optionalClosestEntity.isEmpty()) {
-                notFoundEntities.add(precursorEntity);
-                continue;
-            }
-            createCorrectEntity(precursorEntity, optionalClosestEntity.get());
-        }
-
-        tempEntitiesByValue.values()
-                .stream()
-                .flatMap(Collection::stream)
-                .forEach(TextEntity::removeFromGraph);
-
-        return notFoundEntities;
-    }
-
-
-    /**
-     * Deletes the temp Entity and creates a RedactionEntity with correct values, based on the given parameters.
-     *
-     * @param precursorEntity The entity identifier for the RedactionEntity.
-     * @param closestEntity   The closest Boundary to the RedactionEntity.
-     */
-    public static TextEntity createCorrectEntity(PrecursorEntity precursorEntity, TextEntity closestEntity) {
-
-        return createCorrectEntity(precursorEntity, closestEntity, false);
-    }
-
-
-    public static TextEntity createCorrectEntity(PrecursorEntity precursorEntity, TextEntity closestEntity, boolean generateId) {
-
-        TextEntity correctEntity;
-        if (generateId) {
-            correctEntity = TextEntity.initialEntityNode(closestEntity.getTextRange(),
-                                                         precursorEntity.type(),
-                                                         precursorEntity.getEntityType(),
-                                                         closestEntity.getDeepestFullyContainingNode());
-        } else {
-            correctEntity = TextEntity.initialEntityNode(closestEntity.getTextRange(), precursorEntity.type(), precursorEntity.getEntityType(), precursorEntity.getId());
-        }
-        correctEntity.setDeepestFullyContainingNode(closestEntity.getDeepestFullyContainingNode());
-        correctEntity.setIntersectingNodes(new ArrayList<>(closestEntity.getIntersectingNodes()));
-        correctEntity.setDuplicateTextRanges(new ArrayList<>(closestEntity.getDuplicateTextRanges()));
-        correctEntity.setPages(new HashSet<>(closestEntity.getPages()));
-
-        correctEntity.setValue(closestEntity.getValue());
-        correctEntity.setTextAfter(closestEntity.getTextAfter());
-        correctEntity.setTextBefore(closestEntity.getTextBefore());
-
-        correctEntity.getIntersectingNodes()
-                .forEach(n -> n.getEntities().add(correctEntity));
-        correctEntity.getPages()
-                .forEach(page -> page.getEntities().add(correctEntity));
-
-        correctEntity.addMatchedRules(precursorEntity.getMatchedRuleList());
-        correctEntity.setDictionaryEntry(precursorEntity.isDictionaryEntry());
-        correctEntity.setDossierDictionaryEntry(precursorEntity.isDossierDictionaryEntry());
-        correctEntity.getManualOverwrite().addChanges(precursorEntity.getManualOverwrite().getManualChangeLog());
-        correctEntity.addEngines(precursorEntity.getEngines());
-        return correctEntity;
-    }
-
-}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/ImportedRedactionEntryService.java

@@ -1,60 +0,0 @@
-package com.iqser.red.service.redaction.v1.server.service.document;
-
-import java.util.Collection;
-import java.util.Collections;
-import java.util.List;
-import java.util.stream.Stream;
-
-import org.springframework.stereotype.Service;
-
-import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedactions;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.BaseAnnotation;
-import com.iqser.red.service.redaction.v1.server.model.PrecursorEntity;
-import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
-
-import io.micrometer.observation.annotation.Observed;
-import lombok.RequiredArgsConstructor;
-import lombok.extern.slf4j.Slf4j;
-
-@Slf4j
-@Service
-@RequiredArgsConstructor
-public class ImportedRedactionEntryService {
-
-    private final EntityFromPrecursorCreationService entityFromPrecursorCreationService;
-
-
-    @Observed(name = "ImportedRedactionEntryService", contextualName = "add-imported-redaction-entries")
-    public List<PrecursorEntity> addImportedEntriesAndReturnNotFoundEntries(AnalyzeRequest analyzeRequest, ImportedRedactions importedRedactions, Document document) {
-
-        List<PrecursorEntity> notFoundImportedEntries = Collections.emptyList();
-        if (!importedRedactions.getImportedRedactions().isEmpty()) {
-            notFoundImportedEntries = entityFromPrecursorCreationService.createEntitiesIfFoundAndReturnNotFoundEntries(importedRedactions, document);
-            log.info("Added Imported entries for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
-        }
-
-        if (notFoundImportedEntries.isEmpty()) {
-            return Collections.emptyList();
-        }
-        List<BaseAnnotation> manualChanges = allManualChangesExceptAdd(analyzeRequest.getManualRedactions());
-        for (PrecursorEntity notFoundManualRedactionEntry : notFoundImportedEntries) {
-            manualChanges.stream()
-                    .filter(change -> change.getAnnotationId().equals(notFoundManualRedactionEntry.getId()))
-                    .forEach(change -> notFoundManualRedactionEntry.getManualOverwrite().addChange(change));
-        }
-
-        return notFoundImportedEntries;
-    }
-
-
-    private List<BaseAnnotation> allManualChangesExceptAdd(ManualRedactions manualRedactions) {
-
-        return Stream.of(manualRedactions.getForceRedactions(),
-                manualRedactions.getResizeRedactions(),
-                manualRedactions.getRecategorizations(),
-                manualRedactions.getIdsToRemove(),
-                manualRedactions.getLegalBasisChanges()).flatMap(Collection::stream).map(baseAnnotation -> (BaseAnnotation) baseAnnotation).toList();
-    }
-}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/IntersectingNodeVisitor.java

@@ -1,31 +0,0 @@
-package com.iqser.red.service.redaction.v1.server.service.document;
-
-import java.util.HashSet;
-import java.util.Set;
-
-import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
-import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
-
-import lombok.Getter;
-
-public class IntersectingNodeVisitor implements NodeVisitor {
-
-    @Getter
-    private Set<SemanticNode> intersectingNodes;
-    private final TextRange textRange;
-
-    public IntersectingNodeVisitor(TextRange textRange) {
-
-        this.textRange = textRange;
-        this.intersectingNodes = new HashSet<>();
-    }
-
-    @Override
-    public void visit(SemanticNode node) {
-
-        if (node.getTextRange().intersects(textRange)) {
-            intersectingNodes.add(node);
-        }
-    }
-
-}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/ManualEntityCreationService.java

@@ -0,0 +1,238 @@
+package com.iqser.red.service.redaction.v1.server.service.document;
+
+import static java.lang.String.format;
+import static java.util.stream.Collectors.groupingBy;
+
+import java.awt.geom.Rectangle2D;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Comparator;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog;
+import com.iqser.red.service.redaction.v1.server.model.ManualEntity;
+import com.iqser.red.service.redaction.v1.server.model.RectangleWithPage;
+import com.iqser.red.service.redaction.v1.server.model.dictionary.SearchImplementation;
+import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
+import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
+import com.iqser.red.service.redaction.v1.server.model.document.entity.PositionOnPage;
+import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.Page;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
+
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+@Service
+public class ManualEntityCreationService {
+
+    private static final double MATCH_THRESHOLD = 5; // Is compared to the sum of distances in pdf coordinates for each corner of the bounding box of the entities
+    private final EntityCreationService entityCreationService;
+
+
+    @Autowired
+    public ManualEntityCreationService(EntityEnrichmentService entityEnrichmentService) {
+
+        entityCreationService = new EntityCreationService(entityEnrichmentService);
+    }
+
+
+    public List<ManualEntity> toRedactionEntity(RedactionLog redactionLog, SemanticNode node) {
+
+        List<ManualEntity> manualEntities = redactionLog.getRedactionLogEntry().stream().map(ManualEntity::fromRedactionLogEntry).toList();
+        return toRedactionEntity(manualEntities, node);
+    }
+
+
+    public List<ManualEntity> createRedactionEntitiesIfFoundAndReturnNotFoundEntries(Set<ManualRedactionEntry> manualRedactionEntries, SemanticNode node) {
+
+        List<ManualEntity> manualEntities = manualRedactionEntries.stream()
+                .filter(manualRedactionEntry -> !(manualRedactionEntry.isAddToDictionary() || manualRedactionEntry.isAddToDossierDictionary()))
+                .map(ManualEntity::fromManualRedactionEntry)
+                .toList();
+
+        return toRedactionEntity(manualEntities, node);
+    }
+
+
+    private List<ManualEntity> toRedactionEntity(List<ManualEntity> manualEntities, SemanticNode node) {
+
+        Set<Integer> pageNumbers = manualEntities.stream().flatMap(entry -> entry.getEntityPosition().stream().map(RectangleWithPage::pageNumber)).collect(Collectors.toSet());
+        Set<String> entryValues = manualEntities.stream().map(ManualEntity::getValue).map(String::toLowerCase).collect(Collectors.toSet());
+
+        Map<String, List<TextEntity>> tempEntitiesByValue = findAllPossibleEntitiesAndGroupByValue(node, pageNumbers, entryValues);
+
+        List<ManualEntity> notFoundManualEntities = new LinkedList<>();
+        for (ManualEntity manualEntity : manualEntities) {
+            Optional<TextEntity> optionalRedactionEntity = findClosestEntityAndReturnEmptyIfNotFound(manualEntity, tempEntitiesByValue);
+            if (optionalRedactionEntity.isEmpty()) {
+                notFoundManualEntities.add(manualEntity);
+                continue;
+            }
+            createCorrectEntity(manualEntity, node, optionalRedactionEntity.get().getTextRange());
+        }
+        tempEntitiesByValue.values().stream().flatMap(Collection::stream).forEach(TextEntity::removeFromGraph);
+        return notFoundManualEntities;
+    }
+
+
+    /**
+     * Deletes the temp Entity and creates a RedactionEntity with correct values, based on the given parameters.
+     *
+     * @param manualEntity The entity identifier for the RedactionEntity.
+     * @param node             The SemanticNode associated with the RedactionEntity.
+     * @param closestTextRange  The closest Boundary to the RedactionEntity.
+     */
+    private void createCorrectEntity(ManualEntity manualEntity, SemanticNode node, TextRange closestTextRange) {
+
+        TextEntity correctEntity = entityCreationService.forceByBoundary(closestTextRange, manualEntity.getType(), manualEntity.getEntityType(), node);
+
+        if (manualEntity.isApplied()) {
+            correctEntity.apply(manualEntity.getRuleIdentifier(), manualEntity.getReason(), manualEntity.getLegalBasis());
+        } else {
+            correctEntity.skip(manualEntity.getRuleIdentifier(), manualEntity.getReason());
+        }
+        correctEntity.setDictionaryEntry(manualEntity.isDictionaryEntry());
+        correctEntity.setDossierDictionaryEntry(manualEntity.isDossierDictionaryEntry());
+        correctEntity.getManualOverwrite().addChanges(manualEntity.getManualOverwrite().getManualChangeLog());
+
+
+        // AnnotationIds must match the IDs in the add requests, or comments break. Maybe think about migrating IDs on the fly!
+        List<PositionOnPage> redactionPositionsWithIdOfManualOnPage = new ArrayList<>(correctEntity.getPositionsOnPagePerPage().size());
+        for (PositionOnPage positionOnPage : correctEntity.getPositionsOnPagePerPage()) {
+            redactionPositionsWithIdOfManualOnPage.add(new PositionOnPage(manualEntity.getId(), positionOnPage.getPage(), positionOnPage.getRectanglePerLine()));
+        }
+        correctEntity.setPositionsOnPagePerPage(redactionPositionsWithIdOfManualOnPage);
+    }
+
+
+    private Optional<TextEntity> findClosestEntityAndReturnEmptyIfNotFound(ManualEntity identifier, Map<String, List<TextEntity>> entitiesWithSameValue) {
+
+        List<TextEntity> possibleEntities = entitiesWithSameValue.get(identifier.getValue().toLowerCase(Locale.ROOT));
+
+        if (entityIdentifierValueNotFound(possibleEntities)) {
+            log.warn("Entity could not be created with identifier: {}, due to the value {} not being found anywhere.", identifier, identifier.getValue());
+            return Optional.empty();
+        }
+
+        Optional<TextEntity> optionalClosestEntity = possibleEntities.stream()
+                .filter(entity -> pagesMatch(entity, identifier.getEntityPosition()))
+                .min(Comparator.comparingDouble(entity -> calculateMinDistance(identifier.getEntityPosition(), entity)));
+
+        if (optionalClosestEntity.isEmpty()) {
+            log.warn("No Entity with value {} found on page {}", identifier.getValue(), identifier.getEntityPosition());
+            return Optional.empty();
+        }
+
+        TextEntity closestEntity = optionalClosestEntity.get();
+        double distance = calculateMinDistance(identifier.getEntityPosition(), closestEntity);
+        if (distance > MATCH_THRESHOLD) {
+            log.warn(format("Distance to closest found entity is %.2f and therefore higher than the threshold of %.2f for \n%s \n%s",
+                    distance,
+                    MATCH_THRESHOLD,
+                    identifier.getEntityPosition(),
+                    closestEntity.getPositionsOnPagePerPage()));
+            return Optional.empty();
+        }
+
+        return Optional.of(closestEntity);
+    }
+
+
+    private static boolean entityIdentifierValueNotFound(List<TextEntity> possibleEntities) {
+
+        return possibleEntities == null || possibleEntities.isEmpty();
+    }
+
+
+    private Map<String, List<TextEntity>> findAllPossibleEntitiesAndGroupByValue(SemanticNode node, Set<Integer> pageNumbers, Set<String> entryValues) {
+
+        if (!pageNumbers.stream().allMatch(node::onPage)) {
+            throw new IllegalArgumentException(format("SemanticNode \"%s\" does not contain these pages %s, it has pages: %s",
+                    node,
+                    pageNumbers.stream().filter(pageNumber -> !node.onPage(pageNumber)).toList(),
+                    node.getPages()));
+        }
+        SearchImplementation searchImplementation = new SearchImplementation(entryValues, true);
+
+        return searchImplementation.getBoundaries(node.getTextBlock(), node.getTextRange())
+                .stream()
+                .map(boundary -> entityCreationService.forceByBoundary(boundary, "temp", EntityType.ENTITY, node))
+                .collect(groupingBy(entity -> entity.getValue().toLowerCase(Locale.ROOT)));
+    }
+
+
+    private static boolean allValuesFound(Map<String, List<TextEntity>> entitiesByValue, Set<String> entryValues) {
+
+        return entitiesByValue.keySet().equals(entryValues);
+    }
+
+
+    private static boolean pagesMatch(TextEntity entity, List<RectangleWithPage> originalPositions) {
+
+        Set<Integer> entityPageNumbers = entity.getPositionsOnPagePerPage().stream().map(PositionOnPage::getPage).map(Page::getNumber).collect(Collectors.toSet());
+        Set<Integer> originalPageNumbers = originalPositions.stream().map(RectangleWithPage::pageNumber).collect(Collectors.toSet());
+        return entityPageNumbers.containsAll(originalPageNumbers);
+    }
+
+
+    private double calculateMinDistance(List<RectangleWithPage> originalPositions, TextEntity entity) {
+
+        if (originalPositions.size() != countRectangles(entity)) {
+            return Double.MAX_VALUE;
+        }
+        return originalPositions.stream()
+                .mapToDouble(rectangleWithPage -> calculateMinDistancePerRectangle(entity, rectangleWithPage.pageNumber(), rectangleWithPage.rectangle2D()))
+                .sum();
+    }
+
+
+    private static long countRectangles(TextEntity entity) {
+
+        return entity.getPositionsOnPagePerPage().stream().mapToLong(redactionPosition -> redactionPosition.getRectanglePerLine().size()).sum();
+    }
+
+
+    private double calculateMinDistancePerRectangle(TextEntity entity, int pageNumber, Rectangle2D originalRectangle) {
+
+        return entity.getPositionsOnPagePerPage()
+                .stream()
+                .filter(redactionPosition -> redactionPosition.getPage().getNumber() == pageNumber)
+                .map(PositionOnPage::getRectanglePerLine)
+                .flatMap(Collection::stream)
+                .mapToDouble(rectangle -> calculateDistance(rectangle, originalRectangle))
+                .min()
+                .orElse(Double.MAX_VALUE);
+    }
+
+
+    private double calculateDistance(Rectangle2D rectangle, Rectangle2D rectangle2D) {
+
+        // mirrored coordinates safe comparison
+        double minX1 = Math.min(rectangle.getMinX(), rectangle.getMaxX());
+        double minY1 = Math.min(rectangle.getMinY(), rectangle.getMaxY());
+        double maxX1 = Math.max(rectangle.getMinX(), rectangle.getMaxX());
+        double maxY1 = Math.max(rectangle.getMinY(), rectangle.getMaxY());
+
+        double minX2 = Math.min(rectangle2D.getMinX(), rectangle2D.getMaxX());
+        double minY2 = Math.min(rectangle2D.getMinY(), rectangle2D.getMaxY());
+        double maxX2 = Math.max(rectangle2D.getMinX(), rectangle2D.getMaxX());
+        double maxY2 = Math.max(rectangle2D.getMinY(), rectangle2D.getMaxY());
+
+        return Math.abs(minX1 - minX2) //
+                + Math.abs(minY1 - minY2) //
+                + Math.abs(maxX1 - maxX2) //
+                + Math.abs(maxY1 - maxY2);
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/ManualRedactionEntryService.java

@@ -11,9 +11,8 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequ
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.BaseAnnotation;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
-import com.iqser.red.service.redaction.v1.server.model.PrecursorEntity;
+import com.iqser.red.service.redaction.v1.server.model.ManualEntity;
 
-import io.micrometer.observation.annotation.Observed;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
 
@@ -22,24 +21,22 @@ import lombok.extern.slf4j.Slf4j;
 @RequiredArgsConstructor
 public class ManualRedactionEntryService {
 
-    private final EntityFromPrecursorCreationService entityFromPrecursorCreationService;
+    private final ManualEntityCreationService manualEntityCreationService;
 
 
-    @Observed(name = "ManualRedactionEntryService", contextualName = "add-manual-redaction-entries")
-    public List<PrecursorEntity> addManualRedactionEntriesAndReturnNotFoundEntries(AnalyzeRequest analyzeRequest, Document document, String dossierTemplateId) {
+    public List<ManualEntity> addManualRedactionEntriesAndReturnNotFoundEntries(AnalyzeRequest analyzeRequest, Document document) {
 
-        List<PrecursorEntity> notFoundManualRedactionEntries = Collections.emptyList();
+        List<ManualEntity> notFoundManualRedactionEntries = Collections.emptyList();
         if (analyzeRequest.getManualRedactions() != null) {
-            notFoundManualRedactionEntries = entityFromPrecursorCreationService.createEntitiesIfFoundAndReturnNotFoundEntries(analyzeRequest.getManualRedactions(),
-                    document, dossierTemplateId);
+            notFoundManualRedactionEntries = manualEntityCreationService.createRedactionEntitiesIfFoundAndReturnNotFoundEntries(analyzeRequest.getManualRedactions()
+                    .getEntriesToAdd(), document);
             log.info("Added Manual redaction entries for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
         }
-
         if (notFoundManualRedactionEntries.isEmpty()) {
             return Collections.emptyList();
         }
         List<BaseAnnotation> manualChanges = allManualChangesExceptAdd(analyzeRequest.getManualRedactions());
-        for (PrecursorEntity notFoundManualRedactionEntry : notFoundManualRedactionEntries) {
+        for (ManualEntity notFoundManualRedactionEntry : notFoundManualRedactionEntries) {
             manualChanges.stream()
                     .filter(change -> change.getAnnotationId().equals(notFoundManualRedactionEntry.getId()))
                     .forEach(change -> notFoundManualRedactionEntry.getManualOverwrite().addChange(change));

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/NodeVisitor.java

@@ -1,8 +0,0 @@
-package com.iqser.red.service.redaction.v1.server.service.document;
-
-import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
-
-public interface NodeVisitor {
-
-    void visit(SemanticNode node);
-}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/SectionFinderService.java

@@ -13,9 +13,6 @@ import org.springframework.stereotype.Service;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedaction;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedactions;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;
@@ -43,40 +40,27 @@ import lombok.extern.slf4j.Slf4j;
 public class SectionFinderService {
 
     @Timed("redactmanager_findSectionsToReanalyse")
-    public Set<Integer> findSectionsToReanalyse(DictionaryIncrement dictionaryIncrement,
-                                                EntityLog entityLog,
-                                                Document document,
-                                                AnalyzeRequest analyzeRequest,
-                                                ImportedRedactions importedRedactions) {
+    public Set<Integer> findSectionsToReanalyse(DictionaryIncrement dictionaryIncrement, EntityLog entityLog, Document document, AnalyzeRequest analyzeRequest) {
 
         long start = System.currentTimeMillis();
         Set<String> relevantManuallyModifiedAnnotationIds = getRelevantManuallyModifiedAnnotationIds(analyzeRequest.getManualRedactions());
         Set<Integer> sectionsToReanalyse = new HashSet<>();
         for (EntityLogEntry entry : entityLog.getEntityLogEntry()) {
             if (relevantManuallyModifiedAnnotationIds.contains(entry.getId())) {
-                if (entry.getContainingNodeId().isEmpty()) {
-                    continue; // Empty list means either Entity has not been found or it is between main sections. Thus, this might lead to wrong reanalysis.
-                }
-                sectionsToReanalyse.add(entry.getContainingNodeId()
-                                                .get(0));
+                sectionsToReanalyse.add(entry.getSectionNumber());
             }
         }
 
-        var dictionaryIncrementsSearch = new SearchImplementation(dictionaryIncrement.getValues()
-                                                                          .stream()
-                                                                          .map(DictionaryIncrementValue::getValue)
-                                                                          .toList(), true);
+        var dictionaryIncrementsSearch = new SearchImplementation(dictionaryIncrement.getValues().stream().map(DictionaryIncrementValue::getValue).collect(Collectors.toList()),
+                true);
 
-        document.streamChildren()
-                .forEach(mainNode -> {
-                    if (dictionaryIncrementsSearch.atLeastOneMatches(mainNode.getTextBlock().getSearchText())) {
-                        sectionsToReanalyse.add(mainNode.getTreeId()
-                                                        .get(0));
-                    }
-                });
+        document.streamChildren().forEach(mainNode -> {
+            if (dictionaryIncrementsSearch.atLeastOneMatches(mainNode.getTextBlock().getSearchText())) {
+                sectionsToReanalyse.add(mainNode.getTreeId().get(0));
+            }
+        });
 
         Set<Integer> relevantPagesForReanalysis = getRelevantPageNumbersForAddRedactions(analyzeRequest);
-        relevantPagesForReanalysis.addAll(getRelevantPageNumbersForImportedRedactions(importedRedactions, relevantManuallyModifiedAnnotationIds));
         if (!relevantPagesForReanalysis.isEmpty()) {
             sectionsToReanalyse.addAll(getSectionNumbersOnPages(document, relevantPagesForReanalysis));
         }
@@ -92,11 +76,8 @@ public class SectionFinderService {
         return document.getPages()
                 .stream()
                 .filter(page -> relevantPagesForReanalysis.contains(page.getNumber()))
-                .flatMap(page -> Stream.concat(page.getMainBody()
-                                                       .stream()
-                                                       .filter(node -> node.getType().equals(NodeType.SECTION)), Stream.of(page.getHeader(), page.getFooter())))
-                .map(node -> node.getTreeId()
-                        .get(0))
+                .flatMap(page -> Stream.concat(page.getMainBody().stream().filter(node -> node.getType().equals(NodeType.SECTION)), Stream.of(page.getHeader(), page.getFooter())))
+                .map(node -> node.getTreeId().get(0))
                 .toList();
     }
 
@@ -107,7 +88,8 @@ public class SectionFinderService {
             return Collections.emptySet();
         }
 
-        return analyzeRequest.getManualRedactions().getEntriesToAdd()
+        return analyzeRequest.getManualRedactions()
+                .getEntriesToAdd()
                 .stream()
                 .filter(addRedaction -> !(addRedaction.isAddToDictionary() || addRedaction.isAddToDossierDictionary()))
                 .map(ManualRedactionEntry::getPositions)
@@ -117,44 +99,17 @@ public class SectionFinderService {
     }
 
 
-    private static Set<Integer> getRelevantPageNumbersForImportedRedactions(ImportedRedactions importedRedactions, Set<String> idsWithUnprocessedManualChanges) {
-
-        if (importedRedactions == null || importedRedactions.getImportedRedactions() == null) {
-            return Collections.emptySet();
-        }
-
-        return importedRedactions.getImportedRedactions()
-                .stream()
-                .filter(importedRedaction -> idsWithUnprocessedManualChanges.contains(importedRedaction.getId()))
-                .map(ImportedRedaction::getPositions)
-                .flatMap(Collection::stream)
-                .map(Position::getPageNumber)
-                .collect(Collectors.toSet());
-    }
-
-
     private static Set<String> getRelevantManuallyModifiedAnnotationIds(ManualRedactions manualRedactions) {
 
         if (manualRedactions == null) {
             return new HashSet<>();
         }
 
-        return Stream.concat(manualRedactions.getResizeRedactions()
-                                     .stream()
-                                     .map(ManualResizeRedaction::getAnnotationId),
-                             Stream.concat(manualRedactions.getLegalBasisChanges()
-                                                   .stream()
-                                                   .map(ManualLegalBasisChange::getAnnotationId),
-                                           Stream.concat(manualRedactions.getRecategorizations()
-                                                                 .stream()
-                                                                 .map(ManualRecategorization::getAnnotationId),
-                                                         Stream.concat(manualRedactions.getIdsToRemove()
-                                                                               .stream()
-                                                                               .map(IdRemoval::getAnnotationId),
-                                                                       manualRedactions.getForceRedactions()
-                                                                               .stream()
-                                                                               .map(ManualForceRedaction::getAnnotationId)))))
-                .collect(Collectors.toSet());
+        return Stream.concat(manualRedactions.getResizeRedactions().stream().map(ManualResizeRedaction::getAnnotationId),
+                Stream.concat(manualRedactions.getLegalBasisChanges().stream().map(ManualLegalBasisChange::getAnnotationId),
+                        Stream.concat(manualRedactions.getRecategorizations().stream().map(ManualRecategorization::getAnnotationId),
+                                Stream.concat(manualRedactions.getIdsToRemove().stream().map(IdRemoval::getAnnotationId),
+                                        manualRedactions.getForceRedactions().stream().map(ManualForceRedaction::getAnnotationId))))).collect(Collectors.toSet());
     }
 
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/SemanticNodeComparators.java

@@ -1,46 +0,0 @@
-package com.iqser.red.service.redaction.v1.server.service.document;
-
-import java.util.Comparator;
-import java.util.List;
-
-import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
-
-public abstract class SemanticNodeComparators implements Comparator<SemanticNode> {
-
-    public static SemanticNodeComparators first() {
-
-        return new FirstSemanticNode();
-    }
-
-    public static class FirstSemanticNode extends SemanticNodeComparators {
-
-        @Override
-        public int compare(SemanticNode semanticNode, SemanticNode otherSemanticNode) {
-
-            List<Integer> treeId = semanticNode.getTreeId();
-            List<Integer> otherTreeId = otherSemanticNode.getTreeId();
-
-            int prefixLength = Math.min(treeId.size(), otherTreeId.size());
-            // Compare ids one by one
-            for (int i = 0; i < prefixLength; i++) {
-                int id1 = treeId.get(i);
-                int id2 = otherTreeId.get(i);
-
-                // If ids are different, return the comparison result
-                if (id1 != id2) {
-                    return Integer.compare(id1, id2);
-                }
-            }
-
-            // If all prefix ids are equal, shorter treeId is first
-            if (treeId.size() != otherTreeId.size()) {
-                return Integer.compare(treeId.size(), otherTreeId.size());
-            }
-
-            // Lists are equal
-            return 0;
-        }
-
-    }
-
-}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/ComponentDroolsExecutionService.java

@@ -16,12 +16,9 @@ import org.springframework.stereotype.Service;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState;
 import com.iqser.red.service.redaction.v1.server.RedactionServiceSettings;
 import com.iqser.red.service.redaction.v1.server.model.component.Component;
 import com.iqser.red.service.redaction.v1.server.model.component.Entity;
-import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
-import com.iqser.red.service.redaction.v1.server.service.document.ComponentComparator;
 import com.iqser.red.service.redaction.v1.server.service.document.ComponentCreationService;
 import com.iqser.red.service.redaction.v1.server.utils.exception.DroolsTimeoutException;
 
@@ -39,17 +36,13 @@ public class ComponentDroolsExecutionService {
     RedactionServiceSettings settings;
 
 
-    public List<Component> executeRules(KieContainer kieContainer, EntityLog entityLog, Document document, List<FileAttribute> fileAttributes) {
+    public List<Component> executeRules(KieContainer kieContainer, EntityLog entityLog, List<FileAttribute> fileAttributes) {
 
         KieSession kieSession = kieContainer.newKieSession();
         ComponentCreationService componentCreationService = new ComponentCreationService(kieSession);
 
         kieSession.setGlobal("componentCreationService", componentCreationService);
-        entityLog.getEntityLogEntry()
-                .stream()
-                .filter(entityLogEntry -> entityLogEntry.getState().equals(EntryState.APPLIED))
-                .map(entry -> Entity.fromEntityLogEntry(entry, document))
-                .forEach(kieSession::insert);
+        entityLog.getEntityLogEntry().stream().map(Entity::fromEntityLogEntry).forEach(kieSession::insert);
         fileAttributes.stream().filter(f -> f.getValue() != null).forEach(kieSession::insert);
 
         CompletableFuture<Void> completableFuture = CompletableFuture.supplyAsync(() -> {
@@ -66,12 +59,11 @@ public class ComponentDroolsExecutionService {
             }
             throw new RuntimeException(e);
         } catch (InterruptedException e) {
-            kieSession.dispose();
             throw new RuntimeException(e);
         }
 
         List<FileAttribute> resultingFileAttributes = getFileAttributes(kieSession);
-        List<Component> components = getComponents(kieSession).stream().sorted(ComponentComparator.first()).toList();
+        List<Component> components = getComponents(kieSession);
         kieSession.dispose();
         return components;
     }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/DroolsSyntaxValidationService.java

@@ -1,198 +1,91 @@
 package com.iqser.red.service.redaction.v1.server.service.drools;
 
-import java.util.ArrayList;
-import java.util.Comparator;
 import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.regex.Pattern;
 import java.util.stream.Collectors;
 
-import org.drools.drl.parser.DroolsParserException;
 import org.kie.api.builder.KieBuilder;
 import org.kie.api.builder.Message;
 import org.springframework.stereotype.Service;
 
-import com.google.common.collect.Sets;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType;
-import com.iqser.red.service.redaction.v1.model.DroolsSyntaxDeprecatedWarnings;
 import com.iqser.red.service.redaction.v1.model.DroolsSyntaxErrorMessage;
 import com.iqser.red.service.redaction.v1.model.DroolsSyntaxValidation;
 import com.iqser.red.service.redaction.v1.model.RuleValidationModel;
-import com.iqser.red.service.redaction.v1.server.DeprecatedElementsFinder;
-import com.iqser.red.service.redaction.v1.server.model.dictionary.SearchImplementation;
-import com.iqser.red.service.redaction.v1.server.model.drools.BasicQuery;
 import com.iqser.red.service.redaction.v1.server.model.drools.BasicRule;
-import com.iqser.red.service.redaction.v1.server.model.drools.RuleClass;
 import com.iqser.red.service.redaction.v1.server.model.drools.RuleFileBluePrint;
-import com.iqser.red.service.redaction.v1.server.model.drools.RuleUnit;
 import com.iqser.red.service.redaction.v1.server.storage.RuleManagementResources;
 
 import lombok.RequiredArgsConstructor;
 import lombok.SneakyThrows;
-import lombok.extern.slf4j.Slf4j;
 
 @Service
 @RequiredArgsConstructor
-@Slf4j
 public class DroolsSyntaxValidationService {
 
-    private static final Pattern allowedImportsPattern = Pattern.compile("^(?:import\\s+static\\s+)?(?:import\\s+)?(?:com\\.knecon\\.fforesight|com\\.iqser\\.red)\\..*;$");
-
     private final KieContainerCreationService kieContainerCreationService;
 
-    private final DeprecatedElementsFinder deprecatedElementsFinder;
-
 
     @SneakyThrows
     public DroolsSyntaxValidation testRules(RuleValidationModel rules) {
 
-        DroolsSyntaxValidation customDroolsSyntaxValidation;
-        try {
-            customDroolsSyntaxValidation = buildCustomDroolsSyntaxValidation(rules.getRulesString(), RuleFileType.valueOf(rules.getRuleFileType()));
-        } catch (DroolsParserException e) {
-            // this means the parser could not parse the file at all. In this case use drools compiler only as it will return useful error messages.
-            customDroolsSyntaxValidation = new DroolsSyntaxValidation();
-        }
+        DroolsSyntaxValidation customDroolsSyntaxValidation = buildCustomDroolsSyntaxValidation(rules.getRulesString(), RuleFileType.valueOf(rules.getRuleFileType()));
         DroolsSyntaxValidation droolsCompilerSyntaxValidation = buildDroolsCompilerSyntaxValidation(rules);
         droolsCompilerSyntaxValidation.getDroolsSyntaxErrorMessages().addAll(customDroolsSyntaxValidation.getDroolsSyntaxErrorMessages());
-        droolsCompilerSyntaxValidation.getDroolsSyntaxDeprecatedWarnings().addAll(customDroolsSyntaxValidation.getDroolsSyntaxDeprecatedWarnings());
         return droolsCompilerSyntaxValidation;
     }
 
-    private DroolsSyntaxDeprecatedWarnings getWarningsForDeprecatedImports(RuleFileBluePrint ruleFileBluePrint) {
 
-        if (!deprecatedElementsFinder.getDeprecatedClasses().isEmpty()) {
-            String imports = ruleFileBluePrint.getImports();
-            SearchImplementation classesSearchImplementation = deprecatedElementsFinder.getClassesSearchImplementation();
-            List<SearchImplementation.MatchPosition> matches = classesSearchImplementation.getMatches(imports);
-            if (!matches.isEmpty()) {
-                String sb = "Following imports are deprecated: \n" + matches.stream()
-                        .map(m -> imports.substring(m.startIndex(), m.endIndex()))
-                        .collect(Collectors.joining("\n"));
-                return DroolsSyntaxDeprecatedWarnings.builder().line(ruleFileBluePrint.getImportLine()).column(0).message(sb)
-                                            .build();
-            }
-        }
-        return null;
-    }
-
-    private List<DroolsSyntaxDeprecatedWarnings> getWarningsForDeprecatedRules(RuleFileBluePrint ruleFileBluePrint) {
-
-        List<DroolsSyntaxDeprecatedWarnings> warningMessages = new ArrayList<>();
-
-        // checks the rules for deprecateMethods
-        if (!deprecatedElementsFinder.getDeprecatedMethods().isEmpty()) {
-            SearchImplementation methodsSearchImplementation = deprecatedElementsFinder.getMethodsSearchImplementation();
-            Map<String, String> deprecatedMethodsSignatureMap = deprecatedElementsFinder.getDeprecatedMethodsSignaturesMap();
-
-            for (RuleClass ruleClass : ruleFileBluePrint.getRuleClasses()) {
-                for (RuleUnit ruleUnit : ruleClass.ruleUnits()) {
-                    for (BasicRule basicRule : ruleUnit.rules()) {
-                        List<SearchImplementation.MatchPosition> matches = methodsSearchImplementation.getMatches(basicRule.getCode());
-                        if (!matches.isEmpty()) {
-                            String warningMessage = matches.stream()
-                                    .map(m -> basicRule.getCode().substring(m.startIndex(), m.endIndex()))
-                                    .distinct()
-                                    .map(dm -> String.format("Method %s might be deprecated because of \n %s \n", dm, deprecatedMethodsSignatureMap.get(dm)))
-                                    .collect(Collectors.joining("\n"));
-                            warningMessages.add(DroolsSyntaxDeprecatedWarnings.builder().line(basicRule.getLine()).column(0).message(warningMessage)
-                                                        .build());
-                        }
-                    }
-                }
-            }
-
-        }
-
-        return warningMessages.stream()
-                .sorted(Comparator.comparingInt(DroolsSyntaxDeprecatedWarnings::getLine))
-                .collect(Collectors.toList());
-    }
-
-
-    private DroolsSyntaxValidation buildCustomDroolsSyntaxValidation(String ruleString, RuleFileType ruleFileType) throws DroolsParserException {
+    private DroolsSyntaxValidation buildCustomDroolsSyntaxValidation(String ruleString, RuleFileType ruleFileType) {
 
         RuleFileBluePrint ruleFileBluePrint = RuleFileParser.buildBluePrintFromRulesString(ruleString);
-
         DroolsSyntaxValidation customSyntaxValidation = ruleFileBluePrint.getDroolsSyntaxValidation();
 
-        // find deprecated elements in the ruleFileBluePrint
-        DroolsSyntaxDeprecatedWarnings warningMessageForImports = getWarningsForDeprecatedImports(ruleFileBluePrint);
-        if (warningMessageForImports != null) {
-            customSyntaxValidation.getDroolsSyntaxDeprecatedWarnings().add(warningMessageForImports);
-        }
-        customSyntaxValidation.getDroolsSyntaxDeprecatedWarnings().addAll(getWarningsForDeprecatedRules(ruleFileBluePrint));
-
         RuleFileBluePrint baseRuleFileBluePrint = switch (ruleFileType) {
-            case ENTITY -> RuleFileParser.buildBluePrintFromRulesString(RuleManagementResources.getBaseRuleFileString());
-            case COMPONENT -> RuleFileParser.buildBluePrintFromRulesString(RuleManagementResources.getBaseComponentRuleFileString());
+            case ENTITY -> RuleFileParser.buildBluePrintFromRulesString(RuleManagementResources.getBaseRuleFileInputStreamString());
+            case COMPONENT -> RuleFileParser.buildBluePrintFromRulesString(RuleManagementResources.getBaseCopmonentRuleFileInputStreamString());
         };
 
-        if (!importsAreValid(baseRuleFileBluePrint, ruleFileBluePrint)) {
+        if (!ruleFileBluePrint.getImports().equals(baseRuleFileBluePrint.getImports())) {
             customSyntaxValidation.getDroolsSyntaxErrorMessages()
                     .add(DroolsSyntaxErrorMessage.builder()
-                                 .line(ruleFileBluePrint.getImportLine())
-                                 .column(0)
-                                 .message(String.format("Changing the imports is not allowed! Must be: %n%s", baseRuleFileBluePrint.getImports()))
-                                 .build());
+                            .line(ruleFileBluePrint.getImportLine())
+                            .column(0)
+                            .message(String.format("Changing the imports is not allowed! Must be: %n%s", baseRuleFileBluePrint.getImports()))
+                            .build());
+        }
+        if (!ruleFileBluePrint.getQueries().equals(baseRuleFileBluePrint.getQueries())) {
+            customSyntaxValidation.getDroolsSyntaxErrorMessages()
+                    .add(DroolsSyntaxErrorMessage.builder()
+                            .line(ruleFileBluePrint.getQueriesLine())
+                            .column(0)
+                            .message(String.format("Changing the queries is not allowed! Must be: %n%s", baseRuleFileBluePrint.getQueries()))
+                            .build());
         }
         if (!ruleFileBluePrint.getGlobals().equals(baseRuleFileBluePrint.getGlobals())) {
             customSyntaxValidation.getDroolsSyntaxErrorMessages()
                     .add(DroolsSyntaxErrorMessage.builder()
-                                 .line(ruleFileBluePrint.getGlobalsLine())
-                                 .column(0)
-                                 .message(String.format("Changing the globals is not allowed! Must be: %n%s", baseRuleFileBluePrint.getGlobals()))
-                                 .build());
+                            .line(ruleFileBluePrint.getGlobalsLine())
+                            .column(0)
+                            .message(String.format("Changing the globals is not allowed! Must be: %n%s", baseRuleFileBluePrint.getGlobals()))
+                            .build());
         }
-        baseRuleFileBluePrint.getQueries()
-                .forEach(basicQuery -> {
-                    if (!validateQueryIsPresent(basicQuery, ruleFileBluePrint)) {
-                        customSyntaxValidation.getDroolsSyntaxErrorMessages()
-                                .add(DroolsSyntaxErrorMessage.builder()
-                                             .line(basicQuery.getLine())
-                                             .column(0)
-                                             .message(String.format("Changing or removing the query %s is not allowed! Must be: %n%s", basicQuery.getName(), basicQuery.getCode()))
-                                             .build());
-                    }
-                });
-        if (ruleFileType.equals(RuleFileType.ENTITY)) {
-            String requiredAgendaGroup = "LOCAL_DICTIONARY_ADDS";
-            if (!validateAgendaGroupIsPresent(ruleFileBluePrint, requiredAgendaGroup)) {
+        baseRuleFileBluePrint.streamAllRules().forEach(basicRule -> {
+            if (!validateBaseRulesArePresent(basicRule, ruleFileBluePrint)) {
                 customSyntaxValidation.getDroolsSyntaxErrorMessages()
-                        .add(DroolsSyntaxErrorMessage.builder().line(0).column(0).message(String.format("At least one rule with Agenda-Group '%s' required!", requiredAgendaGroup))
-                                     .build());
+                        .add(DroolsSyntaxErrorMessage.builder()
+                                .line(basicRule.getLine())
+                                .column(0)
+                                .message(String.format("Changing or removing the rule %s is not allowed! Must be: %n%s", basicRule.getName(), basicRule.getCode()))
+                                .build());
             }
-        }
+        });
         return customSyntaxValidation;
     }
 
 
-    private boolean validateAgendaGroupIsPresent(RuleFileBluePrint ruleFileBluePrint, String agendaGroupName) {
+    private static boolean validateBaseRulesArePresent(BasicRule basicRule, RuleFileBluePrint ruleFileBluePrint) {
 
-        return ruleFileBluePrint.streamAllRules()
-                .anyMatch(basicRule -> basicRule.getAgendaGroup().equals(agendaGroupName));
-    }
-
-
-    private boolean importsAreValid(RuleFileBluePrint baseRuleFileBluePrint, RuleFileBluePrint ruleFileBluePrint) {
-
-        // imports may shrink, but not add anything new!
-        Set<String> baseImports = baseRuleFileBluePrint.getImportSplitByKeyword();
-        Set<String> imports = ruleFileBluePrint.getImportSplitByKeyword();
-
-        Set<String> additionalImports = Sets.difference(imports, baseImports);
-
-        return additionalImports.stream()
-                .allMatch(additionalImport -> allowedImportsPattern.matcher(additionalImport).matches());
-    }
-
-
-    private static boolean validateQueryIsPresent(BasicQuery queryToCheckFor, RuleFileBluePrint ruleFileBluePrint) {
-
-        return ruleFileBluePrint.getQueries()
-                .stream()
-                .anyMatch(query -> query.getName().equals(queryToCheckFor.getName()) && query.getCode().equals(queryToCheckFor.getCode()));
+        return ruleFileBluePrint.findRulesByIdentifier(basicRule.getIdentifier()).stream().allMatch(otherRule -> otherRule.getCode().equals(basicRule.getCode()));
     }
 
 
@@ -201,28 +94,24 @@ public class DroolsSyntaxValidationService {
         var versionId = System.currentTimeMillis();
         var testRules = "test-rules";
         KieBuilder kieBuilder = kieContainerCreationService.registerNewKieContainerVersion(testRules,
-                                                                                           versionId,
-                                                                                           rules.getRulesString(),
-                                                                                           RuleFileType.valueOf(rules.getRuleFileType()));
+                versionId,
+                rules.getRulesString(), RuleFileType.valueOf(rules.getRuleFileType()));
         return buildDroolsCompilerSyntaxValidation(kieBuilder);
     }
 
 
+
     private DroolsSyntaxValidation buildDroolsCompilerSyntaxValidation(KieBuilder kieBuilder) {
 
         List<Message> errorMessages = kieBuilder.getResults().getMessages(Message.Level.ERROR);
-        List<DroolsSyntaxErrorMessage> droolsSyntaxErrorMessages = errorMessages.stream()
-                .map(this::buildDroolsSyntaxErrorMessage)
-                .collect(Collectors.toList());
-        return DroolsSyntaxValidation.builder().droolsSyntaxErrorMessages(droolsSyntaxErrorMessages)
-                .build();
+        List<DroolsSyntaxErrorMessage> droolsSyntaxErrorMessages = errorMessages.stream().map(this::buildDroolsSyntaxErrorMessage).collect(Collectors.toList());
+        return DroolsSyntaxValidation.builder().droolsSyntaxErrorMessages(droolsSyntaxErrorMessages).build();
     }
 
 
     private DroolsSyntaxErrorMessage buildDroolsSyntaxErrorMessage(Message message) {
 
-        return DroolsSyntaxErrorMessage.builder().line(message.getLine()).column(message.getColumn()).message(message.getText())
-                .build();
+        return DroolsSyntaxErrorMessage.builder().line(message.getLine()).column(message.getColumn()).message(message.getText()).build();
     }
 
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/EntityDroolsExecutionService.java

@@ -16,8 +16,8 @@ import org.springframework.stereotype.Service;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.BaseAnnotation;
 import com.iqser.red.service.redaction.v1.server.RedactionServiceSettings;
+import com.iqser.red.service.redaction.v1.server.utils.exception.DroolsTimeoutException;
 import com.iqser.red.service.redaction.v1.server.model.NerEntities;
 import com.iqser.red.service.redaction.v1.server.model.dictionary.Dictionary;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
@@ -25,11 +25,8 @@ import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNo
 import com.iqser.red.service.redaction.v1.server.service.ManualChangesApplicationService;
 import com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService;
 import com.iqser.red.service.redaction.v1.server.service.document.EntityEnrichmentService;
-import com.iqser.red.service.redaction.v1.server.utils.exception.DroolsTimeoutException;
 
 import io.micrometer.core.annotation.Timed;
-import io.micrometer.observation.ObservationRegistry;
-import io.micrometer.observation.annotation.Observed;
 import lombok.AccessLevel;
 import lombok.RequiredArgsConstructor;
 import lombok.experimental.FieldDefaults;
@@ -42,13 +39,11 @@ import lombok.extern.slf4j.Slf4j;
 public class EntityDroolsExecutionService {
 
     EntityEnrichmentService entityEnrichmentService;
-    ObservationRegistry observationRegistry;
-    ManualChangesApplicationService manualChangesApplicationService;
+
     RedactionServiceSettings settings;
 
 
     @Timed("redactmanager_executeRules")
-    @Observed(name = "EntityDroolsExecutionService", contextualName = "execute-entity-rules")
     public List<FileAttribute> executeRules(KieContainer kieContainer,
                                             Document document,
                                             Dictionary dictionary,
@@ -56,19 +51,11 @@ public class EntityDroolsExecutionService {
                                             ManualRedactions manualRedactions,
                                             NerEntities nerEntities) {
 
-        return executeRules(kieContainer,
-                            document,
-                            document.streamChildren()
-                                    .toList(),
-                            dictionary,
-                            fileAttributes,
-                            manualRedactions,
-                            nerEntities);
+        return executeRules(kieContainer, document, document.streamChildren().toList(), dictionary, fileAttributes, manualRedactions, nerEntities);
     }
 
 
     @Timed("redactmanager_executeRules")
-    @Observed(name = "EntityDroolsExecutionService", contextualName = "execute-entity-rules")
     public List<FileAttribute> executeRules(KieContainer kieContainer,
                                             Document document,
                                             List<SemanticNode> sectionsToAnalyze,
@@ -77,10 +64,9 @@ public class EntityDroolsExecutionService {
                                             ManualRedactions manualRedactions,
                                             NerEntities nerEntities) {
 
-        addNumberOfPagesAndSectionsToAnalyseToTrace(document.getNumberOfPages(), sectionsToAnalyze.size());
-
         KieSession kieSession = kieContainer.newKieSession();
         EntityCreationService entityCreationService = new EntityCreationService(entityEnrichmentService, kieSession);
+        ManualChangesApplicationService manualChangesApplicationService = new ManualChangesApplicationService(entityCreationService);
 
         kieSession.setGlobal("document", document);
         kieSession.setGlobal("entityCreationService", entityCreationService);
@@ -88,28 +74,19 @@ public class EntityDroolsExecutionService {
         kieSession.setGlobal("dictionary", dictionary);
 
         kieSession.insert(document);
-
-        document.getEntities()
-                .forEach(kieSession::insert);
-
+        document.getEntities().forEach(kieSession::insert);
         sectionsToAnalyze.forEach(kieSession::insert);
-
-        sectionsToAnalyze.stream()
-                .flatMap(SemanticNode::streamAllSubNodes)
-                .forEach(kieSession::insert);
-
-        document.getPages()
-                .forEach(kieSession::insert);
-
-        fileAttributes.stream()
-                .filter(f -> f.getValue() != null)
-                .forEach(kieSession::insert);
+        sectionsToAnalyze.stream().flatMap(SemanticNode::streamAllSubNodes).forEach(kieSession::insert);
+        document.getPages().forEach(kieSession::insert);
+        fileAttributes.stream().filter(f -> f.getValue() != null).forEach(kieSession::insert);
 
         if (manualRedactions != null) {
-            manualRedactions.buildAll()
-                    .stream()
-                    .filter(BaseAnnotation::isLocal)
-                    .forEach(kieSession::insert);
+            manualRedactions.getResizeRedactions().forEach(kieSession::insert);
+            manualRedactions.getRecategorizations().forEach(kieSession::insert);
+            manualRedactions.getEntriesToAdd().forEach(kieSession::insert);
+            manualRedactions.getForceRedactions().forEach(kieSession::insert);
+            manualRedactions.getIdsToRemove().forEach(kieSession::insert);
+            manualRedactions.getLegalBasisChanges().forEach(kieSession::insert);
         }
 
         kieSession.insert(nerEntities);
@@ -122,16 +99,14 @@ public class EntityDroolsExecutionService {
         });
 
         try {
-            completableFuture.orTimeout(settings.getDroolsExecutionTimeoutSecs(), TimeUnit.SECONDS)
-                    .get();
+            completableFuture.orTimeout(settings.getDroolsExecutionTimeoutSecs(), TimeUnit.SECONDS).get();
         } catch (ExecutionException e) {
             kieSession.dispose();
-            if (e.getCause() instanceof TimeoutException) {
+            if(e.getCause() instanceof TimeoutException){
                 throw new DroolsTimeoutException(e, false, RuleFileType.ENTITY);
             }
             throw new RuntimeException(e);
         } catch (InterruptedException e) {
-            kieSession.dispose();
             throw new RuntimeException(e);
         }
 
@@ -151,13 +126,4 @@ public class EntityDroolsExecutionService {
         return fileAttributes;
     }
 
-
-    private void addNumberOfPagesAndSectionsToAnalyseToTrace(int numberOfPages, int sectionsToAnalyse) {
-
-        if (observationRegistry.getCurrentObservation() != null) {
-            observationRegistry.getCurrentObservation().highCardinalityKeyValue("numberOfPages", String.valueOf(numberOfPages));
-            observationRegistry.getCurrentObservation().highCardinalityKeyValue("sectionsToAnalyse", String.valueOf(sectionsToAnalyse));
-        }
-    }
-
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/KieContainerCreationService.java

@@ -20,7 +20,6 @@ import com.iqser.red.service.redaction.v1.server.utils.exception.DroolsTimeoutEx
 import com.knecon.fforesight.tenantcommons.TenantContext;
 
 import feign.FeignException;
-import io.micrometer.observation.annotation.Observed;
 import lombok.RequiredArgsConstructor;
 
 @Service
@@ -30,25 +29,22 @@ public class KieContainerCreationService {
     private final RulesClient rulesClient;
 
 
-    @Observed(name = "KieContainerCreationService",
-            contextualName = "get-kie-container")
     public KieWrapper getLatestKieContainer(String dossierTemplateId, RuleFileType ruleFileType) {
 
+        long version = -1;
         try {
-            long version = rulesClient.getVersion(dossierTemplateId, ruleFileType);
-            if (version == -1) {
-                return KieWrapper.empty();
-            }
-            return new KieWrapper(getKieContainer(dossierTemplateId, version, ruleFileType), version);
+            version = rulesClient.getVersion(dossierTemplateId, ruleFileType);
         } catch (FeignException fe) {
             if (fe.status() == HttpStatus.UNPROCESSABLE_ENTITY.value()) {
                 throw new DroolsTimeoutException(fe.getCause(), true, ruleFileType);
             }
             if (fe.status() == HttpStatus.NOT_FOUND.value()) {
-                return KieWrapper.empty();
+                return new KieWrapper(null, -1);
             }
-            throw new RuntimeException(fe);
         }
+
+        return new KieWrapper(getKieContainer(dossierTemplateId, version, ruleFileType), version);
+
     }
 
 
@@ -77,7 +73,7 @@ public class KieContainerCreationService {
 
         var rules = rulesClient.getRules(dossierTemplateId, ruleFileType);
         if (rules == null || StringUtils.isEmpty(rules.getValue())) {
-            throw new IllegalArgumentException(ruleFileType.name() + " rules cannot be empty.");
+            throw new IllegalArgumentException("Entity rules cannot be empty.");
         }
 
         registerNewKieContainerVersion(dossierTemplateId, version, rules.getValue(), ruleFileType);