extended-entity-finding: increase threshold

wip: prototype extended EntityFindingUtility
Merge branch 'RED-10194' into 'master'
2024-10-22 15:09:38 +02:00 · 2024-10-22 15:08:51 +02:00 · 2024-10-16 12:04:36 +02:00 · 2024-10-16 11:38:59 +02:00 · 2024-10-15 10:59:35 +02:00 · 2024-10-15 10:59:34 +02:00
371 changed files with 13904 additions and 3817908 deletions
@@ -14,13 +14,17 @@ deploy JavaDoc:
  script:
    - echo "Building JavaDoc with gradle version ${BUILDVERSION}"
    - gradle -Pversion=${BUILDVERSION} publish
+    - echo "BUILDVERSION=$(echo ${BUILDVERSION})" >> variables.env
+  artifacts:
+    reports:
+        dotenv: variables.env
  rules:
    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
    - if: $CI_COMMIT_BRANCH =~ /^release/
    - if: $CI_COMMIT_TAG

-generateJavaDoc:
-  stage: build
+generate JavaDoc:
+  stage: deploy
  tags:
    - dind
  script:
@@ -35,14 +39,39 @@ generateJavaDoc:
    - if: $CI_COMMIT_TAG

 pages:
-  stage: build
+  stage: deploy
  needs:
-    - generateJavaDoc
+    - generate JavaDoc
+    - deploy JavaDoc
+    - calculate minor version
+  pages:
+    path_prefix: "$BUILDVERSION"
  script:
    - mkdir public
    - mv redaction-service-v1/redaction-service-server-v1/javadoc/* public/
+    - URL=$(echo $BUILDVERSION | sed -e 's|\.|-|g')
+    - echo "Deploying to ${CI_PAGES_URL}/${URL}"
  rules:
    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
  artifacts:
    paths:
      - public
+
+publish JavaDoc to azure:
+  image: rclone/rclone:1.67.0
+  tags:
+    - dind
+  stage: deploy
+  when: manual
+  variables:
+    VERSION_NAME: "latest"
+  needs:
+    - generate JavaDoc
+  script:
+    - echo "Deploy JavaDoc with version ${VERSION_NAME} to prod"
+    - rclone delete azurejavadocs:/$RCLONE_CONFIG_AZUREJAVADOCS_CONTAINER/${VERSION_NAME}
+    - rclone copy redaction-service-v1/redaction-service-server-v1/javadoc/ azurejavadocs:/$RCLONE_CONFIG_AZUREJAVADOCS_CONTAINER/javadoc/${VERSION_NAME}/
+  rules:
+    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
+    - if: $CI_COMMIT_BRANCH =~ /^release/
+    - if: $CI_COMMIT_TAG
@@ -28,6 +28,8 @@ tasks.named<Test>("test") {
    reports {
        junitXml.outputLocation.set(layout.buildDirectory.dir("reports/junit"))
    }
+    minHeapSize = "512m"
+    maxHeapSize = "2048m"
 }

 tasks.test {
@@ -9,11 +9,14 @@ gradle assemble
 # Get the current Git branch
 branch=$(git rev-parse --abbrev-ref HEAD)

+# Replace any slashes (e.g., in 'feature/' or 'release/') with a hyphen
+cleaned_branch=$(echo "$branch" | sed 's/\//_/g')
+
 # Get the short commit hash (first 5 characters)
 commit_hash=$(git rev-parse --short=5 HEAD)

 # Combine branch and commit hash
-buildName="${USER}-${branch}-${commit_hash}"
+buildName="${USER}-${cleaned_branch}-${commit_hash}"

 gradle bootBuildImage --publishImage -PbuildbootDockerHostNetwork=true -Pversion=${buildName}

@@ -4,7 +4,7 @@ plugins {
 }

 description = "redaction-service-api-v1"
-val persistenceServiceVersion = "2.439.0"
+val persistenceServiceVersion = "2.587.0"

 dependencies {
    implementation("org.springframework:spring-web:6.0.12")
@@ -2,12 +2,18 @@ package com.iqser.red.service.redaction.v1.model;

 public class QueueNames {

-    public static final String REDACTION_QUEUE = "redactionQueue";
-    public static final String REDACTION_PRIORITY_QUEUE = "redactionPriorityQueue";
-    public static final String REDACTION_ANALYSIS_RESPONSE_QUEUE = "redactionAnalysisResponseQueue";
-    public static final String REDACTION_DQL = "redactionDQL";
+    public static final String REDACTION_REQUEST_QUEUE_PREFIX = "redaction_request";
+    public static final String REDACTION_REQUEST_EXCHANGE = "redaction_request_exchange";
+    public static final String REDACTION_PRIORITY_REQUEST_QUEUE_PREFIX = "redaction_priority_request";
+    public static final String REDACTION_PRIORITY_REQUEST_EXCHANGE = "redaction_priority_request_exchange";
+    public static final String REDACTION_RESPONSE_EXCHANGE = "redaction_response_exchange";
+    public static final String REDACTION_DLQ = "redaction_error";

-    public static final String MIGRATION_QUEUE = "migrationQueue";
+    public static final String SEARCH_TERM_OCCURRENCES_RESPONSE_EXCHANGE = "search_bulk_local_term_response_exchange";
+    public static final String SEARCH_BULK_LOCAL_TERM_DLQ = "search_bulk_local_term_error";
+
+
+    public static final String MIGRATION_REQUEST_QUEUE = "migrationQueue";
    public static final String MIGRATION_RESPONSE_QUEUE = "migrationResponseQueue";
    public static final String MIGRATION_DLQ = "migrationDLQ";

@@ -12,14 +12,16 @@ plugins {
 description = "redaction-service-server-v1"


-val layoutParserVersion = "0.141.0"
+val layoutParserVersion = "0.181.0"
 val jacksonVersion = "2.15.2"
 val droolsVersion = "9.44.0.Final"
 val pdfBoxVersion = "3.0.0"
-val persistenceServiceVersion = "2.444.0"
+val persistenceServiceVersion = "2.587.0"
+val llmServiceVersion = "1.11.0"
 val springBootStarterVersion = "3.1.5"
 val springCloudVersion = "4.0.4"
 val testContainersVersion = "1.19.7"
+val tomcatVersion = "10.1.18"

 configurations {
    all {
@@ -29,29 +31,43 @@ configurations {
    }
 }

+configurations.all {
+    resolutionStrategy {
+        force("com.google.protobuf:protobuf-java:4.27.1")
+    }
+}
+
 dependencies {

    implementation(project(":redaction-service-api-v1")) { exclude(group = "com.iqser.red.service", module = "persistence-service-internal-api-v1") }
    implementation("com.iqser.red.service:persistence-service-internal-api-v1:${persistenceServiceVersion}") { exclude(group = "org.springframework.boot") }
    implementation("com.iqser.red.service:persistence-service-shared-mongo-v1:${persistenceServiceVersion}")
+    {
+        exclude(group = "com.knecon.fforesight", module = "tenant-commons")
+    }
    implementation("com.knecon.fforesight:layoutparser-service-internal-api:${layoutParserVersion}")
-
+    implementation("com.knecon.fforesight:llm-service-api:${llmServiceVersion}")
    implementation("com.iqser.red.commons:spring-commons:6.2.0")
    implementation("com.iqser.red.commons:metric-commons:2.3.0")

    implementation("com.iqser.red.commons:dictionary-merge-commons:1.5.0")
-    implementation("com.iqser.red.commons:storage-commons:2.45.0")
-    implementation("com.knecon.fforesight:tenant-commons:0.24.0")
+    implementation("com.iqser.red.commons:storage-commons:2.50.0")
+    implementation("com.knecon.fforesight:tenant-commons:0.30.0")
+    implementation("com.knecon.fforesight:keycloak-commons:0.30.0") {
+        exclude(group = "com.knecon.fforesight", module = "tenant-commons")
+    }
    implementation("com.knecon.fforesight:tracing-commons:0.5.0")
+    implementation("com.knecon.fforesight:lifecycle-commons:0.7.0")

    implementation("com.fasterxml.jackson.module:jackson-module-afterburner:${jacksonVersion}")
    implementation("com.fasterxml.jackson.datatype:jackson-datatype-jsr310:${jacksonVersion}")
-    implementation("org.ahocorasick:ahocorasick:0.6.3")
+    implementation("org.ahocorasick:ahocorasick:0.7.3")
    implementation("org.javassist:javassist:3.29.2-GA")

    implementation("org.drools:drools-engine:${droolsVersion}")
    implementation("org.drools:drools-mvel:${droolsVersion}")
    implementation("org.kie:kie-spring:7.74.1.Final")
+    implementation("com.google.protobuf:protobuf-java:4.27.1")

    implementation("org.locationtech.jts:jts-core:1.19.0")

@@ -60,8 +76,13 @@ dependencies {
    implementation("org.springframework.boot:spring-boot-starter-cache:${springBootStarterVersion}")
    implementation("org.springframework.boot:spring-boot-starter-data-redis:${springBootStarterVersion}")

+    implementation("org.springframework.boot:spring-boot-starter-websocket:${springBootStarterVersion}")
+    implementation("org.springframework.security:spring-security-messaging:6.1.3")
+    implementation("org.apache.tomcat:tomcat-websocket:${tomcatVersion}")
+    implementation("org.apache.tomcat.embed:tomcat-embed-core:${tomcatVersion}")
+
    implementation("net.logstash.logback:logstash-logback-encoder:7.4")
-    implementation("ch.qos.logback:logback-classic")
+    api("ch.qos.logback:logback-classic")

    implementation("org.reflections:reflections:0.10.2")

@@ -83,6 +104,7 @@ dependencies {
            module = "persistence-service-shared-api-v1"
        )
    }
+    testImplementation("com.pdftron:PDFNet:10.11.0")
 }

 dependencyManagement {
@@ -107,6 +129,8 @@ tasks.named<BootBuildImage>("bootBuildImage") {
        "BPE_APPEND_JAVA_TOOL_OPTIONS",
        "-XX:MaxMetaspaceSize=1g -Dfile.encoding=UTF-8 -Dkie.repository.project.cache.size=50 -Dkie.repository.project.versions.cache.size=5"
    )
+    environment.put("BPE_DEFAULT_LANG", "en_US.utf8")
+    environment.put("BPE_DEFAULT_LANG", "en_US.utf8") // java.text.Normalizer does not care for file.encoding

    imageName.set("nexus.knecon.com:5001/red/${project.name}")// must build image with same name always, otherwise the builder will not know which image to use as cache. DO NOT CHANGE!
    if (project.hasProperty("buildbootDockerHostNetwork")) {
@@ -130,18 +154,19 @@ tasks.named<BootBuildImage>("bootBuildImage") {
    }
 }

-fun parseDroolsImports(droolsFilePath: String): List<String> {
-
+fun parseDroolsImports(vararg droolsFilePaths: String): List<String> {
    val imports = mutableListOf<String>()
    val importPattern = Regex("^import\\s+(com\\.iqser\\.red\\.service\\.redaction\\.v1\\.[\\w.]+);")
    val desiredPrefix = "com.iqser.red.service.redaction.v1"

-    File(droolsFilePath).forEachLine { line ->
-        importPattern.find(line)?.let { matchResult ->
-            val importPath = matchResult.groupValues[1].trim()
-            if (importPath.startsWith(desiredPrefix)) {
-                val formattedPath = importPath.replace('.', '/')
-                imports.add("$formattedPath.java")
+    droolsFilePaths.forEach { filePath ->
+        File(filePath).forEachLine { line ->
+            importPattern.find(line)?.let { matchResult ->
+                val importPath = matchResult.groupValues[1].trim()
+                if (importPath.startsWith(desiredPrefix)) {
+                    val formattedPath = importPath.replace('.', '/')
+                    imports.add("$formattedPath.java")
+                }
            }
        }
    }
@@ -149,7 +174,11 @@ fun parseDroolsImports(droolsFilePath: String): List<String> {
    return imports
 }

-val droolsImports = parseDroolsImports("redaction-service-v1/redaction-service-server-v1/src/main/resources/drools/all_rules_documine.drl")
+// Combine imports from both drools files
+val droolsImports = parseDroolsImports(
+    "redaction-service-v1/redaction-service-server-v1/src/main/resources/drools/all_rules_documine.drl",
+    "redaction-service-v1/redaction-service-server-v1/src/main/resources/drools/base_component_rules.drl"
+)

 tasks.register("generateJavaDoc", Javadoc::class) {

@@ -163,8 +192,6 @@ tasks.register("generateJavaDoc", Javadoc::class) {

    options.memberLevel = JavadocMemberLevel.PUBLIC
    (options as StandardJavadocDocletOptions).apply {
-        header = "Redaction Service ${project.version}"
-        footer = "Redaction Service ${project.version}"
        title = "API Documentation for Redaction Service ${project.version}"
    }
 }
@@ -9,10 +9,12 @@ import org.springframework.boot.autoconfigure.jdbc.DataSourceAutoConfiguration;
 import org.springframework.boot.autoconfigure.liquibase.LiquibaseAutoConfiguration;
 import org.springframework.boot.autoconfigure.mongo.MongoAutoConfiguration;
 import org.springframework.boot.autoconfigure.security.servlet.SecurityAutoConfiguration;
+import org.springframework.boot.autoconfigure.task.TaskExecutionAutoConfiguration;
 import org.springframework.boot.context.properties.EnableConfigurationProperties;
 import org.springframework.cache.annotation.EnableCaching;
 import org.springframework.cloud.openfeign.EnableFeignClients;
 import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.EnableAspectJAutoProxy;
 import org.springframework.context.annotation.Import;
 import org.springframework.data.mongodb.repository.config.EnableMongoRepositories;

@@ -20,6 +22,8 @@ import com.iqser.red.service.dictionarymerge.commons.DictionaryMergeService;
 import com.iqser.red.service.persistence.service.v1.api.shared.mongo.SharedMongoAutoConfiguration;
 import com.iqser.red.service.redaction.v1.server.client.RulesClient;
 import com.iqser.red.storage.commons.StorageAutoConfiguration;
+import com.knecon.fforesight.keycloakcommons.DefaultKeyCloakCommonsAutoConfiguration;
+import com.knecon.fforesight.lifecyclecommons.LifecycleAutoconfiguration;
 import com.knecon.fforesight.mongo.database.commons.MongoDatabaseCommonsAutoConfiguration;
 import com.knecon.fforesight.mongo.database.commons.liquibase.EnableMongoLiquibase;
 import com.knecon.fforesight.tenantcommons.MultiTenancyAutoConfiguration;
@@ -32,13 +36,14 @@ import lombok.extern.slf4j.Slf4j;

@Slf4j
@EnableCaching
-@ImportAutoConfiguration({MultiTenancyAutoConfiguration.class, SharedMongoAutoConfiguration.class})
+@ImportAutoConfiguration({MultiTenancyAutoConfiguration.class, SharedMongoAutoConfiguration.class, DefaultKeyCloakCommonsAutoConfiguration.class, LifecycleAutoconfiguration.class})
@Import({MetricsConfiguration.class, StorageAutoConfiguration.class, MongoDatabaseCommonsAutoConfiguration.class})
@EnableFeignClients(basePackageClasses = RulesClient.class)
@EnableConfigurationProperties(RedactionServiceSettings.class)
@EnableMongoRepositories(basePackages = "com.iqser.red.service.persistence")
@EnableMongoLiquibase
@SpringBootApplication(exclude = {SecurityAutoConfiguration.class, ManagementWebSecurityAutoConfiguration.class, DataSourceAutoConfiguration.class, LiquibaseAutoConfiguration.class, MongoAutoConfiguration.class, MongoDataAutoConfiguration.class})
+@EnableAspectJAutoProxy
 public class Application {

    public static void main(String[] args) {
@@ -22,6 +22,10 @@ public class RedactionServiceSettings {

    private boolean nerServiceEnabled = true;

+    private boolean azureNerServiceEnabled;
+
+    private boolean llmNerServiceEnabled;
+
    private boolean priorityMode;

    private long dictionaryCacheMaximumSize = 100;
@@ -34,6 +38,10 @@ public class RedactionServiceSettings {

    private boolean annotationMode;

+    private boolean droolsDebug;
+
+    private boolean protobufJsonFallback = true;
+

    public int getDroolsExecutionTimeoutSecs(int numberOfPages) {

@@ -0,0 +1,10 @@
+package com.iqser.red.service.redaction.v1.server.client;
+
+import org.springframework.cloud.openfeign.FeignClient;
+
+import com.iqser.red.service.persistence.service.v1.api.internal.resources.DateFormatsResource;
+
+@FeignClient(name = "DateFormatsResource", url = "${persistence-service.url}")
+public interface DateFormatsClient extends DateFormatsResource {
+
+}
@@ -15,5 +15,6 @@ public class EntityRecognitionEntity {
    private int startOffset;
    private int endOffset;
    private String type;
+    private Double confidence;

 }
@@ -0,0 +1,42 @@
+package com.iqser.red.service.redaction.v1.server.config;
+
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.data.redis.connection.RedisConnectionFactory;
+import org.springframework.data.redis.listener.PatternTopic;
+import org.springframework.data.redis.listener.RedisMessageListenerContainer;
+import org.springframework.data.redis.listener.adapter.MessageListenerAdapter;
+import org.springframework.messaging.simp.SimpMessagingTemplate;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.iqser.red.service.redaction.v1.server.service.websocket.RedisPubsubReceiver;
+
+import lombok.RequiredArgsConstructor;
+
+@Configuration
+@RequiredArgsConstructor
+public class RedisPubsubConfiguration {
+
+    private final SimpMessagingTemplate template;
+    private final ObjectMapper mapper;
+    private final RedisConnectionFactory connectionFactory;
+
+    @Bean
+    public RedisPubsubReceiver redisPubsubReceiver() {
+        return new RedisPubsubReceiver(template, mapper);
+    }
+
+    @Bean
+    public MessageListenerAdapter redisPubsubListenerAdapter() {
+        return new MessageListenerAdapter(redisPubsubReceiver(), "receiveMessage");
+    }
+
+    @Bean
+    public RedisMessageListenerContainer redisPubsubContainer() {
+        RedisMessageListenerContainer container = new RedisMessageListenerContainer();
+        container.setConnectionFactory(connectionFactory);
+        container.addMessageListener(redisPubsubListenerAdapter(), new PatternTopic("redaction-service-websocket-messages"));
+        return container;
+    }
+
+}
@@ -0,0 +1,95 @@
+package com.iqser.red.service.redaction.v1.server.config;
+
+import java.util.Collections;
+import java.util.Optional;
+
+import org.apache.tomcat.websocket.server.WsSci;
+import org.springframework.boot.web.embedded.tomcat.TomcatContextCustomizer;
+import org.springframework.boot.web.embedded.tomcat.TomcatServletWebServerFactory;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.messaging.Message;
+import org.springframework.messaging.MessageChannel;
+import org.springframework.messaging.simp.config.ChannelRegistration;
+import org.springframework.messaging.simp.config.MessageBrokerRegistry;
+import org.springframework.messaging.simp.stomp.StompCommand;
+import org.springframework.messaging.simp.stomp.StompHeaderAccessor;
+import org.springframework.messaging.support.ChannelInterceptor;
+import org.springframework.messaging.support.MessageHeaderAccessor;
+import org.springframework.security.authentication.AuthenticationManager;
+import org.springframework.security.oauth2.server.resource.authentication.BearerTokenAuthenticationToken;
+import org.springframework.security.oauth2.server.resource.authentication.JwtAuthenticationToken;
+import org.springframework.web.socket.config.annotation.EnableWebSocketMessageBroker;
+import org.springframework.web.socket.config.annotation.StompEndpointRegistry;
+import org.springframework.web.socket.config.annotation.WebSocketMessageBrokerConfigurer;
+
+import com.knecon.fforesight.keycloakcommons.security.TenantAuthenticationManagerResolver;
+
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+@Configuration
+@EnableWebSocketMessageBroker
+@RequiredArgsConstructor
+public class WebSocketConfiguration implements WebSocketMessageBrokerConfigurer {
+
+    private final TenantAuthenticationManagerResolver tenantAuthenticationManagerResolver;
+
+
+    @Override
+    public void configureMessageBroker(MessageBrokerRegistry config) {
+
+        config.enableSimpleBroker("/topic");
+        config.setApplicationDestinationPrefixes("/app");
+    }
+
+
+    @Override
+    public void registerStompEndpoints(StompEndpointRegistry registry) {
+
+        registry.addEndpoint("/api/rules-logging/rulesocket").setAllowedOrigins("*");
+    }
+
+
+    @Override
+    public void configureClientInboundChannel(ChannelRegistration registration) {
+
+        // https://docs.spring.io/spring-framework/reference/web/websocket/stomp/authentication-token-based.html
+        registration.interceptors(new ChannelInterceptor() {
+            @Override
+            public Message<?> preSend(Message<?> message, MessageChannel channel) {
+
+                StompHeaderAccessor accessor = MessageHeaderAccessor.getAccessor(message, StompHeaderAccessor.class);
+                if (StompCommand.CONNECT.equals(accessor.getCommand())) {
+                    Optional.ofNullable(accessor.getNativeHeader("Authorization"))
+                            .ifPresent(ah -> {
+                                String bearerToken = ah.get(0).replace("Bearer ", "");
+                                log.info("Received bearer token {}", bearerToken);
+                                AuthenticationManager authenticationManager = tenantAuthenticationManagerResolver.resolve(bearerToken);
+                                JwtAuthenticationToken token = (JwtAuthenticationToken) authenticationManager.authenticate(new BearerTokenAuthenticationToken(bearerToken));
+                                accessor.setUser(token);
+                            });
+                }
+                return message;
+            }
+        });
+    }
+
+
+    @Bean
+    public TomcatServletWebServerFactory tomcatContainerFactory() {
+
+        TomcatServletWebServerFactory factory = new TomcatServletWebServerFactory();
+        factory.setTomcatContextCustomizers(Collections.singletonList(tomcatContextCustomizer()));
+        return factory;
+    }
+
+
+    @Bean
+    public TomcatContextCustomizer tomcatContextCustomizer() {
+
+        return context -> context.addServletContainerInitializer(new WsSci(), null);
+    }
+
+}
@@ -0,0 +1,88 @@
+package com.iqser.red.service.redaction.v1.server.config;
+
+import java.util.Optional;
+
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.messaging.Message;
+import org.springframework.messaging.simp.SimpMessageType;
+import org.springframework.messaging.simp.stomp.StompHeaderAccessor;
+import org.springframework.security.config.annotation.web.messaging.MessageSecurityMetadataSourceRegistry;
+import org.springframework.security.config.annotation.web.socket.AbstractSecurityWebSocketMessageBrokerConfigurer;
+import org.springframework.security.oauth2.server.resource.authentication.JwtAuthenticationToken;
+
+import com.knecon.fforesight.keycloakcommons.security.TokenUtils;
+
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+@Configuration
+public class WebSocketSecurityConfiguration extends AbstractSecurityWebSocketMessageBrokerConfigurer {
+
+    @Value("${cors.enabled:false}")
+    private boolean corsEnabled;
+
+
+    @Override
+    protected void configureInbound(MessageSecurityMetadataSourceRegistry messages) {
+
+        messages.simpTypeMatchers(SimpMessageType.HEARTBEAT, SimpMessageType.UNSUBSCRIBE, SimpMessageType.DISCONNECT)
+                .permitAll()
+                .simpTypeMatchers(SimpMessageType.CONNECT)
+                .anonymous() // this is intended, see WebSocketConfiguration.configureClientInboundChannel
+                .simpTypeMatchers(SimpMessageType.SUBSCRIBE)
+                .access("@tenantWebSocketSecurityMatcher.checkCanSubscribeTo(authentication,message)")
+                .anyMessage()
+                .denyAll();
+    }
+
+
+    @Override
+    protected boolean sameOriginDisabled() {
+
+        return corsEnabled;
+    }
+
+
+    @Bean
+    public TenantWebSocketSecurityMatcher tenantWebSocketSecurityMatcher() {
+
+        return new TenantWebSocketSecurityMatcher();
+    }
+
+
+    public class TenantWebSocketSecurityMatcher {
+
+        public boolean checkCanSubscribeTo(JwtAuthenticationToken authentication, Message<?> message) {
+
+            var targetedTenant = extractTenantId(message);
+            var currentTenant = getCurrentTenant(authentication);
+            return targetedTenant.isPresent() && currentTenant.isPresent() && currentTenant.get().equals(targetedTenant.get());
+        }
+
+
+        private Optional<String> getCurrentTenant(JwtAuthenticationToken authentication) {
+
+            if (authentication != null && authentication.getToken() != null && authentication.getToken().getTokenValue() != null) {
+                return Optional.of(TokenUtils.toTenant(authentication.getToken().getTokenValue()));
+            } else {
+                return Optional.empty();
+            }
+        }
+
+    }
+
+    private Optional<String> extractTenantId(Message<?> message) {
+
+        StompHeaderAccessor sha = StompHeaderAccessor.wrap(message);
+        String topic = sha.getDestination();
+        if (topic == null) {
+            return Optional.empty();
+        }
+
+        String tenant = topic.split("/")[2];
+        return Optional.of(tenant);
+    }
+
+}
@@ -0,0 +1,12 @@
+package com.iqser.red.service.redaction.v1.server.configuraton;
+
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.context.annotation.Configuration;
+
+import com.knecon.fforesight.tenantcommons.queue.TenantMessagingConfiguration;
+
+@Configuration
+@ConditionalOnProperty(prefix = "redaction-service", name = "priorityMode", havingValue = "true")
+public class PriorityTenantMessagingConfigurationImpl extends TenantMessagingConfiguration {
+
+}
@@ -0,0 +1,12 @@
+package com.iqser.red.service.redaction.v1.server.configuraton;
+
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.context.annotation.Configuration;
+
+import com.knecon.fforesight.tenantcommons.queue.TenantMessagingConfiguration;
+
+@Configuration
+@ConditionalOnProperty(prefix = "redaction-service", name = "priorityMode", havingValue = "false")
+public class TenantMessagingConfigurationImpl extends TenantMessagingConfiguration {
+
+}
@@ -0,0 +1,36 @@
+package com.iqser.red.service.redaction.v1.server.logger;
+
+import lombok.AllArgsConstructor;
+import lombok.Getter;
+import lombok.NoArgsConstructor;
+import lombok.Setter;
+import lombok.ToString;
+
+@AllArgsConstructor
+@NoArgsConstructor
+@Getter
+@ToString
+public final class Context {
+
+    private String fileId;
+    private String dossierId;
+    private String dossierTemplateId;
+    @Setter
+    private long ruleVersion;
+    @Setter
+    private long dateFormatsVersion;
+    private int analysisNumber;
+    private String tenantId;
+
+
+    public Context(String fileId, String dossierId, String dossierTemplateId, long ruleVersion, int analysisNumber, String tenantId) {
+
+        this.fileId = fileId;
+        this.dossierId = dossierId;
+        this.dossierTemplateId = dossierTemplateId;
+        this.ruleVersion = ruleVersion;
+        this.analysisNumber = analysisNumber;
+        this.tenantId = tenantId;
+    }
+
+}
@@ -0,0 +1,70 @@
+package com.iqser.red.service.redaction.v1.server.logger;
+
+import org.kie.api.definition.rule.Rule;
+import org.kie.api.event.rule.DefaultRuleRuntimeEventListener;
+import org.kie.api.event.rule.ObjectDeletedEvent;
+import org.kie.api.event.rule.ObjectInsertedEvent;
+import org.kie.api.event.rule.ObjectUpdatedEvent;
+
+import lombok.AllArgsConstructor;
+
+@AllArgsConstructor
+public class ObjectTrackingEventListener extends DefaultRuleRuntimeEventListener {
+
+    RulesLogger logger;
+
+
+    @Override
+    public void objectInserted(ObjectInsertedEvent event) {
+
+        if (!logger.isObjectTrackingActive()) {
+            return;
+        }
+
+        if (event.getRule() == null) {
+            logger.logObjectTracking("ObjectInsertedEvent:{} has been inserted", event.getObject());
+            return;
+        }
+
+        logger.logObjectTracking("ObjectInsertedEvent:{}: {} has been inserted", formatRuleName(event.getRule()), event.getObject());
+    }
+
+
+    @Override
+    public void objectDeleted(ObjectDeletedEvent event) {
+
+        if (!logger.isObjectTrackingActive()) {
+            return;
+        }
+        if (event.getRule() == null) {
+            logger.logObjectTracking("ObjectDeletedEvent: {} has been deleted", event.getOldObject());
+            return;
+        }
+        logger.logObjectTracking("ObjectDeletedEvent: {}: {} has been deleted", formatRuleName(event.getRule()), event.getOldObject());
+    }
+
+
+    @Override
+    public void objectUpdated(ObjectUpdatedEvent event) {
+
+        if (!logger.isObjectTrackingActive()) {
+            return;
+        }
+        if (event.getRule() == null) {
+            logger.logObjectTracking("ObjectUpdatedEvent:{} has been updated", event.getObject());
+            return;
+        }
+        logger.logObjectTracking("ObjectUpdatedEvent:{}: {} has been updated", formatRuleName(event.getRule()), event.getObject());
+    }
+
+
+    public static String formatRuleName(Rule rule) {
+
+        String name = rule.getName();
+        if (name.length() > 20) {
+            return name.substring(0, 20) + "...";
+        }
+        return name;
+    }
+
+}
@@ -0,0 +1,45 @@
+package com.iqser.red.service.redaction.v1.server.logger;
+
+import java.time.OffsetDateTime;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.Getter;
+import lombok.NoArgsConstructor;
+import lombok.ToString;
+
+@NoArgsConstructor
+@AllArgsConstructor
+@Data
+@Builder
+@ToString
+public class RuleLogEvent {
+
+    private String tenantId;
+    private String fileId;
+    private String dossierId;
+    private String dossierTemplateId;
+    private long ruleVersion;
+    private int analysisNumber;
+    private OffsetDateTime timeStamp;
+    private LogLevel logLevel;
+    private String message;
+
+}
+
+@Getter
+enum LogLevel {
+    INFO("INFO"),
+    WARN("WARN"),
+    ERROR("ERROR");
+
+    private final String name;
+
+
+    LogLevel(String name) {
+
+        this.name = name;
+    }
+
+}
@@ -0,0 +1,174 @@
+package com.iqser.red.service.redaction.v1.server.logger;
+
+import java.time.OffsetDateTime;
+
+import org.slf4j.helpers.MessageFormatter;
+
+import com.iqser.red.service.redaction.v1.server.service.websocket.WebSocketService;
+
+import lombok.Getter;
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+
+/**
+ * This class provides logging functionality specifically for rules execution
+ * in a Drools context. It is designed to log messages with different log levels
+ * (INFO, WARN, ERROR) and formats messages using a placeholder-based approach
+ * similar to popular logging frameworks like SLF4J. <p>
+ * <p>
+ * Log messages can include placeholders (i.e., `{}`), which will be replaced by
+ * the corresponding arguments when the message is formatted. <p>
+ * <p>
+ * Example usage:
+ * <pre>
+ *     logger.info("Message with placeholder {}", object);
+ * </pre>
+ */
+@Slf4j
+@RequiredArgsConstructor
+public class RulesLogger {
+
+    private final WebSocketService webSocketService;
+    private final Context context;
+    @Getter
+    private boolean objectTrackingActive;
+    @Getter
+    private boolean agendaTrackingActive;
+
+
+    /**
+     * Logs a message at the INFO level.
+     *
+     * @param message The log message containing optional placeholders (i.e., `{}`).
+     * @param args    The arguments to replace the placeholders in the message.
+     */
+    public void info(String message, Object... args) {
+
+        log(LogLevel.INFO, message, args);
+    }
+
+
+    /**
+     * Logs a message at the WARN level.
+     *
+     * @param message The log message containing optional placeholders (i.e., `{}`).
+     * @param args    The arguments to replace the placeholders in the message.
+     */
+    public void warn(String message, Object... args) {
+
+        log(LogLevel.WARN, message, args);
+    }
+
+
+    /**
+     * Logs a message at the INFO level, if object tracking has been activated.
+     *
+     * @param message The log message containing optional placeholders (i.e., `{}`).
+     * @param args    The arguments to replace the placeholders in the message.
+     */
+    public void logObjectTracking(String message, Object... args) {
+
+        if (objectTrackingActive) {
+            info(message, args);
+        }
+    }
+
+
+    /**
+     * If object tracking is enabled, the RulesLogger will log all inserted/retracted/updated events.
+     * Initial value is disabled.
+     */
+    public void enableObjectTracking() {
+
+        objectTrackingActive = true;
+    }
+
+
+    /**
+     * If object tracking is disabled, the RulesLogger won't log any inserted/retracted/updated events.
+     * Initial value is disabled.
+     */
+    public void disableObjectTracking() {
+
+        objectTrackingActive = false;
+    }
+
+
+    /**
+     * Logs a message at the INFO level, if agenda tracking has been activated.
+     *
+     * @param message The log message containing optional placeholders (i.e., `{}`).
+     * @param args    The arguments to replace the placeholders in the message.
+     */
+    public void logAgendaTracking(String message, Object... args) {
+
+        if (agendaTrackingActive) {
+            info(message, args);
+        }
+
+    }
+
+
+    /**
+     * If agenda tracking is enabled, the RulesLogger will log each firing Rule with its name, objects and metadata.
+     * Initial value is disabled.
+     */
+    public void enableAgendaTracking() {
+
+        agendaTrackingActive = true;
+    }
+
+
+    /**
+     * If agenda tracking is disabled, the RulesLogger won't log any rule firings.
+     * Initial value is disabled.
+     */
+    public void disableAgendaTracking() {
+
+        agendaTrackingActive = false;
+    }
+
+
+    /**
+     * Logs a message at the ERROR level, including an exception.
+     *
+     * @param throwable The exception to log.
+     * @param message   The log message containing optional placeholders (i.e., `{}`).
+     * @param args      The arguments to replace the placeholders in the message.
+     */
+    public void error(Throwable throwable, String message, Object... args) {
+
+        log(LogLevel.ERROR, message + " Exception: " + throwable.toString(), args);
+    }
+
+
+    private void log(LogLevel logLevel, String message, Object... args) {
+
+        var formattedMessage = formatMessage(message, args);
+        switch (logLevel) {
+            case INFO -> log.info(message, args);
+            case WARN -> log.warn(message, args);
+            case ERROR -> log.error(message, args);
+        }
+        var ruleLog = RuleLogEvent.builder()
+                .tenantId(context.getTenantId())
+                .ruleVersion(context.getRuleVersion())
+                .fileId(context.getFileId())
+                .analysisNumber(context.getAnalysisNumber())
+                .dossierId(context.getDossierId())
+                .dossierTemplateId(context.getDossierTemplateId())
+                .message(formattedMessage)
+                .logLevel(logLevel)
+                .timeStamp(OffsetDateTime.now())
+                .build();
+
+        webSocketService.sendLogEvent(ruleLog);
+    }
+
+
+    private String formatMessage(String message, Object... args) {
+
+        return MessageFormatter.arrayFormat(message, args).getMessage();
+    }
+
+}
@@ -0,0 +1,65 @@
+package com.iqser.red.service.redaction.v1.server.logger;
+
+import java.util.Map;
+
+import org.kie.api.definition.rule.Rule;
+import org.kie.api.event.rule.AfterMatchFiredEvent;
+import org.kie.api.event.rule.DefaultAgendaEventListener;
+import org.kie.api.event.rule.MatchCreatedEvent;
+
+import lombok.AllArgsConstructor;
+
+@AllArgsConstructor
+public class TrackingAgendaEventListener extends DefaultAgendaEventListener {
+
+    private RulesLogger logger;
+
+
+    @Override
+    public void matchCreated(MatchCreatedEvent event) {
+
+        if (logger.isAgendaTrackingActive()) {
+            logger.logAgendaTracking(event.toString());
+        }
+    }
+
+
+    @Override
+    public void afterMatchFired(AfterMatchFiredEvent event) {
+
+        if (!logger.isAgendaTrackingActive()) {
+            return;
+        }
+
+        Rule rule = event.getMatch().getRule();
+
+        String ruleName = formatRuleName(rule);
+        Map<String, Object> ruleMetaDataMap = rule.getMetaData();
+
+        StringBuilder sb = new StringBuilder("AfterMatchFiredEvent: " + ruleName);
+
+        if (event.getMatch().getObjects() != null && !event.getMatch().getObjects().isEmpty()) {
+            sb.append(", ").append(event.getMatch().getObjects().size()).append(" objects: ");
+            for (Object object : event.getMatch().getObjects()) {
+                sb.append(object).append(", ");
+            }
+            sb.delete(sb.length() - 2, sb.length());
+        }
+
+        if (!ruleMetaDataMap.isEmpty()) {
+            sb.append("\n  With [").append(ruleMetaDataMap.size()).append("] meta-data:");
+            for (String key : ruleMetaDataMap.keySet()) {
+                sb.append("\n    key=").append(key).append(", value=").append(ruleMetaDataMap.get(key));
+            }
+        }
+
+        logger.logAgendaTracking(sb.toString());
+    }
+
+
+    public static String formatRuleName(Rule rule) {
+
+        return ObjectTrackingEventListener.formatRuleName(rule);
+    }
+
+}
@@ -1,438 +0,0 @@
-package com.iqser.red.service.redaction.v1.server.migration;
-
-import java.time.OffsetDateTime;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-import java.util.stream.Collectors;
-
-import org.springframework.stereotype.Service;
-
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Change;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ChangeType;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ManualChange;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ManualRedactionType;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Point;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
-import com.iqser.red.service.redaction.v1.server.service.DictionaryService;
-
-import lombok.AllArgsConstructor;
-import lombok.Data;
-import lombok.RequiredArgsConstructor;
-import lombok.extern.slf4j.Slf4j;
-
-@Slf4j
-@Service
-@RequiredArgsConstructor
-public class LegacyRedactionLogMergeService {
-
-    private final DictionaryService dictionaryService;
-
-
-    public RedactionLog addManualAddEntriesAndRemoveSkippedImported(RedactionLog redactionLog, ManualRedactions manualRedactions, String dossierTemplateId) {
-
-        Set<String> skippedImportedRedactions = new HashSet<>();
-        log.info("Adding manual add Entries and removing skipped or imported entries");
-        if (manualRedactions != null) {
-
-            var manualRedactionLogEntries = addManualAddEntries(manualRedactions.getEntriesToAdd(), redactionLog.getAnalysisNumber());
-
-            redactionLog.getRedactionLogEntry().addAll(manualRedactionLogEntries);
-
-            var manualRedactionWrappers = createManualRedactionWrappers(manualRedactions);
-
-            for (RedactionLogEntry entry : redactionLog.getRedactionLogEntry()) {
-
-                if (entry.isImported()) {
-                    processRedactionLogEntry(manualRedactionWrappers.stream()
-                                                     .filter(ManualRedactionWrapper::isApproved)
-                                                     .filter(mr -> entry.getId().equals(mr.getId()))
-                                                     .collect(Collectors.toList()), entry, dossierTemplateId);
-                    if (!entry.isRedacted()) {
-                        skippedImportedRedactions.add(entry.getId());
-                    }
-                }
-
-            }
-
-        }
-
-        Set<String> processedIds = new HashSet<>();
-        redactionLog.getRedactionLogEntry().removeIf(entry -> {
-            if (entry.isFalsePositive()) {
-                return true;
-            }
-            if (entry.getImportedRedactionIntersections() != null) {
-                entry.getImportedRedactionIntersections().removeAll(skippedImportedRedactions);
-                if (!entry.getImportedRedactionIntersections().isEmpty() && (!entry.isImage() || entry.isImage() && !(entry.getType().equals("image") || entry.getType()
-                        .equals("ocr")))) {
-                    return true;
-                }
-            }
-            if (processedIds.contains(entry.getId())) {
-                log.info("Duplicate annotation found with id {}", entry.getId());
-                return true;
-            }
-            processedIds.add(entry.getId());
-            return false;
-        });
-        return redactionLog;
-    }
-
-
-    public long getNumberOfAffectedAnnotations(ManualRedactions manualRedactions) {
-
-        return createManualRedactionWrappers(manualRedactions).stream()
-                .map(ManualRedactionWrapper::getId)
-                .distinct()
-                .count();
-    }
-
-
-    private List<ManualRedactionWrapper> createManualRedactionWrappers(ManualRedactions manualRedactions) {
-
-        List<ManualRedactionWrapper> manualRedactionWrappers = new ArrayList<>();
-
-        manualRedactions.getRecategorizations()
-                .forEach(item -> {
-                    if (item.getSoftDeletedTime() == null) {
-                        manualRedactionWrappers.add(new ManualRedactionWrapper(item.getAnnotationId(), item.getRequestDate(), item, item.isApproved()));
-                    }
-                });
-
-        manualRedactions.getIdsToRemove()
-                .forEach(item -> {
-                    if (item.getSoftDeletedTime() == null) {
-                        manualRedactionWrappers.add(new ManualRedactionWrapper(item.getAnnotationId(), item.getRequestDate(), item, item.isApproved()));
-                    }
-                });
-
-        manualRedactions.getForceRedactions()
-                .forEach(item -> {
-                    if (item.getSoftDeletedTime() == null) {
-                        manualRedactionWrappers.add(new ManualRedactionWrapper(item.getAnnotationId(), item.getRequestDate(), item, item.isApproved()));
-                    }
-                });
-
-        manualRedactions.getLegalBasisChanges()
-                .forEach(item -> {
-                    if (item.getSoftDeletedTime() == null) {
-                        manualRedactionWrappers.add(new ManualRedactionWrapper(item.getAnnotationId(), item.getRequestDate(), item, item.isApproved()));
-                    }
-                });
-
-        manualRedactions.getResizeRedactions()
-                .forEach(item -> {
-                    if (item.getSoftDeletedTime() == null) {
-                        manualRedactionWrappers.add(new ManualRedactionWrapper(item.getAnnotationId(), item.getRequestDate(), item, item.isApproved()));
-                    }
-                });
-
-        Collections.sort(manualRedactionWrappers);
-
-        return manualRedactionWrappers;
-    }
-
-
-    private void processRedactionLogEntry(List<ManualRedactionWrapper> manualRedactionWrappers, RedactionLogEntry redactionLogEntry, String dossierTemplateId) {
-
-        manualRedactionWrappers.forEach(mrw -> {
-
-            Object item = mrw.getItem();
-            if (item instanceof ManualRecategorization imageRecategorization) {
-                processManualImageRecategorization(redactionLogEntry, dossierTemplateId, imageRecategorization);
-            }
-
-            if (item instanceof IdRemoval manualRemoval) {
-                processIdRemoval(redactionLogEntry, manualRemoval);
-            }
-
-            if (item instanceof ManualForceRedaction manualForceRedact) {
-                processManualForceRedaction(redactionLogEntry, dossierTemplateId, manualForceRedact);
-            }
-
-            if (item instanceof ManualLegalBasisChange manualLegalBasisChange) {
-                processManualLegalBasisChange(redactionLogEntry, manualLegalBasisChange);
-            }
-
-            if (item instanceof ManualResizeRedaction manualResizeRedact) {
-                processManualResizeRedaction(redactionLogEntry, manualResizeRedact);
-            }
-
-        });
-
-    }
-
-
-    private void processManualImageRecategorization(RedactionLogEntry redactionLogEntry, String dossierTemplateId, ManualRecategorization imageRecategorization) {
-
-        String manualOverrideReason = null;
-        if (imageRecategorization.getStatus().equals(AnnotationStatus.APPROVED)) {
-
-            redactionLogEntry.setType(imageRecategorization.getType());
-            redactionLogEntry.setSection("Image:" + redactionLogEntry.getType());
-
-            if (dictionaryService.isHint(imageRecategorization.getType(), dossierTemplateId)) {
-                redactionLogEntry.setRedacted(false);
-                redactionLogEntry.setHint(true);
-            } else {
-                redactionLogEntry.setHint(false);
-            }
-
-            manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", recategorized by manual override");
-        } else if (imageRecategorization.getStatus().equals(AnnotationStatus.REQUESTED)) {
-            manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", requested to recategorize");
-        }
-
-        if (manualOverrideReason != null) {
-            redactionLogEntry.setReason(manualOverrideReason);
-        }
-
-        redactionLogEntry.getManualChanges()
-                .add(ManualChange.from(imageRecategorization)
-                             .withManualRedactionType(ManualRedactionType.RECATEGORIZE)
-                             .withChange("type", imageRecategorization.getType())
-                             .withChange("section", imageRecategorization.getSection())
-                             .withChange("legalBasis", imageRecategorization.getLegalBasis())
-                             .withChange("value", imageRecategorization.getValue()));
-    }
-
-
-    private String mergeReasonIfNecessary(String currentReason, String addition) {
-
-        if (currentReason != null) {
-            if (!currentReason.contains(addition)) {
-                return currentReason + addition;
-            }
-            return currentReason;
-        } else {
-            return "";
-        }
-    }
-
-
-    private void processIdRemoval(RedactionLogEntry redactionLogEntry, IdRemoval manualRemoval) {
-
-        boolean isApprovedRedaction = manualRemoval.getStatus().equals(AnnotationStatus.APPROVED);
-        if (isApprovedRedaction && manualRemoval.isRemoveFromDictionary() && isBasedOnDictionaryOnly(redactionLogEntry)) {
-            log.debug("Skipping merge for dictionary-modifying entry");
-        } else {
-            String manualOverrideReason = null;
-            if (isApprovedRedaction) {
-                redactionLogEntry.setRedacted(false);
-                manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", removed by manual override");
-                redactionLogEntry.setHint(false);
-            } else if (manualRemoval.getStatus().equals(AnnotationStatus.REQUESTED)) {
-                manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", requested to remove");
-            }
-
-            if (manualOverrideReason != null) {
-                redactionLogEntry.setReason(manualOverrideReason);
-            }
-        }
-
-        redactionLogEntry.getManualChanges()
-                .add(ManualChange.from(manualRemoval)
-                             .withManualRedactionType(manualRemoval.isRemoveFromDictionary() ? ManualRedactionType.REMOVE_FROM_DICTIONARY : ManualRedactionType.REMOVE_LOCALLY));
-    }
-
-
-    private boolean isBasedOnDictionaryOnly(RedactionLogEntry redactionLogEntry) {
-
-        return redactionLogEntry.getEngines().contains(Engine.DICTIONARY) && redactionLogEntry.getEngines().size() == 1;
-    }
-
-
-    private void processManualForceRedaction(RedactionLogEntry redactionLogEntry, String dossierTemplateId, ManualForceRedaction manualForceRedact) {
-
-        String manualOverrideReason = null;
-        var dictionaryIsHint = dictionaryService.isHint(redactionLogEntry.getType(), dossierTemplateId);
-        if (manualForceRedact.getStatus().equals(AnnotationStatus.APPROVED)) {
-            // Forcing a skipped hint should result in a hint
-            if (dictionaryIsHint) {
-                redactionLogEntry.setHint(true);
-            } else {
-                redactionLogEntry.setRedacted(true);
-            }
-            manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", forced by manual override");
-            redactionLogEntry.setLegalBasis(manualForceRedact.getLegalBasis());
-        } else if (manualForceRedact.getStatus().equals(AnnotationStatus.REQUESTED)) {
-            manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", requested to force " + (dictionaryIsHint ? "hint" : "redact"));
-            redactionLogEntry.setLegalBasis(manualForceRedact.getLegalBasis());
-        }
-
-        if (manualOverrideReason != null) {
-            redactionLogEntry.setReason(manualOverrideReason);
-        }
-
-        var manualChange = ManualChange.from(manualForceRedact).withManualRedactionType(dictionaryIsHint ? ManualRedactionType.FORCE_HINT : ManualRedactionType.FORCE_REDACT);
-
-        redactionLogEntry.getManualChanges().add(manualChange);
-    }
-
-
-    private void processManualLegalBasisChange(RedactionLogEntry redactionLogEntry, ManualLegalBasisChange manualLegalBasisChange) {
-
-        String manualOverrideReason = null;
-        if (manualLegalBasisChange.getStatus().equals(AnnotationStatus.APPROVED)) {
-            manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", legal basis was manually changed");
-            redactionLogEntry.setLegalBasis(manualLegalBasisChange.getLegalBasis());
-            redactionLogEntry.setRedacted(true);
-            if (manualLegalBasisChange.getSection() != null) {
-                redactionLogEntry.setSection(manualLegalBasisChange.getSection());
-            }
-            if (redactionLogEntry.isRectangle() && manualLegalBasisChange.getValue() != null) {
-                redactionLogEntry.setValue(manualLegalBasisChange.getValue());
-            }
-        } else if (manualLegalBasisChange.getStatus().equals(AnnotationStatus.REQUESTED)) {
-            manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", legal basis change requested");
-        }
-
-        if (manualOverrideReason != null) {
-            redactionLogEntry.setReason(manualOverrideReason);
-        }
-
-        var manualChange = ManualChange.from(manualLegalBasisChange).withManualRedactionType(ManualRedactionType.LEGAL_BASIS_CHANGE);
-        manualChange.withChange("legalBasis", manualLegalBasisChange.getLegalBasis());
-        if (manualLegalBasisChange.getSection() != null) {
-            manualChange.withChange("section", manualLegalBasisChange.getSection());
-        }
-        if (redactionLogEntry.isRectangle() && manualLegalBasisChange.getValue() != null) {
-            manualChange.withChange("value", manualLegalBasisChange.getValue());
-        }
-        redactionLogEntry.getManualChanges().add(manualChange);
-    }
-
-
-    private void processManualResizeRedaction(RedactionLogEntry redactionLogEntry, ManualResizeRedaction manualResizeRedact) {
-
-        String manualOverrideReason = null;
-        if (manualResizeRedact.getStatus().equals(AnnotationStatus.APPROVED)) {
-            redactionLogEntry.setPositions(convertPositions(manualResizeRedact.getPositions()));
-            if (!"signature".equalsIgnoreCase(redactionLogEntry.getType()) && !"logo".equalsIgnoreCase(redactionLogEntry.getType())) {
-                redactionLogEntry.setValue(manualResizeRedact.getValue());
-            }
-            // This is for backwards compatibility, now the text after/before is calculated during reanalysis because we need to find dict entries on positions where entries are resized to smaller.
-            if (manualResizeRedact.getTextBefore() != null || manualResizeRedact.getTextAfter() != null) {
-                redactionLogEntry.setTextBefore(manualResizeRedact.getTextBefore());
-                redactionLogEntry.setTextAfter(manualResizeRedact.getTextAfter());
-            }
-            manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", resized by manual override");
-        } else if (manualResizeRedact.getStatus().equals(AnnotationStatus.REQUESTED)) {
-            manualOverrideReason = mergeReasonIfNecessary(redactionLogEntry.getReason(), ", requested to resize redact");
-            redactionLogEntry.setPositions(convertPositions(manualResizeRedact.getPositions()));
-
-            // This is for backwards compatibility, now the text after/before is calculated during reanalysis because we need to find dict entries on positions where entries are resized to smaller.
-            if (manualResizeRedact.getTextBefore() != null || manualResizeRedact.getTextAfter() != null) {
-                redactionLogEntry.setTextBefore(manualResizeRedact.getTextBefore());
-                redactionLogEntry.setTextAfter(manualResizeRedact.getTextAfter());
-            }
-        }
-
-        redactionLogEntry.setReason(manualOverrideReason);
-        redactionLogEntry.getManualChanges()
-                .add(ManualChange.from(manualResizeRedact).withManualRedactionType(ManualRedactionType.RESIZE).withChange("value", manualResizeRedact.getValue()));
-    }
-
-
-    public List<RedactionLogEntry> addManualAddEntries(Set<ManualRedactionEntry> manualAdds, int analysisNumber) {
-
-        List<RedactionLogEntry> redactionLogEntries = new ArrayList<>();
-
-        for (ManualRedactionEntry manualRedactionEntry : manualAdds) {
-
-            if (shouldCreateManualEntry(manualRedactionEntry)) {
-                RedactionLogEntry redactionLogEntry = createRedactionLogEntry(manualRedactionEntry, manualRedactionEntry.getAnnotationId(), analysisNumber);
-                redactionLogEntry.setPositions(convertPositions(manualRedactionEntry.getPositions()));
-                redactionLogEntry.setTextBefore(manualRedactionEntry.getTextBefore());
-                redactionLogEntry.setTextAfter(manualRedactionEntry.getTextAfter());
-
-                redactionLogEntries.add(redactionLogEntry);
-            }
-        }
-
-        return redactionLogEntries;
-    }
-
-
-    private List<Rectangle> convertPositions(List<com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle> positions) {
-
-        return positions.stream()
-                .map(pos -> new Rectangle(new Point(pos.getTopLeftX(), pos.getTopLeftY()), pos.getWidth(), pos.getHeight(), pos.getPage()))
-                .collect(Collectors.toList());
-    }
-
-
-    @SuppressWarnings("PMD.UselessParentheses")
-    private boolean shouldCreateManualEntry(ManualRedactionEntry manualRedactionEntry) {
-
-        if (!manualRedactionEntry.isApproved()) {
-            return false;
-        }
-
-        return (!manualRedactionEntry.isAddToDictionary() && !manualRedactionEntry.isAddToDossierDictionary()) || ((manualRedactionEntry.isAddToDictionary()
-                                                                                                                    || manualRedactionEntry.isAddToDossierDictionary())
-                                                                                                                   && manualRedactionEntry.getProcessedDate() == null);
-    }
-
-
-    private RedactionLogEntry createRedactionLogEntry(ManualRedactionEntry manualRedactionEntry, String id, int analysisNumber) {
-
-        var addToDictionary = manualRedactionEntry.isAddToDictionary() || manualRedactionEntry.isAddToDossierDictionary();
-
-        var change = ManualChange.from(manualRedactionEntry).withManualRedactionType(addToDictionary ? ManualRedactionType.ADD_TO_DICTIONARY : ManualRedactionType.ADD_LOCALLY);
-        List<ManualChange> changeList = new ArrayList<>();
-        changeList.add(change);
-
-        return RedactionLogEntry.builder()
-                .id(id)
-                .reason(manualRedactionEntry.getReason())
-                .isDictionaryEntry(manualRedactionEntry.isAddToDictionary())
-                .isDossierDictionaryEntry(manualRedactionEntry.isAddToDossierDictionary())
-                .legalBasis(manualRedactionEntry.getLegalBasis())
-                .value(manualRedactionEntry.getValue())
-                .sourceId(manualRedactionEntry.getSourceId())
-                .section(manualRedactionEntry.getSection())
-                .type(manualRedactionEntry.getType())
-                .redacted(true)
-                .isHint(false)
-                .sectionNumber(-1)
-                .rectangle(manualRedactionEntry.isRectangle())
-                .manualChanges(changeList)
-                .changes(List.of(new Change(analysisNumber + 1, ChangeType.ADDED, manualRedactionEntry.getRequestDate())))
-                .build();
-    }
-
-
-    @Data
-    @AllArgsConstructor
-    private static class ManualRedactionWrapper implements Comparable<ManualRedactionWrapper> {
-
-        private String id;
-        private OffsetDateTime date;
-        private Object item;
-        private boolean approved;
-
-
-        @Override
-        public int compareTo(ManualRedactionWrapper o) {
-
-            return this.date.compareTo(o.date);
-        }
-
-    }
-
-}
@@ -1,75 +0,0 @@
-package com.iqser.red.service.redaction.v1.server.migration;
-
-import java.lang.reflect.Field;
-import java.util.Comparator;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.stream.Collectors;
-
-import org.springframework.stereotype.Service;
-
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
-
-import lombok.SneakyThrows;
-
-@Service
-@SuppressWarnings("PMD")
-public class LegacyVersion0MigrationService {
-
-    public RedactionLog mergeDuplicateAnnotationIds(RedactionLog redactionLog) {
-
-        List<RedactionLogEntry> mergedEntries = new LinkedList<>();
-        Map<String, List<RedactionLogEntry>> entriesById = redactionLog.getRedactionLogEntry()
-                .stream()
-                .collect(Collectors.groupingBy(RedactionLogEntry::getId));
-        for (List<RedactionLogEntry> entries : entriesById.values()) {
-
-            if (entries.isEmpty()) {
-                continue;
-            }
-
-            if (entries.size() == 1) {
-                mergedEntries.add(entries.get(0));
-                continue;
-            }
-
-            List<RedactionLogEntry> sortedEntries = entries.stream()
-                    .sorted(Comparator.comparing(entry -> entry.getChanges()
-                            .get(0).getDateTime()))
-                    .toList();
-
-            RedactionLogEntry initialEntry = sortedEntries.get(0);
-            for (RedactionLogEntry entry : sortedEntries.subList(1, sortedEntries.size())) {
-                copyNonNullFields(entry, initialEntry);
-            }
-            mergedEntries.add(initialEntry);
-        }
-
-        redactionLog.setRedactionLogEntry(mergedEntries);
-        return redactionLog;
-    }
-
-
-    @SneakyThrows
-    public static void copyNonNullFields(RedactionLogEntry source, RedactionLogEntry destination) {
-
-        if (source == null || destination == null) {
-            throw new IllegalArgumentException("Source and destination objects must not be null");
-        }
-
-        Class<?> sourceClass = source.getClass();
-
-        Field[] sourceFields = sourceClass.getDeclaredFields();
-
-        for (Field field : sourceFields) {
-            field.setAccessible(true);
-            Object value = field.get(source);
-            if (value != null) {
-                field.set(destination, value);
-            }
-        }
-    }
-
-}
@@ -1,101 +0,0 @@
-package com.iqser.red.service.redaction.v1.server.migration;
-
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-
-import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ChangeType;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualChange;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Change;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ManualRedactionType;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
-
-public class MigrationMapper {
-
-    public static com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Change toEntityLogChanges(Change change) {
-
-        return new com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Change(change.getAnalysisNumber(),
-                                                                                                              toEntityLogType(change.getType()),
-                                                                                                              change.getDateTime());
-    }
-
-
-    public static com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualChange toEntityLogManualChanges(com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ManualChange manualChange) {
-
-        return new com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualChange(toManualRedactionType(manualChange.getManualRedactionType()),
-                                                                                                                    manualChange.getProcessedDate(),
-                                                                                                                    manualChange.getRequestedDate(),
-                                                                                                                    manualChange.getUserId(),
-                                                                                                                    manualChange.getPropertyChanges());
-    }
-
-
-    public static ChangeType toEntityLogType(com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ChangeType type) {
-
-        return switch (type) {
-            case ADDED -> ChangeType.ADDED;
-            case REMOVED -> ChangeType.REMOVED;
-            case CHANGED -> ChangeType.CHANGED;
-        };
-    }
-
-
-    public static com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType toManualRedactionType(ManualRedactionType manualRedactionType) {
-
-        return switch (manualRedactionType) {
-            case ADD_LOCALLY -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.ADD;
-            case ADD_TO_DICTIONARY -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.ADD_TO_DICTIONARY;
-            case REMOVE_LOCALLY -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.REMOVE;
-            case REMOVE_FROM_DICTIONARY -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.REMOVE_FROM_DICTIONARY;
-            case FORCE_REDACT -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.FORCE;
-            case FORCE_HINT -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.FORCE;
-            case RECATEGORIZE -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.RECATEGORIZE;
-            case LEGAL_BASIS_CHANGE -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.LEGAL_BASIS_CHANGE;
-            case RESIZE -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.RESIZE;
-        };
-    }
-
-
-    public static com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine toEntityLogEngine(Engine engine) {
-
-        return switch (engine) {
-            case DICTIONARY -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine.DICTIONARY;
-            case NER -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine.NER;
-            case RULE -> com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine.RULE;
-        };
-    }
-
-
-    public static Set<com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine> getMigratedEngines(RedactionLogEntry entry) {
-
-        Set<com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine> engines = new HashSet<>();
-
-        if (entry.isImported()) {
-            engines.add(com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine.IMPORTED);
-        }
-
-        if (entry.getEngines() == null) {
-            return engines;
-        }
-        entry.getEngines()
-                .stream()
-                .map(MigrationMapper::toEntityLogEngine)
-                .forEach(engines::add);
-
-        return engines;
-    }
-
-
-    public List<ManualChange> migrateManualChanges(List<com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ManualChange> manualChanges) {
-
-        if (manualChanges == null) {
-            return Collections.emptyList();
-        }
-        return manualChanges.stream()
-                .map(MigrationMapper::toEntityLogManualChanges)
-                .toList();
-    }
-
-}
@@ -1,96 +0,0 @@
-package com.iqser.red.service.redaction.v1.server.migration;
-
-import static com.iqser.red.service.redaction.v1.model.QueueNames.MIGRATION_QUEUE;
-import static com.iqser.red.service.redaction.v1.model.QueueNames.MIGRATION_RESPONSE_QUEUE;
-
-import org.springframework.amqp.core.Message;
-import org.springframework.amqp.rabbit.annotation.RabbitHandler;
-import org.springframework.amqp.rabbit.annotation.RabbitListener;
-import org.springframework.amqp.rabbit.core.RabbitTemplate;
-import org.springframework.stereotype.Service;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog;
-import com.iqser.red.service.redaction.v1.model.MigrationRequest;
-import com.iqser.red.service.redaction.v1.model.MigrationResponse;
-import com.iqser.red.service.redaction.v1.server.model.MigratedEntityLog;
-import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
-import com.iqser.red.service.redaction.v1.server.service.DictionaryService;
-import com.iqser.red.service.redaction.v1.server.service.document.DocumentGraphMapper;
-import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
-
-import lombok.AccessLevel;
-import lombok.RequiredArgsConstructor;
-import lombok.SneakyThrows;
-import lombok.experimental.FieldDefaults;
-import lombok.extern.slf4j.Slf4j;
-
-@Slf4j
-@Service
-@RequiredArgsConstructor
-@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
-public class MigrationMessageReceiver {
-
-    ObjectMapper objectMapper;
-    RedactionLogToEntityLogMigrationService redactionLogToEntityLogMigrationService;
-    RedactionStorageService redactionStorageService;
-    LegacyRedactionLogMergeService legacyRedactionLogMergeService;
-    LegacyVersion0MigrationService legacyVersion0MigrationService;
-    RabbitTemplate rabbitTemplate;
-    DictionaryService dictionaryService;
-
-
-    @SneakyThrows
-    @RabbitHandler
-    @RabbitListener(queues = MIGRATION_QUEUE)
-    public void receiveMigrationRequest(Message message) {
-
-        MigrationRequest migrationRequest = objectMapper.readValue(message.getBody(), MigrationRequest.class);
-        log.info("--------------------------------------------------------------------");
-        log.info("Starting redactionLog to entityLog migration for dossierId {} and fileId {}", migrationRequest.getDossierId(), migrationRequest.getFileId());
-
-        dictionaryService.updateDictionary(migrationRequest.getDossierTemplateId(), migrationRequest.getDossierId());
-
-        Document document = DocumentGraphMapper.toDocumentGraph(redactionStorageService.getDocumentData(migrationRequest.getDossierId(), migrationRequest.getFileId()));
-        RedactionLog redactionLog = redactionStorageService.getRedactionLog(migrationRequest.getDossierId(), migrationRequest.getFileId());
-
-        if (redactionLog.getAnalysisVersion() == 0) {
-            redactionLog = legacyVersion0MigrationService.mergeDuplicateAnnotationIds(redactionLog);
-        } else {
-            redactionLog = legacyRedactionLogMergeService.addManualAddEntriesAndRemoveSkippedImported(redactionLog,
-                                                                                                      migrationRequest.getManualRedactions(),
-                                                                                                      migrationRequest.getDossierTemplateId());
-        }
-
-        MigratedEntityLog migratedEntityLog = redactionLogToEntityLogMigrationService.migrate(redactionLog,
-                                                                                              document,
-                                                                                              migrationRequest.getDossierTemplateId(),
-                                                                                              migrationRequest.getManualRedactions(),
-                                                                                              migrationRequest.getFileId(),
-                                                                                              migrationRequest.getEntitiesWithComments(),
-                                                                                              migrationRequest.isFileIsApproved());
-
-        log.info("Storing migrated entityLog and ids to migrate in DB for file {}", migrationRequest.getFileId());
-
-        redactionStorageService.storeObject(migrationRequest.getDossierId(), migrationRequest.getFileId(), FileType.ENTITY_LOG, migratedEntityLog.getEntityLog());
-        redactionStorageService.storeObject(migrationRequest.getDossierId(), migrationRequest.getFileId(), FileType.MIGRATED_IDS, migratedEntityLog.getMigratedIds());
-
-        sendFinished(MigrationResponse.builder().dossierId(migrationRequest.getDossierId()).fileId(migrationRequest.getFileId()).build());
-        log.info("Migrated {} redactionLog entries, found {} annotation ids for migration in the db, {} new manual entries, for dossierId {} and fileId {}",
-                 migratedEntityLog.getEntityLog().getEntityLogEntry().size(),
-                 migratedEntityLog.getMigratedIds().getMappings().size(),
-                 migratedEntityLog.getMigratedIds().getManualRedactionEntriesToAdd().size(),
-                 migrationRequest.getDossierId(),
-                 migrationRequest.getFileId());
-        log.info("");
-    }
-
-
-    @SneakyThrows
-    public void sendFinished(MigrationResponse migrationResponse) {
-
-        rabbitTemplate.convertAndSend(MIGRATION_RESPONSE_QUEUE, migrationResponse);
-    }
-
-}
@@ -1,354 +0,0 @@
-package com.iqser.red.service.redaction.v1.server.migration;
-
-import java.awt.geom.Rectangle2D;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Optional;
-import java.util.Set;
-import java.util.function.Function;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
-
-import org.springframework.stereotype.Service;
-
-import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogLegalBasis;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.migration.MigratedIds;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.BaseAnnotation;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogLegalBasis;
-import com.iqser.red.service.redaction.v1.server.model.MigratedEntityLog;
-import com.iqser.red.service.redaction.v1.server.model.MigrationEntity;
-import com.iqser.red.service.redaction.v1.server.model.PrecursorEntity;
-import com.iqser.red.service.redaction.v1.server.model.RectangleWithPage;
-import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
-import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
-import com.iqser.red.service.redaction.v1.server.model.document.nodes.Image;
-import com.iqser.red.service.redaction.v1.server.model.document.nodes.ImageType;
-import com.iqser.red.service.redaction.v1.server.service.DictionaryService;
-import com.iqser.red.service.redaction.v1.server.service.ManualChangesApplicationService;
-import com.iqser.red.service.redaction.v1.server.service.document.EntityFindingUtility;
-import com.iqser.red.service.redaction.v1.server.service.document.EntityFromPrecursorCreationService;
-import com.iqser.red.service.redaction.v1.server.utils.IdBuilder;
-import com.iqser.red.service.redaction.v1.server.utils.MigratedIdsCollector;
-
-import lombok.AccessLevel;
-import lombok.RequiredArgsConstructor;
-import lombok.experimental.FieldDefaults;
-import lombok.extern.slf4j.Slf4j;
-
-@Slf4j
-@Service
-@RequiredArgsConstructor
-@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
-//TODO: remove this, once the migration is done
-public class RedactionLogToEntityLogMigrationService {
-
-    private static final double MATCH_THRESHOLD = 10;
-    EntityFindingUtility entityFindingUtility;
-    DictionaryService dictionaryService;
-    ManualChangesApplicationService manualChangesApplicationService;
-
-
-    public MigratedEntityLog migrate(RedactionLog redactionLog,
-                                     Document document,
-                                     String dossierTemplateId,
-                                     ManualRedactions manualRedactions,
-                                     String fileId,
-                                     Set<String> entitiesWithComments,
-                                     boolean fileIsApproved) {
-
-        log.info("Migrating entities for file {}", fileId);
-        List<MigrationEntity> entitiesToMigrate = calculateMigrationEntitiesFromRedactionLog(redactionLog, document, dossierTemplateId, fileId);
-
-        MigratedIds migratedIds = entitiesToMigrate.stream()
-                .collect(new MigratedIdsCollector());
-
-        log.info("applying manual changes to migrated entities for file {}", fileId);
-        applyLocalProcessedManualChanges(entitiesToMigrate, manualRedactions, fileIsApproved);
-
-        EntityLog entityLog = new EntityLog();
-        entityLog.setAnalysisNumber(redactionLog.getAnalysisNumber());
-        entityLog.setRulesVersion(redactionLog.getRulesVersion());
-        entityLog.setDictionaryVersion(redactionLog.getDictionaryVersion());
-        entityLog.setDossierDictionaryVersion(redactionLog.getDossierDictionaryVersion());
-        entityLog.setLegalBasisVersion(redactionLog.getLegalBasisVersion());
-        entityLog.setAnalysisVersion(redactionLog.getAnalysisVersion());
-        entityLog.setLegalBasis(redactionLog.getLegalBasis()
-                                        .stream()
-                                        .map(RedactionLogToEntityLogMigrationService::toEntityLogLegalBasis)
-                                        .toList());
-
-        Map<String, String> oldToNewIDMapping = migratedIds.buildOldToNewMapping();
-
-        log.info("Writing migrated entities to entityLog for file {}", fileId);
-        entityLog.setEntityLogEntry(entitiesToMigrate.stream()
-                                            .map(migrationEntity -> migrationEntity.toEntityLogEntry(oldToNewIDMapping))
-                                            .toList());
-
-        if (getNumberOfApprovedEntries(redactionLog, document.getNumberOfPages()) != entityLog.getEntityLogEntry().size()) {
-            String message = String.format("Not all entities have been found during the migration redactionLog has %d entries and new entityLog %d",
-                                           redactionLog.getRedactionLogEntry().size(),
-                                           entityLog.getEntityLogEntry().size());
-            log.error(message);
-            throw new AssertionError(message);
-        }
-
-        Set<String> entitiesWithUnprocessedChanges = manualRedactions.buildAll()
-                .stream()
-                .filter(manualRedaction -> manualRedaction.getProcessedDate() == null)
-                .map(BaseAnnotation::getAnnotationId)
-                .collect(Collectors.toSet());
-
-        MigratedIds idsToMigrateInDb = entitiesToMigrate.stream()
-                .filter(migrationEntity -> migrationEntity.hasManualChangesOrComments(entitiesWithComments, entitiesWithUnprocessedChanges))
-                .filter(m -> !m.getOldId().equals(m.getNewId()))
-                .collect(new MigratedIdsCollector());
-
-        List<ManualRedactionEntry> manualRedactionEntriesToAdd = entitiesToMigrate.stream()
-                .filter(MigrationEntity::needsManualEntry)
-                .map(MigrationEntity::buildManualRedactionEntry)
-                .toList();
-
-        idsToMigrateInDb.setManualRedactionEntriesToAdd(manualRedactionEntriesToAdd);
-
-        return new MigratedEntityLog(idsToMigrateInDb, entityLog);
-    }
-
-
-    private void applyLocalProcessedManualChanges(List<MigrationEntity> entitiesToMigrate, ManualRedactions manualRedactions, boolean fileIsApproved) {
-
-        if (manualRedactions == null) {
-            return;
-        }
-        Map<String, List<BaseAnnotation>> manualChangesPerAnnotationId;
-
-        if (fileIsApproved) {
-            manualChangesPerAnnotationId = manualRedactions.buildAll()
-                    .stream()
-                    .filter(manualChange -> (manualChange.getProcessedDate() != null && manualChange.isLocal()) //
-                                            // unprocessed dict change of type IdRemoval or ManualResize must be applied for approved documents
-                                            || (manualChange.getProcessedDate() == null && !manualChange.isLocal() //
-                                                && (manualChange instanceof IdRemoval || manualChange instanceof ManualResizeRedaction)))
-                    .map(this::convertPendingDictChangesToLocal)
-                    .collect(Collectors.groupingBy(BaseAnnotation::getAnnotationId));
-        } else {
-            manualChangesPerAnnotationId = manualRedactions.buildAll()
-                    .stream()
-                    .filter(manualChange -> manualChange.getProcessedDate() != null)
-                    .filter(BaseAnnotation::isLocal)
-                    .collect(Collectors.groupingBy(BaseAnnotation::getAnnotationId));
-        }
-
-        entitiesToMigrate.forEach(migrationEntity -> migrationEntity.applyManualChanges(manualChangesPerAnnotationId.getOrDefault(migrationEntity.getOldId(),
-                                                                                                                                  Collections.emptyList()),
-                                                                                        manualChangesApplicationService));
-
-    }
-
-
-    private BaseAnnotation convertPendingDictChangesToLocal(BaseAnnotation baseAnnotation) {
-
-        if (baseAnnotation.getProcessedDate() != null) {
-            return baseAnnotation;
-        }
-
-        if (baseAnnotation.isLocal()) {
-            return baseAnnotation;
-        }
-
-        if (baseAnnotation instanceof ManualResizeRedaction manualResizeRedaction) {
-            manualResizeRedaction.setAddToAllDossiers(false);
-            manualResizeRedaction.setUpdateDictionary(false);
-        } else if (baseAnnotation instanceof IdRemoval idRemoval) {
-            idRemoval.setRemoveFromAllDossiers(false);
-            idRemoval.setRemoveFromDictionary(false);
-        }
-
-        return baseAnnotation;
-    }
-
-
-    private long getNumberOfApprovedEntries(RedactionLog redactionLog, int numberOfPages) {
-
-        return redactionLog.getRedactionLogEntry()
-                .stream()
-                .filter(redactionLogEntry -> isOnExistingPage(redactionLogEntry, numberOfPages))
-                .count();
-    }
-
-
-    private List<MigrationEntity> calculateMigrationEntitiesFromRedactionLog(RedactionLog redactionLog, Document document, String dossierTemplateId, String fileId) {
-
-        List<MigrationEntity> images = getImageBasedMigrationEntities(redactionLog, document, fileId, dossierTemplateId);
-        List<MigrationEntity> textMigrationEntities = getTextBasedMigrationEntities(redactionLog, document, dossierTemplateId, fileId);
-        return Stream.of(textMigrationEntities.stream(), images.stream())
-                .flatMap(Function.identity())
-                .toList();
-    }
-
-
-    private static EntityLogLegalBasis toEntityLogLegalBasis(RedactionLogLegalBasis redactionLogLegalBasis) {
-
-        return new EntityLogLegalBasis(redactionLogLegalBasis.getName(), redactionLogLegalBasis.getDescription(), redactionLogLegalBasis.getReason());
-    }
-
-
-    private List<MigrationEntity> getImageBasedMigrationEntities(RedactionLog redactionLog, Document document, String fileId, String dossierTemplateId) {
-
-        List<Image> images = document.streamAllImages()
-                .collect(Collectors.toList());
-
-        List<RedactionLogEntry> redactionLogImages = redactionLog.getRedactionLogEntry()
-                .stream()
-                .filter(RedactionLogEntry::isImage)
-                .toList();
-
-        List<MigrationEntity> migrationEntities = new LinkedList<>();
-        for (RedactionLogEntry redactionLogImage : redactionLogImages) {
-            List<RectangleWithPage> imagePositions = redactionLogImage.getPositions()
-                    .stream()
-                    .map(RectangleWithPage::fromRedactionLogRectangle)
-                    .toList();
-            assert imagePositions.size() == 1;
-            Optional<Image> optionalClosestImage = images.stream()
-                    .filter(image -> image.onPage(redactionLogImage.getPositions()
-                                                          .get(0).getPage()))
-                    .min(Comparator.comparingDouble(image -> EntityFindingUtility.calculateDistance(image.getPosition(), imagePositions.get(0).rectangle2D())))
-                    .filter(image -> EntityFindingUtility.calculateDistance(image.getPosition(), imagePositions.get(0).rectangle2D()) <= MATCH_THRESHOLD);
-
-            Image closestImage;
-            if (optionalClosestImage.isEmpty()) { // if no fitting image can be found create a new one with the previous values!
-                closestImage = buildImageDirectly(document, redactionLogImage);
-            } else {
-                closestImage = optionalClosestImage.get();
-                images.remove(closestImage);
-            }
-
-            String ruleIdentifier;
-            String reason = Optional.ofNullable(redactionLogImage.getReason())
-                    .orElse("");
-            if (redactionLogImage.getMatchedRule().isBlank() || redactionLogImage.getMatchedRule() == null) {
-                ruleIdentifier = "OLDIMG.0.0";
-            } else {
-                ruleIdentifier = "OLDIMG." + redactionLogImage.getMatchedRule() + ".0";
-            }
-
-            if (redactionLogImage.lastChangeIsRemoved()) {
-                closestImage.remove(ruleIdentifier, reason);
-            } else if (redactionLogImage.isRedacted()) {
-                closestImage.apply(ruleIdentifier, reason, redactionLogImage.getLegalBasis());
-            } else {
-                closestImage.skip(ruleIdentifier, reason);
-            }
-            migrationEntities.add(MigrationEntity.fromRedactionLogImage(redactionLogImage, closestImage, fileId, dictionaryService, dossierTemplateId));
-        }
-        return migrationEntities;
-    }
-
-
-    private static Image buildImageDirectly(Document document, RedactionLogEntry redactionLogImage) {
-
-        Image image = Image.builder()
-                .documentTree(document.getDocumentTree())
-                .imageType(ImageType.fromString(redactionLogImage.getType()))
-                .transparent(redactionLogImage.isImageHasTransparency())
-                .page(document.getPages()
-                              .stream()
-                              .filter(p -> p.getNumber() == redactionLogImage.getPositions()
-                                      .get(0).getPage())
-                              .findFirst()
-                              .orElseThrow())
-                .position(toRectangle2D(redactionLogImage.getPositions()
-                                                .get(0)))
-                .build();
-
-        List<Integer> treeId = document.getDocumentTree().createNewMainEntryAndReturnId(image);
-        image.setTreeId(treeId);
-        image.setId(IdBuilder.buildId(image.getPages(),
-                                      image.getBBox().values()
-                                              .stream()
-                                              .toList(),
-                                      "",
-                                      ""));
-
-        return image;
-    }
-
-
-    private static Rectangle2D toRectangle2D(Rectangle rect) {
-
-        return new Rectangle2D.Double(rect.getTopLeft().getX(), rect.getTopLeft().getY(), rect.getWidth(), rect.getHeight());
-    }
-
-
-    private List<MigrationEntity> getTextBasedMigrationEntities(RedactionLog redactionLog, Document document, String dossierTemplateId, String fileId) {
-
-        List<MigrationEntity> entitiesToMigrate = redactionLog.getRedactionLogEntry()
-                .stream()
-                .filter(redactionLogEntry -> !redactionLogEntry.isImage())
-                .filter(redactionLogEntry -> isOnExistingPage(redactionLogEntry, document.getNumberOfPages()))
-                .map(entry -> MigrationEntity.fromRedactionLogEntry(entry, fileId, dictionaryService, dossierTemplateId))
-                .toList();
-
-        List<PrecursorEntity> precursorEntities = entitiesToMigrate.stream()
-                .map(MigrationEntity::getPrecursorEntity)
-                .toList();
-
-        log.info("Finding all possible entities");
-        Map<String, List<TextEntity>> tempEntitiesByValue = entityFindingUtility.findAllPossibleEntitiesAndGroupByValue(document, precursorEntities);
-
-        for (MigrationEntity migrationEntity : entitiesToMigrate) {
-            Optional<TextEntity> optionalTextEntity = entityFindingUtility.findClosestEntityAndReturnEmptyIfNotFound(migrationEntity.getPrecursorEntity(),
-                                                                                                                     tempEntitiesByValue,
-                                                                                                                     MATCH_THRESHOLD);
-
-            if (optionalTextEntity.isEmpty()) {
-                migrationEntity.setMigratedEntity(migrationEntity.getPrecursorEntity());
-                migrationEntity.setOldId(migrationEntity.getPrecursorEntity().getId());
-                migrationEntity.setNewId(migrationEntity.getPrecursorEntity().getId());
-                continue;
-            }
-
-            TextEntity migratedEntity = EntityFromPrecursorCreationService.createCorrectEntity(migrationEntity.getPrecursorEntity(), optionalTextEntity.get(), true);
-
-            migrationEntity.setMigratedEntity(migratedEntity);
-            migrationEntity.setOldId(migrationEntity.getPrecursorEntity().getId());
-            migrationEntity.setNewId(migratedEntity.getId());
-        }
-
-        tempEntitiesByValue.values()
-                .stream()
-                .flatMap(Collection::stream)
-                .forEach(TextEntity::removeFromGraph);
-
-        return entitiesToMigrate;
-    }
-
-
-    private boolean isOnExistingPage(RedactionLogEntry redactionLogEntry, int numberOfPages) {
-
-        var pages = redactionLogEntry.getPositions()
-                .stream()
-                .map(Rectangle::getPage)
-                .collect(Collectors.toSet());
-
-        for (int page : pages) {
-            if (page > numberOfPages) {
-                return false;
-            }
-        }
-        return true;
-    }
-
-}
@@ -1,469 +0,0 @@
-package com.iqser.red.service.redaction.v1.server.model;
-
-import static com.iqser.red.service.redaction.v1.server.service.EntityLogCreatorService.buildEntryState;
-import static com.iqser.red.service.redaction.v1.server.service.EntityLogCreatorService.buildEntryType;
-
-import java.awt.geom.Rectangle2D;
-import java.time.OffsetDateTime;
-import java.util.Collections;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Locale;
-import java.util.Map;
-import java.util.Optional;
-import java.util.Set;
-import java.util.stream.Collectors;
-
-import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Change;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ChangeType;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryType;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualChangeFactory;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.BaseAnnotation;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntryType;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ManualRedactionType;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
-import com.iqser.red.service.redaction.v1.server.migration.MigrationMapper;
-import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
-import com.iqser.red.service.redaction.v1.server.model.document.entity.IEntity;
-import com.iqser.red.service.redaction.v1.server.model.document.entity.ManualChangeOverwrite;
-import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
-import com.iqser.red.service.redaction.v1.server.model.document.nodes.Image;
-import com.iqser.red.service.redaction.v1.server.model.document.nodes.ImageType;
-import com.iqser.red.service.redaction.v1.server.service.DictionaryService;
-import com.iqser.red.service.redaction.v1.server.service.ManualChangesApplicationService;
-
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.Data;
-import lombok.RequiredArgsConstructor;
-import lombok.extern.slf4j.Slf4j;
-
-@Slf4j
-@Data
-@Builder
-@AllArgsConstructor
-@RequiredArgsConstructor
-public final class MigrationEntity {
-
-    private final PrecursorEntity precursorEntity;
-    private final RedactionLogEntry redactionLogEntry;
-    private final DictionaryService dictionaryService;
-    private final String dossierTemplateId;
-    private IEntity migratedEntity;
-    private String oldId;
-    private String newId;
-    private String fileId;
-
-    @Builder.Default
-    List<BaseAnnotation> manualChanges = new LinkedList<>();
-
-
-    public static MigrationEntity fromRedactionLogEntry(RedactionLogEntry redactionLogEntry, String fileId, DictionaryService dictionaryService, String dossierTemplateId) {
-
-        boolean hint = dictionaryService.isHint(redactionLogEntry.getType(), dossierTemplateId);
-        PrecursorEntity precursorEntity = createPrecursorEntity(redactionLogEntry, hint);
-        if (precursorEntity.getEntityType().equals(EntityType.HINT) && !redactionLogEntry.isHint() && !redactionLogEntry.isRedacted()) {
-            precursorEntity.ignore(precursorEntity.getRuleIdentifier(), precursorEntity.getReason());
-        } else if (redactionLogEntry.lastChangeIsRemoved()) {
-            precursorEntity.remove(precursorEntity.getRuleIdentifier(), precursorEntity.getReason());
-        } else if (lastManualChangeIsRemove(redactionLogEntry)) {
-            precursorEntity.ignore(precursorEntity.getRuleIdentifier(), precursorEntity.getReason());
-        } else if (precursorEntity.isApplied() && redactionLogEntry.isRecommendation()) {
-            precursorEntity.skip(precursorEntity.getRuleIdentifier(), precursorEntity.getReason());
-        } else if (precursorEntity.isApplied()) {
-            precursorEntity.apply(precursorEntity.getRuleIdentifier(), precursorEntity.getReason(), precursorEntity.getLegalBasis());
-        } else {
-            precursorEntity.skip(precursorEntity.getRuleIdentifier(), precursorEntity.getReason());
-        }
-
-        return MigrationEntity.builder()
-                .precursorEntity(precursorEntity)
-                .redactionLogEntry(redactionLogEntry)
-                .oldId(redactionLogEntry.getId())
-                .fileId(fileId)
-                .dictionaryService(dictionaryService)
-                .dossierTemplateId(dossierTemplateId)
-                .build();
-    }
-
-
-    public static MigrationEntity fromRedactionLogImage(RedactionLogEntry redactionLogImage,
-                                                        Image image,
-                                                        String fileId,
-                                                        DictionaryService dictionaryService,
-                                                        String dossierTemplateId) {
-
-        return MigrationEntity.builder()
-                .redactionLogEntry(redactionLogImage)
-                .migratedEntity(image)
-                .oldId(redactionLogImage.getId())
-                .newId(image.getId())
-                .fileId(fileId)
-                .dictionaryService(dictionaryService)
-                .dossierTemplateId(dossierTemplateId)
-                .build();
-    }
-
-
-    private static boolean lastManualChangeIsRemove(RedactionLogEntry redactionLogEntry) {
-
-        if (redactionLogEntry.getManualChanges() == null) {
-            return false;
-        }
-
-        return redactionLogEntry.getManualChanges()
-                .stream()
-                .reduce((a, b) -> b)
-                .map(m -> m.getManualRedactionType().equals(ManualRedactionType.REMOVE_LOCALLY))
-                .orElse(false);
-    }
-
-
-    public static PrecursorEntity createPrecursorEntity(RedactionLogEntry redactionLogEntry, boolean hint) {
-
-        String ruleIdentifier = buildRuleIdentifier(redactionLogEntry);
-        List<RectangleWithPage> rectangleWithPages = redactionLogEntry.getPositions()
-                .stream()
-                .map(RectangleWithPage::fromRedactionLogRectangle)
-                .toList();
-        EntityType entityType = getEntityType(redactionLogEntry, hint);
-        return PrecursorEntity.builder()
-                .id(redactionLogEntry.getId())
-                .value(redactionLogEntry.getValue())
-                .entityPosition(rectangleWithPages)
-                .ruleIdentifier(ruleIdentifier)
-                .reason(Optional.ofNullable(redactionLogEntry.getReason())
-                                .orElse(""))
-                .legalBasis(redactionLogEntry.getLegalBasis())
-                .type(redactionLogEntry.getType())
-                .section(redactionLogEntry.getSection())
-                .engines(MigrationMapper.getMigratedEngines(redactionLogEntry))
-                .entityType(entityType)
-                .applied(redactionLogEntry.isRedacted())
-                .isDictionaryEntry(redactionLogEntry.isDictionaryEntry())
-                .isDossierDictionaryEntry(redactionLogEntry.isDossierDictionaryEntry())
-                .rectangle(redactionLogEntry.isRectangle())
-                .manualOverwrite(new ManualChangeOverwrite(entityType))
-                .build();
-    }
-
-
-    private static String buildRuleIdentifier(RedactionLogEntry redactionLogEntry) {
-
-        String ruleIdentifier;
-        if (redactionLogEntry.getMatchedRule() != null) {
-            ruleIdentifier = "OLD." + redactionLogEntry.getMatchedRule() + ".0";
-        } else {
-            ruleIdentifier = "MAN.5.0"; // pure ManualRedactions used to have no matched rule
-        }
-        return ruleIdentifier;
-    }
-
-
-    private static EntityType getEntityType(RedactionLogEntry redactionLogEntry, boolean hint) {
-
-        if (hint) {
-            return EntityType.HINT;
-        }
-        if (redactionLogEntry.isFalsePositive()) {
-            return EntityType.FALSE_POSITIVE;
-        }
-        if (redactionLogEntry.isHint()) {
-            return EntityType.HINT;
-        }
-        if (redactionLogEntry.isRecommendation()) {
-            return EntityType.RECOMMENDATION;
-        }
-        return EntityType.ENTITY;
-    }
-
-
-    public EntityLogEntry toEntityLogEntry(Map<String, String> oldToNewIdMapping) {
-
-        EntityLogEntry entityLogEntry;
-        if (migratedEntity instanceof Image image) {
-            entityLogEntry = createEntityLogEntry(image);
-        } else if (migratedEntity instanceof TextEntity textEntity) {
-            entityLogEntry = createEntityLogEntry(textEntity);
-        } else if (migratedEntity instanceof PrecursorEntity entity) {
-            entityLogEntry = createEntityLogEntry(entity);
-        } else {
-            throw new UnsupportedOperationException("Unknown subclass " + migratedEntity.getClass());
-        }
-
-        entityLogEntry.setManualChanges(ManualChangeFactory.toLocalManualChangeList(migratedEntity.getManualOverwrite().getManualChangeLog(), true));
-        entityLogEntry.setColor(redactionLogEntry.getColor());
-        entityLogEntry.setChanges(redactionLogEntry.getChanges()
-                                          .stream()
-                                          .map(MigrationMapper::toEntityLogChanges)
-                                          .toList());
-        entityLogEntry.setReference(migrateSetOfIds(redactionLogEntry.getReference(), oldToNewIdMapping));
-        entityLogEntry.setImportedRedactionIntersections(migrateSetOfIds(redactionLogEntry.getImportedRedactionIntersections(), oldToNewIdMapping));
-        entityLogEntry.setEngines(MigrationMapper.getMigratedEngines(redactionLogEntry));
-
-        if (entityLogEntry.getEntryType().equals(EntryType.HINT) && lastManualChangeIsRemoveLocally(entityLogEntry)) {
-            entityLogEntry.setState(EntryState.IGNORED);
-        }
-
-        if (redactionLogEntry.isImported() && redactionLogEntry.getValue() == null) {
-            entityLogEntry.setValue("Imported Redaction");
-        }
-
-        if(entityLogEntry.getChanges() != null
-           && !entityLogEntry.getChanges().isEmpty()
-           && entityLogEntry.getChanges().stream().map(Change::getType).toList().get(entityLogEntry.getChanges().size() - 1).equals(ChangeType.REMOVED)) {
-            entityLogEntry.setState(EntryState.REMOVED);
-        }
-
-        return entityLogEntry;
-    }
-
-
-    private static boolean lastManualChangeIsRemoveLocally(EntityLogEntry entityLogEntry) {
-
-        return entityLogEntry.getManualChanges()
-                .stream()
-                .reduce((a, b) -> b)
-                .filter(mc -> mc.getManualRedactionType().equals(com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.REMOVE))
-                .isPresent();
-    }
-
-
-    private Set<String> migrateSetOfIds(Set<String> ids, Map<String, String> oldToNewIdMapping) {
-
-        if (ids == null) {
-            return Collections.emptySet();
-        }
-        return ids.stream()
-                .map(oldToNewIdMapping::get)
-                .collect(Collectors.toSet());
-    }
-
-
-    public EntityLogEntry createEntityLogEntry(Image image) {
-
-        String imageType = image.getImageType().equals(ImageType.OTHER) ? "image" : image.getImageType().toString().toLowerCase(Locale.ENGLISH);
-        List<Position> positions = getPositionsFromOverride(image).orElse(List.of(new Position(image.getPosition(), image.getPage().getNumber())));
-        return EntityLogEntry.builder()
-                .id(image.getId())
-                .value(image.getValue())
-                .type(imageType)
-                .reason(image.buildReasonWithManualChangeDescriptions())
-                .legalBasis(image.getManualOverwrite().getLegalBasis()
-                                    .orElse(redactionLogEntry.getLegalBasis()))
-                .matchedRule(image.getMatchedRule().getRuleIdentifier().toString())
-                .dictionaryEntry(false)
-                .positions(positions)
-                .containingNodeId(image.getTreeId())
-                .closestHeadline(image.getHeadline().getTextBlock().getSearchText())
-                .section(image.getManualOverwrite().getSection()
-                                 .orElse(redactionLogEntry.getSection()))
-                .textAfter(redactionLogEntry.getTextAfter())
-                .textBefore(redactionLogEntry.getTextBefore())
-                .imageHasTransparency(image.isTransparent())
-                .state(buildEntryState(image))
-                .entryType(dictionaryService.isHint(imageType, dossierTemplateId) ? EntryType.IMAGE_HINT : EntryType.IMAGE)
-                .build();
-
-    }
-
-
-    public EntityLogEntry createEntityLogEntry(PrecursorEntity precursorEntity) {
-
-        return EntityLogEntry.builder()
-                .id(precursorEntity.getId())
-                .reason(precursorEntity.buildReasonWithManualChangeDescriptions())
-                .legalBasis(precursorEntity.getManualOverwrite().getLegalBasis()
-                                    .orElse(redactionLogEntry.getLegalBasis()))
-                .value(precursorEntity.value())
-                .type(precursorEntity.type())
-                .state(buildEntryState(precursorEntity))
-                .entryType(buildEntryType(precursorEntity))
-                .section(precursorEntity.getManualOverwrite().getSection()
-                                 .orElse(redactionLogEntry.getSection()))
-                .textAfter(redactionLogEntry.getTextAfter())
-                .textBefore(redactionLogEntry.getTextBefore())
-                .containingNodeId(Collections.emptyList())
-                .closestHeadline("")
-                .matchedRule(precursorEntity.getMatchedRule().getRuleIdentifier().toString())
-                .dictionaryEntry(precursorEntity.isDictionaryEntry())
-                .dossierDictionaryEntry(precursorEntity.isDossierDictionaryEntry())
-                .startOffset(-1)
-                .endOffset(-1)
-                .positions(precursorEntity.getManualOverwrite().getPositions()
-                                   .orElse(precursorEntity.getEntityPosition())
-                                   .stream()
-                                   .map(entityPosition -> new Position(entityPosition.rectangle2D(), entityPosition.pageNumber()))
-                                   .toList())
-                .engines(Collections.emptySet())
-                .build();
-    }
-
-
-    public EntityLogEntry createEntityLogEntry(TextEntity entity) {
-
-        assert entity.getPositionsOnPagePerPage().size() == 1;
-        List<Position> rectanglesPerLine = getRectanglesPerLine(entity);
-        return EntityLogEntry.builder()
-                .id(entity.getId())
-                .positions(rectanglesPerLine)
-                .reason(entity.buildReasonWithManualChangeDescriptions())
-                .legalBasis(entity.getManualOverwrite().getLegalBasis()
-                                    .orElse(redactionLogEntry.getLegalBasis()))
-                .value(entity.getManualOverwrite().getValue()
-                               .orElse(entity.getMatchedRule().isWriteValueWithLineBreaks() ? entity.getValueWithLineBreaks() : entity.getValue()))
-                .type(entity.type())
-                .section(entity.getManualOverwrite().getSection()
-                                 .orElse(redactionLogEntry.getSection()))
-                .textAfter(entity.getTextAfter())
-                .textBefore(entity.getTextBefore())
-                .containingNodeId(entity.getDeepestFullyContainingNode().getTreeId())
-                .closestHeadline(entity.getDeepestFullyContainingNode().getHeadline().getTextBlock().getSearchText())
-                .matchedRule(entity.getMatchedRule().getRuleIdentifier().toString())
-                .dictionaryEntry(entity.isDictionaryEntry())
-                .startOffset(entity.getTextRange().start())
-                .endOffset(entity.getTextRange().end())
-                .dossierDictionaryEntry(entity.isDossierDictionaryEntry())
-                .engines(entity.getEngines() != null ? entity.getEngines() : Collections.emptySet())
-                .state(buildEntryState(entity))
-                .entryType(buildEntryType(entity))
-                .build();
-    }
-
-
-    private static List<Position> getRectanglesPerLine(TextEntity entity) {
-
-        return getPositionsFromOverride(entity).orElse(entity.getPositionsOnPagePerPage()
-                                                               .get(0).getRectanglePerLine()
-                                                               .stream()
-                                                               .map(rectangle2D -> new Position(rectangle2D,
-                                                                                                entity.getPositionsOnPagePerPage()
-                                                                                                        .get(0).getPage().getNumber()))
-                                                               .toList());
-    }
-
-
-    private static Optional<List<Position>> getPositionsFromOverride(IEntity entity) {
-
-        return entity.getManualOverwrite().getPositions()
-                .map(rects -> rects.stream()
-                        .map(r -> new Position(r.rectangle2D(), r.pageNumber()))
-                        .toList());
-    }
-
-
-    public boolean hasManualChangesOrComments(Set<String> entitiesWithComments, Set<String> entitiesWithUnprocessedChanges) {
-
-        return !(redactionLogEntry.getManualChanges() == null || redactionLogEntry.getManualChanges().isEmpty()) || //
-               !(redactionLogEntry.getComments() == null || redactionLogEntry.getComments().isEmpty()) //
-               || hasManualChanges() || entitiesWithComments.contains(oldId) || entitiesWithUnprocessedChanges.contains(oldId);
-    }
-
-
-    public boolean hasManualChanges() {
-
-        return !manualChanges.isEmpty();
-    }
-
-
-    public void applyManualChanges(List<BaseAnnotation> manualChangesToApply, ManualChangesApplicationService manualChangesApplicationService) {
-
-        manualChanges.addAll(manualChangesToApply);
-        manualChangesToApply.forEach(manualChange -> {
-            if (manualChange instanceof ManualResizeRedaction manualResizeRedaction && migratedEntity instanceof TextEntity textEntity) {
-                manualResizeRedaction.setAnnotationId(newId);
-                manualChangesApplicationService.resize(textEntity, manualResizeRedaction);
-            } else if (manualChange instanceof ManualRecategorization manualRecategorization && migratedEntity instanceof Image image) {
-                image.setImageType(ImageType.fromString(manualRecategorization.getType()));
-                migratedEntity.getManualOverwrite().addChange(manualChange);
-            } else {
-                migratedEntity.getManualOverwrite().addChange(manualChange);
-            }
-        });
-    }
-
-
-    public ManualRedactionEntry buildManualRedactionEntry() {
-
-        assert hasManualChanges();
-
-        // currently we need to insert a manual redaction entry, whenever an entity has been resized.
-        String user = manualChanges.stream()
-                .filter(mc -> mc instanceof ManualResizeRedaction)
-                .findFirst()
-                .orElse(manualChanges.get(0)).getUser();
-
-        OffsetDateTime requestDate = manualChanges.get(0).getRequestDate();
-
-        return ManualRedactionEntry.builder()
-                .annotationId(newId)
-                .fileId(fileId)
-                .user(user)
-                .requestDate(requestDate)
-                .type(redactionLogEntry.getType())
-                .value(redactionLogEntry.getValue())
-                .reason(redactionLogEntry.getReason())
-                .legalBasis(redactionLogEntry.getLegalBasis())
-                .section(redactionLogEntry.getSection())
-                .rectangle(false)
-                .addToDictionary(false)
-                .addToDossierDictionary(false)
-                .positions(buildPositions(migratedEntity))
-                .textAfter(redactionLogEntry.getTextAfter())
-                .textBefore(redactionLogEntry.getTextBefore())
-                .dictionaryEntryType(DictionaryEntryType.ENTRY)
-                .build();
-    }
-
-
-    private List<Rectangle> buildPositions(IEntity entity) {
-
-        if (entity instanceof TextEntity textEntity) {
-
-            var positionsOnPage = textEntity.getPositionsOnPagePerPage()
-                    .get(0);
-            return positionsOnPage.getRectanglePerLine()
-                    .stream()
-                    .map(p -> new Rectangle((float) p.getX(), (float) p.getY(), (float) p.getWidth(), (float) p.getHeight(), positionsOnPage.getPage().getNumber()))
-                    .toList();
-        }
-        if (entity instanceof PrecursorEntity pEntity) {
-
-            return pEntity.getManualOverwrite().getPositions()
-                    .orElse(pEntity.getEntityPosition())
-                    .stream()
-                    .map(p -> new Rectangle((float) p.rectangle2D().getX(),
-                                            (float) p.rectangle2D().getY(),
-                                            (float) p.rectangle2D().getWidth(),
-                                            (float) p.rectangle2D().getHeight(),
-                                            p.pageNumber()))
-                    .toList();
-        }
-        if (entity instanceof Image image) {
-
-            Rectangle2D position = image.getManualOverwrite().getPositions()
-                    .map(p -> p.get(0).rectangle2D())
-                    .orElse(image.getPosition());
-
-            return List.of(new Rectangle((float) position.getX(), (float) position.getY(), (float) position.getWidth(), (float) position.getHeight(), image.getPage().getNumber()));
-
-        } else {
-            throw new UnsupportedOperationException();
-        }
-    }
-
-
-    public boolean needsManualEntry() {
-
-        return manualChanges.stream()
-                       .anyMatch(mc -> mc instanceof ManualResizeRedaction && !((ManualResizeRedaction) mc).getUpdateDictionary()) && !(migratedEntity instanceof Image);
-    }
-
-}
@@ -1,5 +1,6 @@
 package com.iqser.red.service.redaction.v1.server.model;

+import java.util.ArrayList;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.stream.Stream;
@@ -17,7 +18,7 @@ import lombok.experimental.FieldDefaults;
 */
@Getter
@AllArgsConstructor
-@FieldDefaults(level = AccessLevel.PRIVATE, makeFinal = true)
+@FieldDefaults(level = AccessLevel.PRIVATE)
 public class NerEntities {

    List<NerEntity> nerEntityList;
@@ -29,6 +30,14 @@ public class NerEntities {
    }


+    public void merge(NerEntities other) {
+
+        List<NerEntity> mergedList = new ArrayList<>(nerEntityList);
+        mergedList.addAll(other.getNerEntityList());
+        nerEntityList = mergedList;
+    }
+
+
    /**
     * Checks if there are any entities of a specified type.
     *
@@ -58,8 +67,19 @@ public class NerEntities {
    /**
     * Represents a single NER entity with its value, text range, and type.
     */
-    public record NerEntity(String value, TextRange textRange, String type) {
+    public record NerEntity(String value, TextRange textRange, String type, Double confidence, Engine engine) {
+
+        public NerEntity(String value, TextRange textRange, String type) {
+
+            this(value, textRange, type, null, Engine.NER);
+        }

    }

+    public enum Engine {
+        NER,
+        CLOUD_NER,
+        LLM_NER
+    }
+
 }
@@ -120,7 +120,8 @@ public class PrecursorEntity implements IEntity {
        EntityType entityType = getEntityType(entryType);
        String value = Optional.ofNullable(importedRedaction.getValue())
                .orElse("");
-        return PrecursorEntity.builder()
+
+        PrecursorEntityBuilder precursorEntityBuilder = PrecursorEntity.builder()
                .id(importedRedaction.getId())
                .value(value)
                .entityPosition(rectangleWithPages)
@@ -130,14 +131,21 @@ public class PrecursorEntity implements IEntity {
                                    .orElse(""))
                .type(Optional.ofNullable(importedRedaction.getType())
                              .orElse(IMPORTED_REDACTION_TYPE))
-                .section(importedRedaction.getManualOverwriteSection())
+                .section(Optional.ofNullable(importedRedaction.getSection())
+                                 .orElse(""))
                .entityType(entityType)
                .isDictionaryEntry(false)
                .isDossierDictionaryEntry(false)
+                .manualOverwrite(new ManualChangeOverwrite(entityType))
                .rectangle(value.isBlank() || entryType.equals(EntryType.IMAGE) || entryType.equals(EntryType.IMAGE_HINT) || entryType.equals(EntryType.AREA))
-                .manualOverwrite(new ManualChangeOverwrite(entityType, importedRedaction.getManualOverwriteSection()))
-                .engines(Set.of(Engine.IMPORTED))
-                .build();
+                .engines(Set.of(Engine.IMPORTED));
+
+        if (importedRedaction.getManualOverwriteSection() != null && !importedRedaction.getManualOverwriteSection().isEmpty()) {
+            precursorEntityBuilder.section(importedRedaction.getManualOverwriteSection())
+                    .manualOverwrite(new ManualChangeOverwrite(entityType, importedRedaction.getManualOverwriteSection()));
+        }
+
+        return precursorEntityBuilder.build();
    }


@@ -172,6 +180,22 @@ public class PrecursorEntity implements IEntity {
    }


+    /**
+     * @return true when this entity is of EntityType ENTITY or HINT
+     */
+    public boolean validEntityType() {
+
+        return entityType.equals(EntityType.ENTITY) || entityType.equals(EntityType.HINT);
+    }
+
+
+    @Override
+    public boolean valid() {
+
+        return active() && validEntityType();
+    }
+
+
    private static EntityType getEntityType(EntryType entryType) {

        switch (entryType) {
@@ -31,6 +31,9 @@ public class Dictionary {

    @Getter
    private List<DictionaryModel> dictionaryModels;
+    // todo: dossier and dossier template level DictionaryModels override each other
+    // at the moment there are no problems because they always have the same rank / hint information
+    // but it should be changed so that the localAccessMap contains all models
    private Map<String, DictionaryModel> localAccessMap = new HashMap<>();

    @Getter
@@ -11,6 +11,7 @@ import java.util.stream.Stream;
 import org.ahocorasick.trie.Trie;

 import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
+import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;

 import lombok.Data;

@@ -18,6 +19,7 @@ import lombok.Data;
 public class SearchImplementation {

    private boolean ignoreCase;
+    private boolean ignoreWhiteSpace;
    private List<String> values;

    private Pattern pattern;
@@ -40,13 +42,22 @@ public class SearchImplementation {
    }


+    public SearchImplementation(Collection<String> values, boolean ignoreCase, boolean ignoreWhiteSpace) {
+
+        this.values = new ArrayList<>(values);
+        this.ignoreCase = ignoreCase;
+        this.ignoreWhiteSpace = ignoreWhiteSpace;
+        this.createSearchImplementation();
+    }
+
+
    private void createSearchImplementation() {

        if (this.values.isEmpty()) {
            return;
        }

-        if (this.values.size() == 1) {
+        if (this.values.size() == 1 && !this.ignoreWhiteSpace) {
            var text = this.values.iterator().next();

            if (this.ignoreCase) {
@@ -59,6 +70,9 @@ public class SearchImplementation {
            if (this.ignoreCase) {
                builder.ignoreCase();
            }
+            if (this.ignoreWhiteSpace) {
+                builder.ignoreWhiteSpace();
+            }

            builder.addKeywords(this.values);

@@ -104,6 +118,12 @@ public class SearchImplementation {
    }


+    public Stream<TextRange> getBoundaries(TextBlock textBlock) {
+
+        return getBoundaries(textBlock, textBlock.getTextRange());
+    }
+
+
    public Stream<TextRange> getBoundaries(CharSequence text, TextRange region) {

        if (this.values.isEmpty()) {
@@ -0,0 +1,75 @@
+package com.iqser.red.service.redaction.v1.server.model.document;
+
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Set;
+import java.util.function.BiConsumer;
+import java.util.function.BinaryOperator;
+import java.util.function.Function;
+import java.util.function.Supplier;
+import java.util.stream.Collector;
+import java.util.stream.Stream;
+
+import com.iqser.red.service.redaction.v1.server.model.document.textblock.ConcatenatedTextBlock;
+import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
+
+import lombok.NoArgsConstructor;
+
+@NoArgsConstructor
+public class ConsecutiveTextBlockCollector implements Collector<TextBlock, List<ConcatenatedTextBlock>, List<TextBlock>> {
+
+    @Override
+    public Supplier<List<ConcatenatedTextBlock>> supplier() {
+
+        return LinkedList::new;
+    }
+
+
+    @Override
+    public BiConsumer<List<ConcatenatedTextBlock>, TextBlock> accumulator() {
+
+        return (existingList, textBlock) -> {
+            if (existingList.isEmpty()) {
+                ConcatenatedTextBlock ctb = ConcatenatedTextBlock.empty();
+                ctb.concat(textBlock);
+                existingList.add(ctb);
+                return;
+            }
+
+            ConcatenatedTextBlock prevBlock = existingList.get(existingList.size() - 1);
+
+            if (prevBlock.getTextRange().end() == textBlock.getTextRange().start()) {
+                prevBlock.concat(textBlock);
+            } else {
+                ConcatenatedTextBlock ctb = ConcatenatedTextBlock.empty();
+                ctb.concat(textBlock);
+                existingList.add(ctb);
+            }
+        };
+    }
+
+
+    @Override
+    public BinaryOperator<List<ConcatenatedTextBlock>> combiner() {
+
+        return (list1, list2) -> Stream.concat(list1.stream(), list2.stream())
+                .toList();
+    }
+
+
+    @Override
+    public Function<List<ConcatenatedTextBlock>, List<TextBlock>> finisher() {
+
+        return a -> a.stream()
+                .map(tb -> (TextBlock) tb)
+                .toList();
+    }
+
+
+    @Override
+    public Set<Characteristics> characteristics() {
+
+        return Set.of(Characteristics.IDENTITY_FINISH);
+    }
+
+}
@@ -1,11 +1,13 @@
 package com.iqser.red.service.redaction.v1.server.model.document;

+import static com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentStructureProto.DocumentStructure;
+
 import java.io.Serializable;

-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPage;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPositionData;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentStructure;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentTextData;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPageProto.AllDocumentPages;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPositionDataProto.AllDocumentPositionData;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentStructureWrapper;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentTextDataProto.AllDocumentTextData;

 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
@@ -21,9 +23,15 @@ import lombok.experimental.FieldDefaults;
@FieldDefaults(level = AccessLevel.PRIVATE)
 public class DocumentData implements Serializable {

-    DocumentPage[] documentPages;
-    DocumentTextData[] documentTextData;
-    DocumentPositionData[] documentPositionData;
-    DocumentStructure documentStructure;
+    AllDocumentPages documentPages;
+    AllDocumentTextData documentTextData;
+    AllDocumentPositionData documentPositionData;
+    DocumentStructureWrapper documentStructureWrapper;
+
+
+    public DocumentStructure getDocumentStructure() {
+
+        return documentStructureWrapper.getDocumentStructure();
+    }

 }
@@ -288,8 +288,8 @@ public class DocumentTree {
        if (treeId.isEmpty()) {
            return root;
        }
-        Entry entry = root.children.get(treeId.get(0));
-        for (int id : treeId.subList(1, treeId.size())) {
+        Entry entry = root;
+        for (int id : treeId) {
            entry = entry.children.get(id);
        }
        return entry;
@@ -52,6 +52,17 @@ public interface IEntity {
    String type();


+    /**
+     * An Entity is valid, when it active and not a false recommendation, a false positive or a dictionary removal.
+     *
+     * @return true, if the entity is valid, false otherwise/
+     */
+    default boolean valid() {
+
+        return active();
+    }
+
+
    /**
     * Calculates the length of the entity's value.
     *
@@ -24,6 +24,10 @@ import lombok.Data;
 import lombok.EqualsAndHashCode;
 import lombok.experimental.FieldDefaults;

+/**
+ * Represents a text entity within a document, characterized by its text range, type, entity type,
+ * and associated metadata like matched rules, pages, and engines.
+ */
@Data
@Builder
@AllArgsConstructor
@@ -194,25 +198,32 @@ public class TextEntity implements IEntity {

    public boolean containedBy(TextEntity textEntity) {

-        return this.textRange.containedBy(textEntity.getTextRange()) //
-               || duplicateTextRanges.stream()
-                       .anyMatch(duplicateTextRange -> duplicateTextRange.containedBy(textEntity.textRange)) //
-               || duplicateTextRanges.stream()
-                       .anyMatch(duplicateTextRange -> textEntity.getDuplicateTextRanges()
-                               .stream()
-                               .anyMatch(duplicateTextRange::containedBy));
+        return textEntity.contains(this);
    }


    public boolean contains(TextEntity textEntity) {

-        return this.textRange.contains(textEntity.getTextRange()) //
-               || duplicateTextRanges.stream()
-                       .anyMatch(duplicateTextRange -> duplicateTextRange.contains(textEntity.textRange)) //
-               || duplicateTextRanges.stream()
-                       .anyMatch(duplicateTextRange -> textEntity.getDuplicateTextRanges()
-                               .stream()
-                               .anyMatch(duplicateTextRange::contains));
+        if (this.textRange.contains(textEntity.getTextRange())) {
+            return true;
+        }
+
+        List<TextRange> textEntityDuplicateRanges = textEntity.getDuplicateTextRanges();
+        // use optimized indexed loops for extra performance boost
+        for (int i = 0, duplicateTextRangesSize = duplicateTextRanges.size(); i < duplicateTextRangesSize; i++) {
+            TextRange duplicateTextRange = duplicateTextRanges.get(i);
+            if (duplicateTextRange.contains(textEntity.getTextRange())) {
+                return true;
+            }
+            for (int j = 0, textEntityDuplicateRangesSize = textEntityDuplicateRanges.size(); j < textEntityDuplicateRangesSize; j++) {
+                TextRange otherRange = textEntityDuplicateRanges.get(j);
+                if (duplicateTextRange.contains(otherRange)) {
+                    return true;
+                }
+            }
+        }
+
+        return false;
    }


@@ -278,6 +289,21 @@ public class TextEntity implements IEntity {
    }


+    /**
+     * @return true when this entity is of EntityType ENTITY or HINT
+     */
+    public boolean validEntityType() {
+
+        return entityType.equals(EntityType.ENTITY) || entityType.equals(EntityType.HINT);
+    }
+
+
+    public boolean valid() {
+
+        return active() && validEntityType();
+    }
+
+
    @Override
    public String value() {

@@ -9,7 +9,7 @@ import java.util.Set;
 import com.iqser.red.service.redaction.v1.server.model.document.DocumentTree;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngineProto.LayoutEngine;

 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
@@ -9,6 +9,7 @@ import java.util.Set;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;

+import com.iqser.red.service.redaction.v1.server.model.document.ConsecutiveTextBlockCollector;
 import com.iqser.red.service.redaction.v1.server.model.document.DocumentTree;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;

@@ -63,8 +64,8 @@ public class Document extends AbstractSemanticNode {
     *
     * @return A list of main sections within the document
     * @deprecated This method is marked for removal.
-     *  Use {@link #streamChildrenOfType(NodeType)} instead,
-     *  or {@link #getChildrenOfTypeSectionOrSuperSection()} which returns children of type SECTION as well as SUPER_SECTION.
+     * Use {@link #streamChildrenOfType(NodeType)} instead,
+     * or {@link #getChildrenOfTypeSectionOrSuperSection()} which returns children of type SECTION as well as SUPER_SECTION.
     */
    @Deprecated(forRemoval = true)
    public List<Section> getMainSections() {
@@ -1,19 +1,9 @@
 package com.iqser.red.service.redaction.v1.server.model.document.nodes;

-import java.awt.geom.Rectangle2D;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-import com.iqser.red.service.redaction.v1.server.model.document.DocumentTree;
-import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;

 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
-import lombok.Builder;
 import lombok.Data;
 import lombok.EqualsAndHashCode;
 import lombok.NoArgsConstructor;
@@ -1,19 +1,9 @@
 package com.iqser.red.service.redaction.v1.server.model.document.nodes;

-import java.awt.geom.Rectangle2D;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
-import com.iqser.red.service.redaction.v1.server.model.document.DocumentTree;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;

 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
-import lombok.Builder;
 import lombok.Data;
 import lombok.EqualsAndHashCode;
 import lombok.NoArgsConstructor;
@@ -1,20 +1,10 @@
 package com.iqser.red.service.redaction.v1.server.model.document.nodes;

-import java.awt.geom.Rectangle2D;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-import com.iqser.red.service.redaction.v1.server.model.document.DocumentTree;
-import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.AtomicTextBlock;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;

 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
-import lombok.Builder;
 import lombok.Data;
 import lombok.EqualsAndHashCode;
 import lombok.NoArgsConstructor;
@@ -71,6 +61,11 @@ public class Headline extends AbstractSemanticNode {
    }


+    /**
+     * Extracts the SectionIdentifier from the text of this headline.
+     *
+     * @return The SectionIdentifier, with which the headline starts.
+     */
    @Override
    public SectionIdentifier getSectionIdentifier() {

@@ -3,22 +3,16 @@ package com.iqser.red.service.redaction.v1.server.model.document.nodes;
 import java.awt.geom.Rectangle2D;
 import java.util.Collections;
 import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
 import java.util.Locale;
 import java.util.Map;
 import java.util.PriorityQueue;
 import java.util.Set;

-import com.iqser.red.service.redaction.v1.server.model.document.DocumentTree;
 import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.IEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.ManualChangeOverwrite;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule;
-import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
-import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlockCollector;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;

 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
@@ -3,8 +3,10 @@ package com.iqser.red.service.redaction.v1.server.model.document.nodes;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
+import java.util.stream.Stream;

 import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
+import com.iqser.red.service.redaction.v1.server.model.document.textblock.AtomicTextBlock;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlockCollector;

@@ -35,7 +37,7 @@ public class Page {
    Integer width;
    Integer rotation;

-    List<SemanticNode> mainBody;
+    List<AtomicTextBlock> textBlocksOnPage;
    Header header;
    Footer footer;

@@ -53,17 +55,67 @@ public class Page {
     */
    public TextBlock getMainBodyTextBlock() {

-        return mainBody.stream()
-                .filter(SemanticNode::isLeaf)
-                .map(SemanticNode::getTextBlock)
+        return textBlocksOnPage.stream()
+                .filter(atb -> !atb.isEmpty())
                .collect(new TextBlockCollector());
    }


+    /**
+     * Retrieves the highest SemanticNodes, which appear only on this page. It is achieved by traversing the DocumentTree up, until a SemanticNode's direct parent is no longer exclusively on this page.
+     *
+     * @return A list which contains the highes SemanticNodes, which appear only on this page.
+     */
+    public List<SemanticNode> getMainBody() {
+
+        return textBlocksOnPage.stream()
+                .map(AtomicTextBlock::getParent)
+                .map(this::getHighestParentOnlyOnPage)
+                .distinct()
+                .toList();
+    }
+
+
+    /**
+     * Retrieves the highest SemanticNodes which are present on the page. There might be multiples, as two or more Main Sections start on a page.
+     * This is achieved by traversing up the document tree and returning all SemanticNodes whose direct parent is the Document
+     *
+     * @return A list of the highest SemanticNodes present on this page
+     */
+    public Stream<SemanticNode> streamHighestSemanticNodesOnPage() {
+
+        return textBlocksOnPage.stream()
+                .map(AtomicTextBlock::getParent)
+                .map(this::getHighestSemanticNodeOnPage)
+                .distinct();
+    }
+
+
+    private SemanticNode getHighestParentOnlyOnPage(SemanticNode node) {
+
+        SemanticNode currentNode = node;
+        while (currentNode.hasParent() && currentNode.getParent().onlyOnPage(this)) {
+            currentNode = currentNode.getParent();
+        }
+        return currentNode;
+    }
+
+
+    private SemanticNode getHighestSemanticNodeOnPage(SemanticNode node) {
+
+        SemanticNode currentNode = node;
+        while (currentNode.hasParent() //
+               && !currentNode.getParent().getType().equals(NodeType.DOCUMENT)) {
+            currentNode = currentNode.getParent();
+        }
+        return currentNode;
+    }
+
+
    @Override
    public String toString() {

-        return String.valueOf(number);
+        return "Page: " + number;
    }

 }
@@ -1,19 +1,9 @@
 package com.iqser.red.service.redaction.v1.server.model.document.nodes;

-import java.awt.geom.Rectangle2D;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-import com.iqser.red.service.redaction.v1.server.model.document.DocumentTree;
-import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;

 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
-import lombok.Builder;
 import lombok.Data;
 import lombok.EqualsAndHashCode;
 import lombok.experimental.FieldDefaults;
@@ -1,23 +1,9 @@
 package com.iqser.red.service.redaction.v1.server.model.document.nodes;

-import java.awt.geom.Rectangle2D;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
-import com.iqser.red.service.redaction.v1.server.model.document.DocumentTree;
-import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
-import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlockCollector;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
-
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
-import lombok.Builder;
 import lombok.Data;
 import lombok.EqualsAndHashCode;
-import lombok.RequiredArgsConstructor;
 import lombok.experimental.FieldDefaults;
 import lombok.experimental.SuperBuilder;
 import lombok.extern.slf4j.Slf4j;
@@ -33,7 +19,6 @@ import lombok.extern.slf4j.Slf4j;
@EqualsAndHashCode(onlyExplicitlyIncluded = true, callSuper = true)
 public class Section extends AbstractSemanticNode {

-
    @Override
    public NodeType getType() {

@@ -53,6 +38,11 @@ public class Section extends AbstractSemanticNode {
    }


+    /**
+     * Returns the SectionIdentifier from the headline obtained by the getHeadline() method.
+     *
+     * @return the SectionIdentifier of the associated Headline
+     */
    @Override
    public SectionIdentifier getSectionIdentifier() {

@@ -60,7 +50,6 @@ public class Section extends AbstractSemanticNode {
    }


-
    @Override
    public String toString() {

@@ -100,5 +89,4 @@ public class Section extends AbstractSemanticNode {
        return streamAllSubNodesOfType(NodeType.HEADLINE).anyMatch(h -> h.containsStringIgnoreCase(value));
    }

-
 }
@@ -3,29 +3,36 @@ package com.iqser.red.service.redaction.v1.server.model.document.nodes;
 import java.util.Collections;
 import java.util.LinkedList;
 import java.util.List;
+import java.util.Locale;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;

 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
+import lombok.Getter;
 import lombok.experimental.FieldDefaults;

 /**
- * Represents a unique identifier for a section within a document.
+ * Represents the textual identifier sometimes present in a Headline. For example, given the headline 3.1 Results, the section identifier is 3.1.
+ * Keep in mind, this identifier must not be unique in a single document, as there might be multiple headlines starting with the same textual identifier.
 */
@AllArgsConstructor
@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
 public class SectionIdentifier {

-    static Pattern numericalIdentifierPattern = Pattern.compile("^[\\s]?(\\d+)[\\s.,;]?(\\d+)?[\\s.,;]?(\\d+)?[\\s.,;]?(\\d+)?");
+    public static Pattern numericalIdentifierPattern = Pattern.compile("^[\\s]?(\\d+)[\\s.,;]?(\\d+)?[\\s.,;]?(\\d+)?[\\s.,;]?(\\d+)?");
+    public static Pattern alphanumericIdentifierPattern = Pattern.compile("^[\\s]?[A-Za-z][\\s.,;]?(\\d+)[\\s.,;]?(\\d+)?[\\s.,;]?(\\d+)?[\\s.,;]?(\\d+)?[\\s.,;]?");

-    private enum Format {
+    protected enum Format {
        EMPTY,
        NUMERICAL,
+        ALPHANUMERIC,
        DOCUMENT
    }

+    @Getter
    Format format;
+    @Getter
    String identifierString;
    List<Integer> identifiers;
    boolean asChild;
@@ -47,6 +54,10 @@ public class SectionIdentifier {
        if (numericalIdentifierMatcher.find()) {
            return buildNumericalSectionIdentifier(headline, numericalIdentifierMatcher);
        }
+        Matcher alphanumericIdentifierMatcher = alphanumericIdentifierPattern.matcher(headline);
+        if (alphanumericIdentifierMatcher.find()) {
+            return buildAlphanumericSectionIdentifier(headline, alphanumericIdentifierMatcher);
+        }
        // more formats here
        return SectionIdentifier.empty();
    }
@@ -105,8 +116,34 @@ public class SectionIdentifier {
    }


+    private static SectionIdentifier buildAlphanumericSectionIdentifier(String headline, Matcher alphanumericIdentifierMatcher) {
+
+        String identifierString = headline.substring(alphanumericIdentifierMatcher.start(), alphanumericIdentifierMatcher.end());
+
+        String alphanumericIdentifier = alphanumericIdentifierMatcher.group(0).substring(0, 1).toUpperCase(Locale.ENGLISH);
+        int mappedCharacterValue = alphanumericIdentifier.charAt(0) - 'A' + 1;
+        List<Integer> identifiers = new LinkedList<>();
+        identifiers.add(mappedCharacterValue);
+
+        for (int i = 1; i <= 3; i++) {
+            String numericalIdentifier = alphanumericIdentifierMatcher.group(i);
+            if (numericalIdentifier == null || numericalIdentifier.equals("0") || numericalIdentifier.isEmpty() || numericalIdentifier.isBlank()) {
+                break;
+            }
+            identifiers.add(Integer.parseInt(numericalIdentifier.trim()));
+        }
+
+        return new SectionIdentifier(Format.ALPHANUMERIC,
+                                     // Changed format to reflect alphanumeric
+                                     identifierString,
+                                     identifiers.stream()
+                                             .toList(),
+                                     false);
+    }
+
+
    /**
-     * Determines if the current section is the parent of the given section.
+     * Determines if the current SectionIdentifier is the parent of the given SectionIdentifier.
     *
     * @param sectionIdentifier The section identifier to compare against.
     * @return true if the current section is the parent of the given section, false otherwise.
@@ -155,4 +192,30 @@ public class SectionIdentifier {
        return identifierString;
    }

+
+    /**
+     * @return true, when no identifier could be found
+     */
+    public boolean isEmpty() {
+
+        return this.format.equals(Format.EMPTY);
+    }
+
+
+    /**
+     * The level of a SectionIdentifier corresponds with the count of identifiers. E.g. 1.1 is level 2, and 1. is level 1
+     *
+     * @return the level of the Headline
+     */
+    public int level() {
+
+        return identifiers.size();
+    }
+
+
+    protected List<Integer> getIdentifiers() {
+
+        return identifiers;
+    }
+
 }
@@ -14,8 +14,10 @@ import java.util.Set;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;

+import com.iqser.red.service.redaction.v1.server.model.document.ConsecutiveTextBlockCollector;
 import com.iqser.red.service.redaction.v1.server.model.document.DocumentTree;
 import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
+import com.iqser.red.service.redaction.v1.server.model.document.entity.IEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.AtomicTextBlock;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
@@ -23,7 +25,7 @@ import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBl
 import com.iqser.red.service.redaction.v1.server.service.document.NodeVisitor;
 import com.iqser.red.service.redaction.v1.server.utils.RectangleTransformations;
 import com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngineProto.LayoutEngine;

 public interface SemanticNode {

@@ -50,6 +52,23 @@ public interface SemanticNode {
    }


+    /**
+     * Searches all Nodes located underneath this Node in the DocumentTree that are found on the given pages.
+     * Then consecutive AtomicTextBlocks are concatenated where possible and the list of the resulting TextBlocks is returned.
+     *
+     * @return List of TextBlocks containing all AtomicTextBlocks that are located under this Node on the given pages.
+     */
+    default List<TextBlock> getTextBlocksByPageNumbers(Set<Integer> pageNumbers) {
+
+        return streamAllSubNodes().filter(SemanticNode::isLeaf)
+                .map(SemanticNode::getTextBlock)
+                .map(TextBlock::getAtomicTextBlocks)
+                .flatMap(List::stream)
+                .filter(atb -> pageNumbers.contains(atb.getPage().getNumber()))
+                .collect(new ConsecutiveTextBlockCollector());
+    }
+
+
    /**
     * Any Node maintains its own Set of Entities.
     * This Set contains all Entities whose TextRange intersects the TextRange of this node.
@@ -59,6 +78,20 @@ public interface SemanticNode {
    Set<TextEntity> getEntities();


+    /**
+     * A view of the Entity Set of this SemanticNode including only the active (APPLIED or SKIPPED) Entities which are of a valid type (ENTITY or HINT).
+     * This is used for all functions, which check for the existence of an Entity, such as hasEntityOfType().
+     *
+     * @return Set of valid TextEntities
+     */
+    default Stream<TextEntity> streamValidEntities() {
+
+        return getEntities().stream()
+                .filter(IEntity::active)
+                .filter(TextEntity::validEntityType);
+    }
+
+
    /**
     * Each AtomicTextBlock is assigned a page, so to get the pages this node appears on, it collects the PageNodes from each AtomicTextBlock belonging to this node's TextBlock.
     *
@@ -149,7 +182,7 @@ public interface SemanticNode {


    /**
-     * Returns a SectionIdentifier, such that it acts as a child of the first Headline associated with this SemanticNode.
+     * Returns the SectionIdentifier as a child of the SectionIdentifier returned by the getHeadline() method.
     *
     * @return The SectionIdentifier from the first Headline.
     */
@@ -259,9 +292,7 @@ public interface SemanticNode {
     */
    default boolean hasEntitiesOfType(String type) {

-        return getEntities().stream()
-                .filter(TextEntity::active)
-                .anyMatch(redactionEntity -> redactionEntity.type().equals(type));
+        return streamValidEntities().anyMatch(redactionEntity -> redactionEntity.type().equals(type));
    }


@@ -274,10 +305,8 @@ public interface SemanticNode {
     */
    default boolean hasEntitiesOfAnyType(String... types) {

-        return getEntities().stream()
-                .filter(TextEntity::active)
-                .anyMatch(redactionEntity -> Arrays.stream(types)
-                        .anyMatch(type -> redactionEntity.type().equals(type)));
+        return streamValidEntities().anyMatch(redactionEntity -> Arrays.stream(types)
+                .anyMatch(type -> redactionEntity.type().equals(type)));
    }


@@ -290,9 +319,7 @@ public interface SemanticNode {
     */
    default boolean hasEntitiesOfAllTypes(String... types) {

-        return getEntities().stream()
-                .filter(TextEntity::active)
-                .map(TextEntity::type)
+        return streamValidEntities().map(TextEntity::type)
                .collect(Collectors.toUnmodifiableSet())
                .containsAll(Arrays.stream(types)
                                     .toList());
@@ -301,31 +328,28 @@ public interface SemanticNode {

    /**
     * Returns a List of Entities in this SemanticNode which are of the provided type such as "CBI_author".
-     * Ignores Entity with ignored == true or removed == true.
+     * Ignores Entity which are not active or of a removal type ignored == true or removed == true.
     *
     * @param type string representing the type of entities to return
     * @return List of RedactionEntities of any the type
     */
    default List<TextEntity> getEntitiesOfType(String type) {

-        return getEntities().stream()
-                .filter(TextEntity::active)
-                .filter(redactionEntity -> redactionEntity.type().equals(type))
+        return streamValidEntities().filter(redactionEntity -> redactionEntity.type().equals(type))
                .toList();
    }


    /**
     * Returns a List of Entities in this SemanticNode which have any of the provided types such as "CBI_author".
-     * Ignores Entity with ignored == true or removed == true.
+     * Ignores Entity that are not valid.
     *
     * @param types A list of strings representing the types of entities to return
     * @return List of RedactionEntities of any provided type
     */
    default List<TextEntity> getEntitiesOfType(List<String> types) {

-        return getEntities().stream()
-                .filter(TextEntity::active)
+        return streamValidEntities()//
                .filter(redactionEntity -> redactionEntity.isAnyType(types))
                .toList();
    }
@@ -333,15 +357,14 @@ public interface SemanticNode {

    /**
     * Returns a List of Entities in this SemanticNode which have any of the provided types.
-     * Ignores Entity with the ignored flag set to true or the removed flag set to true.
+     * Ignores Entity that are not valid.
     *
     * @param types A list of strings representing the types of entities to return
     * @return List of RedactionEntities that match any of the provided types
     */
    default List<TextEntity> getEntitiesOfType(String... types) {

-        return getEntities().stream()
-                .filter(TextEntity::active)
+        return streamValidEntities()//
                .filter(redactionEntity -> redactionEntity.isAnyType(Arrays.stream(types)
                                                                             .toList()))
                .toList();
@@ -445,7 +468,7 @@ public interface SemanticNode {
     */
    default boolean containsStringIgnoreCase(String string) {

-        return getTextBlock().getSearchText().toLowerCase(Locale.ROOT).contains(string.toLowerCase(Locale.ROOT));
+        return getTextBlock().getSearchTextLowerCase().contains(string.toLowerCase(Locale.ROOT));
    }


@@ -756,13 +779,12 @@ public interface SemanticNode {


    /**
-     * TODO: this produces unwanted results for sections spanning multiple columns.
     * Computes the Union of the bounding boxes of all children recursively.
     *
     * @return The union of the BoundingBoxes of all children
     */
    private Map<Page, Rectangle2D> getBBoxFromChildren() {
-
+        //TODO: this produces unwanted results for sections spanning multiple columns.
        Map<Page, Rectangle2D> bBoxPerPage = new HashMap<>();
        List<Map<Page, Rectangle2D>> childrenBBoxes = streamChildren().map(SemanticNode::getBBox)
                .toList();
@@ -808,4 +830,17 @@ public interface SemanticNode {
        streamChildren().forEach(childNode -> childNode.accept(visitor));
    }

+
+    /**
+     * Checks wether this SemanticNode appears on a single page only, and if that page is the provided one.
+     *
+     * @param page the page to check
+     * @return true, when SemanticNode is on a single page only and the page is the provided page. Otherwise, false.
+     */
+    default boolean onlyOnPage(Page page) {
+
+        Set<Page> pages = getPages();
+        return pages.size() == 1 && pages.contains(page);
+    }
+
 }
@@ -2,7 +2,6 @@ package com.iqser.red.service.redaction.v1.server.model.document.nodes;

 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
-import lombok.Builder;
 import lombok.Data;
 import lombok.EqualsAndHashCode;
 import lombok.experimental.FieldDefaults;
@@ -39,6 +38,11 @@ public class SuperSection extends AbstractSemanticNode {
    }


+    /**
+     * Returns the SectionIdentifier from the headline obtained by the getHeadline() method.
+     *
+     * @return the SectionIdentifier of the associated Headline
+     */
    @Override
    public SectionIdentifier getSectionIdentifier() {

@@ -46,7 +50,6 @@ public class SuperSection extends AbstractSemanticNode {
    }


-
    @Override
    public String toString() {

@@ -86,5 +89,4 @@ public class SuperSection extends AbstractSemanticNode {
        return streamAllSubNodesOfType(NodeType.HEADLINE).anyMatch(h -> h.containsStringIgnoreCase(value));
    }

-
 }
@@ -17,7 +17,7 @@ import com.iqser.red.service.redaction.v1.server.model.document.DocumentTree;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlockCollector;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngineProto.LayoutEngine;

 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
@@ -74,8 +74,7 @@ public class Table implements SemanticNode {
        return IntStream.range(0, numberOfRows).boxed()
                .filter(row -> rowContainsStringsIgnoreCase(row, strings))
                .flatMap(this::streamRow)
-                .map(TableCell::getEntities)
-                .flatMap(Collection::stream);
+                .flatMap(TableCell::streamValidEntities);
    }


@@ -135,11 +134,11 @@ public class Table implements SemanticNode {


    /**
-     * Streams all entities in this table, that appear in a row, which contains at least one entity with any of the provided types.
+     * Streams all entities in this table, that appear in a row, which contains at least one valid entity with any of the provided types.
     * Ignores Entity with ignored == true or removed == true.
     *
     * @param types type strings to check whether a row contains an entity like them
-     * @return Stream of all entities in this table, that appear in a row, which contains at least one entity with any of the provided types.
+     * @return Stream of all entities in this table, that appear in a row, which contains at least one valid entity with any of the provided types.
     */
    public Stream<TextEntity> streamEntitiesWhereRowContainsEntitiesOfType(List<String> types) {

@@ -192,30 +191,26 @@ public class Table implements SemanticNode {


    /**
-     * Streams all Entities in the given row.
+     * Streams all valid Entities in the given row.
     *
     * @param rowNumber the row number to look for
     * @return stream of TextEntities occurring in row
     */
    public Stream<TextEntity> streamTextEntitiesInRow(int rowNumber) {

-        return streamRow(rowNumber).map(TableCell::getEntities)
-                .flatMap(Collection::stream)
-                .filter(TextEntity::active);
+        return streamRow(rowNumber).flatMap(TableCell::streamValidEntities);
    }


    /**
-     * Streams all Entities in the given col.
+     * Streams all valid Entities in the given col.
     *
     * @param colNumber the column number to look for
     * @return stream of TextEntities occurring in row
     */
    public Stream<TextEntity> streamTextEntitiesInCol(int colNumber) {

-        return streamCol(colNumber).map(TableCell::getEntities)
-                .flatMap(Collection::stream)
-                .filter(TextEntity::active);
+        return streamCol(colNumber).flatMap(TableCell::streamValidEntities);
    }


@@ -269,6 +264,7 @@ public class Table implements SemanticNode {

        return streamHeaders().filter(tableCellNode -> tableCellNode.getTextBlock().getSearchText().contains(header))
                .map(TableCell::getCol)
+                .distinct()
                .flatMap(this::streamCol)
                .filter(tableCellNode -> !tableCellNode.isHeader());
    }
@@ -2,20 +2,13 @@ package com.iqser.red.service.redaction.v1.server.model.document.nodes;

 import java.awt.geom.Rectangle2D;
 import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
 import java.util.Map;
-import java.util.Set;

-import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
-import com.iqser.red.service.redaction.v1.server.model.document.DocumentTree;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlockCollector;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;

 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
-import lombok.Builder;
 import lombok.Data;
 import lombok.EqualsAndHashCode;
 import lombok.experimental.FieldDefaults;
@@ -42,7 +35,6 @@ public class TableCell extends AbstractSemanticNode {
    TextBlock textBlock;


-
    @Override
    public Map<Page, Rectangle2D> getBBox() {

@@ -1,11 +1,12 @@
 package com.iqser.red.service.redaction.v1.server.model.document.textblock;

+import static com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPositionDataProto.DocumentPositionData;
 import static java.lang.String.format;

 import java.awt.geom.Rectangle2D;
+import java.lang.ref.SoftReference;
 import java.text.BreakIterator;
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
@@ -19,8 +20,8 @@ import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Page;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
 import com.iqser.red.service.redaction.v1.server.utils.RectangleTransformations;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPositionData;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentTextData;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPositionDataProto.DocumentPositionData.Position;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentTextDataProto.DocumentTextData;

 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
@@ -43,9 +44,11 @@ public class AtomicTextBlock implements TextBlock {
    //string coordinates
    TextRange textRange;
    String searchText;
-    List<String> words;
    List<Integer> lineBreaks;

+    SoftReference<String> searchTextLowerCaseCache;
+    SoftReference<List<String>> wordsCache;
+
    //position coordinates
    List<Integer> stringIdxToPositionIdx;
    @Getter
@@ -86,20 +89,18 @@ public class AtomicTextBlock implements TextBlock {
                .page(page)
                .textRange(new TextRange(atomicTextBlockData.getStart(), atomicTextBlockData.getEnd()))
                .searchText(atomicTextBlockData.getSearchText())
-                .lineBreaks(Arrays.stream(atomicTextBlockData.getLineBreaks()).boxed()
-                                    .toList())
-                .stringIdxToPositionIdx(Arrays.stream(atomicPositionBlockData.getStringIdxToPositionIdx()).boxed()
-                                                .toList())
-                .positions(toRectangle2DList(atomicPositionBlockData.getPositions()))
+                .lineBreaks(atomicTextBlockData.getLineBreaksList())
+                .stringIdxToPositionIdx(atomicPositionBlockData.getStringIdxToPositionIdxList())
+                .positions(toRectangle2DList(atomicPositionBlockData.getPositionsList()))
                .parent(parent)
                .build();
    }


-    private static List<Rectangle2D> toRectangle2DList(float[][] positions) {
+    private static List<Rectangle2D> toRectangle2DList(List<Position> positions) {

-        return Arrays.stream(positions)
-                .map(floatArr -> (Rectangle2D) new Rectangle2D.Float(floatArr[0], floatArr[1], floatArr[2], floatArr[3]))
+        return positions.stream()
+                .map(pos -> (Rectangle2D) new Rectangle2D.Float(pos.getValue(0), pos.getValue(1), pos.getValue(2), pos.getValue(3)))
                .toList();
    }

@@ -121,8 +122,31 @@ public class AtomicTextBlock implements TextBlock {
    }


+    @Override
+    public String getSearchTextLowerCase() {
+
+        String text = null;
+        if (searchTextLowerCaseCache != null) {
+            text = searchTextLowerCaseCache.get();
+        }
+
+        if (text == null) {
+            text = getSearchText().toLowerCase(Locale.ENGLISH);
+            searchTextLowerCaseCache = new SoftReference<>(text);
+        }
+
+        return text;
+    }
+
+
    public List<String> getWords() {

+        List<String> words = null;
+
+        if (wordsCache != null) {
+            words = wordsCache.get();
+        }
+
        if (words == null) {
            words = new ArrayList<>();
            BreakIterator iterator = BreakIterator.getWordInstance(Locale.ENGLISH);
@@ -131,6 +155,7 @@ public class AtomicTextBlock implements TextBlock {
            for (int end = iterator.next(); end != BreakIterator.DONE; start = end, end = iterator.next()) {
                words.add(searchText.substring(start, end));
            }
+            wordsCache = new SoftReference<>(words);
        }
        return words;
    }
@@ -239,7 +264,7 @@ public class AtomicTextBlock implements TextBlock {
    }


-    private List<Integer> getAllLineBreaksInBoundary(TextRange textRange) {
+    protected List<Integer> getAllLineBreaksInBoundary(TextRange textRange) {

        return getLineBreaks().stream()
                .map(linebreak -> linebreak + this.textRange.start())
@@ -3,11 +3,13 @@ package com.iqser.red.service.redaction.v1.server.model.document.textblock;
 import static java.lang.String.format;

 import java.awt.geom.Rectangle2D;
+import java.lang.ref.SoftReference;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.LinkedList;
 import java.util.List;
-import java.util.Collection;
+import java.util.Locale;
 import java.util.Map;
 import java.util.stream.Stream;

@@ -25,6 +27,7 @@ public class ConcatenatedTextBlock implements TextBlock {
    List<AtomicTextBlock> atomicTextBlocks;
    String searchText;
    TextRange textRange;
+    SoftReference<String> searchTextLowerCaseCache;


    public static ConcatenatedTextBlock empty() {
@@ -100,6 +103,23 @@ public class ConcatenatedTextBlock implements TextBlock {
    }


+    @Override
+    public String getSearchTextLowerCase() {
+
+        String text = null;
+        if (searchTextLowerCaseCache != null) {
+            text = searchTextLowerCaseCache.get();
+        }
+
+        if (text == null) {
+            text = getSearchText().toLowerCase(Locale.ENGLISH);
+            searchTextLowerCaseCache = new SoftReference<>(text);
+        }
+
+        return text;
+    }
+
+
    @Override
    public List<String> getWords() {

@@ -0,0 +1,126 @@
+package com.iqser.red.service.redaction.v1.server.model.document.textblock;
+
+import java.awt.geom.Rectangle2D;
+import java.text.BreakIterator;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+
+import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.Page;
+
+import lombok.AccessLevel;
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.experimental.FieldDefaults;
+
+@Data
+@AllArgsConstructor
+@FieldDefaults(level = AccessLevel.PRIVATE)
+public class PartialTextBlock implements TextBlock {
+
+    AtomicTextBlock atomicTextBlock;
+    TextRange textRange;
+
+
+    @Override
+    public String getSearchText() {
+
+        return atomicTextBlock.getSearchText().substring(textRange.start(), textRange.end());
+    }
+
+
+    @Override
+    public String getSearchTextLowerCase() {
+
+        return atomicTextBlock.getSearchTextLowerCase().substring(textRange.start(), textRange.end());
+    }
+
+
+    @Override
+    public List<String> getWords() {
+
+        String searchText = getSearchText();
+
+        List<String> words = new ArrayList<>();
+        BreakIterator iterator = BreakIterator.getWordInstance(Locale.ENGLISH);
+        iterator.setText(getSearchText());
+        int start = iterator.first();
+        for (int end = iterator.next(); end != BreakIterator.DONE; start = end, end = iterator.next()) {
+            words.add(searchText.substring(start, end));
+        }
+        return words;
+    }
+
+
+    @Override
+    public List<AtomicTextBlock> getAtomicTextBlocks() {
+
+        return List.of(atomicTextBlock);
+    }
+
+
+    @Override
+    public int getNextLinebreak(int fromIndex) {
+
+        return atomicTextBlock.getNextLinebreak(fromIndex);
+    }
+
+
+    @Override
+    public int getPreviousLinebreak(int fromIndex) {
+
+        return atomicTextBlock.getPreviousLinebreak(fromIndex);
+    }
+
+
+    @Override
+    public TextRange getLineTextRange(int lineNumber) {
+
+        return atomicTextBlock.getLineTextRange(lineNumber);
+    }
+
+
+    @Override
+    public List<Integer> getLineBreaks() {
+
+        return atomicTextBlock.getAllLineBreaksInBoundary(textRange);
+    }
+
+
+    @Override
+    public Rectangle2D getPosition(int stringIdx) {
+
+        return atomicTextBlock.getPosition(stringIdx);
+    }
+
+
+    @Override
+    public List<Rectangle2D> getPositions(TextRange stringTextRange) {
+
+        return atomicTextBlock.getPositions(stringTextRange);
+    }
+
+
+    @Override
+    public Map<Page, List<Rectangle2D>> getPositionsPerPage(TextRange stringTextRange) {
+
+        return atomicTextBlock.getPositionsPerPage(stringTextRange);
+    }
+
+
+    @Override
+    public String subSequenceWithLineBreaks(TextRange textRange) {
+
+        return atomicTextBlock.subSequenceWithLineBreaks(textRange);
+    }
+
+
+    @Override
+    public int numberOfLines() {
+
+        return getLineBreaks().size() + 1;
+    }
+
+}
@@ -19,6 +19,9 @@ public interface TextBlock extends CharSequence {
    String getSearchText();


+    String getSearchTextLowerCase();
+
+
    List<String> getWords();


@@ -1,6 +1,5 @@
 package com.iqser.red.service.redaction.v1.server.queue;

-import static com.iqser.red.service.redaction.v1.model.QueueNames.REDACTION_QUEUE;

 import org.springframework.amqp.core.Message;
 import org.springframework.amqp.rabbit.annotation.RabbitHandler;
@@ -13,11 +12,13 @@ import lombok.extern.slf4j.Slf4j;
@RequiredArgsConstructor
 public class MessageReceiver {

+    public static final String REDACTION_REQUEST_LISTENER_ID = "redaction-request-listener";
+
    private final RedactionMessageReceiver redactionMessageReceiver;


    @RabbitHandler
-    @RabbitListener(queues = REDACTION_QUEUE, concurrency = "1")
+    @RabbitListener(id = REDACTION_REQUEST_LISTENER_ID, concurrency = "1")
    public void receiveAnalyzeRequest(Message message) {

        redactionMessageReceiver.receiveAnalyzeRequest(message, false);
@@ -1,12 +1,8 @@
 package com.iqser.red.service.redaction.v1.server.queue;

-import static com.iqser.red.service.redaction.v1.model.QueueNames.MIGRATION_DLQ;
-import static com.iqser.red.service.redaction.v1.model.QueueNames.MIGRATION_QUEUE;
-import static com.iqser.red.service.redaction.v1.model.QueueNames.REDACTION_ANALYSIS_RESPONSE_QUEUE;
-import static com.iqser.red.service.redaction.v1.model.QueueNames.REDACTION_DQL;
-import static com.iqser.red.service.redaction.v1.model.QueueNames.REDACTION_PRIORITY_QUEUE;
-import static com.iqser.red.service.redaction.v1.model.QueueNames.REDACTION_QUEUE;
+import static com.iqser.red.service.redaction.v1.model.QueueNames.*;

+import org.springframework.amqp.core.DirectExchange;
 import org.springframework.amqp.core.Queue;
 import org.springframework.amqp.core.QueueBuilder;
 import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
@@ -23,6 +19,27 @@ public class MessagingConfiguration {
    public static final String X_ERROR_INFO_TIMESTAMP_HEADER = "x-error-message-timestamp";


+    @Bean
+    public DirectExchange redactionRequestExchange() {
+
+        return new DirectExchange(REDACTION_REQUEST_EXCHANGE);
+    }
+
+
+    @Bean
+    public DirectExchange redactionPriorityRequestExchange() {
+
+        return new DirectExchange(REDACTION_PRIORITY_REQUEST_EXCHANGE);
+    }
+
+
+    @Bean
+    public Queue redactionDLQ() {
+
+        return QueueBuilder.durable(REDACTION_DLQ).build();
+    }
+
+
    @Bean
    @ConditionalOnProperty(prefix = "redaction-service", name = "priorityMode", havingValue = "false")
    public MessageReceiver messageReceiver(RedactionMessageReceiver redactionMessageReceiver) {
@@ -42,7 +59,7 @@ public class MessagingConfiguration {
    @Bean
    public Queue migrationQueue() {

-        return QueueBuilder.durable(MIGRATION_QUEUE).withArgument("x-dead-letter-exchange", "").withArgument("x-dead-letter-routing-key", MIGRATION_DLQ).maxPriority(2).build();
+        return QueueBuilder.durable(MIGRATION_REQUEST_QUEUE).withArgument("x-dead-letter-exchange", "").withArgument("x-dead-letter-routing-key", MIGRATION_DLQ).maxPriority(2).build();
    }


@@ -52,40 +69,4 @@ public class MessagingConfiguration {
        return QueueBuilder.durable(MIGRATION_DLQ).build();
    }

-
-    @Bean
-    public Queue redactionQueue() {
-
-        return QueueBuilder.durable(REDACTION_QUEUE).withArgument("x-dead-letter-exchange", "").withArgument("x-dead-letter-routing-key", REDACTION_DQL).maxPriority(2).build();
-    }
-
-
-    @Bean
-    public Queue redactionPriorityQueue() {
-
-        return QueueBuilder.durable(REDACTION_PRIORITY_QUEUE)
-                .withArgument("x-dead-letter-exchange", "")
-                .withArgument("x-dead-letter-routing-key", REDACTION_DQL)
-                .maxPriority(2)
-                .build();
-    }
-
-
-    @Bean
-    public Queue redactionAnalysisResponseQueue() {
-
-        return QueueBuilder.durable(REDACTION_ANALYSIS_RESPONSE_QUEUE)
-                .withArgument("x-dead-letter-exchange", "")
-                .withArgument("x-dead-letter-routing-key", REDACTION_DQL)
-                .maxPriority(2)
-                .build();
-    }
-
-
-    @Bean
-    public Queue redactionDeadLetterQueue() {
-
-        return QueueBuilder.durable(REDACTION_DQL).build();
-    }
-
 }
@@ -1,7 +1,5 @@
 package com.iqser.red.service.redaction.v1.server.queue;

-import static com.iqser.red.service.redaction.v1.model.QueueNames.REDACTION_PRIORITY_QUEUE;
-
 import org.springframework.amqp.core.Message;
 import org.springframework.amqp.rabbit.annotation.RabbitHandler;
 import org.springframework.amqp.rabbit.annotation.RabbitListener;
@@ -13,11 +11,13 @@ import lombok.extern.slf4j.Slf4j;
@RequiredArgsConstructor
 public class PriorityMessageReceiver {

+    public static final String REDACTION_PRIORITY_REQUEST_LISTENER_ID = "redaction-priority-request-listener";
+
    private final RedactionMessageReceiver redactionMessageReceiver;


    @RabbitHandler
-    @RabbitListener(queues = REDACTION_PRIORITY_QUEUE, concurrency = "1")
+    @RabbitListener(id = REDACTION_PRIORITY_REQUEST_LISTENER_ID, concurrency = "1")
    public void receiveAnalyzeRequest(Message message) {

        redactionMessageReceiver.receiveAnalyzeRequest(message, true);
@@ -0,0 +1,68 @@
+package com.iqser.red.service.redaction.v1.server.queue;
+
+import static com.iqser.red.service.redaction.v1.model.QueueNames.*;
+import java.util.Map;
+import java.util.Set;
+
+import org.springframework.amqp.rabbit.annotation.RabbitHandler;
+import org.springframework.amqp.rabbit.annotation.RabbitListener;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.boot.context.event.ApplicationReadyEvent;
+import org.springframework.context.event.EventListener;
+import org.springframework.stereotype.Service;
+
+import com.knecon.fforesight.tenantcommons.TenantProvider;
+import com.knecon.fforesight.tenantcommons.model.TenantCreatedEvent;
+import com.knecon.fforesight.tenantcommons.model.TenantQueueConfiguration;
+import com.knecon.fforesight.tenantcommons.model.TenantResponse;
+import com.knecon.fforesight.tenantcommons.queue.RabbitQueueFromExchangeService;
+import com.knecon.fforesight.tenantcommons.queue.TenantExchangeMessageReceiver;
+
+@Service
+@ConditionalOnProperty(prefix = "redaction-service", name = "priorityMode", havingValue = "true")
+public class PriorityTenantExchangeMessageReceiverImpl extends TenantExchangeMessageReceiver {
+
+    public PriorityTenantExchangeMessageReceiverImpl(RabbitQueueFromExchangeService rabbitQueueService, TenantProvider tenantProvider) {
+
+        super(rabbitQueueService, tenantProvider);
+    }
+
+
+    @Override
+    protected Set<TenantQueueConfiguration> getTenantQueueConfigs() {
+
+        return Set.of(TenantQueueConfiguration.builder()
+                              .listenerId(PriorityMessageReceiver.REDACTION_PRIORITY_REQUEST_LISTENER_ID)
+                              .exchangeName(REDACTION_PRIORITY_REQUEST_EXCHANGE)
+                              .queuePrefix(REDACTION_PRIORITY_REQUEST_QUEUE_PREFIX)
+                              .dlqName(REDACTION_DLQ)
+                              .arguments(Map.of("x-max-priority", 2))
+                              .build());
+    }
+
+
+    @EventListener(ApplicationReadyEvent.class)
+    public void onApplicationReady() {
+
+        System.out.println("application ready invoked");
+        super.initializeQueues();
+    }
+
+
+    @RabbitHandler
+    @RabbitListener(queues = "#{priorityTenantMessagingConfigurationImpl.getTenantCreatedQueueName()}")
+    public void reactToTenantCreation(TenantCreatedEvent tenantCreatedEvent) {
+
+        super.reactToTenantCreation(tenantCreatedEvent);
+    }
+
+
+    @RabbitHandler
+    @RabbitListener(queues = "#{priorityTenantMessagingConfigurationImpl.getTenantDeletedQueueName()}")
+    public void reactToTenantDeletion(TenantResponse tenantResponse) {
+
+        super.reactToTenantDeletion(tenantResponse);
+
+    }
+
+}
@@ -1,8 +1,6 @@
 package com.iqser.red.service.redaction.v1.server.queue;

-import static com.iqser.red.service.redaction.v1.model.QueueNames.REDACTION_DQL;
-import static com.iqser.red.service.redaction.v1.model.QueueNames.REDACTION_PRIORITY_QUEUE;
-import static com.iqser.red.service.redaction.v1.model.QueueNames.REDACTION_QUEUE;
+import static com.iqser.red.service.redaction.v1.model.QueueNames.*;
 import static com.iqser.red.service.redaction.v1.server.queue.MessagingConfiguration.X_ERROR_INFO_HEADER;
 import static com.iqser.red.service.redaction.v1.server.queue.MessagingConfiguration.X_ERROR_INFO_TIMESTAMP_HEADER;
 import static java.lang.String.format;
@@ -24,6 +22,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemp
 import com.iqser.red.service.redaction.v1.server.client.FileStatusProcessingUpdateClient;
 import com.iqser.red.service.redaction.v1.server.client.RulesClient;
 import com.iqser.red.service.redaction.v1.server.service.AnalyzeService;
+import com.iqser.red.service.redaction.v1.server.service.DocumentSearchService;
 import com.iqser.red.service.redaction.v1.server.service.UnprocessedChangesService;
 import com.iqser.red.service.redaction.v1.server.utils.exception.DroolsTimeoutException;

@@ -41,6 +40,7 @@ public class RedactionMessageReceiver {
    private final FileStatusProcessingUpdateClient fileStatusProcessingUpdateClient;
    private final RulesClient rulesClient;
    private final UnprocessedChangesService unprocessedChangesService;
+    private final DocumentSearchService documentSearchService;


    @SneakyThrows
@@ -56,7 +56,7 @@ public class RedactionMessageReceiver {
            fileStatusProcessingUpdateClient.analysisFailed(analyzeRequest.getDossierId(),
                                                            analyzeRequest.getFileId(),
                                                            new FileErrorInfo(errorMessage,
-                                                                              priority ? REDACTION_PRIORITY_QUEUE : REDACTION_QUEUE,
+                                                                              priority ? REDACTION_PRIORITY_REQUEST_EXCHANGE : REDACTION_REQUEST_EXCHANGE,
                                                                              "redaction-service",
                                                                              OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS)));
            throw new AmqpRejectAndDontRequeueException(errorMessage);
@@ -101,6 +101,19 @@ public class RedactionMessageReceiver {
                             format("%.2f", result.getDuration() / 1000.0));
                    log.info("----------------------------------------------------------------------------------");
                    break;
+
+                case REANALYSE_COMPONENTS_ONLY:
+                    log.info("------------------------------Components Only Reanalysis------------------------------------------");
+                    log.info("Starting Components Only Reanalysis for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
+                    log.debug(analyzeRequest.getManualRedactions().toString());
+                    result = analyzeService.reanalyzeComponentsOnly(analyzeRequest);
+                    log.info("Successfully reanalyzed the components for dossier {} file {} took: {} s",
+                             analyzeRequest.getDossierId(),
+                             analyzeRequest.getFileId(),
+                             format("%.2f", result.getDuration() / 1000.0));
+                    log.info("----------------------------------------------------------------------------------");
+                    break;
+
                case SURROUNDING_TEXT_ANALYSIS:
                    log.info("------------------------------Surrounding Text Analysis------------------------------------------");
                    log.info("Starting Surrounding Text Analysis for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
@@ -111,6 +124,27 @@ public class RedactionMessageReceiver {
                    shouldRespond = false;
                    break;

+                case IMPORTED_REDACTIONS_ONLY:
+                    log.info("------------------------------Imported Redactions Analysis Only------------------------------------------");
+                    log.info("Starting Imported Redactions Analysis Only for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
+                    log.debug(analyzeRequest.getManualRedactions().toString());
+                    result = analyzeService.analyzeImportedRedactionsOnly(analyzeRequest);
+                    log.info("Successful Imported Redactions Analysis Only dossier {} file {}", analyzeRequest.getDossierId(), analyzeRequest.getFileId());
+                    log.info("-------------------------------------------------------------------------------------------------");
+                    break;
+
+                case SEARCH_BULK_LOCAL_TERM:
+                    log.info("------------------------------Search Term occurrences for bulk local add ------------------------------------------");
+                    log.info("Starting term search for {} for file {} in dossier {}",
+                             analyzeRequest.getBulkLocalRequest().getSearchTerm(),
+                             analyzeRequest.getFileId(),
+                             analyzeRequest.getDossierId());
+                    documentSearchService.searchTermOccurrences(analyzeRequest);
+                    log.info("Successfully located all term occurrences dossier {} file {} ", analyzeRequest.getDossierId(), analyzeRequest.getFileId());
+                    log.info("-------------------------------------------------------------------------------------------------");
+                    shouldRespond = false;
+                    break;
+
                default:
                    throw new IllegalArgumentException("Unknown MessageType: " + analyzeRequest.getMessageType());
            }
@@ -136,12 +170,15 @@ public class RedactionMessageReceiver {
        var timestamp = OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS);
        fileStatusProcessingUpdateClient.analysisFailed(analyzeRequest.getDossierId(),
                                                        analyzeRequest.getFileId(),
-                                                        new FileErrorInfo(e.getMessage(), priority ? REDACTION_PRIORITY_QUEUE : REDACTION_QUEUE, "redaction-service", timestamp));
+                                                        new FileErrorInfo(e.getMessage(),
+                                                                          priority ? REDACTION_PRIORITY_REQUEST_EXCHANGE : REDACTION_REQUEST_EXCHANGE,
+                                                                          "redaction-service",
+                                                                          timestamp));
    }


    @RabbitHandler
-    @RabbitListener(queues = REDACTION_DQL)
+    @RabbitListener(queues = REDACTION_DLQ)
    public void receiveAnalyzeRequestDLQ(Message in) throws IOException {

        var analyzeRequest = objectMapper.readValue(in.getBody(), AnalyzeRequest.class);
@@ -152,7 +189,7 @@ public class RedactionMessageReceiver {
        log.info("Failed to process analyze request, errorCause: {}, timestamp: {}", errorCause, timestamp);
        fileStatusProcessingUpdateClient.analysisFailed(analyzeRequest.getDossierId(),
                                                        analyzeRequest.getFileId(),
-                                                        new FileErrorInfo(errorCause, REDACTION_DQL, "redaction-service", timestamp));
+                                                        new FileErrorInfo(errorCause, REDACTION_DLQ, "redaction-service", timestamp));
    }

 }
@@ -0,0 +1,68 @@
+package com.iqser.red.service.redaction.v1.server.queue;
+
+import static com.iqser.red.service.redaction.v1.model.QueueNames.*;
+import java.util.Map;
+import java.util.Set;
+
+import org.springframework.amqp.rabbit.annotation.RabbitHandler;
+import org.springframework.amqp.rabbit.annotation.RabbitListener;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.boot.context.event.ApplicationReadyEvent;
+import org.springframework.context.event.EventListener;
+import org.springframework.stereotype.Service;
+
+import com.knecon.fforesight.tenantcommons.TenantProvider;
+import com.knecon.fforesight.tenantcommons.model.TenantCreatedEvent;
+import com.knecon.fforesight.tenantcommons.model.TenantQueueConfiguration;
+import com.knecon.fforesight.tenantcommons.model.TenantResponse;
+import com.knecon.fforesight.tenantcommons.queue.RabbitQueueFromExchangeService;
+import com.knecon.fforesight.tenantcommons.queue.TenantExchangeMessageReceiver;
+
+@Service
+@ConditionalOnProperty(prefix = "redaction-service", name = "priorityMode", havingValue = "false")
+public class TenantExchangeMessageReceiverImpl extends TenantExchangeMessageReceiver {
+
+    public TenantExchangeMessageReceiverImpl(RabbitQueueFromExchangeService rabbitQueueService, TenantProvider tenantProvider) {
+
+        super(rabbitQueueService, tenantProvider);
+    }
+
+
+    @Override
+    protected Set<TenantQueueConfiguration> getTenantQueueConfigs() {
+
+        return Set.of(TenantQueueConfiguration.builder()
+                              .listenerId(MessageReceiver.REDACTION_REQUEST_LISTENER_ID)
+                              .exchangeName(REDACTION_REQUEST_EXCHANGE)
+                              .queuePrefix(REDACTION_REQUEST_QUEUE_PREFIX)
+                              .dlqName(REDACTION_DLQ)
+                              .arguments(Map.of("x-max-priority", 2))
+                              .build());
+    }
+
+
+    @EventListener(ApplicationReadyEvent.class)
+    public void onApplicationReady() {
+
+        System.out.println("application ready invoked");
+        super.initializeQueues();
+    }
+
+
+    @RabbitHandler
+    @RabbitListener(queues = "#{tenantMessagingConfigurationImpl.getTenantCreatedQueueName()}")
+    public void reactToTenantCreation(TenantCreatedEvent tenantCreatedEvent) {
+
+        super.reactToTenantCreation(tenantCreatedEvent);
+    }
+
+
+    @RabbitHandler
+    @RabbitListener(queues = "#{tenantMessagingConfigurationImpl.getTenantDeletedQueueName()}")
+    public void reactToTenantDeletion(TenantResponse tenantResponse) {
+
+        super.reactToTenantDeletion(tenantResponse);
+
+    }
+
+}
@@ -0,0 +1,145 @@
+package com.iqser.red.service.redaction.v1.server.service;
+
+import java.util.List;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.CompletableFuture;
+
+import org.springframework.stereotype.Service;
+
+import com.iqser.gin4.commons.metrics.meters.FunctionTimerValues;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeResult;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentLog;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogChanges;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
+import com.iqser.red.service.redaction.v1.server.RedactionServiceSettings;
+import com.iqser.red.service.redaction.v1.server.logger.Context;
+import com.iqser.red.service.redaction.v1.server.model.KieWrapper;
+import com.iqser.red.service.redaction.v1.server.model.component.Component;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
+import com.iqser.red.service.redaction.v1.server.service.components.ComponentLogCreatorService;
+import com.iqser.red.service.redaction.v1.server.service.drools.ComponentDroolsExecutionService;
+import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
+
+import io.micrometer.core.annotation.Timed;
+import io.micrometer.observation.annotation.Observed;
+import lombok.AccessLevel;
+import lombok.RequiredArgsConstructor;
+import lombok.experimental.FieldDefaults;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+@Service
+@RequiredArgsConstructor
+@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
+public class AnalysisFinalizationService {
+
+    RedactionStorageService redactionStorageService;
+    FunctionTimerValues redactmanagerAnalyzePagewiseValues;
+    ComponentLogCreatorService componentLogCreatorService;
+    RedactionServiceSettings redactionServiceSettings;
+    ComponentDroolsExecutionService componentDroolsExecutionService;
+
+
+    @Timed("redactmanager_finalizeAnalysis")
+    @Observed(name = "AnalysisFinalizationService", contextualName = "finalize-analysis")
+    public AnalyzeResult finalizeAnalysis(AnalyzeRequest analyzeRequest,
+                                          long startTime,
+                                          KieWrapper kieWrapperComponentRules,
+                                          EntityLogChanges entityLogChanges,
+                                          Document document,
+                                          int numberOfPages,
+                                          boolean isReanalysis,
+                                          Set<FileAttribute> addedFileAttributes,
+                                          Context context) {
+
+        EntityLog entityLog = entityLogChanges.getEntityLog();
+
+        // as workaround for duplicate key exceptions occurring due to simultaneous analyses and reanalyses save instead of insert is used
+        // also analysis numbers should be incremented in every follow-up request, so checking if the log exists is not needed
+        if (!redactionStorageService.entityLogExists(analyzeRequest.getDossierId(), analyzeRequest.getFileId())) {
+            redactionStorageService.saveEntityLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), entityLog);
+
+        } else {
+            redactionStorageService.updateEntityLogWithoutEntries(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), entityLog);
+
+            if (!entityLogChanges.getNewEntityLogEntries().isEmpty()) {
+                redactionStorageService.saveEntityLogEntries(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), entityLogChanges.getNewEntityLogEntries());
+            }
+            if (!entityLogChanges.getUpdatedEntityLogEntries().isEmpty()) {
+                redactionStorageService.updateEntityLogEntries(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), entityLogChanges.getUpdatedEntityLogEntries());
+            }
+        }
+
+        log.info("Created entity log for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
+
+        context.setRuleVersion(kieWrapperComponentRules.rulesVersion());
+        Optional<ComponentLog> componentLog = computeComponentsWhenRulesArePresent(analyzeRequest, kieWrapperComponentRules, document, addedFileAttributes, entityLog, context);
+
+        long duration = System.currentTimeMillis() - startTime;
+
+        redactmanagerAnalyzePagewiseValues.increase(numberOfPages, duration);
+
+        return AnalyzeResult.builder()
+                .dossierId(analyzeRequest.getDossierId())
+                .fileId(analyzeRequest.getFileId())
+                .duration(duration)
+                .numberOfPages(numberOfPages)
+                .hasUpdates(entityLogChanges.hasChanges())
+                .analysisVersion(redactionServiceSettings.getAnalysisVersion())
+                .analysisNumber(analyzeRequest.getAnalysisNumber())
+                .rulesVersion(entityLog.getRulesVersion())
+                .componentRulesVersion(kieWrapperComponentRules.rulesVersion())
+                .dateFormatsVersion(componentLog.map(ComponentLog::getDateFormatsVersion)
+                                            .orElse(-1L))
+                .dictionaryVersion(entityLog.getDictionaryVersion())
+                .legalBasisVersion(entityLog.getLegalBasisVersion())
+                .dossierDictionaryVersion(entityLog.getDossierDictionaryVersion())
+                .wasReanalyzed(isReanalysis)
+                .manualRedactions(analyzeRequest.getManualRedactions())
+                .addedFileAttributes(addedFileAttributes)
+                .usedComponentMappings(analyzeRequest.getComponentMappings())
+                .build();
+    }
+
+
+    private Optional<ComponentLog> computeComponentsWhenRulesArePresent(AnalyzeRequest analyzeRequest,
+                                                                        KieWrapper kieWrapperComponentRules,
+                                                                        Document document,
+                                                                        Set<FileAttribute> addedFileAttributes,
+                                                                        EntityLog entityLog,
+                                                                        Context context) {
+
+        if (!kieWrapperComponentRules.isPresent()) {
+            return Optional.empty();
+        }
+
+        // We need the latest EntityLog entries for components rules execution
+        entityLog.setEntityLogEntry(redactionStorageService.getEntityLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId()).getEntityLogEntry());
+
+        List<Component> components = componentDroolsExecutionService.executeRules(kieWrapperComponentRules.container(),
+                                                                                  entityLog,
+                                                                                  document,
+                                                                                  addedFileAttributes,
+                                                                                  analyzeRequest.getComponentMappings(),
+                                                                                  context);
+
+        log.info("Finished component rule execution for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
+
+        ComponentLog componentLog = componentLogCreatorService.buildComponentLog(analyzeRequest.getAnalysisNumber(),
+                                                                                 components,
+                                                                                 kieWrapperComponentRules.rulesVersion(),
+                                                                                 analyzeRequest.getComponentDefinitions(),
+                                                                                 context.getDateFormatsVersion());
+
+        redactionStorageService.saveComponentLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), componentLog);
+
+        log.info("Stored component log for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
+
+        return Optional.of(componentLog);
+    }
+
+}
@@ -0,0 +1,464 @@
+package com.iqser.red.service.redaction.v1.server.service;
+
+import static com.iqser.red.service.redaction.v1.server.service.document.SectionFinderService.getRelevantManuallyModifiedAnnotationIds;
+import static org.springframework.boot.autoconfigure.task.TaskExecutionAutoConfiguration.APPLICATION_TASK_EXECUTOR_BEAN_NAME;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ExecutionException;
+import java.util.function.Supplier;
+import java.util.stream.Collectors;
+
+import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.core.task.TaskExecutor;
+import org.springframework.stereotype.Service;
+
+import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedactions;
+import com.iqser.red.service.redaction.v1.server.RedactionServiceSettings;
+import com.iqser.red.service.redaction.v1.server.client.model.NerEntitiesModel;
+import com.iqser.red.service.redaction.v1.server.model.KieWrapper;
+import com.iqser.red.service.redaction.v1.server.model.NerEntities;
+import com.iqser.red.service.redaction.v1.server.model.PrecursorEntity;
+import com.iqser.red.service.redaction.v1.server.model.dictionary.Dictionary;
+import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryIncrement;
+import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryVersion;
+import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
+import com.iqser.red.service.redaction.v1.server.service.document.DocumentGraphMapper;
+import com.iqser.red.service.redaction.v1.server.service.document.ImportedRedactionEntryService;
+import com.iqser.red.service.redaction.v1.server.service.document.ManualRedactionEntryService;
+import com.iqser.red.service.redaction.v1.server.service.document.NerEntitiesAdapter;
+import com.iqser.red.service.redaction.v1.server.service.document.SectionFinderService;
+import com.iqser.red.service.redaction.v1.server.service.drools.KieContainerCreationService;
+import com.iqser.red.service.redaction.v1.server.storage.ObservedStorageService;
+import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
+
+import io.micrometer.core.annotation.Timed;
+import io.micrometer.observation.annotation.Observed;
+import lombok.AccessLevel;
+import lombok.SneakyThrows;
+import lombok.experimental.FieldDefaults;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+@Service
+@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
+public class AnalysisPreparationService {
+
+    KieContainerCreationService kieContainerCreationService;
+    ObservedStorageService observedStorageService;
+    RedactionStorageService redactionStorageService;
+    RedactionServiceSettings redactionServiceSettings;
+    ManualRedactionEntryService manualRedactionEntryService;
+    ImportedRedactionEntryService importedRedactionEntryService;
+    DictionaryService dictionaryService;
+    SectionFinderService sectionFinderService;
+    TaskExecutor taskExecutor;
+
+
+    public AnalysisPreparationService(KieContainerCreationService kieContainerCreationService,
+                                      ObservedStorageService observedStorageService,
+                                      RedactionStorageService redactionStorageService,
+                                      RedactionServiceSettings redactionServiceSettings,
+                                      ManualRedactionEntryService manualRedactionEntryService,
+                                      ImportedRedactionEntryService importedRedactionEntryService,
+                                      DictionaryService dictionaryService,
+                                      SectionFinderService sectionFinderService,
+                                      @Qualifier(APPLICATION_TASK_EXECUTOR_BEAN_NAME) TaskExecutor taskExecutor) {
+
+        this.kieContainerCreationService = kieContainerCreationService;
+        this.observedStorageService = observedStorageService;
+        this.redactionStorageService = redactionStorageService;
+        this.redactionServiceSettings = redactionServiceSettings;
+        this.manualRedactionEntryService = manualRedactionEntryService;
+        this.importedRedactionEntryService = importedRedactionEntryService;
+        this.dictionaryService = dictionaryService;
+        this.sectionFinderService = sectionFinderService;
+        this.taskExecutor = taskExecutor;
+    }
+
+
+    @SneakyThrows
+    @Timed("redactmanager_getAnalysisData")
+    @Observed(name = "AnalysisPreparationService", contextualName = "get-analysis-data")
+    public AnalysisData getAnalysisData(AnalyzeRequest analyzeRequest) {
+
+        CompletableFuture<KieWrapper> kieWrapperComponentRulesFuture = CompletableFuture.supplyAsync(() -> getKieWrapper(analyzeRequest, RuleFileType.COMPONENT), taskExecutor);
+
+        CompletableFuture<KieWrapper> kieWrapperEntityRulesFuture = CompletableFuture.supplyAsync(() -> getKieWrapper(analyzeRequest, RuleFileType.ENTITY), taskExecutor);
+
+        CompletableFuture<Document> documentFuture = CompletableFuture.supplyAsync(() -> getDocument(analyzeRequest), taskExecutor);
+
+        CompletableFuture<ImportedRedactions> importedRedactionsFuture = CompletableFuture.supplyAsync(() -> getImportedRedactions(analyzeRequest), taskExecutor);
+
+        CompletableFuture<NerEntities> nerEntitiesFuture = documentFuture.thenApplyAsync((document) -> getNerEntities(analyzeRequest, document), taskExecutor);
+
+        CompletableFuture.allOf(kieWrapperEntityRulesFuture, kieWrapperComponentRulesFuture, documentFuture, importedRedactionsFuture, nerEntitiesFuture).join();
+
+        Dictionary dictionary = getDictionary(analyzeRequest);
+
+        Document document = documentFuture.get();
+        ImportedRedactions importedRedactions = importedRedactionsFuture.get();
+
+        List<PrecursorEntity> notFoundManualRedactionEntries = manualRedactionEntryService.addManualRedactionEntriesAndReturnNotFoundEntries(analyzeRequest,
+                                                                                                                                             document,
+                                                                                                                                             analyzeRequest.getDossierTemplateId());
+
+        List<PrecursorEntity> notFoundImportedEntries = importedRedactionEntryService.addImportedEntriesAndReturnNotFoundEntries(analyzeRequest, importedRedactions, document);
+
+        return new AnalysisData(kieWrapperEntityRulesFuture.get(),
+                                kieWrapperComponentRulesFuture.get(),
+                                document,
+                                importedRedactions,
+                                dictionary,
+                                notFoundManualRedactionEntries,
+                                notFoundImportedEntries,
+                                nerEntitiesFuture.get());
+    }
+
+
+    @SneakyThrows
+    @Timed("redactmanager_getReanalysisSetupData")
+    @Observed(name = "AnalysisPreparationService", contextualName = "get-reanalysis-setup-data")
+    public ReanalysisSetupData getReanalysisSetupData(AnalyzeRequest analyzeRequest) {
+
+        return getReanalysisSetupData(analyzeRequest, () -> getEntityLogWithoutEntries(analyzeRequest));
+    }
+
+
+    @SneakyThrows
+    public ReanalysisSetupData getReanalysisSetupDataForComponentsOnlyReanalyze(AnalyzeRequest analyzeRequest) {
+
+        return getReanalysisSetupData(analyzeRequest, () -> getEntityLog(analyzeRequest));
+    }
+
+
+    @SneakyThrows
+    private ReanalysisSetupData getReanalysisSetupData(AnalyzeRequest analyzeRequest, Supplier<EntityLog> entityLogSupplier) {
+
+        CompletableFuture<EntityLog> entityLogFuture = CompletableFuture.supplyAsync(entityLogSupplier, taskExecutor);
+        CompletableFuture<Document> documentFuture = CompletableFuture.supplyAsync(() -> getDocument(analyzeRequest), taskExecutor);
+
+        CompletableFuture.allOf(entityLogFuture, documentFuture).join();
+
+        return new ReanalysisSetupData(entityLogFuture.get(), documentFuture.get());
+    }
+
+
+    @SneakyThrows
+    @Timed("redactmanager_getReanalysisInitialProcessingData")
+    @Observed(name = "AnalysisPreparationService", contextualName = "get-reanalysis-initial-processing-data")
+    public ReanalysisInitialProcessingData getReanalysisInitialProcessingData(AnalyzeRequest analyzeRequest, ReanalysisSetupData reanalysisSetupData) {
+
+        CompletableFuture<ImportedRedactions> importedRedactionsFuture = CompletableFuture.supplyAsync(() -> getImportedRedactions(analyzeRequest), taskExecutor);
+
+        CompletableFuture<SectionsToReanalyzeData> incrementAndSectionsToReanalyzeFuture = importedRedactionsFuture.thenApplyAsync((importedRedactions) -> {
+            DictionaryIncrement dictionaryIncrement = getDictionaryIncrement(analyzeRequest, reanalysisSetupData);
+            return getDictionaryIncrementAndSectionsToReanalyze(analyzeRequest, dictionaryIncrement, reanalysisSetupData, importedRedactions);
+
+        }, taskExecutor);
+
+        CompletableFuture<KieWrapper> kieWrapperComponentRulesFuture = CompletableFuture.supplyAsync(() -> getKieWrapper(analyzeRequest, RuleFileType.COMPONENT), taskExecutor);
+
+        CompletableFuture<KieWrapper> kieWrapperEntityRulesFuture = CompletableFuture.supplyAsync(() -> getKieWrapper(analyzeRequest, RuleFileType.ENTITY), taskExecutor);
+
+        CompletableFuture.allOf(importedRedactionsFuture, incrementAndSectionsToReanalyzeFuture, kieWrapperComponentRulesFuture, kieWrapperEntityRulesFuture).join();
+
+        return new ReanalysisInitialProcessingData(importedRedactionsFuture.get(),
+                                                   incrementAndSectionsToReanalyzeFuture.get().dictionaryIncrement(),
+                                                   incrementAndSectionsToReanalyzeFuture.get().sectionsToReanalyseIds(),
+                                                   incrementAndSectionsToReanalyzeFuture.get().sectionsToReanalyze(),
+                                                   kieWrapperComponentRulesFuture.get(),
+                                                   kieWrapperEntityRulesFuture.get());
+    }
+
+
+    @SneakyThrows
+    @Timed("redactmanager_getReanalysisFinalProcessingData")
+    @Observed(name = "AnalysisPreparationService", contextualName = "get-reanalysis-final-processing-data")
+    public ReanalysisFinalProcessingData getReanalysisFinalProcessingData(AnalyzeRequest analyzeRequest,
+                                                                          ReanalysisSetupData reanalysisSetupData,
+                                                                          ReanalysisInitialProcessingData reanalysisInitialProcessingData) {
+
+        CompletableFuture<NerEntities> nerEntitiesFuture = CompletableFuture.supplyAsync(() -> getNerEntitiesFiltered(analyzeRequest,
+                                                                                                                      reanalysisSetupData.document,
+                                                                                                                      reanalysisInitialProcessingData.sectionsToReanalyseIds),
+                                                                                         taskExecutor);
+
+        CompletableFuture<DictionaryAndNotFoundEntries> dictionaryAndNotFoundEntriesCompletableFuture = CompletableFuture.supplyAsync(() -> {
+            Dictionary dictionary = getDictionary(analyzeRequest);
+            NotFoundEntries notFoundEntries = getNotFoundEntries(analyzeRequest, reanalysisSetupData.document(), reanalysisInitialProcessingData.importedRedactions());
+            return new DictionaryAndNotFoundEntries(dictionary, notFoundEntries.notFoundManualRedactionEntries(), notFoundEntries.notFoundImportedEntries());
+        }, taskExecutor);
+
+        CompletableFuture.allOf(nerEntitiesFuture, dictionaryAndNotFoundEntriesCompletableFuture).join();
+
+        return new ReanalysisFinalProcessingData(nerEntitiesFuture.get(),
+                                                 dictionaryAndNotFoundEntriesCompletableFuture.get().dictionary(),
+                                                 dictionaryAndNotFoundEntriesCompletableFuture.get().notFoundManualRedactionEntries(),
+                                                 dictionaryAndNotFoundEntriesCompletableFuture.get().notFoundImportedEntries());
+    }
+
+
+    public KieWrapper getKieWrapper(AnalyzeRequest analyzeRequest, RuleFileType ruleFileType) {
+
+        KieWrapper kieWrapperComponentRules = kieContainerCreationService.getLatestKieContainer(analyzeRequest.getDossierTemplateId(), ruleFileType);
+        log.info("Updated {} Rules to Version {} for file {} in dossier {}",
+                 ruleFileType,
+                 kieWrapperComponentRules.rulesVersion(),
+                 analyzeRequest.getFileId(),
+                 analyzeRequest.getDossierId());
+        return kieWrapperComponentRules;
+    }
+
+
+    private Document getDocument(AnalyzeRequest analyzeRequest) {
+
+        Document document = DocumentGraphMapper.toDocumentGraph(observedStorageService.getDocumentData(analyzeRequest.getDossierId(), analyzeRequest.getFileId()));
+        log.info("Loaded Document Graph for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
+        return document;
+    }
+
+
+    private ImportedRedactions getImportedRedactions(AnalyzeRequest analyzeRequest) {
+
+        ImportedRedactions importedRedactions = redactionStorageService.getImportedRedactions(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
+        log.info("Loaded Imported Redactions for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
+        return importedRedactions;
+    }
+
+
+    private NerEntities getNerEntities(AnalyzeRequest analyzeRequest, Document document) {
+
+        NerEntities nerEntities = getEntityRecognitionEntities(analyzeRequest, document);
+        log.info("Loaded Ner Entities for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
+        return nerEntities;
+    }
+
+
+    private NerEntities getNerEntitiesFiltered(AnalyzeRequest analyzeRequest, Document document, Set<Integer> sectionsToReanalyseIds) {
+
+        NerEntities nerEntities = getEntityRecognitionEntitiesFilteredBySectionIds(analyzeRequest, document, sectionsToReanalyseIds);
+        log.info("Loaded Ner Entities for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
+        return nerEntities;
+    }
+
+
+    private Dictionary getDictionary(AnalyzeRequest analyzeRequest) {
+
+        dictionaryService.updateDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId());
+        Dictionary dictionary = dictionaryService.getDeepCopyDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId());
+        log.info("Updated Dictionaries for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
+        return dictionary;
+    }
+
+
+    private NotFoundEntries getNotFoundEntries(AnalyzeRequest analyzeRequest, Document document, ImportedRedactions importedRedactions) {
+
+        var notFoundManualRedactionEntries = manualRedactionEntryService.addManualRedactionEntriesAndReturnNotFoundEntries(analyzeRequest,
+                                                                                                                           document,
+                                                                                                                           analyzeRequest.getDossierTemplateId());
+        var notFoundImportedEntries = importedRedactionEntryService.addImportedEntriesAndReturnNotFoundEntries(analyzeRequest, importedRedactions, document);
+        return new NotFoundEntries(notFoundManualRedactionEntries, notFoundImportedEntries);
+    }
+
+
+    private NerEntities getEntityRecognitionEntities(AnalyzeRequest analyzeRequest, Document document) {
+
+        NerEntities nerEntities;
+        if (redactionServiceSettings.isNerServiceEnabled()) {
+            nerEntities = NerEntitiesAdapter.toNerEntities(redactionStorageService.getNerEntities(analyzeRequest.getDossierId(), analyzeRequest.getFileId()),
+                                                           document,
+                                                           NerEntities.Engine.NER);
+        } else {
+            nerEntities = new NerEntities(Collections.emptyList());
+        }
+        if (redactionServiceSettings.isAzureNerServiceEnabled()) {
+            NerEntitiesModel azureNerEntitiesModel = redactionStorageService.getAzureNerEntities(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
+            NerEntities azureNerEntities = NerEntitiesAdapter.toNerEntities(azureNerEntitiesModel, document, NerEntities.Engine.CLOUD_NER);
+            nerEntities.merge(azureNerEntities);
+        }
+        if (redactionServiceSettings.isLlmNerServiceEnabled()) {
+            NerEntities llmNerEntities = getLlmNerEntities(analyzeRequest);
+            nerEntities.merge(llmNerEntities);
+        }
+        return nerEntities;
+    }
+
+
+    private NerEntities getLlmNerEntities(AnalyzeRequest analyzeRequest) {
+
+        return new NerEntities(redactionStorageService.getLlmNerEntities(analyzeRequest.getDossierId(), analyzeRequest.getFileId()).getEntities()
+                                       .stream()
+                                       .map(e -> new NerEntities.NerEntity(e.getValue(),
+                                                                           new TextRange(e.getStartOffset(), e.getEndOffset()),
+                                                                           e.getType(),
+                                                                           null,
+                                                                           NerEntities.Engine.LLM_NER))
+                                       .toList());
+    }
+
+
+    private EntityLog getEntityLogWithoutEntries(AnalyzeRequest analyzeRequest) {
+
+        EntityLog entityLogWithoutEntries = redactionStorageService.getEntityLogWithoutEntries(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
+        log.info("Loaded previous entity log without entries for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
+        return entityLogWithoutEntries;
+    }
+
+
+    private EntityLog getEntityLog(AnalyzeRequest analyzeRequest) {
+
+        EntityLog entityLog = redactionStorageService.getEntityLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
+        log.info("Loaded full entity log for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
+        return entityLog;
+    }
+
+
+    private SectionsToReanalyzeData getDictionaryIncrementAndSectionsToReanalyze(AnalyzeRequest analyzeRequest,
+                                                                                 DictionaryIncrement dictionaryIncrement,
+                                                                                 ReanalysisSetupData reanalysisSetupData,
+                                                                                 ImportedRedactions importedRedactions) {
+
+        Set<String> relevantManuallyModifiedAnnotationIds = getRelevantManuallyModifiedAnnotationIds(analyzeRequest.getManualRedactions());
+
+        Set<Integer> sectionsToReanalyseIds = redactionStorageService.findIdsOfSectionsToReanalyse(analyzeRequest.getDossierId(),
+                                                                                                   analyzeRequest.getFileId(),
+                                                                                                   relevantManuallyModifiedAnnotationIds);
+        sectionsToReanalyseIds.addAll(getSectionsToReanalyseIds(analyzeRequest,
+                                                                reanalysisSetupData.document(),
+                                                                dictionaryIncrement,
+                                                                importedRedactions,
+                                                                relevantManuallyModifiedAnnotationIds));
+
+        List<SemanticNode> sectionsToReAnalyse = getSectionsToReAnalyse(reanalysisSetupData.document(), sectionsToReanalyseIds);
+        log.info("{} Sections to reanalyze found for file {} in dossier {}", sectionsToReanalyseIds.size(), analyzeRequest.getFileId(), analyzeRequest.getDossierId());
+        return new SectionsToReanalyzeData(dictionaryIncrement, sectionsToReanalyseIds, sectionsToReAnalyse);
+    }
+
+
+    private DictionaryIncrement getDictionaryIncrement(AnalyzeRequest analyzeRequest, ReanalysisSetupData reanalysisSetupData) {
+
+        return dictionaryService.getDictionaryIncrements(analyzeRequest.getDossierTemplateId(),
+                                                         new DictionaryVersion(reanalysisSetupData.entityLog().getDictionaryVersion(),
+                                                                               reanalysisSetupData.entityLog().getDossierDictionaryVersion()),
+                                                         analyzeRequest.getDossierId());
+    }
+
+
+    private NerEntities getEntityRecognitionEntitiesFilteredBySectionIds(AnalyzeRequest analyzeRequest, Document document, Set<Integer> sectionsToReanalyseIds) {
+
+        NerEntities nerEntities;
+        if (redactionServiceSettings.isNerServiceEnabled()) {
+            NerEntitiesModel nerEntitiesModel = redactionStorageService.getNerEntities(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
+            nerEntitiesModel = filterNerEntitiesModelBySectionIds(sectionsToReanalyseIds, nerEntitiesModel);
+            nerEntities = NerEntitiesAdapter.toNerEntities(nerEntitiesModel, document, NerEntities.Engine.NER);
+        } else {
+            nerEntities = new NerEntities(Collections.emptyList());
+        }
+        if (redactionServiceSettings.isAzureNerServiceEnabled()) {
+            NerEntitiesModel azureNerEntitiesModel = redactionStorageService.getAzureNerEntities(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
+            azureNerEntitiesModel = filterNerEntitiesModelBySectionIds(sectionsToReanalyseIds, azureNerEntitiesModel);
+            NerEntities azureNerEntities = NerEntitiesAdapter.toNerEntities(azureNerEntitiesModel, document, NerEntities.Engine.CLOUD_NER);
+            nerEntities.merge(azureNerEntities);
+        }
+        if (redactionServiceSettings.isLlmNerServiceEnabled()) {
+            NerEntities llmNerEntities = getLlmNerEntities(analyzeRequest);
+            nerEntities.merge(llmNerEntities);
+        }
+        return nerEntities;
+    }
+
+
+    private static NerEntitiesModel filterNerEntitiesModelBySectionIds(Set<Integer> sectionsToReanalyseIds, NerEntitiesModel nerEntitiesModel) {
+
+        return new NerEntitiesModel(nerEntitiesModel.getData().entrySet()
+                                            .stream() //
+                                            .filter(entry -> sectionsToReanalyseIds.contains(getSuperSectionID(entry.getKey()))) //
+                                            .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)));
+    }
+
+
+    private static Integer getSuperSectionID(String section) {
+
+        return NerEntitiesAdapter.sectionNumberToTreeId(section)
+                .get(0);
+
+    }
+
+
+    private static List<SemanticNode> getSectionsToReAnalyse(Document document, Set<Integer> sectionsToReanalyseIds) {
+
+        return document.streamChildren()
+                .filter(section -> sectionsToReanalyseIds.contains(section.getTreeId()
+                                                                           .get(0)))
+                .collect(Collectors.toList());
+    }
+
+
+    private Set<Integer> getSectionsToReanalyseIds(AnalyzeRequest analyzeRequest,
+                                                   Document document,
+                                                   DictionaryIncrement dictionaryIncrement,
+                                                   ImportedRedactions importedRedactions,
+                                                   Set<String> relevantManuallyModifiedAnnotationIds) {
+
+        return sectionFinderService.findSectionsToReanalyse(dictionaryIncrement, document, analyzeRequest, importedRedactions, relevantManuallyModifiedAnnotationIds);
+    }
+
+
+    private record DictionaryAndNotFoundEntries(Dictionary dictionary, List<PrecursorEntity> notFoundManualRedactionEntries, List<PrecursorEntity> notFoundImportedEntries) {
+
+    }
+
+    private record NotFoundEntries(List<PrecursorEntity> notFoundManualRedactionEntries, List<PrecursorEntity> notFoundImportedEntries) {
+
+    }
+
+    private record SectionsToReanalyzeData(DictionaryIncrement dictionaryIncrement, Set<Integer> sectionsToReanalyseIds, List<SemanticNode> sectionsToReanalyze) {
+
+    }
+
+    public record AnalysisData(
+            KieWrapper kieWrapperEntityRules,
+            KieWrapper kieWrapperComponentRules,
+            Document document,
+            ImportedRedactions importedRedactions,
+            Dictionary dictionary,
+            List<PrecursorEntity> notFoundManualRedactionEntries,
+            List<PrecursorEntity> notFoundImportedEntries,
+            NerEntities nerEntities
+    ) {
+
+    }
+
+    public record ReanalysisSetupData(
+            EntityLog entityLog, Document document
+    ) {
+
+    }
+
+    public record ReanalysisInitialProcessingData(
+            ImportedRedactions importedRedactions,
+            DictionaryIncrement dictionaryIncrement,
+            Set<Integer> sectionsToReanalyseIds,
+            List<SemanticNode> sectionsToReAnalyse,
+            KieWrapper kieWrapperComponentRules,
+            KieWrapper kieWrapperEntityRules
+    ) {
+
+    }
+
+    public record ReanalysisFinalProcessingData(
+            NerEntities nerEntities, Dictionary dictionary, List<PrecursorEntity> notFoundManualRedactionEntries, List<PrecursorEntity> notFoundImportedEntries
+    ) {
+
+    }
+
+}
@@ -1,13 +1,14 @@
 package com.iqser.red.service.redaction.v1.server.service;

-import static com.iqser.red.service.redaction.v1.server.service.document.SectionFinderService.getRelevantManuallyModifiedAnnotationIds;
+import static com.iqser.red.service.redaction.v1.server.service.AnalysisPreparationService.ReanalysisFinalProcessingData;
+import static com.iqser.red.service.redaction.v1.server.service.AnalysisPreparationService.ReanalysisInitialProcessingData;
+import static com.iqser.red.service.redaction.v1.server.service.AnalysisPreparationService.ReanalysisSetupData;

 import java.util.ArrayList;
 import java.util.Collection;
-import java.util.Collections;
 import java.util.HashSet;
 import java.util.List;
-import java.util.Map;
+import java.util.Optional;
 import java.util.Set;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
@@ -23,29 +24,20 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileTyp
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentLog;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogChanges;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedactions;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedLegalBases;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.mapper.ImportedLegalBasisMapper;
+import com.iqser.red.service.persistence.service.v1.api.shared.mongo.service.EntityLogMongoService;
 import com.iqser.red.service.redaction.v1.server.RedactionServiceSettings;
-import com.iqser.red.service.redaction.v1.server.client.model.NerEntitiesModel;
+import com.iqser.red.service.redaction.v1.server.logger.Context;
 import com.iqser.red.service.redaction.v1.server.model.KieWrapper;
-import com.iqser.red.service.redaction.v1.server.model.NerEntities;
 import com.iqser.red.service.redaction.v1.server.model.component.Component;
-import com.iqser.red.service.redaction.v1.server.model.dictionary.Dictionary;
-import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryIncrement;
-import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryVersion;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
-import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
+import com.iqser.red.service.redaction.v1.server.service.AnalysisPreparationService.AnalysisData;
 import com.iqser.red.service.redaction.v1.server.service.components.ComponentLogCreatorService;
-import com.iqser.red.service.redaction.v1.server.service.document.DocumentGraphMapper;
-import com.iqser.red.service.redaction.v1.server.service.document.ImportedRedactionEntryService;
-import com.iqser.red.service.redaction.v1.server.service.document.ManualRedactionEntryService;
-import com.iqser.red.service.redaction.v1.server.service.document.NerEntitiesAdapter;
-import com.iqser.red.service.redaction.v1.server.service.document.SectionFinderService;
 import com.iqser.red.service.redaction.v1.server.service.drools.ComponentDroolsExecutionService;
 import com.iqser.red.service.redaction.v1.server.service.drools.EntityDroolsExecutionService;
-import com.iqser.red.service.redaction.v1.server.service.drools.KieContainerCreationService;
-import com.iqser.red.service.redaction.v1.server.storage.ObservedStorageService;
 import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
+import com.knecon.fforesight.tenantcommons.TenantContext;

 import io.micrometer.core.annotation.Timed;
 import io.micrometer.observation.annotation.Observed;
@@ -61,21 +53,15 @@ import lombok.extern.slf4j.Slf4j;
@RequiredArgsConstructor
 public class AnalyzeService {

-    DictionaryService dictionaryService;
    EntityDroolsExecutionService entityDroolsExecutionService;
-    ComponentDroolsExecutionService componentDroolsExecutionService;
-    KieContainerCreationService kieContainerCreationService;
    DictionarySearchService dictionarySearchService;
    EntityLogCreatorService entityLogCreatorService;
-    ComponentLogCreatorService componentLogCreatorService;
+    EntityLogMongoService entityLogMongoService;
    RedactionStorageService redactionStorageService;
-    RedactionServiceSettings redactionServiceSettings;
    NotFoundImportedEntitiesService notFoundImportedEntitiesService;
-    SectionFinderService sectionFinderService;
-    ManualRedactionEntryService manualRedactionEntryService;
-    ImportedRedactionEntryService importedRedactionEntryService;
-    ObservedStorageService observedStorageService;
-    FunctionTimerValues redactmanagerAnalyzePagewiseValues;
+    AnalysisPreparationService analysisPreparationService;
+    AnalysisFinalizationService analysisFinalizationService;
+    ImportedLegalBasisMapper importedLegalBasisMapper = ImportedLegalBasisMapper.INSTANCE;


    @Timed("redactmanager_reanalyze")
@@ -84,303 +70,223 @@ public class AnalyzeService {
    public AnalyzeResult reanalyze(@RequestBody AnalyzeRequest analyzeRequest) {

        long startTime = System.currentTimeMillis();
-        EntityLog entityLogWithoutEntries = redactionStorageService.getEntityLogWithoutEntries(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
-        log.info("Loaded previous entity log for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());

-        Document document = DocumentGraphMapper.toDocumentGraph(observedStorageService.getDocumentData(analyzeRequest.getDossierId(), analyzeRequest.getFileId()));
-        log.info("Loaded Document Graph for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
-
-        ImportedRedactions importedRedactions = redactionStorageService.getImportedRedactions(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
-        log.info("Loaded Imported Redactions for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
+        ReanalysisSetupData setupData = analysisPreparationService.getReanalysisSetupData(analyzeRequest);
+        Context context = new Context(analyzeRequest.getFileId(),
+                                      analyzeRequest.getDossierId(),
+                                      analyzeRequest.getDossierTemplateId(),
+                                      0,
+                                      analyzeRequest.getAnalysisNumber(),
+                                      TenantContext.getTenantId());

        // not yet ready for reanalysis
-        if (entityLogWithoutEntries == null || document == null || document.getNumberOfPages() == 0) {
+        if (setupData.entityLog() == null || setupData.document() == null || setupData.document().getNumberOfPages() == 0) {
            return analyze(analyzeRequest);
        }

-        DictionaryIncrement dictionaryIncrement = dictionaryService.getDictionaryIncrements(analyzeRequest.getDossierTemplateId(),
-                                                                                            new DictionaryVersion(entityLogWithoutEntries.getDictionaryVersion(),
-                                                                                                                  entityLogWithoutEntries.getDossierDictionaryVersion()),
-                                                                                            analyzeRequest.getDossierId());
+        ReanalysisInitialProcessingData initialProcessingData = analysisPreparationService.getReanalysisInitialProcessingData(analyzeRequest, setupData);

-        Set<String> relevantManuallyModifiedAnnotationIds = getRelevantManuallyModifiedAnnotationIds(analyzeRequest.getManualRedactions());
+        if (initialProcessingData.sectionsToReAnalyse().isEmpty()) {

-        Set<Integer> sectionsToReanalyseIds = redactionStorageService.findIdsOfSectionsToReanalyse(analyzeRequest.getDossierId(),
-                                                                                                   analyzeRequest.getFileId(),
-                                                                                                   relevantManuallyModifiedAnnotationIds);
-        sectionsToReanalyseIds.addAll(getSectionsToReanalyseIds(analyzeRequest, document, dictionaryIncrement, importedRedactions, relevantManuallyModifiedAnnotationIds));
-
-        List<SemanticNode> sectionsToReAnalyse = getSectionsToReAnalyse(document, sectionsToReanalyseIds);
-        log.info("{} Sections to reanalyze found for file {} in dossier {}", sectionsToReanalyseIds.size(), analyzeRequest.getFileId(), analyzeRequest.getDossierId());
-
-        if (sectionsToReAnalyse.isEmpty()) {
-
-            EntityLogChanges entityLogChanges = entityLogCreatorService.updateVersionsAndReturnChanges(entityLogWithoutEntries,
-                                                                                                       dictionaryIncrement.getDictionaryVersion(),
+            EntityLogChanges entityLogChanges = entityLogCreatorService.updateVersionsAndReturnChanges(setupData.entityLog(),
+                                                                                                       initialProcessingData.dictionaryIncrement().getDictionaryVersion(),
                                                                                                       analyzeRequest,
                                                                                                       new ArrayList<>(),
                                                                                                       new ArrayList<>());

-            return finalizeAnalysis(analyzeRequest,
-                                    startTime,
-                                    kieContainerCreationService.getLatestKieContainer(analyzeRequest.getDossierTemplateId(), RuleFileType.COMPONENT),
-                                    entityLogChanges,
-                                    document,
-                                    document.getNumberOfPages(),
-                                    true,
-                                    Collections.emptySet());
+            return analysisFinalizationService.finalizeAnalysis(analyzeRequest,
+                                                                startTime,
+                                                                initialProcessingData.kieWrapperComponentRules(),
+                                                                entityLogChanges,
+                                                                setupData.document(),
+                                                                setupData.document().getNumberOfPages(),
+                                                                true,
+                                                                new HashSet<>(),
+                                                                context);
        }

-        KieWrapper kieWrapperEntityRules = kieContainerCreationService.getLatestKieContainer(analyzeRequest.getDossierTemplateId(), RuleFileType.ENTITY);
-        log.info("Updated entity rules to version {} for file {} in dossier {}", kieWrapperEntityRules.rulesVersion(), analyzeRequest.getFileId(), analyzeRequest.getDossierId());
+        ReanalysisFinalProcessingData finalProcessingData = analysisPreparationService.getReanalysisFinalProcessingData(analyzeRequest, setupData, initialProcessingData);

-        NerEntities nerEntities = getEntityRecognitionEntitiesFilteredBySectionIds(analyzeRequest, document, sectionsToReanalyseIds);
-        log.info("Loaded Ner Entities for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
+        dictionarySearchService.addDictionaryEntities(finalProcessingData.dictionary(), initialProcessingData.sectionsToReAnalyse());
+        log.info("Finished Dictionary Search for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());

-        var notFoundManualRedactionEntries = manualRedactionEntryService.addManualRedactionEntriesAndReturnNotFoundEntries(analyzeRequest,
-                                                                                                                           document,
-                                                                                                                           analyzeRequest.getDossierTemplateId());
-        var notFoundImportedEntries = importedRedactionEntryService.addImportedEntriesAndReturnNotFoundEntries(analyzeRequest, importedRedactions, document);
-        var notFoundManualOrImportedEntries = Stream.of(notFoundManualRedactionEntries, notFoundImportedEntries)
+        var notFoundManualOrImportedEntries = Stream.of(finalProcessingData.notFoundManualRedactionEntries(), finalProcessingData.notFoundImportedEntries())
                .flatMap(Collection::stream)
                .collect(Collectors.toList());

-        Dictionary dictionary = dictionaryService.getDeepCopyDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId());
-        log.info("Updated Dictionaries for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
-
-        dictionarySearchService.addDictionaryEntities(dictionary, sectionsToReAnalyse);
-        log.info("Finished Dictionary Search for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
-
        // we could add the imported redactions similar to the manual redactions here as well for additional processing
-        List<FileAttribute> allFileAttributes = entityDroolsExecutionService.executeRules(kieWrapperEntityRules.container(),
-                                                                                          document,
-                                                                                          sectionsToReAnalyse,
-                                                                                          dictionary,
+        context.setRuleVersion(initialProcessingData.kieWrapperEntityRules().rulesVersion());
+        List<FileAttribute> allFileAttributes = entityDroolsExecutionService.executeRules(initialProcessingData.kieWrapperEntityRules().container(),
+                                                                                          setupData.document(),
+                                                                                          initialProcessingData.sectionsToReAnalyse(),
+                                                                                          finalProcessingData.dictionary(),
                                                                                          analyzeRequest.getFileAttributes(),
                                                                                          analyzeRequest.getManualRedactions(),
-                                                                                          nerEntities);
+                                                                                          finalProcessingData.nerEntities(),
+                                                                                          context);
        log.info("Finished entity rule execution for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());

        EntityLogChanges entityLogChanges = entityLogCreatorService.updatePreviousEntityLog(analyzeRequest,
-                                                                                            document,
-                                                                                            entityLogWithoutEntries,
+                                                                                            setupData.document(),
+                                                                                            setupData.entityLog(),
                                                                                            notFoundManualOrImportedEntries,
-                                                                                            sectionsToReanalyseIds,
-                                                                                            dictionary.getVersion());
+                                                                                            initialProcessingData.sectionsToReanalyseIds(),
+                                                                                            finalProcessingData.dictionary().getVersion());

-        notFoundImportedEntitiesService.processEntityLog(entityLogChanges.getEntityLog(), analyzeRequest, notFoundImportedEntries);
+        notFoundImportedEntitiesService.processEntityLog(entityLogChanges.getEntityLog(), analyzeRequest, finalProcessingData.notFoundImportedEntries());

-        return finalizeAnalysis(analyzeRequest,
-                                startTime,
-                                kieContainerCreationService.getLatestKieContainer(analyzeRequest.getDossierTemplateId(), RuleFileType.COMPONENT),
-                                entityLogChanges,
-                                document,
-                                document.getNumberOfPages(),
-                                true,
-                                new HashSet<>(allFileAttributes));
+        return analysisFinalizationService.finalizeAnalysis(analyzeRequest,
+                                                            startTime,
+                                                            initialProcessingData.kieWrapperComponentRules(),
+                                                            entityLogChanges,
+                                                            setupData.document(),
+                                                            setupData.document().getNumberOfPages(),
+                                                            true,
+                                                            new HashSet<>(allFileAttributes),
+                                                            context);
    }


+    @SneakyThrows
+    @Timed("redactmanager_reanalyzeComponentsOnly")
+    @Observed(name = "AnalyzeService", contextualName = "reanalyzeComponentsOnly")
+    public AnalyzeResult reanalyzeComponentsOnly(AnalyzeRequest analyzeRequest) {
+
+        long startTime = System.currentTimeMillis();
+
+        ReanalysisSetupData setupData = analysisPreparationService.getReanalysisSetupDataForComponentsOnlyReanalyze(analyzeRequest);
+
+        Context context = new Context(analyzeRequest.getFileId(),
+                                      analyzeRequest.getDossierId(),
+                                      analyzeRequest.getDossierTemplateId(),
+                                      0,
+                                      analyzeRequest.getAnalysisNumber(),
+                                      TenantContext.getTenantId());
+
+        Optional<EntityLog> entityLog = entityLogMongoService.findEntityLogByDossierIdAndFileId(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
+
+        // not yet ready for reanalysis
+        if (entityLog.isEmpty() || setupData.document() == null || setupData.document().getNumberOfPages() == 0) {
+            return analyze(analyzeRequest);
+        }
+
+        KieWrapper kieWrapperComponentRules = analysisPreparationService.getKieWrapper(analyzeRequest, RuleFileType.COMPONENT);
+
+        EntityLogChanges entityLogChanges = EntityLogChanges.builder()
+                .newEntityLogEntries(new ArrayList<>())
+                .updatedEntityLogEntries(new ArrayList<>())
+                .entityLog(entityLog.get())
+                .build();
+
+        return analysisFinalizationService.finalizeAnalysis(analyzeRequest,
+                                                            startTime,
+                                                            kieWrapperComponentRules,
+                                                            entityLogChanges,
+                                                            setupData.document(),
+                                                            setupData.document().getNumberOfPages(),
+                                                            true,
+                                                            new HashSet<>(analyzeRequest.getFileAttributes()),
+                                                            context);
+    }
+
+
+    @SneakyThrows
    @Timed("redactmanager_analyze")
    @Observed(name = "AnalyzeService", contextualName = "analyze")
    public AnalyzeResult analyze(AnalyzeRequest analyzeRequest) {

        long startTime = System.currentTimeMillis();

-        var kieWrapperEntityRules = kieContainerCreationService.getLatestKieContainer(analyzeRequest.getDossierTemplateId(), RuleFileType.ENTITY);
-        log.info("Updated Rules to Version {} for file {} in dossier {}", kieWrapperEntityRules.rulesVersion(), analyzeRequest.getFileId(), analyzeRequest.getDossierId());
+        AnalysisData analysisData = analysisPreparationService.getAnalysisData(analyzeRequest);

-        var kieWrapperComponentRules = kieContainerCreationService.getLatestKieContainer(analyzeRequest.getDossierTemplateId(), RuleFileType.COMPONENT);
-        log.info("Updated Rules to Version {} for file {} in dossier {}", kieWrapperEntityRules.rulesVersion(), analyzeRequest.getFileId(), analyzeRequest.getDossierId());
+        Context context = new Context(analyzeRequest.getFileId(),
+                                      analyzeRequest.getDossierId(),
+                                      analyzeRequest.getDossierTemplateId(),
+                                      analysisData.kieWrapperEntityRules().rulesVersion(),
+                                      analyzeRequest.getAnalysisNumber(),
+                                      TenantContext.getTenantId());

-        Document document = DocumentGraphMapper.toDocumentGraph(observedStorageService.getDocumentData(analyzeRequest.getDossierId(), analyzeRequest.getFileId()));
-        log.info("Loaded Document Graph for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
-
-        ImportedRedactions importedRedactions = redactionStorageService.getImportedRedactions(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
-        log.info("Loaded Imported Redactions for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
-
-        NerEntities nerEntities = getEntityRecognitionEntities(analyzeRequest, document);
-        log.info("Loaded Ner Entities for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
-
-        dictionaryService.updateDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId());
-        Dictionary dictionary = dictionaryService.getDeepCopyDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId());
-        log.info("Updated Dictionaries for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
-
-        var notFoundManualRedactionEntries = manualRedactionEntryService.addManualRedactionEntriesAndReturnNotFoundEntries(analyzeRequest,
-                                                                                                                           document,
-                                                                                                                           analyzeRequest.getDossierTemplateId());
-        var notFoundImportedEntries = importedRedactionEntryService.addImportedEntriesAndReturnNotFoundEntries(analyzeRequest, importedRedactions, document);
-        var notFoundManualOrImportedEntries = Stream.of(notFoundManualRedactionEntries, notFoundImportedEntries)
+        var notFoundManualOrImportedEntries = Stream.of(analysisData.notFoundManualRedactionEntries(), analysisData.notFoundImportedEntries())
                .flatMap(Collection::stream)
                .collect(Collectors.toList());

-        dictionarySearchService.addDictionaryEntities(dictionary, document);
+        dictionarySearchService.addDictionaryEntities(analysisData.dictionary(), analysisData.document());
        log.info("Finished Dictionary Search for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());

        // we could add the imported redactions similar to the manual redactions here as well for additional processing
-        List<FileAttribute> allFileAttributes = entityDroolsExecutionService.executeRules(kieWrapperEntityRules.container(),
-                                                                                          document,
-                                                                                          dictionary,
+        List<FileAttribute> allFileAttributes = entityDroolsExecutionService.executeRules(analysisData.kieWrapperEntityRules().container(),
+                                                                                          analysisData.document(),
+                                                                                          analysisData.dictionary(),
                                                                                          analyzeRequest.getFileAttributes(),
                                                                                          analyzeRequest.getManualRedactions(),
-                                                                                          nerEntities);
+                                                                                          analysisData.nerEntities(),
+                                                                                          context);
        log.info("Finished entity rule execution for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());

        EntityLogChanges entityLogChanges = entityLogCreatorService.createInitialEntityLog(analyzeRequest,
-                                                                                           document,
+                                                                                           analysisData.document(),
                                                                                           notFoundManualOrImportedEntries,
-                                                                                           dictionary.getVersion(),
-                                                                                           kieWrapperEntityRules.rulesVersion());
+                                                                                           analysisData.dictionary().getVersion(),
+                                                                                           analysisData.kieWrapperEntityRules().rulesVersion());

-        notFoundImportedEntitiesService.processEntityLog(entityLogChanges.getEntityLog(), analyzeRequest, notFoundImportedEntries);
+        notFoundImportedEntitiesService.processEntityLog(entityLogChanges.getEntityLog(), analyzeRequest, analysisData.notFoundImportedEntries());

-        return finalizeAnalysis(analyzeRequest,
-                                startTime,
-                                kieWrapperComponentRules,
-                                entityLogChanges,
-                                document,
-                                document.getNumberOfPages(),
-                                false,
-                                new HashSet<>(allFileAttributes));
+        return analysisFinalizationService.finalizeAnalysis(analyzeRequest,
+                                                            startTime,
+                                                            analysisData.kieWrapperComponentRules(),
+                                                            entityLogChanges,
+                                                            analysisData.document(),
+                                                            analysisData.document().getNumberOfPages(),
+                                                            false,
+                                                            new HashSet<>(allFileAttributes),
+                                                            context);
    }


-    private AnalyzeResult finalizeAnalysis(AnalyzeRequest analyzeRequest,
-                                           long startTime,
-                                           KieWrapper kieWrapperComponentRules,
-                                           EntityLogChanges entityLogChanges,
-                                           Document document,
-                                           int numberOfPages,
-                                           boolean isReanalysis,
-                                           Set<FileAttribute> addedFileAttributes) {
+    @Timed("redactmanager_analyzeImportedRedactionsOnly")
+    @Observed(name = "AnalyzeService", contextualName = "analyzeImportedRedactionsOnly")
+    public AnalyzeResult analyzeImportedRedactionsOnly(AnalyzeRequest analyzeRequest) {

-        EntityLog entityLog = entityLogChanges.getEntityLog();
+        long startTime = System.currentTimeMillis();

-        // as workaround for duplicate key exceptions occurring due to simultaneous analyses and reanalyses save instead of insert is used
-        // also analysis numbers should be incremented in every follow-up request, so checking if the log exists is not needed
-        if (!redactionStorageService.entityLogExists(analyzeRequest.getDossierId(), analyzeRequest.getFileId())) {
-            redactionStorageService.saveEntityLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), entityLog);
+        AnalysisData analysisData = analysisPreparationService.getAnalysisData(analyzeRequest);

-        } else {
-            redactionStorageService.updateEntityLogWithoutEntries(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), entityLog);
+        Context context = new Context(analyzeRequest.getFileId(),
+                                      analyzeRequest.getDossierId(),
+                                      analyzeRequest.getDossierTemplateId(),
+                                      analysisData.kieWrapperEntityRules().rulesVersion(),
+                                      analyzeRequest.getAnalysisNumber(),
+                                      TenantContext.getTenantId());

-            if (!entityLogChanges.getNewEntityLogEntries().isEmpty()) {
-                redactionStorageService.saveEntityLogEntries(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), entityLogChanges.getNewEntityLogEntries());
-            }
-            if (!entityLogChanges.getUpdatedEntityLogEntries().isEmpty()) {
-                redactionStorageService.updateEntityLogEntries(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), entityLogChanges.getUpdatedEntityLogEntries());
-            }
-        }
-
-        log.info("Created entity log for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
-
-        computeComponentsWhenRulesArePresent(analyzeRequest, kieWrapperComponentRules, document, addedFileAttributes, entityLog);
-
-        long duration = System.currentTimeMillis() - startTime;
-
-        redactmanagerAnalyzePagewiseValues.increase(numberOfPages, duration);
-
-        return AnalyzeResult.builder()
-                .dossierId(analyzeRequest.getDossierId())
-                .fileId(analyzeRequest.getFileId())
-                .duration(duration)
-                .numberOfPages(numberOfPages)
-                .hasUpdates(entityLogChanges.hasChanges())
-                .analysisVersion(redactionServiceSettings.getAnalysisVersion())
-                .analysisNumber(analyzeRequest.getAnalysisNumber())
-                .rulesVersion(entityLog.getRulesVersion())
-                .componentRulesVersion(kieWrapperComponentRules.rulesVersion())
-                .dictionaryVersion(entityLog.getDictionaryVersion())
-                .legalBasisVersion(entityLog.getLegalBasisVersion())
-                .dossierDictionaryVersion(entityLog.getDossierDictionaryVersion())
-                .wasReanalyzed(isReanalysis)
-                .manualRedactions(analyzeRequest.getManualRedactions())
-                .addedFileAttributes(addedFileAttributes)
-                .usedComponentMappings(analyzeRequest.getComponentMappings())
-                .build();
-    }
-
-
-    private void computeComponentsWhenRulesArePresent(AnalyzeRequest analyzeRequest,
-                                                      KieWrapper kieWrapperComponentRules,
-                                                      Document document,
-                                                      Set<FileAttribute> addedFileAttributes,
-                                                      EntityLog entityLog) {
-
-        if (!kieWrapperComponentRules.isPresent()) {
-            return;
-        }
-
-        // We need the latest EntityLog entries for components rules execution
-        entityLog.setEntityLogEntry(redactionStorageService.getEntityLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId()).getEntityLogEntry());
-
-        List<Component> components = componentDroolsExecutionService.executeRules(kieWrapperComponentRules.container(),
-                                                                                  entityLog,
-                                                                                  document,
-                                                                                  addedFileAttributes,
-                                                                                  analyzeRequest.getComponentMappings());
-
-        log.info("Finished component rule execution for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
-
-        ComponentLog componentLog = componentLogCreatorService.buildComponentLog(analyzeRequest.getAnalysisNumber(), components, kieWrapperComponentRules.rulesVersion());
-
-        redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.COMPONENT_LOG, componentLog);
-
-        log.info("Stored component log for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
-    }
-
-
-    private static List<SemanticNode> getSectionsToReAnalyse(Document document, Set<Integer> sectionsToReanalyseIds) {
-
-        return document.streamChildren()
-                .filter(section -> sectionsToReanalyseIds.contains(section.getTreeId()
-                                                                           .get(0)))
+        var notFoundManualOrImportedEntries = Stream.of(analysisData.notFoundManualRedactionEntries(), analysisData.notFoundImportedEntries())
+                .flatMap(Collection::stream)
                .collect(Collectors.toList());
-    }

+        ImportedLegalBases importedLegalBases = redactionStorageService.getImportedLegalBases(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
+        log.info("Loaded Imported Legal Bases for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());

-    private Set<Integer> getSectionsToReanalyseIds(AnalyzeRequest analyzeRequest,
-                                                   Document document,
-                                                   DictionaryIncrement dictionaryIncrement,
-                                                   ImportedRedactions importedRedactions,
-                                                   Set<String> relevantManuallyModifiedAnnotationIds) {
+        EntityLogChanges entityLogChanges = entityLogCreatorService.createInitialEntityLog(analyzeRequest,
+                                                                                           analysisData.document(),
+                                                                                           notFoundManualOrImportedEntries,
+                                                                                           analysisData.dictionary().getVersion(),
+                                                                                           analysisData.kieWrapperEntityRules().rulesVersion());
+        entityLogChanges.getEntityLog()
+                .setLegalBasis(importedLegalBases.getImportedLegalBases()
+                                       .stream()
+                                       .map(importedLegalBasisMapper::toEntityLogLegalBasis)
+                                       .toList());

-        return sectionFinderService.findSectionsToReanalyse(dictionaryIncrement, document, analyzeRequest, importedRedactions, relevantManuallyModifiedAnnotationIds);
-    }
+        notFoundImportedEntitiesService.processEntityLog(entityLogChanges.getEntityLog(), analyzeRequest, analysisData.notFoundImportedEntries());

-
-    private NerEntities getEntityRecognitionEntitiesFilteredBySectionIds(AnalyzeRequest analyzeRequest, Document document, Set<Integer> sectionsToReanalyseIds) {
-
-        NerEntities nerEntities;
-        if (redactionServiceSettings.isNerServiceEnabled()) {
-            NerEntitiesModel nerEntitiesModel = redactionStorageService.getNerEntities(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
-            nerEntitiesModel = filterNerEntitiesModelBySectionIds(sectionsToReanalyseIds, nerEntitiesModel);
-            nerEntities = NerEntitiesAdapter.toNerEntities(nerEntitiesModel, document);
-        } else {
-            nerEntities = new NerEntities(Collections.emptyList());
-        }
-        return nerEntities;
-    }
-
-
-    private static NerEntitiesModel filterNerEntitiesModelBySectionIds(Set<Integer> sectionsToReanalyseIds, NerEntitiesModel nerEntitiesModel) {
-
-        return new NerEntitiesModel(nerEntitiesModel.getData().entrySet()
-                                            .stream() //
-                                            .filter(entry -> sectionsToReanalyseIds.contains(entry.getKey())) //
-                                            .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)));
-    }
-
-
-    private NerEntities getEntityRecognitionEntities(AnalyzeRequest analyzeRequest, Document document) {
-
-        NerEntities nerEntities;
-        if (redactionServiceSettings.isNerServiceEnabled()) {
-            nerEntities = NerEntitiesAdapter.toNerEntities(redactionStorageService.getNerEntities(analyzeRequest.getDossierId(), analyzeRequest.getFileId()), document);
-        } else {
-            nerEntities = new NerEntities(Collections.emptyList());
-        }
-        return nerEntities;
+        return analysisFinalizationService.finalizeAnalysis(analyzeRequest,
+                                                            startTime,
+                                                            analysisData.kieWrapperComponentRules(),
+                                                            entityLogChanges,
+                                                            analysisData.document(),
+                                                            analysisData.document().getNumberOfPages(),
+                                                            false,
+                                                            new HashSet<>(),
+                                                            context);
    }

 }
@@ -39,14 +39,27 @@ public class DictionarySearchService {
    @Observed(name = "DictionarySearchService", contextualName = "add-dictionary-entries")
    public void addDictionaryEntities(Dictionary dictionary, SemanticNode node) {

-        for (DictionaryModel model : dictionary.getDictionaryModels()) {
-            bySearchImplementationAsDictionary(model.getEntriesSearch(), model.getType(), model.isHint() ? EntityType.HINT : EntityType.ENTITY, node, model.isDossierDictionary());
-            bySearchImplementationAsDictionary(model.getFalsePositiveSearch(), model.getType(), EntityType.FALSE_POSITIVE, node, model.isDossierDictionary());
-            bySearchImplementationAsDictionary(model.getFalseRecommendationsSearch(), model.getType(), EntityType.FALSE_RECOMMENDATION, node, model.isDossierDictionary());
-            if (model.isDossierDictionary()) {
-                bySearchImplementationAsDictionary(model.getDeletionEntriesSearch(), model.getType(), EntityType.DICTIONARY_REMOVAL, node, model.isDossierDictionary());
-            }
-        }
+        dictionary.getDictionaryModels()
+                .stream()
+                .parallel()
+                .forEach(model -> {
+                    synchronized (node) {
+                        bySearchImplementationAsDictionary(model.getEntriesSearch(),
+                                                           model.getType(),
+                                                           model.isHint() ? EntityType.HINT : EntityType.ENTITY,
+                                                           node,
+                                                           model.isDossierDictionary());
+                        bySearchImplementationAsDictionary(model.getFalsePositiveSearch(), model.getType(), EntityType.FALSE_POSITIVE, node, model.isDossierDictionary());
+                        bySearchImplementationAsDictionary(model.getFalseRecommendationsSearch(),
+                                                           model.getType(),
+                                                           EntityType.FALSE_RECOMMENDATION,
+                                                           node,
+                                                           model.isDossierDictionary());
+                        if (model.isDossierDictionary()) {
+                            bySearchImplementationAsDictionary(model.getDeletionEntriesSearch(), model.getType(), EntityType.DICTIONARY_REMOVAL, node, model.isDossierDictionary());
+                        }
+                    }
+                });
    }


@@ -58,7 +71,7 @@ public class DictionarySearchService {

        Set<Engine> engines = isDossierDictionaryEntry ? Set.of(Engine.DOSSIER_DICTIONARY) : Set.of(Engine.DICTIONARY);
        EntityCreationService entityCreationService = new EntityCreationService(entityEnrichmentService);
-        searchImplementation.getBoundaries(node.getTextBlock(), node.getTextRange())
+        searchImplementation.getBoundaries(node.getTextBlock())
                .filter(boundary -> entityCreationService.isValidEntityTextRange(node.getTextBlock(), boundary))
                .forEach(bounds -> entityCreationService.byTextRangeWithEngine(bounds, type, entityType, node, engines)
                        .ifPresent(entity -> {
@@ -1,7 +1,6 @@
 package com.iqser.red.service.redaction.v1.server.service;

 import java.awt.Color;
-import java.util.ArrayList;
 import java.util.Comparator;
 import java.util.HashSet;
 import java.util.LinkedList;
@@ -24,6 +23,7 @@ import com.iqser.red.service.dictionarymerge.commons.DictionaryEntry;
 import com.iqser.red.service.dictionarymerge.commons.DictionaryEntryModel;
 import com.iqser.red.service.dictionarymerge.commons.DictionaryMergeService;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.configuration.Colors;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type;
 import com.iqser.red.service.redaction.v1.server.RedactionServiceSettings;
 import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
 import com.iqser.red.service.redaction.v1.server.model.dictionary.Dictionary;
@@ -98,8 +98,7 @@ public class DictionaryService {
            updateDictionaryEntry(dossierTemplateId, dossierDictionaryVersion, getVersion(dossierDictionary), dossierId);
        }

-        return DictionaryVersion.builder().dossierTemplateVersion(dossierTemplateDictionaryVersion).dossierVersion(dossierDictionaryVersion)
-                .build();
+        return DictionaryVersion.builder().dossierTemplateVersion(dossierTemplateDictionaryVersion).dossierVersion(dossierDictionaryVersion).build();
    }


@@ -168,13 +167,18 @@ public class DictionaryService {
        try {
            DictionaryRepresentation dictionaryRepresentation = new DictionaryRepresentation();

-            var typeResponse = dossierId == null ? dictionaryClient.getAllTypesForDossierTemplate(dossierTemplateId, true) : dictionaryClient.getAllTypesForDossier(dossierId,
-                                                                                                                                                                    true);
+            var typeResponse = dossierId == null ? dictionaryClient.getAllTypesForDossierTemplate(dossierTemplateId, currentVersion, true) : dictionaryClient.getAllTypesForDossier(
+                    dossierId,
+                    currentVersion,
+                    true);
            if (CollectionUtils.isNotEmpty(typeResponse)) {

+                String tenantId = TenantContext.getTenantId();
                List<DictionaryModel> dictionary = typeResponse.stream()
+                        .parallel()
                        .map(t -> {

+                            TenantContext.setTenantId(tenantId);
                            Optional<DictionaryModel> optionalOldModel;
                            if (dossierId == null) {
                                var representation = getDossierTemplateDictionary(dossierTemplateId);
@@ -194,7 +198,7 @@ public class DictionaryService {
                            Set<DictionaryEntryModel> falsePositives = new HashSet<>();
                            Set<DictionaryEntryModel> falseRecommendations = new HashSet<>();

-                            DictionaryEntries newEntries = getEntries(t.getId(), currentVersion);
+                            DictionaryEntries newEntries = mapEntries(t);

                            var newValues = newEntries.getEntries()
                                    .stream()
@@ -309,9 +313,7 @@ public class DictionaryService {
    }


-    private DictionaryEntries getEntries(String typeId, Long fromVersion) {
-
-        var type = dictionaryClient.getDictionaryForType(typeId, fromVersion);
+    private DictionaryEntries mapEntries(Type type) {

        Set<DictionaryEntryModel> entries = type.getEntries() != null ? new HashSet<>(type.getEntries()
                                                                                              .stream()
@@ -335,7 +337,7 @@ public class DictionaryService {
                  entries.size(),
                  falsePositives.size(),
                  falseRecommendations.size(),
-                  typeId);
+                  entries);
        return new DictionaryEntries(entries, falsePositives, falseRecommendations);
    }

@@ -401,7 +403,9 @@ public class DictionaryService {
        return new Dictionary(mergedDictionaries.stream()
                                      .sorted(Comparator.comparingInt(DictionaryModel::getRank).reversed())
                                      .collect(Collectors.toList()),
-                              DictionaryVersion.builder().dossierTemplateVersion(dossierTemplateRepresentation.getDictionaryVersion()).dossierVersion(dossierDictionaryVersion)
+                              DictionaryVersion.builder()
+                                      .dossierTemplateVersion(dossierTemplateRepresentation.getDictionaryVersion())
+                                      .dossierVersion(dossierDictionaryVersion)
                                      .build());
    }

@@ -0,0 +1,77 @@
+package com.iqser.red.service.redaction.v1.server.service;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.Set;
+
+import org.springframework.amqp.rabbit.core.RabbitTemplate;
+import org.springframework.stereotype.Service;
+
+import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.BulkLocalRequest;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.BulkLocalResponse;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.FoundTerm;
+import com.iqser.red.service.redaction.v1.model.QueueNames;
+import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
+import com.iqser.red.service.redaction.v1.server.service.document.DocumentGraphMapper;
+import com.iqser.red.service.redaction.v1.server.service.document.EntityFindingUtility;
+import com.iqser.red.service.redaction.v1.server.storage.ObservedStorageService;
+import com.knecon.fforesight.tenantcommons.TenantContext;
+
+import io.micrometer.observation.annotation.Observed;
+import lombok.AccessLevel;
+import lombok.RequiredArgsConstructor;
+import lombok.experimental.FieldDefaults;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+@Service
+@FieldDefaults(level = AccessLevel.PRIVATE)
+@RequiredArgsConstructor
+public class DocumentSearchService {
+
+    final RabbitTemplate rabbitTemplate;
+    final ObservedStorageService observedStorageService;
+    final EntityFindingUtility entityFindingUtility;
+    final EntityLogCreatorService entityLogCreatorService;
+
+
+    @Observed(name = "DocumentSearchService", contextualName = "search-term-occurrences")
+    public void searchTermOccurrences(AnalyzeRequest analyzeRequest) {
+
+        BulkLocalRequest bulkLocalRequest = analyzeRequest.getBulkLocalRequest();
+        String searchTerm = bulkLocalRequest.getSearchTerm();
+        Set<Integer> pageNumbers = bulkLocalRequest.getPageNumbers();
+
+        Document document = DocumentGraphMapper.toDocumentGraph(observedStorageService.getDocumentData(analyzeRequest.getDossierId(), analyzeRequest.getFileId()));
+
+        List<TextEntity> possibleEntities;
+        if (pageNumbers == null || pageNumbers.isEmpty()) {
+            possibleEntities = entityFindingUtility.findAllPossibleEntitiesByValue(document, searchTerm, bulkLocalRequest.isCaseSensitive());
+        } else {
+            possibleEntities = entityFindingUtility.findAllPossibleEntitiesByValueOnPages(document, searchTerm, bulkLocalRequest.isCaseSensitive(), pageNumbers);
+        }
+
+        List<FoundTerm> foundTerms = possibleEntities.stream()
+                .flatMap(entity -> entityLogCreatorService.toEntityLogEntries(entity, 0, Collections.emptyMap())
+                        .stream())
+                .map(entityLogEntry -> new FoundTerm(entityLogEntry.getPositions(), entityLogEntry.getValue()))
+                .toList();
+
+        BulkLocalResponse bulkLocalResponse = BulkLocalResponse.builder()
+                .dossierId(analyzeRequest.getDossierId())
+                .fileId(analyzeRequest.getFileId())
+                .type(bulkLocalRequest.getType())
+                .reason(bulkLocalRequest.getReason())
+                .legalBasis(bulkLocalRequest.getLegalBasis())
+                .section(bulkLocalRequest.getSection())
+                .foundTerms(foundTerms)
+                .comment(bulkLocalRequest.getComment())
+                .userId(bulkLocalRequest.getUserId())
+                .build();
+
+        rabbitTemplate.convertAndSend(QueueNames.SEARCH_TERM_OCCURRENCES_RESPONSE_EXCHANGE, TenantContext.getTenantId(), bulkLocalResponse);
+    }
+
+}
@@ -2,7 +2,9 @@ package com.iqser.red.service.redaction.v1.server.service;

 import java.time.OffsetDateTime;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.List;
+import java.util.Objects;
 import java.util.Optional;
 import java.util.Set;
 import java.util.stream.Collectors;
@@ -13,8 +15,14 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ChangeType;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualChange;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.PropertyChange;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ChangeFactory;
+import com.iqser.red.service.redaction.v1.server.utils.EntityLogEntryDiffChecker;

 import io.micrometer.core.annotation.Timed;
+import io.micrometer.observation.annotation.Observed;
 import lombok.AccessLevel;
 import lombok.RequiredArgsConstructor;
 import lombok.experimental.FieldDefaults;
@@ -27,11 +35,12 @@ import lombok.extern.slf4j.Slf4j;
 public class EntityChangeLogService {

    @Timed("redactmanager_computeChanges")
+    @Observed(name = "EntityChangeLogService", contextualName = "compute-changes")
    public EntryChanges computeChanges(List<EntityLogEntry> previousEntityLogEntries, List<EntityLogEntry> newEntityLogEntries, int analysisNumber) {

        var now = OffsetDateTime.now();
        if (previousEntityLogEntries.isEmpty()) {
-            newEntityLogEntries.forEach(entry -> entry.getChanges().add(new Change(analysisNumber, ChangeType.ADDED, now)));
+            newEntityLogEntries.forEach(entry -> entry.getChanges().add(new Change(analysisNumber, ChangeType.ADDED, now, Collections.emptyMap())));
            return new EntryChanges(newEntityLogEntries, new ArrayList<>());
        }

@@ -42,7 +51,7 @@ public class EntityChangeLogService {
                    .filter(entry -> entry.getId().equals(entityLogEntry.getId()))
                    .findAny();
            if (optionalPreviousEntity.isEmpty()) {
-                entityLogEntry.getChanges().add(new Change(analysisNumber, ChangeType.ADDED, now));
+                entityLogEntry.getChanges().add(new Change(analysisNumber, ChangeType.ADDED, now, Collections.emptyMap()));
                toInsert.add(entityLogEntry);
                continue;
            }
@@ -51,10 +60,9 @@ public class EntityChangeLogService {
            entityLogEntry.getChanges().addAll(previousEntity.getChanges());

            if (!previousEntity.equals(entityLogEntry)) {
-                if(!previousEntity.getState().equals(entityLogEntry.getState())) {
-                    ChangeType changeType = calculateChangeType(entityLogEntry.getState(), previousEntity.getState());
-                    entityLogEntry.getChanges().add(new Change(analysisNumber, changeType, now));
-                }
+                List<PropertyChange> propertyChanges = EntityLogEntryDiffChecker.compareEntityLogEntries(previousEntity, entityLogEntry);
+                Change change = calculateChange(previousEntity, entityLogEntry, propertyChanges, analysisNumber);
+                entityLogEntry.getChanges().add(change);
                toUpdate.add(entityLogEntry);
            }
        }
@@ -77,18 +85,43 @@ public class EntityChangeLogService {
                .toList();
        removedEntries.stream()
                .filter(entry -> !entry.getState().equals(EntryState.REMOVED))
-                .peek(entry -> entry.getChanges().add(new Change(analysisNumber, ChangeType.REMOVED, now)))
+                .peek(entry -> entry.getChanges().add(new Change(analysisNumber, ChangeType.REMOVED, now, Collections.emptyMap())))
                .forEach(entry -> entry.setState(EntryState.REMOVED));
        newEntityLogEntries.addAll(removedEntries);
        return removedEntries;
    }


+    private Change calculateChange(EntityLogEntry previousEntry, EntityLogEntry currentEntry, List<PropertyChange> propertyChanges, int analysisNumber) {
+
+        ManualChange previousLastManualChange;
+        ManualChange currentLastManualChange;
+
+        if (previousEntry.getManualChanges().isEmpty() && currentEntry.getManualChanges().isEmpty()) {
+            return systemChange(propertyChanges, analysisNumber, previousEntry.getState(), currentEntry.getState());
+        }
+
+        if (previousEntry.getManualChanges().isEmpty() && !currentEntry.getManualChanges().isEmpty()) {
+            currentLastManualChange = currentEntry.getManualChanges()
+                    .get(currentEntry.getManualChanges().size() - 1);
+            return manualChange(propertyChanges, analysisNumber, currentLastManualChange, currentEntry.getState());
+        }
+
+        previousLastManualChange = previousEntry.getManualChanges()
+                .get(previousEntry.getManualChanges().size() - 1);
+        currentLastManualChange = currentEntry.getManualChanges()
+                .get(currentEntry.getManualChanges().size() - 1);
+
+        if (Objects.equals(previousLastManualChange, currentLastManualChange)) {
+            return systemChange(propertyChanges, analysisNumber, previousEntry.getState(), currentEntry.getState());
+        } else {
+            return manualChange(propertyChanges, analysisNumber, currentLastManualChange, currentEntry.getState());
+        }
+    }
+
+
    private ChangeType calculateChangeType(EntryState state, EntryState previousState) {

-        if (state.equals(previousState)) {
-            throw new IllegalArgumentException("States are equal, can't calculate ChangeType.");
-        }
        if (!isRemoved(previousState) && isRemoved(state)) {
            return ChangeType.REMOVED;
        }
@@ -99,12 +132,45 @@ public class EntityChangeLogService {
    }


+    private Change systemChange(List<PropertyChange> propertyChanges, int analysisNumber, EntryState previousState, EntryState currentState) {
+
+        return ChangeFactory.toChange(calculateChangeType(currentState, previousState), OffsetDateTime.now(), analysisNumber, propertyChanges.toArray(new PropertyChange[0]));
+    }
+
+
+    private Change manualChange(List<PropertyChange> propertyChanges, int analysisNumber, ManualChange manualChange, EntryState entryState) {
+
+        Change change = ChangeFactory.toChange(convertToChangeType(manualChange.getManualRedactionType()),
+                                      manualChange.getRequestedDate(),
+                                      analysisNumber,
+                                      propertyChanges.toArray(new PropertyChange[0]));
+
+        if (change.getType().equals(ChangeType.REMOVED) && entryState.equals(EntryState.IGNORED)) {
+            change.setType(ChangeType.IGNORED);
+        }
+        return change;
+    }
+
+
    private static boolean isRemoved(EntryState state) {

        return (state.equals(EntryState.REMOVED) || state.equals(EntryState.IGNORED));
    }


+    public ChangeType convertToChangeType(ManualRedactionType manualRedactionType) {
+
+        return switch (manualRedactionType) {
+            case ADD_LOCALLY, ADD, ADD_TO_DICTIONARY -> ChangeType.ADDED;
+            case FORCE_REDACT, FORCE_HINT, FORCE -> ChangeType.FORCE_REDACT;
+            case REMOVE_LOCALLY, REMOVE, REMOVE_FROM_DICTIONARY -> ChangeType.REMOVED;
+            case RECATEGORIZE, RECATEGORIZE_IN_DICTIONARY -> ChangeType.RECATEGORIZE;
+            case LEGAL_BASIS_CHANGE -> ChangeType.LEGAL_BASIS_CHANGE;
+            case RESIZE, RESIZE_IN_DICTIONARY -> ChangeType.RESIZED;
+        };
+    }
+
+
    public record EntryChanges(List<EntityLogEntry> inserted, List<EntityLogEntry> updated) {

    }
@@ -2,12 +2,16 @@ package com.iqser.red.service.redaction.v1.server.service;

 import java.util.ArrayList;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Locale;
+import java.util.Map;
 import java.util.Optional;
 import java.util.Set;
+import java.util.concurrent.atomic.AtomicReference;
 import java.util.stream.Collectors;
+import java.util.stream.Stream;

 import org.springframework.stereotype.Service;

@@ -20,9 +24,11 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogLegalBasis;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryType;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualChange;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualChangeFactory;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.legalbasis.LegalBasis;
+import com.iqser.red.service.persistence.service.v1.api.shared.mongo.service.EntityLogMongoService;
 import com.iqser.red.service.redaction.v1.server.RedactionServiceSettings;
 import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient;
 import com.iqser.red.service.redaction.v1.server.model.PrecursorEntity;
@@ -39,7 +45,12 @@ import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNo
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.AtomicTextBlock;
 import com.iqser.red.service.redaction.v1.server.service.EntityChangeLogService.EntryChanges;
 import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
+import com.iqser.red.service.redaction.v1.server.utils.ManualChangesUtils;

+import io.micrometer.core.annotation.Timed;
+import io.micrometer.observation.Observation;
+import io.micrometer.observation.ObservationRegistry;
+import io.micrometer.observation.annotation.Observed;
 import lombok.AccessLevel;
 import lombok.RequiredArgsConstructor;
 import lombok.experimental.FieldDefaults;
@@ -56,6 +67,8 @@ public class EntityLogCreatorService {
    LegalBasisClient legalBasisClient;
    EntityChangeLogService entityChangeLogService;
    RedactionStorageService redactionStorageService;
+    EntityLogMongoService entityLogMongoService;
+    ObservationRegistry observationRegistry;


    private static boolean notFalsePositiveOrFalseRecommendationOrRemoval(TextEntity textEntity) {
@@ -66,13 +79,15 @@ public class EntityLogCreatorService {
    }


+    @Timed("redactmanager_createInitialEntityLog")
+    @Observed(name = "EntityLogCreatorService", contextualName = "create-initial-entity-log")
    public EntityLogChanges createInitialEntityLog(AnalyzeRequest analyzeRequest,
                                                   Document document,
                                                   List<PrecursorEntity> notFoundEntities,
                                                   DictionaryVersion dictionaryVersion,
                                                   long rulesVersion) {

-        List<EntityLogEntry> entityLogEntries = createEntityLogEntries(document, analyzeRequest, notFoundEntities);
+        List<EntityLogEntry> entityLogEntries = createEntityLogEntries(document, analyzeRequest, notFoundEntities, analyzeRequest.getAnalysisNumber());

        List<LegalBasis> legalBasis = legalBasisClient.getLegalBasisMapping(analyzeRequest.getDossierTemplateId());

@@ -105,6 +120,8 @@ public class EntityLogCreatorService {
    }


+    @Timed("redactmanager_updateVersionsAndReturnChanges")
+    @Observed(name = "EntityLogCreatorService", contextualName = "update-versions-and-return-changes")
    public EntityLogChanges updateVersionsAndReturnChanges(EntityLog entityLog,
                                                           DictionaryVersion dictionaryVersion,
                                                           AnalyzeRequest analyzeRequest,
@@ -122,6 +139,8 @@ public class EntityLogCreatorService {
    }


+    @Timed("redactmanager_updatePreviousEntityLog")
+    @Observed(name = "EntityLogCreatorService", contextualName = "update-previous-entity-log")
    public EntityLogChanges updatePreviousEntityLog(AnalyzeRequest analyzeRequest,
                                                    Document document,
                                                    EntityLog entityLogWithoutEntries,
@@ -129,57 +148,88 @@ public class EntityLogCreatorService {
                                                    Set<Integer> sectionsToReanalyseIds,
                                                    DictionaryVersion dictionaryVersion) {

-        List<EntityLogEntry> newEntityLogEntries = createEntityLogEntries(document, analyzeRequest, notFoundEntries).stream()
-                .filter(entry -> entry.getContainingNodeId().isEmpty() || sectionsToReanalyseIds.contains(entry.getContainingNodeId()
-                                                                                                                  .get(0)))
-                .collect(Collectors.toList());
+        AtomicReference<List<EntityLogEntry>> newEntityLogEntriesReference = new AtomicReference<>();
+
+        Observation.createNotStarted("EntityLogCreatorService", observationRegistry)
+                .contextualName("create-new-entity-log-entries")
+                .observe(() -> newEntityLogEntriesReference.set(createEntityLogEntries(document, analyzeRequest, notFoundEntries, analyzeRequest.getAnalysisNumber()).stream()
+                                                                        .filter(entry -> entry.getContainingNodeId().isEmpty()
+                                                                                         || sectionsToReanalyseIds.contains(entry.getContainingNodeId()
+                                                                                                                                    .get(0)))
+                                                                        .collect(Collectors.toList())));

        List<EntityLogEntry> previousEntriesFromReAnalyzedSections = redactionStorageService.findEntriesContainedBySectionsOrNotContained(analyzeRequest.getDossierId(),
                                                                                                                                          analyzeRequest.getFileId(),
                                                                                                                                          sectionsToReanalyseIds);

-        EntryChanges entryChanges = entityChangeLogService.computeChanges(previousEntriesFromReAnalyzedSections, newEntityLogEntries, analyzeRequest.getAnalysisNumber());
+        EntryChanges entryChanges = entityChangeLogService.computeChanges(previousEntriesFromReAnalyzedSections,
+                                                                          newEntityLogEntriesReference.get(),
+                                                                          analyzeRequest.getAnalysisNumber());

        return updateVersionsAndReturnChanges(entityLogWithoutEntries, dictionaryVersion, analyzeRequest, entryChanges.inserted(), entryChanges.updated());
    }


-    private List<EntityLogEntry> createEntityLogEntries(Document document, AnalyzeRequest analyzeRequest, List<PrecursorEntity> notFoundPrecursorEntries) {
+    public List<EntityLogEntry> createEntityLogEntries(Document document, AnalyzeRequest analyzeRequest, List<PrecursorEntity> notFoundPrecursorEntries, int analysisNumber) {

        String dossierTemplateId = analyzeRequest.getDossierTemplateId();

        List<EntityLogEntry> entries = new ArrayList<>();

-        document.getEntities()
+        List<TextEntity> textEntities = document.getEntities()
                .stream()
                .filter(entity -> !entity.getValue().isEmpty())
                .filter(EntityLogCreatorService::notFalsePositiveOrFalseRecommendationOrRemoval)
                .filter(entity -> !entity.removed())
-                .forEach(entityNode -> entries.addAll(toEntityLogEntries(entityNode)));
-        document.streamAllImages()
+                .toList();
+
+        List<Image> images = document.streamAllImages()
                .filter(entity -> !entity.removed())
-                .forEach(imageNode -> entries.add(createEntityLogEntry(imageNode, dossierTemplateId)));
-        notFoundPrecursorEntries.stream()
+                .toList();
+
+        List<PrecursorEntity> notFoundPrecursorEntities = notFoundPrecursorEntries.stream()
                .filter(entity -> !entity.removed())
-                .forEach(precursorEntity -> entries.add(createEntityLogEntry(precursorEntity)));
+                .toList();
+
+        List<String> allIds = new ArrayList<>();
+        allIds.addAll(textEntities.stream()
+                              .flatMap(entity -> entity.getPositionsOnPagePerPage()
+                                      .stream()
+                                      .map(PositionOnPage::getId))
+                              .toList());
+        allIds.addAll(images.stream()
+                              .map(Image::getId)
+                              .toList());
+        allIds.addAll(notFoundPrecursorEntities.stream()
+                              .map(PrecursorEntity::getId)
+                              .toList());
+
+        Map<String, List<ManualChange>> manualChangesMap = getManualChangesByEntityLogIds(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), allIds);
+
+        textEntities.forEach(entityNode -> entries.addAll(toEntityLogEntries(entityNode, analysisNumber, manualChangesMap)));
+
+        images.forEach(imageNode -> entries.add(createEntityLogEntry(imageNode, dossierTemplateId, analysisNumber, manualChangesMap.getOrDefault(imageNode.getId(), new ArrayList<>()))));
+
+        notFoundPrecursorEntities.forEach(precursorEntity -> entries.add(createEntityLogEntry(precursorEntity, analysisNumber, manualChangesMap.getOrDefault(precursorEntity.getId(), new ArrayList<>()))));
+
        return entries;
    }


-    private List<EntityLogEntry> toEntityLogEntries(TextEntity textEntity) {
+    public List<EntityLogEntry> toEntityLogEntries(TextEntity textEntity, int analysisNumber, Map<String, List<ManualChange>> existingManualChangesMap) {

        List<EntityLogEntry> entityLogEntries = new ArrayList<>();

        // split entity into multiple entries if it occurs on multiple pages, since FE can't handle multi page entities
        for (PositionOnPage positionOnPage : textEntity.getPositionsOnPagePerPage()) {

-            EntityLogEntry entityLogEntry = createEntityLogEntry(textEntity);
-
            List<Position> rectanglesPerLine = positionOnPage.getRectanglePerLine()
                    .stream()
                    .map(rectangle2D -> new Position(rectangle2D, positionOnPage.getPage().getNumber()))
                    .toList();

+            EntityLogEntry entityLogEntry = createEntityLogEntry(textEntity, analysisNumber, existingManualChangesMap.getOrDefault(positionOnPage.getId(), new ArrayList<>()));
+
            // set the ID from the positions, since it might contain a "-" with the page number if the entity is split across multiple pages
            entityLogEntry.setId(positionOnPage.getId());
            entityLogEntry.setPositions(rectanglesPerLine);
@@ -190,10 +240,13 @@ public class EntityLogCreatorService {
    }


-    private EntityLogEntry createEntityLogEntry(Image image, String dossierTemplateId) {
+    private EntityLogEntry createEntityLogEntry(Image image, String dossierTemplateId, int analysisNumber, List<ManualChange> existingManualChanges) {

        String imageType = image.getImageType().equals(ImageType.OTHER) ? "image" : image.getImageType().toString().toLowerCase(Locale.ENGLISH);
        boolean isHint = dictionaryService.isHint(imageType, dossierTemplateId);
+
+        List<ManualChange> allManualChanges = ManualChangeFactory.toLocalManualChangeList(image.getManualOverwrite().getManualChangeLog(), true, analysisNumber);
+
        return EntityLogEntry.builder()
                .id(image.getId())
                .value(image.getValue())
@@ -209,7 +262,7 @@ public class EntityLogCreatorService {
 //                                 .orElse(image.getParent().toString()))
                                 .orElse(this.buildSectionString(image.getParent())))
                .imageHasTransparency(image.isTransparent())
-                .manualChanges(ManualChangeFactory.toLocalManualChangeList(image.getManualOverwrite().getManualChangeLog(), true))
+                .manualChanges(ManualChangesUtils.mergeManualChanges(existingManualChanges, allManualChanges))
                .state(buildEntryState(image))
                .entryType(isHint ? EntryType.IMAGE_HINT : EntryType.IMAGE)
                .engines(getEngines(null, image.getManualOverwrite()))
@@ -219,11 +272,13 @@ public class EntityLogCreatorService {
    }


-    private EntityLogEntry createEntityLogEntry(PrecursorEntity precursorEntity) {
+    private EntityLogEntry createEntityLogEntry(PrecursorEntity precursorEntity, int analysisNumber, List<ManualChange> existingManualChanges) {

        String type = precursorEntity.getManualOverwrite().getType()
                .orElse(precursorEntity.getType());
-        boolean isHint = isHint(precursorEntity.getEntityType());
+
+        List<ManualChange> allManualChanges = ManualChangeFactory.toLocalManualChangeList(precursorEntity.getManualOverwrite().getManualChangeLog(), true, analysisNumber);
+
        return EntityLogEntry.builder()
                .id(precursorEntity.getId())
                .reason(precursorEntity.buildReasonWithManualChangeDescriptions())
@@ -253,13 +308,13 @@ public class EntityLogCreatorService {
                //(was .imported(precursorEntity.getEngines() != null && precursorEntity.getEngines().contains(Engine.IMPORTED)))
                .imported(false)
                .reference(Collections.emptySet())
-                .manualChanges(ManualChangeFactory.toLocalManualChangeList(precursorEntity.getManualOverwrite().getManualChangeLog(), true))
+                .manualChanges(ManualChangesUtils.mergeManualChanges(existingManualChanges, allManualChanges))
                .paragraphPageIdx(-1)
                .build();
    }


-    private EntityLogEntry createEntityLogEntry(TextEntity entity) {
+    private EntityLogEntry createEntityLogEntry(TextEntity entity, int analysisNumber, List<ManualChange> existingManualChanges) {

        Set<String> referenceIds = new HashSet<>();
        entity.references()
@@ -270,6 +325,8 @@ public class EntityLogCreatorService {

        EntryType entryType = buildEntryType(entity);

+        List<ManualChange> allManualChanges = ManualChangeFactory.toLocalManualChangeList(entity.getManualOverwrite().getManualChangeLog(), true, analysisNumber);
+
        return EntityLogEntry.builder()
                .reason(entity.buildReasonWithManualChangeDescriptions())
                .legalBasis(entity.legalBasis())
@@ -297,7 +354,7 @@ public class EntityLogCreatorService {
                //(was .imported(entity.getEngines() != null && entity.getEngines().contains(Engine.IMPORTED)))
                .imported(false)
                .reference(referenceIds)
-                .manualChanges(ManualChangeFactory.toLocalManualChangeList(entity.getManualOverwrite().getManualChangeLog(), true))
+                .manualChanges(ManualChangesUtils.mergeManualChanges(existingManualChanges, allManualChanges))
                .state(buildEntryState(entity))
                .entryType(entryType)
                .paragraphPageIdx(determinePageParagraphIndex(entity, entryType))
@@ -341,12 +398,6 @@ public class EntityLogCreatorService {
    }


-    private boolean isHint(EntityType entityType) {
-
-        return entityType.equals(EntityType.HINT);
-    }
-
-
    public static EntryState buildEntryState(IEntity entity) {

        if (entity.applied() && entity.active()) {
@@ -393,7 +444,7 @@ public class EntityLogCreatorService {
    private List<EntityLogLegalBasis> toEntityLogLegalBasis(List<LegalBasis> legalBasis) {

        return legalBasis.stream()
-                .map(l -> new EntityLogLegalBasis(l.getName(), l.getDescription(), l.getReason()))
+                .map(l -> new EntityLogLegalBasis(l.getName(), l.getDescription(), l.getReason(), l.getTechnicalName()))
                .collect(Collectors.toList());
    }

@@ -403,4 +454,17 @@ public class EntityLogCreatorService {
        return node.getType().toString() + ": " + node.getTextBlock().buildSummary();
    }

+
+    public Map<String, List<ManualChange>> getManualChangesByEntityLogIds(String dossierId, String fileId, List<String> ids) {
+
+        Map<String, List<ManualChange>> manualChangesMap = new HashMap<>();
+        List<EntityLogEntry> entityLogEntries = entityLogMongoService.findEntityLogEntriesByIds(dossierId, fileId, ids);
+
+        for (EntityLogEntry entry : entityLogEntries) {
+            manualChangesMap.put(entry.getId(), entry.getManualChanges());
+        }
+
+        return manualChangesMap;
+    }
+
 }
@@ -9,21 +9,22 @@ import java.util.Map;
 import java.util.Set;
 import java.util.stream.Collectors;

+import org.springframework.stereotype.Service;
+
 import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Change;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ChangeType;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ChangeFactory;
+import com.iqser.red.service.redaction.v1.server.RedactionServiceSettings;
 import com.iqser.red.service.redaction.v1.server.model.PrecursorEntity;
 import com.iqser.red.service.redaction.v1.server.model.RectangleWithPage;
-import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
-
-import org.springframework.stereotype.Service;

 import io.micrometer.core.annotation.Timed;
+import io.micrometer.observation.annotation.Observed;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;

@@ -33,9 +34,11 @@ import lombok.extern.slf4j.Slf4j;
 public class NotFoundImportedEntitiesService {

    public static final String IMPORTED_REDACTION_TYPE = "imported_redaction";
+    private final RedactionServiceSettings settings;


    @Timed("redactmanager_processEntityLog")
+    @Observed(name = "NotFoundImportedEntitiesService", contextualName = "process-entity-log")
    public void processEntityLog(EntityLog entityLog, AnalyzeRequest analyzeRequest, List<PrecursorEntity> notFoundEntities) {
        // recreate imported redactions with manual changes
        if (notFoundEntities == null || notFoundEntities.isEmpty()) {
@@ -64,7 +67,7 @@ public class NotFoundImportedEntitiesService {
    }


-    public Map<Integer, List<PrecursorEntity>> mapImportedRedactionsOnPage(List<PrecursorEntity> importedEntities) {
+    private Map<Integer, List<PrecursorEntity>> mapImportedRedactionsOnPage(List<PrecursorEntity> importedEntities) {

        Map<Integer, List<PrecursorEntity>> importedRedactionsMap = new HashMap<>();
        Set<Integer> pageNumbers = importedEntities.stream()
@@ -81,7 +84,7 @@ public class NotFoundImportedEntitiesService {
    }


-    public void addIntersections(EntityLogEntry entityLogEntry, Map<Integer, List<PrecursorEntity>> importedEntitiesMap, int analysisNumber) {
+    private void addIntersections(EntityLogEntry entityLogEntry, Map<Integer, List<PrecursorEntity>> importedEntitiesMap, int analysisNumber) {

        for (Position rectangle : entityLogEntry.getPositions()) {
            if (importedEntitiesMap.containsKey(rectangle.getPageNumber())) {
@@ -94,9 +97,9 @@ public class NotFoundImportedEntitiesService {
                            }
                            entityLogEntry.getImportedRedactionIntersections().add(precursorEntity.getId());

-                            if (entityLogEntry.getState() != EntryState.REMOVED) {
+                            if (entityLogEntry.getState() != EntryState.REMOVED && !settings.isAnnotationMode()) {
                                entityLogEntry.setState(EntryState.REMOVED);
-                                entityLogEntry.getChanges().add(new Change(analysisNumber, ChangeType.REMOVED, OffsetDateTime.now()));
+                                entityLogEntry.getChanges().add(ChangeFactory.toChange(ChangeType.REMOVED, OffsetDateTime.now(), analysisNumber));
                            }
                        }
                    }
@@ -31,6 +31,7 @@ import com.iqser.red.service.redaction.v1.server.service.document.EntityFindingU
 import com.iqser.red.service.redaction.v1.server.service.document.EntityFromPrecursorCreationService;
 import com.iqser.red.service.redaction.v1.server.storage.ObservedStorageService;
 import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
+import com.knecon.fforesight.tenantcommons.TenantContext;

 import io.micrometer.observation.annotation.Observed;
 import lombok.AccessLevel;
@@ -127,7 +128,8 @@ public class UnprocessedChangesService {

        notFoundManualEntities.forEach(manualEntity -> unprocessedManualEntities.add(builDefaultUnprocessedManualEntity(manualEntity)));

-        rabbitTemplate.convertAndSend(QueueNames.REDACTION_ANALYSIS_RESPONSE_QUEUE,
+        rabbitTemplate.convertAndSend(QueueNames.REDACTION_RESPONSE_EXCHANGE,
+                                      TenantContext.getTenantId(),
                                      AnalyzeResponse.builder().fileId(analyzeRequest.getFileId()).unprocessedManualEntities(unprocessedManualEntities).build());
    }

@@ -1,9 +1,15 @@
 package com.iqser.red.service.redaction.v1.server.service.components;

 import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
+import java.util.Locale;
 import java.util.Map;
+import java.util.Set;
+import java.util.function.Function;
+import java.util.stream.Collectors;

 import org.springframework.stereotype.Service;

@@ -12,6 +18,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentLogEntry;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentLogEntryValue;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.component.ComponentDefinition;
 import com.iqser.red.service.redaction.v1.server.model.component.Component;
 import com.iqser.red.service.redaction.v1.server.model.component.Entity;
 import com.iqser.red.service.redaction.v1.server.service.document.ComponentComparator;
@@ -20,7 +27,11 @@ import com.iqser.red.service.redaction.v1.server.service.document.EntityComparat
@Service
 public class ComponentLogCreatorService {

-    public ComponentLog buildComponentLog(int analysisNumber, List<Component> components, long componentRulesVersion) {
+    public ComponentLog buildComponentLog(int analysisNumber,
+                                          List<Component> components,
+                                          long componentRulesVersion,
+                                          List<ComponentDefinition> componentDefinitions,
+                                          long dateFormatsVersion) {

        Map<String, List<ComponentLogEntryValue>> map = new HashMap<>();
        components.stream()
@@ -29,11 +40,23 @@ public class ComponentLogCreatorService {
                    ComponentLogEntryValue componentLogEntryValue = buildComponentLogEntry(component);
                    map.computeIfAbsent(component.getName(), k -> new ArrayList<>()).add(componentLogEntryValue);
                });
-        List<ComponentLogEntry> componentLogComponents = map.entrySet()
+        List<ComponentLogEntry> componentLogEntries = map.entrySet()
                .stream()
-                .map(entry -> new ComponentLogEntry(entry.getKey(), entry.getValue(), false))
-                .toList();
-        return new ComponentLog(analysisNumber, componentRulesVersion, componentLogComponents);
+                .map(entry -> new ComponentLogEntry(entry.getKey(), entry.getValue(), entry.getValue(), false))
+                .collect(Collectors.toList());
+
+        Set<String> existingComponentNames = map.keySet()
+                .stream()
+                .map(s -> s.toLowerCase(Locale.ROOT))
+                .collect(Collectors.toSet());
+        for (ComponentDefinition componentDefinition : componentDefinitions) {
+            String technicalName = componentDefinition.getTechnicalName().toLowerCase(Locale.ROOT);
+            if (!existingComponentNames.contains(technicalName)) {
+                componentLogEntries.add(new ComponentLogEntry(componentDefinition.getDisplayName(), new ArrayList<>(), new ArrayList<>(), false));
+            }
+        }
+
+        return new ComponentLog(analysisNumber, componentRulesVersion, dateFormatsVersion, componentLogEntries);
    }


@@ -41,7 +64,6 @@ public class ComponentLogCreatorService {

        return ComponentLogEntryValue.builder()
                .value(component.getValue())
-                .originalValue(component.getValue())
                .componentRuleId(component.getMatchedRule().toString())
                .valueDescription(component.getValueDescription())
                .componentLogEntityReferences(toComponentEntityReferences(component.getReferences()
@@ -40,7 +40,8 @@ public class ComponentMappingFileSystemCache {
    public File getComponentMappingFile(ComponentMappingMetadata metadata) {

        Path mappingFile = getMappingFileFromMetadata(metadata);
-        Path mappingFileMetaDataFile = mappingFile.resolveSibling(metadata.getName() + METADATA_SUFFIX);
+        Path mappingFileMetaDataFile = getMappingMetadataFileFromMetadata(metadata);
+
        synchronized (ComponentMappingFileSystemCache.class) {

            if (fileExistsAndUpToDate(metadata, mappingFile, mappingFileMetaDataFile)) {
@@ -52,7 +53,7 @@ public class ComponentMappingFileSystemCache {
            InputStreamResource inputStreamResource = storageService.getObject(TenantContext.getTenantId(), metadata.getStorageId());

            Files.write(mappingFile, inputStreamResource.getContentAsByteArray(), StandardOpenOption.WRITE, StandardOpenOption.TRUNCATE_EXISTING, StandardOpenOption.CREATE);
-            
+
            mapper.writeValue(mappingFileMetaDataFile.toFile(), metadata);

            return mappingFile.toFile();
@@ -87,6 +88,13 @@ public class ComponentMappingFileSystemCache {
    }


+    private Path getMappingMetadataFileFromMetadata(ComponentMappingMetadata metadata) {
+
+        Path tenantStem = mappingFileDir.resolve(TenantContext.getTenantId());
+        return tenantStem.resolve(metadata.getStorageId() + METADATA_SUFFIX);
+    }
+
+
    private boolean fileExistsAndUpToDate(ComponentMappingMetadata metadata, Path mappingFile, Path mappingFileMetaDataFile) {

        if (mappingFile.toFile().exists() && mappingFile.toFile().isFile() && mappingFileMetaDataFile.toFile().exists() && mappingFileMetaDataFile.toFile().isFile()) {
@@ -4,6 +4,7 @@ import java.io.File;
 import java.io.FileReader;
 import java.io.Reader;
 import java.nio.charset.Charset;
+import java.time.Duration;
 import java.util.List;

 import org.springframework.stereotype.Service;
@@ -36,7 +37,7 @@ public class ComponentMappingMemoryCache {
    public ComponentMappingMemoryCache(ComponentMappingFileSystemCache componentMappingFileSystemCache) {

        this.fileSystemCache = componentMappingFileSystemCache;
-        cache = CacheBuilder.newBuilder().maximumWeight(MAX_NUMBER_OF_LINES).weigher(COMPONENT_MAPPING_WEIGHER).build();
+        cache = CacheBuilder.newBuilder().maximumWeight(MAX_NUMBER_OF_LINES).weigher(COMPONENT_MAPPING_WEIGHER).expireAfterAccess(Duration.ofDays(1)).build();
    }


@@ -0,0 +1,73 @@
+package com.iqser.red.service.redaction.v1.server.service.components;
+
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+
+import org.springframework.stereotype.Service;
+
+import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive;
+import com.iqser.red.service.redaction.v1.server.client.DateFormatsClient;
+import com.iqser.red.service.redaction.v1.server.utils.DateConverter;
+import com.knecon.fforesight.tenantcommons.TenantContext;
+
+import lombok.AccessLevel;
+import lombok.experimental.FieldDefaults;
+
+@Service
+@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
+public class DateConverterMemoryCache {
+
+    ConcurrentMap<String, DateConverterCacheEntry> cache = new ConcurrentHashMap<>();
+    DateFormatsClient dateFormatsClient;
+
+
+    public DateConverterMemoryCache(DateFormatsClient dateFormatsClient) {
+
+        this.dateFormatsClient = dateFormatsClient;
+    }
+
+
+    public DateConverterCacheEntry getDateConverter(String dossierTemplateId) {
+
+        String tenantId = TenantContext.getTenantId();
+        String cacheKey = buildCacheKey(tenantId, dossierTemplateId);
+
+        DateConverterCacheEntry cacheEntry = cache.get(cacheKey);
+        long latestVersion = dateFormatsClient.getVersion(dossierTemplateId);
+
+        if (cacheEntry != null && cacheEntry.version() >= latestVersion) {
+            return cacheEntry;
+        }
+
+        synchronized (this) {
+            cacheEntry = cache.get(cacheKey);
+            if (cacheEntry != null && cacheEntry.version() >= latestVersion) {
+                return cacheEntry;
+            }
+
+            DateConverter dateConverter = loadDateConverter(dossierTemplateId);
+            DateConverterCacheEntry dateConverterCacheEntry = new DateConverterCacheEntry(dateConverter, latestVersion);
+            cache.put(cacheKey, dateConverterCacheEntry);
+            return dateConverterCacheEntry;
+        }
+    }
+
+
+    private DateConverter loadDateConverter(String dossierTemplateId) {
+
+        JSONPrimitive<String> dateFormats = dateFormatsClient.getDateFormats(dossierTemplateId);
+        return new DateConverter(dateFormats.getValue());
+    }
+
+
+    private static String buildCacheKey(String tenantId, String dossierTemplateId) {
+
+        return tenantId + "/" + dossierTemplateId;
+    }
+
+
+    public record DateConverterCacheEntry(DateConverter dateConverter, long version) {
+
+    }
+
+}
@@ -30,17 +30,23 @@ import com.iqser.red.service.redaction.v1.server.utils.ComponentCreationUtils;
 import com.iqser.red.service.redaction.v1.server.utils.DateConverter;

 import lombok.AccessLevel;
-import lombok.RequiredArgsConstructor;
 import lombok.experimental.FieldDefaults;

-@RequiredArgsConstructor
@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
 public class ComponentCreationService {

    KieSession kieSession;
+    DateConverter dateConverter;
    Set<Entity> referencedEntities = new HashSet<>();


+    public ComponentCreationService(KieSession kieSession, DateConverter dateConverter) {
+
+        this.kieSession = kieSession;
+        this.dateConverter = dateConverter;
+    }
+
+
    /**
     * Joins entity values, and creates a component from the result.
     *
@@ -342,16 +348,7 @@ public class ComponentCreationService {
     */
    public void create(String ruleIdentifier, String name, String value, String valueDescription, Entity reference) {

-        referencedEntities.add(reference);
-        List<Entity> referenceList = new LinkedList<>();
-        referenceList.add(reference);
-        kieSession.insert(Component.builder()
-                                  .matchedRule(RuleIdentifier.fromString(ruleIdentifier))
-                                  .name(name)
-                                  .value(value)
-                                  .valueDescription(valueDescription)
-                                  .references(referenceList)
-                                  .build());
+        create(ruleIdentifier, name, value, valueDescription, List.of(reference));
    }


@@ -399,7 +396,7 @@ public class ComponentCreationService {

        for (Entity entity : entities) {
            String value = entity.getValue();
-            Optional<Date> optionalDate = DateConverter.parseDate(value);
+            Optional<Date> optionalDate = dateConverter.parseDate(value);
            if (optionalDate.isPresent()) {
                dates.add(optionalDate.get());
            } else {
@@ -410,7 +407,7 @@ public class ComponentCreationService {
        String formattedDateStrings = Stream.concat(//
                                                    dates.stream()
                                                            .sorted()
-                                                            .map(date -> DateConverter.convertDate(date, resultFormat)), //
+                                                            .map(date -> dateConverter.convertDate(date, resultFormat)), //
                                                    unparsedDates.stream())//
                .collect(Collectors.joining(", "));

@@ -1,9 +1,11 @@
 package com.iqser.red.service.redaction.v1.server.service.document;

+import static com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPageProto.DocumentPage;
+import static com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPositionDataProto.AllDocumentPositionData;
+import static com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentTextDataProto.AllDocumentTextData;
+import static com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.EntryDataProto.EntryData;

 import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
 import java.util.HashSet;
 import java.util.LinkedList;
 import java.util.List;
@@ -27,10 +29,6 @@ import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.AtomicTextBlock;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlockCollector;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPage;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPositionData;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentStructure;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentTextData;

 import lombok.experimental.UtilityClass;

@@ -43,28 +41,30 @@ public class DocumentGraphMapper {
        DocumentTree documentTree = new DocumentTree(document);
        Context context = new Context(documentData, documentTree);

-        context.pageData.addAll(Arrays.stream(documentData.getDocumentPages())
+        context.pageData.addAll(documentData.getDocumentPages().getDocumentPagesList()
+                                        .stream()
                                        .map(DocumentGraphMapper::buildPage)
                                        .toList());

-        context.documentTree.getRoot().getChildren().addAll(buildEntries(documentData.getDocumentStructure().getRoot().getChildren(), context));
+        context.documentTree.getRoot().getChildren().addAll(buildEntries(documentData.getDocumentStructure().getRoot().getChildrenList(), context));

        document.setDocumentTree(context.documentTree);
        document.setPages(new HashSet<>(context.pageData));
-        document.setNumberOfPages(documentData.getDocumentPages().length);
+        document.setNumberOfPages(documentData.getDocumentPages().getDocumentPagesCount());

        document.setTextBlock(document.getTextBlock());
        return document;
    }


-    private List<DocumentTree.Entry> buildEntries(List<DocumentStructure.EntryData> entries, Context context) {
+    private List<DocumentTree.Entry> buildEntries(List<EntryData> entries, Context context) {

        List<DocumentTree.Entry> newEntries = new ArrayList<>(entries.size());
-        for (DocumentStructure.EntryData entryData : entries) {
+        for (EntryData entryData : entries) {

-            List<Page> pages = Arrays.stream(entryData.getPageNumbers())
-                    .map(pageNumber -> getPage(pageNumber, context))
+            List<Page> pages = entryData.getPageNumbersList()
+                    .stream()
+                    .map(context::getPage)
                    .toList();

            SemanticNode node = switch (entryData.getType()) {
@@ -76,33 +76,30 @@ public class DocumentGraphMapper {
                case FOOTER -> buildFooter(context);
                case TABLE -> buildTable(context, entryData.getProperties());
                case TABLE_CELL -> buildTableCell(context, entryData.getProperties());
-                case IMAGE -> buildImage(context, entryData.getProperties(), entryData.getPageNumbers());
+                case IMAGE -> buildImage(context, entryData.getProperties(), entryData.getPageNumbersList());
                default -> throw new UnsupportedOperationException("Not yet implemented for type " + entryData.getType());
            };

-            if (entryData.getAtomicBlockIds().length > 0) {
-                TextBlock textBlock = toTextBlock(entryData.getAtomicBlockIds(), context, node);
+            if (entryData.getAtomicBlockIdsCount() > 0) {
+                TextBlock textBlock = toTextBlock(entryData.getAtomicBlockIdsList(), context, node);
                node.setLeafTextBlock(textBlock);
+
+                switch (entryData.getType()) {
+                    case HEADER -> pages.forEach(page -> page.setHeader((Header) node));
+                    case FOOTER -> pages.forEach(page -> page.setFooter((Footer) node));
+                    case IMAGE -> pages.forEach(page -> page.getImages().add((Image) node));
+                    default -> textBlock.getAtomicTextBlocks()
+                            .forEach(atb -> atb.getPage().getTextBlocksOnPage().add(atb));
+                }
+
            }
-            List<Integer> treeId = Arrays.stream(entryData.getTreeId()).boxed()
-                    .toList();
-            if (entryData.getEngines() != null) {
-                entryData.getEngines()
-                        .forEach(node::addEngine);
-            } else {
-                entryData.setEngines(Collections.emptySet());
-            }
+            List<Integer> treeId = entryData.getTreeIdList();
+            entryData.getEnginesList()
+                    .forEach(node::addEngine);
            node.setTreeId(treeId);

-            switch (entryData.getType()) {
-                case HEADER -> pages.forEach(page -> page.setHeader((Header) node));
-                case FOOTER -> pages.forEach(page -> page.setFooter((Footer) node));
-                default -> pages.forEach(page -> page.getMainBody().add(node));
-            }
-
-            newEntries.add(DocumentTree.Entry.builder().treeId(treeId).children(buildEntries(entryData.getChildren(), context)).node(node).build());
-        }
-        return newEntries;
+            newEntries.add(DocumentTree.Entry.builder().treeId(treeId).children(buildEntries(entryData.getChildrenList(), context)).node(node).build());
+        } return newEntries;
    }


@@ -112,10 +109,10 @@ public class DocumentGraphMapper {
    }


-    private Image buildImage(Context context, Map<String, String> properties, Long[] pageNumbers) {
+    private Image buildImage(Context context, Map<String, String> properties, List<Long> pageNumbers) {

-        assert pageNumbers.length == 1;
-        Page page = getPage(pageNumbers[0], context);
+        assert pageNumbers.size() == 1;
+        Page page = context.getPage(pageNumbers.get(0));
        var builder = Image.builder();
        PropertiesMapper.parseImageProperties(properties, builder);
        return builder.documentTree(context.documentTree).page(page).build();
@@ -161,13 +158,14 @@ public class DocumentGraphMapper {
        return SuperSection.builder().documentTree(context.documentTree).build();
    }

+
    private Paragraph buildParagraph(Context context, Map<String, String> properties) {

        if (PropertiesMapper.isDuplicateParagraph(properties)) {

            DuplicatedParagraph duplicatedParagraph = DuplicatedParagraph.builder().documentTree(context.documentTree).build();

-            Long[] unsortedTextblockIds = PropertiesMapper.getUnsortedTextblockIds(properties);
+            var unsortedTextblockIds = PropertiesMapper.getUnsortedTextblockIds(properties);
            duplicatedParagraph.setUnsortedLeafTextBlock(toTextBlock(unsortedTextblockIds, context, duplicatedParagraph));
            return duplicatedParagraph;
        }
@@ -176,9 +174,9 @@ public class DocumentGraphMapper {
    }


-    private TextBlock toTextBlock(Long[] atomicTextBlockIds, Context context, SemanticNode parent) {
+    private TextBlock toTextBlock(List<Long> atomicTextBlockIds, Context context, SemanticNode parent) {

-        return Arrays.stream(atomicTextBlockIds)
+        return atomicTextBlockIds.stream()
                .map(atomicTextBlockId -> getAtomicTextBlock(context, parent, atomicTextBlockId))
                .collect(new TextBlockCollector());
    }
@@ -186,24 +184,16 @@ public class DocumentGraphMapper {

    private AtomicTextBlock getAtomicTextBlock(Context context, SemanticNode parent, Long atomicTextBlockId) {

-        return AtomicTextBlock.fromAtomicTextBlockData(context.documentTextData.get(Math.toIntExact(atomicTextBlockId)),
-                                                       context.documentPositionData.get(Math.toIntExact(atomicTextBlockId)),
+        return AtomicTextBlock.fromAtomicTextBlockData(context.documentTextData.getDocumentTextData(Math.toIntExact(atomicTextBlockId)),
+                                                       context.documentPositionData.getDocumentPositionData(Math.toIntExact(atomicTextBlockId)),
                                                       parent,
-                                                       getPage(context.documentTextData.get(Math.toIntExact(atomicTextBlockId)).getPage(), context));
+                                                       context.getPage(context.documentTextData.getDocumentTextData(Math.toIntExact(atomicTextBlockId)).getPage()));
    }


    private Page buildPage(DocumentPage p) {

-        return Page.builder().rotation(p.getRotation()).height(p.getHeight()).width(p.getWidth()).number(p.getNumber()).mainBody(new LinkedList<>()).build();
-    }
-
-
-    private Page getPage(Long pageIndex, Context context) {
-
-        Page page = context.pageData.get(Math.toIntExact(pageIndex) - 1);
-        assert page.getNumber() == Math.toIntExact(pageIndex);
-        return page;
+        return Page.builder().rotation(p.getRotation()).height(p.getHeight()).width(p.getWidth()).number(p.getNumber()).textBlocksOnPage(new LinkedList<>()).build();
    }


@@ -211,21 +201,27 @@ public class DocumentGraphMapper {

        private final DocumentTree documentTree;
        private final List<Page> pageData;
-        private final List<DocumentTextData> documentTextData;
-        private final List<DocumentPositionData> documentPositionData;
+        private final AllDocumentTextData documentTextData;
+        private final AllDocumentPositionData documentPositionData;


        Context(DocumentData documentData, DocumentTree documentTree) {

            this.documentTree = documentTree;
            this.pageData = new ArrayList<>();
-            this.documentTextData = Arrays.stream(documentData.getDocumentTextData())
-                    .toList();
-            this.documentPositionData = Arrays.stream(documentData.getDocumentPositionData())
-                    .toList();
+            this.documentTextData = documentData.getDocumentTextData();
+            this.documentPositionData = documentData.getDocumentPositionData();

        }

+
+        private Page getPage(Long pageIndex) {
+
+            Page page = pageData.get(Math.toIntExact(pageIndex) - 1);
+            assert page.getNumber() == Math.toIntExact(pageIndex);
+            return page;
+        }
+
    }

 }
@@ -475,7 +475,7 @@ public class EntityCreationService {
     */
    public Stream<TextEntity> bySearchImplementation(SearchImplementation searchImplementation, String type, EntityType entityType, SemanticNode node) {

-        return searchImplementation.getBoundaries(node.getTextBlock(), node.getTextRange())
+        return searchImplementation.getBoundaries(node.getTextBlock())
                .filter(boundary -> isValidEntityTextRange(node.getTextBlock(), boundary))
                .map(bounds -> byTextRange(bounds, type, entityType, node))
                .filter(Optional::isPresent)
@@ -496,7 +496,7 @@ public class EntityCreationService {

        TextBlock textBlock = node.getTextBlock();
        SearchImplementation searchImplementation = new SearchImplementation(strings, false);
-        return searchImplementation.getBoundaries(textBlock, node.getTextRange())
+        return searchImplementation.getBoundaries(textBlock)
                .map(boundary -> toLineAfterTextRange(textBlock, boundary))
                .filter(boundary -> isValidEntityTextRange(textBlock, boundary))
                .map(boundary -> byTextRange(boundary, type, entityType, node))
@@ -518,7 +518,7 @@ public class EntityCreationService {

        TextBlock textBlock = node.getTextBlock();
        SearchImplementation searchImplementation = new SearchImplementation(strings, true);
-        return searchImplementation.getBoundaries(textBlock, node.getTextRange())
+        return searchImplementation.getBoundaries(textBlock)
                .map(boundary -> toLineAfterTextRange(textBlock, boundary))
                .filter(boundary -> isValidEntityTextRange(textBlock, boundary))
                .map(boundary -> byTextRange(boundary, type, entityType, node))
@@ -614,21 +614,23 @@ public class EntityCreationService {
    /**
     * Looks across the remaining table row to the right of the provided TableCell if any line intersects the y coordinates of the found text.
     *
-     * @param TextRanges a list of textRanges
+     * @param textRanges a list of textRanges
     * @param tableCell  the table cell
     * @param type       the type
     * @param entityType the entity type
     * @param tableNode  the table node
     * @return a stream of RedactionEntities
     */
-    private Stream<TextEntity> lineAfterBoundariesAcrossColumns(List<TextRange> TextRanges, TableCell tableCell, String type, EntityType entityType, Table tableNode) {
+    private Stream<TextEntity> lineAfterBoundariesAcrossColumns(List<TextRange> textRanges, TableCell tableCell, String type, EntityType entityType, Table tableNode) {

-        return TextRanges.stream()
+        return textRanges.stream()
                .map(boundary -> RectangleTransformations.rectangle2DBBox(tableCell.getTextBlock().getPositions(boundary)))
                .map(bBox -> Pair.of(bBox.getMaxY(), bBox.getMinY()))
                .map(maxMinPair -> tableNode.streamRow(tableCell.getRow())
                        .filter(nextTableCell -> nextTableCell.getCol() > tableCell.getCol())
-                        .map(nextTableCell -> RedactionSearchUtility.findTextRangesOfAllLinesInYRange(maxMinPair.getLeft(), maxMinPair.getRight(), nextTableCell.getTextBlock()))
+                        .map(nextTableCell -> RedactionSearchUtility.findTextRangesOfAllLinesWithCloseYCoordinates(maxMinPair.getLeft(),
+                                                                                                                   maxMinPair.getRight(),
+                                                                                                                   nextTableCell.getTextBlock()))
                        .map(b -> b.trim(tableNode.getTextBlock()))
                        .filter(boundary -> isValidEntityTextRange(tableNode.getTextBlock(), boundary))
                        .map(boundary -> byTextRange(boundary, type, entityType, tableNode))
@@ -1223,6 +1225,49 @@ public class EntityCreationService {
    }


+    /**
+     * Optionally creates a text entity based on a Named Entity Recognition (NER) entity
+     * if the confidence of the entity lies above the given minimal confidence.
+     *
+     * @param nerEntity     The NER entity used for creating the text entity.
+     * @param minConfidence The minimal confidence required
+     * @param entityType    The entity's classification.
+     * @param semanticNode  The semantic node related to the NER entity.
+     * @return An {@link Optional} containing the new {@link TextEntity} based on the NER entity, or {@link Optional#empty()} if not created.
+     */
+    public Optional<TextEntity> optionalByNerEntityWithConfidence(NerEntities.NerEntity nerEntity, Double minConfidence, EntityType entityType, SemanticNode semanticNode) {
+
+        if (nerEntity.confidence() != null && nerEntity.confidence() < minConfidence) {
+            return Optional.empty();
+        }
+        return byTextRangeWithEngine(nerEntity.textRange(), nerEntity.type(), entityType, semanticNode, Set.of(Engine.NER));
+    }
+
+
+    /**
+     * Optionally creates a text entity based on a Named Entity Recognition (NER) entity, with a specified type
+     * if the confidence of the entity lies above the given minimal confidence.
+     *
+     * @param nerEntity     The NER entity used for creating the text entity.
+     * @param minConfidence The minimal confidence required
+     * @param type          Type of the entity.
+     * @param entityType    The entity's classification.
+     * @param semanticNode  The semantic node related to the NER entity.
+     * @return An {@link Optional} containing the new {@link TextEntity} based on the NER entity, or {@link Optional#empty()} if not created.
+     */
+    public Optional<TextEntity> optionalByNerEntityWithConfidence(NerEntities.NerEntity nerEntity,
+                                                                  Double minConfidence,
+                                                                  String type,
+                                                                  EntityType entityType,
+                                                                  SemanticNode semanticNode) {
+
+        if (nerEntity.confidence() != null && nerEntity.confidence() < minConfidence) {
+            return Optional.empty();
+        }
+        return byTextRangeWithEngine(nerEntity.textRange(), type, entityType, semanticNode, Set.of(Engine.NER));
+    }
+
+
    /**
     * Combines multiple NER entities into a single text entity.
     *
@@ -1241,6 +1286,137 @@ public class EntityCreationService {
    }


+    /**
+     * Combines multiple NER entities into a single text entity based on the specified types and minimum parts to combine.
+     *
+     * @param nerEntities       The collection of NER entities to combine.
+     * @param type              The type for the combined entity.
+     * @param entityType        The classification for the combined entity.
+     * @param semanticNode      The semantic node related to these entities.
+     * @param essentialTypes    A set of essential types that must be present in the combination.
+     * @param typesToCombine    A set of types that should be considered for combination.
+     * @param minPartsToCombine The minimum number of parts that must be combined.
+     * @return A stream of combined {@link TextEntity} objects that match the specified criteria.
+     */
+    public Stream<TextEntity> combineNerEntities(NerEntities nerEntities,
+                                                 String type,
+                                                 EntityType entityType,
+                                                 SemanticNode semanticNode,
+                                                 Set<String> essentialTypes,
+                                                 Set<String> typesToCombine,
+                                                 int minPartsToCombine) {
+
+        return NerEntitiesAdapter.combineNerEntitiesOfAllGivenTypes(nerEntities, essentialTypes, typesToCombine, minPartsToCombine)
+                .map(boundary -> byTextRangeWithEngine(boundary, type, entityType, semanticNode, Set.of(Engine.NER)))
+                .filter(Optional::isPresent)
+                .map(Optional::get);
+    }
+
+
+    /**
+     * Combines multiple NER entities into a single text entity based on the specified types, maximum distance between parts, and minimum parts to combine.
+     *
+     * @param nerEntities             The collection of NER entities to combine.
+     * @param type                    The type for the combined entity.
+     * @param entityType              The classification for the combined entity.
+     * @param semanticNode            The semantic node related to these entities.
+     * @param essentialTypes          A set of essential types that must be present in the combination.
+     * @param typesToCombine          A set of types that should be considered for combination.
+     * @param maxDistanceBetweenParts The maximum distance allowed between parts to consider them for combination.
+     * @param minPartsToCombine       The minimum number of parts that must be combined.
+     * @return A stream of combined {@link TextEntity} objects that match the specified criteria.
+     */
+    public Stream<TextEntity> combineNerEntities(NerEntities nerEntities,
+                                                 String type,
+                                                 EntityType entityType,
+                                                 SemanticNode semanticNode,
+                                                 Set<String> essentialTypes,
+                                                 Set<String> typesToCombine,
+                                                 int maxDistanceBetweenParts,
+                                                 int minPartsToCombine) {
+
+        return NerEntitiesAdapter.combineNerEntitiesOfAllGivenTypes(nerEntities, essentialTypes, typesToCombine, maxDistanceBetweenParts, minPartsToCombine)
+                .map(boundary -> byTextRangeWithEngine(boundary, type, entityType, semanticNode, Set.of(Engine.NER)))
+                .filter(Optional::isPresent)
+                .map(Optional::get);
+    }
+
+
+    /**
+     * Combines multiple NER entities into a single text entity based on the specified types, maximum distance between parts, minimum parts to combine, and minimum essential types combined.
+     *
+     * @param nerEntities               The collection of NER entities to combine.
+     * @param type                      The type for the combined entity.
+     * @param entityType                The classification for the combined entity.
+     * @param semanticNode              The semantic node related to these entities.
+     * @param essentialTypes            A set of essential types that must be present in the combination.
+     * @param typesToCombine            A set of types that should be considered for combination.
+     * @param maxDistanceBetweenParts   The maximum distance allowed between parts to consider them for combination.
+     * @param minPartsToCombine         The minimum number of parts that must be combined.
+     * @param minEssentialTypesCombined The minimum number of essential types that must be combined.
+     * @return A stream of combined {@link TextEntity} objects that match the specified criteria.
+     */
+    public Stream<TextEntity> combineNerEntities(NerEntities nerEntities,
+                                                 String type,
+                                                 EntityType entityType,
+                                                 SemanticNode semanticNode,
+                                                 Set<String> essentialTypes,
+                                                 Set<String> typesToCombine,
+                                                 int maxDistanceBetweenParts,
+                                                 int minPartsToCombine,
+                                                 int minEssentialTypesCombined) {
+
+        return NerEntitiesAdapter.combineNerEntitiesOfAllGivenTypes(nerEntities,
+                                                                    essentialTypes,
+                                                                    typesToCombine,
+                                                                    maxDistanceBetweenParts,
+                                                                    minPartsToCombine,
+                                                                    minEssentialTypesCombined)
+                .map(boundary -> byTextRangeWithEngine(boundary, type, entityType, semanticNode, Set.of(Engine.NER)))
+                .filter(Optional::isPresent)
+                .map(Optional::get);
+    }
+
+
+    /**
+     * Combines multiple NER entities into a single text entity based on the specified types, maximum distance between parts, minimum parts to combine, minimum essential types combined, and confidence level.
+     *
+     * @param nerEntities               The collection of NER entities to combine.
+     * @param type                      The type for the combined entity.
+     * @param entityType                The classification for the combined entity.
+     * @param semanticNode              The semantic node related to these entities.
+     * @param essentialTypes            A set of essential types that must be present in the combination.
+     * @param typesToCombine            A set of types that should be considered for combination.
+     * @param maxDistanceBetweenParts   The maximum distance allowed between parts to consider them for combination.
+     * @param minPartsToCombine         The minimum number of parts that must be combined.
+     * @param minEssentialTypesCombined The minimum number of essential types that must be combined.
+     * @param confidence                The confidence level required for combining entities.
+     * @return A stream of combined {@link TextEntity} objects that match the specified criteria and confidence level.
+     */
+    public Stream<TextEntity> combineNerEntitiesWithConfidence(NerEntities nerEntities,
+                                                               String type,
+                                                               EntityType entityType,
+                                                               SemanticNode semanticNode,
+                                                               Set<String> essentialTypes,
+                                                               Set<String> typesToCombine,
+                                                               int maxDistanceBetweenParts,
+                                                               int minPartsToCombine,
+                                                               int minEssentialTypesCombined,
+                                                               Double confidence) {
+
+        return NerEntitiesAdapter.combineNerEntitiesOfAllGivenTypesWithConfidence(nerEntities,
+                                                                                  essentialTypes,
+                                                                                  typesToCombine,
+                                                                                  maxDistanceBetweenParts,
+                                                                                  minPartsToCombine,
+                                                                                  minEssentialTypesCombined,
+                                                                                  confidence)
+                .map(boundary -> byTextRangeWithEngine(boundary, type, entityType, semanticNode, Set.of(Engine.NER)))
+                .filter(Optional::isPresent)
+                .map(Optional::get);
+    }
+
+
    /**
     * Validates if a given text range within a text block represents a valid entity.
     *
@@ -4,6 +4,7 @@ import static java.lang.String.format;
 import static java.util.stream.Collectors.groupingBy;

 import java.awt.geom.Rectangle2D;
+import java.util.ArrayList;
 import java.util.Collections;
 import java.util.Comparator;
 import java.util.List;
@@ -14,20 +15,27 @@ import java.util.Optional;
 import java.util.Set;
 import java.util.stream.Collectors;

+import org.apache.commons.text.similarity.LevenshteinDistance;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;

+import com.google.common.collect.Sets;
 import com.iqser.red.service.redaction.v1.server.model.ClosestEntity;
 import com.iqser.red.service.redaction.v1.server.model.PrecursorEntity;
 import com.iqser.red.service.redaction.v1.server.model.RectangleWithPage;
 import com.iqser.red.service.redaction.v1.server.model.dictionary.SearchImplementation;
+import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.PositionOnPage;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Page;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
-import com.iqser.red.service.redaction.v1.server.service.DictionaryService;
+import com.iqser.red.service.redaction.v1.server.model.document.textblock.AtomicTextBlock;
+import com.iqser.red.service.redaction.v1.server.model.document.textblock.PartialTextBlock;
+import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
 import com.iqser.red.service.redaction.v1.server.utils.RectangleTransformations;
+import com.iqser.red.service.redaction.v1.server.utils.TextNormalizationUtilities;

 import lombok.extern.slf4j.Slf4j;

@@ -35,15 +43,14 @@ import lombok.extern.slf4j.Slf4j;
@Service
 public class EntityFindingUtility {

+    private static final float STRING_SIMILARITY_THRESHOLD = 0.2f;
    EntityCreationService entityCreationService;
-    DictionaryService dictionaryService;


    @Autowired
-    public EntityFindingUtility(EntityEnrichmentService entityEnrichmentService, DictionaryService dictionaryService) {
+    public EntityFindingUtility(EntityEnrichmentService entityEnrichmentService) {

        entityCreationService = new EntityCreationService(entityEnrichmentService);
-        this.dictionaryService = dictionaryService;
    }


@@ -55,7 +62,7 @@ public class EntityFindingUtility {
            return Optional.empty();
        }

-        List<TextEntity> possibleEntities = entitiesWithSameValue.get(precursorEntity.getValue().toLowerCase(Locale.ENGLISH));
+        List<TextEntity> possibleEntities = entitiesWithSameValue.get(TextNormalizationUtilities.removeAllWhitespaces(precursorEntity.getValue().toLowerCase(Locale.ENGLISH)));

        if (entityIdentifierValueNotFound(possibleEntities)) {
            log.info("Entity could not be created with precursorEntity: {}, due to the value {} not being found anywhere.", precursorEntity, precursorEntity.getValue());
@@ -91,6 +98,79 @@ public class EntityFindingUtility {
    }


+    public Optional<TextEntity> findEntityByUnderlyingText(PrecursorEntity precursorEntity, Document document) {
+
+        if (precursorEntity.getEntityPosition().isEmpty()) {
+            return Optional.empty();
+        }
+
+        Optional<Page> optionalPage = document.getPages()
+                .stream()
+                .filter(docPage -> docPage.getNumber()
+                        .equals(precursorEntity.getEntityPosition()
+                                        .get(0).pageNumber()))
+                .findFirst();
+
+        if (optionalPage.isEmpty()) {
+            return Optional.empty();
+        }
+
+        Page page = optionalPage.get();
+        Rectangle2D rect = precursorEntity.getEntityPosition()
+                .stream()
+                .map(RectangleWithPage::rectangle2D)
+                .collect(RectangleTransformations.collectBBox());
+        Optional<AtomicTextBlock> intersectingTbOptional = page.getTextBlocksOnPage()
+                .stream()
+                .filter(tb -> RectangleTransformations.rectangle2DBBox(tb.getPositions()).intersects(rect))
+                .findFirst();
+        if (intersectingTbOptional.isEmpty()) {
+            return Optional.empty();
+        }
+        AtomicTextBlock intersectingTb = intersectingTbOptional.get();
+        List<PartialTextBlock> underlyingTextRuns = findUnderlyingCharacterRuns(intersectingTb, rect);
+
+        for (PartialTextBlock underlyingText : underlyingTextRuns) {
+            int threshold = (int) (Math.min(underlyingText.length(), precursorEntity.length()) * STRING_SIMILARITY_THRESHOLD) + 1;
+            int distance = new LevenshteinDistance(threshold).apply(underlyingText.getSearchText(), precursorEntity.getValue());
+            if (distance >= 0) {
+                return entityCreationService.byTextRangeWithEngine(underlyingText.getTextRange(), "temp", EntityType.ENTITY, document, Collections.emptySet());
+            }
+        }
+        return Optional.empty();
+    }
+
+
+    private static List<PartialTextBlock> findUnderlyingCharacterRuns(AtomicTextBlock intersectingTb, Rectangle2D rect) {
+
+        List<PartialTextBlock> intersectingTextBlocks = new ArrayList<>();
+        int first = -1;
+        int last = -1;
+
+        for (int i = 0; i < intersectingTb.getPositions().size(); i++) {
+            Rectangle2D rectangle2D = intersectingTb.getPosition(i + intersectingTb.getTextRange().start());
+
+            if (rectangle2D.intersects(rect)) {
+                if (first == -1) {
+                    first = i;
+                }
+                last = i;
+            } else if (first != -1) {
+                intersectingTextBlocks.add(new PartialTextBlock(intersectingTb, new TextRange(first, last)));
+
+                first = -1;
+                last = -1;
+            }
+        }
+
+        if (first != -1) {
+            intersectingTextBlocks.add(new PartialTextBlock(intersectingTb, new TextRange(first, last)));
+        }
+
+        return intersectingTextBlocks;
+    }
+
+
    private static boolean entityIdentifierValueNotFound(List<TextEntity> possibleEntities) {

        return possibleEntities == null || possibleEntities.isEmpty();
@@ -172,24 +252,69 @@ public class EntityFindingUtility {

        if (!pageNumbers.stream()
                .allMatch(node::onPage)) {
-            throw new IllegalArgumentException(format("SemanticNode \"%s\" does not contain these pages %s, it has pages: %s",
+            throw new IllegalArgumentException(format("SemanticNode \"%s\" is missing pages %s",
                                                      node,
-                                                      pageNumbers.stream()
-                                                              .filter(pageNumber -> !node.onPage(pageNumber))
-                                                              .toList(),
-                                                      node.getPages()));
+                                                      Sets.difference(pageNumbers,
+                                                                      node.getPages()
+                                                                              .stream()
+                                                                              .map(Page::getNumber)
+                                                                              .collect(Collectors.toSet()))));
        }

        SearchImplementation searchImplementation = new SearchImplementation(entryValues.stream()
                                                                                     .map(String::trim)
-                                                                                     .collect(Collectors.toSet()), true);
+                                                                                     .collect(Collectors.toSet()), true, true);

-        return searchImplementation.getBoundaries(node.getTextBlock(), node.getTextRange())
+        List<TextBlock> textBlocks = node.getTextBlocksByPageNumbers(pageNumbers);
+
+        return textBlocks.stream()
+                .flatMap(searchImplementation::getBoundaries)
                .map(boundary -> entityCreationService.byTextRangeWithEngine(boundary, "temp", EntityType.ENTITY, node, Collections.emptySet()))
                .filter(Optional::isPresent)
                .map(Optional::get)
                .distinct()
-                .collect(groupingBy(entity -> entity.getValue().toLowerCase(Locale.ROOT)));
+                .collect(groupingBy(entity -> TextNormalizationUtilities.removeAllWhitespaces(entity.getValue().toLowerCase(Locale.ROOT))));
+    }
+
+
+    public List<TextEntity> findAllPossibleEntitiesByValueOnPages(Document document, String value, boolean caseSensitive, Set<Integer> pageNumbers) {
+
+        if (!pageNumbers.stream()
+                .allMatch(document::onPage)) {
+            throw new IllegalArgumentException(format("SemanticNode \"%s\" does not contain these pages %s, it has pages: %s",
+                                                      document,
+                                                      pageNumbers.stream()
+                                                              .filter(pageNumber -> !document.onPage(pageNumber))
+                                                              .toList(),
+                                                      document.getPages()));
+        }
+
+        SearchImplementation searchImplementation = new SearchImplementation(value, !caseSensitive);
+
+        List<TextBlock> textBlocks = document.getTextBlocksByPageNumbers(pageNumbers);
+
+        return textBlocks.stream()
+                .flatMap(tb -> searchImplementation.getBoundaries(tb)
+                        .filter(textRange -> entityCreationService.isValidEntityTextRange(tb, textRange)))
+                .map(boundary -> entityCreationService.byTextRangeWithEngine(boundary, "temp", EntityType.ENTITY, document, Collections.emptySet()))
+                .filter(Optional::isPresent)
+                .map(Optional::get)
+                .distinct()
+                .toList();
+    }
+
+
+    public List<TextEntity> findAllPossibleEntitiesByValue(Document document, String value, boolean caseSensitive) {
+
+        SearchImplementation searchImplementation = new SearchImplementation(value, !caseSensitive);
+
+        return searchImplementation.getBoundaries(document.getTextBlock())
+                .filter(textRange -> entityCreationService.isValidEntityTextRange(document.getTextBlock(), textRange))
+                .map(boundary -> entityCreationService.byTextRangeWithEngine(boundary, "temp", EntityType.ENTITY, document, Collections.emptySet()))
+                .filter(Optional::isPresent)
+                .map(Optional::get)
+                .distinct()
+                .toList();
    }

 }
@@ -11,10 +11,12 @@ import java.util.stream.Collectors;

 import org.springframework.stereotype.Service;

+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedactions;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.BaseAnnotation;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;
+import com.iqser.red.service.redaction.v1.server.RedactionServiceSettings;
 import com.iqser.red.service.redaction.v1.server.model.PrecursorEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
@@ -32,9 +34,10 @@ import lombok.extern.slf4j.Slf4j;
@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
 public class EntityFromPrecursorCreationService {

-    static double MATCH_THRESHOLD = 10; // Is compared to the average sum of distances in pdf coordinates for each corner of the bounding box of the entities
+    static double MATCH_THRESHOLD = 100; // Is compared to the average sum of distances in pdf coordinates for each corner of the bounding box of the entities
    EntityFindingUtility entityFindingUtility;
    DictionaryService dictionaryService;
+    RedactionServiceSettings settings;


    public List<PrecursorEntity> createEntitiesIfFoundAndReturnNotFoundEntries(ManualRedactions manualRedactions, SemanticNode node, String dossierTemplateId) {
@@ -92,7 +95,7 @@ public class EntityFromPrecursorCreationService {
                notFoundEntities.add(precursorEntity);
                continue;
            }
-            createCorrectEntity(precursorEntity, optionalClosestEntity.get());
+            createCorrectEntity(precursorEntity, optionalClosestEntity.get(), settings.isAnnotationMode());
        }

        tempEntitiesByValue.values()
@@ -125,12 +128,14 @@ public class EntityFromPrecursorCreationService {
                                                         precursorEntity.getEntityType(),
                                                         closestEntity.getDeepestFullyContainingNode());
        } else {
-            correctEntity = TextEntity.initialEntityNode(closestEntity.getTextRange(),
-                                                         precursorEntity.type(),
-                                                         precursorEntity.getEntityType(),
-                                                         precursorEntity.getId(),
-                                                         precursorEntity.getManualOverwrite().getSection()
-                                                                 .orElse(null));
+            String section = precursorEntity.getManualOverwrite().getSection()
+                    .orElse(null);
+            if ((section == null || section.isBlank()) && precursorEntity.getSection() != null && !precursorEntity.getSection().isBlank() && precursorEntity.getEngines()
+                    .contains(Engine.IMPORTED)) {
+                section = precursorEntity.getSection();
+            }
+
+            correctEntity = TextEntity.initialEntityNode(closestEntity.getTextRange(), precursorEntity.type(), precursorEntity.getEntityType(), precursorEntity.getId(), section);
        }
        correctEntity.setDeepestFullyContainingNode(closestEntity.getDeepestFullyContainingNode());
        correctEntity.setIntersectingNodes(new ArrayList<>(closestEntity.getIntersectingNodes()));
@@ -19,6 +19,7 @@ import com.iqser.red.service.redaction.v1.server.model.document.nodes.NodeType;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;

 import lombok.AccessLevel;
+import lombok.NonNull;
 import lombok.experimental.FieldDefaults;
 import lombok.experimental.UtilityClass;
 import lombok.extern.slf4j.Slf4j;
@@ -35,6 +36,7 @@ public class NerEntitiesAdapter {
    static int MAX_DISTANCE_BETWEEN_PARTS = 20;
    static int MIN_PARTS_TO_COMBINE = 3;
    static boolean ALLOW_DUPLICATES;
+    static int MIN_ESSENTIAL_TYPES_COMBINED;


    /**
@@ -42,14 +44,29 @@ public class NerEntitiesAdapter {
     *
     * @param nerEntitiesModel the Entities just as the NER Service returns them
     * @param document         the document structure, from which the NER Service found the entities
+     * @param engine           the origin of the NerEntities
     * @return a stream of validated entities
     */
+    public NerEntities toNerEntities(NerEntitiesModel nerEntitiesModel, Document document, NerEntities.Engine engine) {
+
+        return new NerEntities(addOffsetsAndFlatten(getStringStartOffsetsForMainSectionsHeadersFooters(document), nerEntitiesModel).map(nerEntityModel -> new NerEntities.NerEntity(
+                        nerEntityModel.getValue(),
+                        new TextRange(nerEntityModel.getStartOffset(), nerEntityModel.getEndOffset()),
+                        nerEntityModel.getType(),
+                        nerEntityModel.getConfidence(),
+                        engine))
+                                       .toList());
+    }
+
+
    public NerEntities toNerEntities(NerEntitiesModel nerEntitiesModel, Document document) {

        return new NerEntities(addOffsetsAndFlatten(getStringStartOffsetsForMainSectionsHeadersFooters(document), nerEntitiesModel).map(nerEntityModel -> new NerEntities.NerEntity(
                        nerEntityModel.getValue(),
                        new TextRange(nerEntityModel.getStartOffset(), nerEntityModel.getEndOffset()),
-                        nerEntityModel.getType()))
+                        nerEntityModel.getType(),
+                        nerEntityModel.getConfidence(),
+                        NerEntities.Engine.NER))
                                       .toList());
    }

@@ -60,12 +77,13 @@ public class NerEntitiesAdapter {
     * The first part must be an EntityRecognitionEntity of an essential type.
     * The resulting list must consist of at least minPartsToCombine parts.
     *
-     * @param nerEntities             already validated entities from the NER Service
-     * @param essentialTypes          the combined entities must contain at least one of these types
-     * @param typesToCombine          all types which should be used to combine, must contain all essentialTypes
-     * @param maxDistanceBetweenParts all parts used to combine should be at most this value apart in string offset coordinates
-     * @param minPartsToCombine       minimum number of parts to combine
-     * @param allowDuplicates         allow combining multiple parts of same type
+     * @param nerEntities               already validated entities from the NER Service
+     * @param essentialTypes            the combined entities must contain at least one of these types
+     * @param typesToCombine            all types which should be used to combine, must contain all essentialTypes
+     * @param maxDistanceBetweenParts   all parts used to combine should be at most this value apart in string offset coordinates
+     * @param minPartsToCombine         minimum number of parts to combine
+     * @param minEssentialTypesCombined minimum number of essential types that must be part of a combination
+     * @param allowDuplicates           allow combining multiple parts of same type
     * @return A Stream of the combined boundaries
     */
    public Stream<TextRange> combineNerEntities(NerEntities nerEntities,
@@ -73,51 +91,107 @@ public class NerEntitiesAdapter {
                                                Set<String> typesToCombine,
                                                int maxDistanceBetweenParts,
                                                int minPartsToCombine,
+                                                int minEssentialTypesCombined,
                                                boolean allowDuplicates) {

-        List<NerEntities.NerEntity> sortedEntities = nerEntities.getNerEntityList()
-                .stream()
-                .filter(entity -> typesToCombine.contains(entity.type()))
-                .sorted(Comparator.comparingInt(entity -> entity.textRange().start()))
-                .toList();
+        return getTextRangeStream(nerEntities, essentialTypes, typesToCombine, maxDistanceBetweenParts, minPartsToCombine, minEssentialTypesCombined, allowDuplicates);
+
+    }
+
+
+    @NonNull
+    private static Stream<TextRange> getTextRangeStream(NerEntities nerEntities,
+                                                        Set<String> essentialTypes,
+                                                        Set<String> typesToCombine,
+                                                        int maxDistanceBetweenParts,
+                                                        int minPartsToCombine,
+                                                        int minEssentialTypesCombined,
+                                                        boolean allowDuplicates) {
+
+        List<NerEntities.NerEntity> sortedEntities = getSortedEntities(nerEntities, typesToCombine);

        if (sortedEntities.isEmpty()) {
            return Stream.empty();
        }

-        List<List<NerEntities.NerEntity>> entityClusters = new LinkedList<>();
-
-        List<NerEntities.NerEntity> startEntitiesOfEssentialType = sortedEntities.stream()
-                .filter(e -> essentialTypes.contains(e.type()))
-                .toList();
-        for (NerEntities.NerEntity startEntity : startEntitiesOfEssentialType) {
-            List<NerEntities.NerEntity> currentCluster = new LinkedList<>();
-            entityClusters.add(currentCluster);
-            int lastEndOffset = startEntity.textRange().end();
-
-            for (NerEntities.NerEntity entity : sortedEntities) {
-                if (entity.textRange().start() < lastEndOffset) {
-                    continue;
-                }
-                if (distanceIsLargerThanMaxDistance(lastEndOffset, entity, maxDistanceBetweenParts) || isDuplicate(currentCluster, entity, allowDuplicates)) {
-                    currentCluster = new LinkedList<>();
-                    entityClusters.add(currentCluster);
-                    currentCluster.add(entity);
-                    lastEndOffset = entity.textRange().end();
-                } else {
-                    currentCluster.add(entity);
-                    lastEndOffset = entity.textRange().end();
-                }
-            }
-        }
+        List<List<NerEntities.NerEntity>> entityClusters = getEntityClusters(essentialTypes,
+                                                                             maxDistanceBetweenParts,
+                                                                             minPartsToCombine,
+                                                                             minEssentialTypesCombined,
+                                                                             allowDuplicates,
+                                                                             sortedEntities);

        return entityClusters.stream()
-                .filter(cluster -> cluster.size() >= minPartsToCombine)
                .map(NerEntitiesAdapter::toContainingBoundary)
                .distinct();
    }


+    private static List<NerEntities.NerEntity> getSortedEntities(NerEntities nerEntities, Set<String> typesToCombine) {
+
+        return nerEntities.getNerEntityList()
+                .stream()
+                .filter(entity -> typesToCombine.contains(entity.type()))
+                .sorted(Comparator.comparingInt(entity -> entity.textRange().start()))
+                .toList();
+    }
+
+
+    private static List<List<NerEntities.NerEntity>> getEntityClusters(Set<String> essentialTypes,
+                                                                       int maxDistanceBetweenParts,
+                                                                       int minPartsToCombine,
+                                                                       int minEssentialTypesCombined,
+                                                                       boolean allowDuplicates,
+                                                                       List<NerEntities.NerEntity> sortedEntities) {
+
+        List<List<NerEntities.NerEntity>> entityClusters = new LinkedList<>();
+        List<NerEntities.NerEntity> currentCluster = new LinkedList<>();
+        int lastEndOffset = -1;
+
+        for (NerEntities.NerEntity entity : sortedEntities) {
+
+            // cluster can be added as it is either duplicated or distance is too large
+            if (!currentCluster.isEmpty() && (distanceIsLargerThanMaxDistance(lastEndOffset, entity, maxDistanceBetweenParts) || isDuplicate(currentCluster,
+                                                                                                                                             entity,
+                                                                                                                                             allowDuplicates))) {
+                entityClusters.add(new LinkedList<>(currentCluster));
+                currentCluster.clear();
+            }
+
+            currentCluster.add(entity);
+            lastEndOffset = entity.textRange().end();
+        }
+
+        // add the last cluster if not empty
+        if (!currentCluster.isEmpty()) {
+            entityClusters.add(currentCluster);
+        }
+
+        return filterClusters(entityClusters, essentialTypes, minPartsToCombine, minEssentialTypesCombined);
+    }
+
+
+    private static List<List<NerEntities.NerEntity>> filterClusters(List<List<NerEntities.NerEntity>> clusters,
+                                                                    Set<String> essentialTypes,
+                                                                    int minPartsToCombine,
+                                                                    int minEssentialTypesCombined) {
+
+        return clusters.stream()
+                .filter(cluster -> cluster.size() >= minPartsToCombine)
+                .filter(cluster -> countEssentialEntities(cluster, essentialTypes) >= minEssentialTypesCombined)
+                .distinct()
+                .toList();
+    }
+
+
+    private static long countEssentialEntities(List<NerEntities.NerEntity> cluster, Set<String> essentialTypes) {
+
+        return cluster.stream()
+                .filter(entity -> essentialTypes.contains(entity.type()))
+                .count();
+    }
+
+
    /**
     * Calls combine NerEntities with the following settings.
     * <p>
@@ -137,6 +211,76 @@ public class NerEntitiesAdapter {
                                  CBI_ADDRESS_TYPES_TO_COMBINE,
                                  MAX_DISTANCE_BETWEEN_PARTS,
                                  MIN_PARTS_TO_COMBINE,
+                                  MIN_ESSENTIAL_TYPES_COMBINED,
+                                  ALLOW_DUPLICATES);
+    }
+
+
+    public Stream<TextRange> combineNerEntitiesOfAllGivenTypes(NerEntities entityRecognitionEntities,
+                                                               Set<String> essentialTypes,
+                                                               Set<String> typesToCombine,
+                                                               int minPartsToCombine) {
+
+        return combineNerEntities(entityRecognitionEntities,
+                                  essentialTypes,
+                                  typesToCombine,
+                                  MAX_DISTANCE_BETWEEN_PARTS,
+                                  minPartsToCombine,
+                                  MIN_ESSENTIAL_TYPES_COMBINED,
+                                  ALLOW_DUPLICATES);
+    }
+
+
+    public Stream<TextRange> combineNerEntitiesOfAllGivenTypes(NerEntities entityRecognitionEntities,
+                                                               Set<String> essentialTypes,
+                                                               Set<String> typesToCombine,
+                                                               int maxDistanceBetweenParts,
+                                                               int minPartsToCombine) {
+
+        return combineNerEntities(entityRecognitionEntities,
+                                  essentialTypes,
+                                  typesToCombine,
+                                  maxDistanceBetweenParts,
+                                  minPartsToCombine,
+                                  MIN_ESSENTIAL_TYPES_COMBINED,
+                                  ALLOW_DUPLICATES);
+    }
+
+
+    public Stream<TextRange> combineNerEntitiesOfAllGivenTypes(NerEntities entityRecognitionEntities,
+                                                               Set<String> essentialTypes,
+                                                               Set<String> typesToCombine,
+                                                               int maxDistanceBetweenParts,
+                                                               int minPartsToCombine,
+                                                               int minEssentialTypesCombined) {
+
+        return combineNerEntities(entityRecognitionEntities,
+                                  essentialTypes,
+                                  typesToCombine,
+                                  maxDistanceBetweenParts,
+                                  minPartsToCombine,
+                                  minEssentialTypesCombined,
+                                  ALLOW_DUPLICATES);
+    }
+
+
+    public Stream<TextRange> combineNerEntitiesOfAllGivenTypesWithConfidence(NerEntities entityRecognitionEntities,
+                                                                             Set<String> essentialTypes,
+                                                                             Set<String> typesToCombine,
+                                                                             int maxDistanceBetweenParts,
+                                                                             int minPartsToCombine,
+                                                                             int minEssentialTypesCombined,
+                                                                             Double confidence) {
+
+        return combineNerEntities(new NerEntities(entityRecognitionEntities.getNerEntityList()
+                                                          .stream()
+                                                          .filter(nerEntity -> nerEntity.confidence() == null || nerEntity.confidence() >= confidence)
+                                                          .toList()),
+                                  essentialTypes,
+                                  typesToCombine,
+                                  maxDistanceBetweenParts,
+                                  minPartsToCombine,
+                                  minEssentialTypesCombined,
                                  ALLOW_DUPLICATES);
    }

@@ -177,7 +321,7 @@ public class NerEntitiesAdapter {
    }


-    private static List<Integer> sectionNumberToTreeId(String sectionNumber) {
+    public static List<Integer> sectionNumberToTreeId(String sectionNumber) {

        return Arrays.stream(sectionNumber.split("\\."))
                .map(Integer::parseInt)
@@ -9,7 +9,7 @@ import com.iqser.red.service.redaction.v1.server.model.document.nodes.Image;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.ImageType;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Table;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentStructure;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentStructureWrapper;

 import lombok.experimental.UtilityClass;

@@ -18,32 +18,32 @@ public class PropertiesMapper {

    public void parseImageProperties(Map<String, String> properties, Image.ImageBuilder builder) {

-        builder.imageType(ImageType.fromString(properties.get(DocumentStructure.ImageProperties.IMAGE_TYPE)));
-        builder.transparent(Boolean.parseBoolean(properties.get(DocumentStructure.ImageProperties.TRANSPARENT)));
-        builder.position(parseRectangle2D(properties.get(DocumentStructure.ImageProperties.POSITION)));
-        builder.id(properties.get(DocumentStructure.ImageProperties.ID));
+        builder.imageType(ImageType.fromString(properties.get(DocumentStructureWrapper.ImageProperties.IMAGE_TYPE)));
+        builder.transparent(Boolean.parseBoolean(properties.get(DocumentStructureWrapper.ImageProperties.TRANSPARENT)));
+        builder.position(parseRectangle2D(properties.get(DocumentStructureWrapper.ImageProperties.POSITION)));
+        builder.id(properties.get(DocumentStructureWrapper.ImageProperties.ID));
    }


    public void parseTableCellProperties(Map<String, String> properties, TableCell.TableCellBuilder builder) {

-        builder.row(Integer.parseInt(properties.get(DocumentStructure.TableCellProperties.ROW)));
-        builder.col(Integer.parseInt(properties.get(DocumentStructure.TableCellProperties.COL)));
-        builder.header(Boolean.parseBoolean(properties.get(DocumentStructure.TableCellProperties.HEADER)));
-        builder.bBox(parseRectangle2D(properties.get(DocumentStructure.TableCellProperties.B_BOX)));
+        builder.row(Integer.parseInt(properties.get(DocumentStructureWrapper.TableCellProperties.ROW)));
+        builder.col(Integer.parseInt(properties.get(DocumentStructureWrapper.TableCellProperties.COL)));
+        builder.header(Boolean.parseBoolean(properties.get(DocumentStructureWrapper.TableCellProperties.HEADER)));
+        builder.bBox(parseRectangle2D(properties.get(DocumentStructureWrapper.TableCellProperties.B_BOX)));
    }


    public void parseTableProperties(Map<String, String> properties, Table.TableBuilder builder) {

-        builder.numberOfRows(Integer.parseInt(properties.get(DocumentStructure.TableProperties.NUMBER_OF_ROWS)));
-        builder.numberOfCols(Integer.parseInt(properties.get(DocumentStructure.TableProperties.NUMBER_OF_COLS)));
+        builder.numberOfRows(Integer.parseInt(properties.get(DocumentStructureWrapper.TableProperties.NUMBER_OF_ROWS)));
+        builder.numberOfCols(Integer.parseInt(properties.get(DocumentStructureWrapper.TableProperties.NUMBER_OF_COLS)));
    }


    private Rectangle2D parseRectangle2D(String bBox) {

-        List<Float> floats = Arrays.stream(bBox.split(DocumentStructure.RECTANGLE_DELIMITER))
+        List<Float> floats = Arrays.stream(bBox.split(DocumentStructureWrapper.RECTANGLE_DELIMITER))
                .map(Float::parseFloat)
                .toList();
        return new Rectangle2D.Float(floats.get(0), floats.get(1), floats.get(2), floats.get(3));
@@ -52,19 +52,19 @@ public class PropertiesMapper {

    public static boolean isDuplicateParagraph(Map<String, String> properties) {

-        return properties.containsKey(DocumentStructure.DuplicateParagraphProperties.UNSORTED_TEXTBLOCK_ID);
+        return properties.containsKey(DocumentStructureWrapper.DuplicateParagraphProperties.UNSORTED_TEXTBLOCK_ID);
    }


-    public static Long[] getUnsortedTextblockIds(Map<String, String> properties) {
+    public static List<Long> getUnsortedTextblockIds(Map<String, String> properties) {

-        return toLongArray(properties.get(DocumentStructure.DuplicateParagraphProperties.UNSORTED_TEXTBLOCK_ID));
+        return toLongList(properties.get(DocumentStructureWrapper.DuplicateParagraphProperties.UNSORTED_TEXTBLOCK_ID));
    }


-    public static Long[] toLongArray(String ids) {
+    public static List<Long> toLongList(String ids) {

-        return Arrays.stream(ids.substring(1, ids.length() - 1).trim().split(",")).map(Long::valueOf).toArray(Long[]::new);
+        return Arrays.stream(ids.substring(1, ids.length() - 1).trim().split(",")).map(Long::valueOf).toList();
    }


@@ -11,7 +11,6 @@ import java.util.stream.Stream;
 import org.springframework.stereotype.Service;

 import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedaction;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedactions;
@@ -27,7 +26,6 @@ import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryIncr
 import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryIncrementValue;
 import com.iqser.red.service.redaction.v1.server.model.dictionary.SearchImplementation;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
-import com.iqser.red.service.redaction.v1.server.model.document.nodes.NodeType;

 import io.micrometer.core.annotation.Timed;
 import lombok.AccessLevel;
@@ -51,7 +49,6 @@ public class SectionFinderService {
        long start = System.currentTimeMillis();
        Set<Integer> sectionsToReanalyse = new HashSet<>();

-
        var dictionaryIncrementsSearch = new SearchImplementation(dictionaryIncrement.getValues()
                                                                          .stream()
                                                                          .map(DictionaryIncrementValue::getValue)
@@ -82,9 +79,7 @@ public class SectionFinderService {
        return document.getPages()
                .stream()
                .filter(page -> relevantPagesForReanalysis.contains(page.getNumber()))
-                .flatMap(page -> Stream.concat(page.getMainBody()
-                                                       .stream()
-                                                       .filter(node -> node.getType().equals(NodeType.SECTION)), Stream.of(page.getHeader(), page.getFooter())))
+                .flatMap(page -> Stream.concat(page.streamHighestSemanticNodesOnPage(), Stream.of(page.getHeader(), page.getFooter())))
                .map(node -> node.getTreeId()
                        .get(0))
                .toList();
@@ -22,13 +22,19 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.component.ComponentMappingMetadata;
 import com.iqser.red.service.redaction.v1.server.RedactionServiceSettings;
+import com.iqser.red.service.redaction.v1.server.logger.Context;
+import com.iqser.red.service.redaction.v1.server.logger.ObjectTrackingEventListener;
+import com.iqser.red.service.redaction.v1.server.logger.RulesLogger;
+import com.iqser.red.service.redaction.v1.server.logger.TrackingAgendaEventListener;
 import com.iqser.red.service.redaction.v1.server.model.component.Component;
 import com.iqser.red.service.redaction.v1.server.model.component.Entity;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
 import com.iqser.red.service.redaction.v1.server.service.components.ComponentMappingMemoryCache;
 import com.iqser.red.service.redaction.v1.server.service.components.ComponentMappingService;
+import com.iqser.red.service.redaction.v1.server.service.components.DateConverterMemoryCache;
 import com.iqser.red.service.redaction.v1.server.service.document.ComponentComparator;
 import com.iqser.red.service.redaction.v1.server.service.document.ComponentCreationService;
+import com.iqser.red.service.redaction.v1.server.service.websocket.WebSocketService;
 import com.iqser.red.service.redaction.v1.server.utils.exception.DroolsTimeoutException;
 import com.knecon.fforesight.tenantcommons.TenantContext;

@@ -44,24 +50,41 @@ import lombok.extern.slf4j.Slf4j;
 public class ComponentDroolsExecutionService {

    public static final String COMPONENT_MAPPING_SERVICE_GLOBAL = "componentMappingService";
+    public static final String RULES_LOGGER_GLOBAL = "logger";

    RedactionServiceSettings settings;
    ComponentMappingMemoryCache componentMappingMemoryCache;
+    DateConverterMemoryCache dateConverterMemoryCache;
+    WebSocketService webSocketService;


    public List<Component> executeRules(KieContainer kieContainer,
                                        EntityLog entityLog,
                                        Document document,
                                        Set<FileAttribute> fileAttributes,
-                                        List<ComponentMappingMetadata> componentMappings) {
+                                        List<ComponentMappingMetadata> componentMappings,
+                                        Context context) {

        KieSession kieSession = kieContainer.newKieSession();
-        ComponentCreationService componentCreationService = new ComponentCreationService(kieSession);
+        DateConverterMemoryCache.DateConverterCacheEntry dateConverterCacheEntry = dateConverterMemoryCache.getDateConverter(context.getDossierTemplateId());
+        context.setDateFormatsVersion(dateConverterCacheEntry.version());
+        ComponentCreationService componentCreationService = new ComponentCreationService(kieSession, dateConverterCacheEntry.dateConverter());
        ComponentMappingService componentMappingService = new ComponentMappingService(componentMappingMemoryCache, componentMappings);
+        RulesLogger logger = new RulesLogger(webSocketService, context);
+        if (settings.isDroolsDebug()) {
+            logger.enableAgendaTracking();
+            logger.enableObjectTracking();
+        }
+        kieSession.addEventListener(new TrackingAgendaEventListener(logger));
+        kieSession.addEventListener(new ObjectTrackingEventListener(logger));

        kieSession.setGlobal("componentCreationService", componentCreationService);

-        if (hasComponentMappingServiceGlobal(kieSession)) {
+        if (hasGlobalWithName(kieSession, RULES_LOGGER_GLOBAL)) {
+            kieSession.setGlobal(RULES_LOGGER_GLOBAL, logger);
+        }
+
+        if (hasGlobalWithName(kieSession, COMPONENT_MAPPING_SERVICE_GLOBAL)) {
            kieSession.setGlobal(COMPONENT_MAPPING_SERVICE_GLOBAL, componentMappingService);
        }

@@ -73,9 +96,10 @@ public class ComponentDroolsExecutionService {
                    entities.add(Entity.fromEntityLogEntry(entry, document, entry.getStartOffset(), entry.getEndOffset()));
                    if (entry.getDuplicatedTextRanges() != null && !entry.getDuplicatedTextRanges().isEmpty()) {
                        entry.getDuplicatedTextRanges()
-                                .forEach(duplicatedTextRange -> {
-                                    entities.add(Entity.fromEntityLogEntry(entry, document, duplicatedTextRange.getStart(), duplicatedTextRange.getEnd()));
-                                });
+                                .forEach(duplicatedTextRange -> entities.add(Entity.fromEntityLogEntry(entry,
+                                                                                                       document,
+                                                                                                       duplicatedTextRange.getStart(),
+                                                                                                       duplicatedTextRange.getEnd())));
                    }
                    return entities.stream();
                })
@@ -94,17 +118,20 @@ public class ComponentDroolsExecutionService {
        });

        try {
-            completableFuture.orTimeout(settings.getDroolsExecutionTimeoutSecs(document.getNumberOfPages()), TimeUnit.SECONDS)
-                    .get();
+            completableFuture.get(settings.getDroolsExecutionTimeoutSecs(document.getNumberOfPages()), TimeUnit.SECONDS);
        } catch (ExecutionException e) {
+            logger.error(e, "Exception during rule execution");
            kieSession.dispose();
            if (e.getCause() instanceof TimeoutException) {
                throw new DroolsTimeoutException(e, false, RuleFileType.COMPONENT);
            }
            throw new RuntimeException(e);
        } catch (InterruptedException e) {
+            logger.error(e, "Exception during rule execution");
            kieSession.dispose();
            throw new RuntimeException(e);
+        } catch (TimeoutException e) {
+            throw new DroolsTimeoutException(e, false, RuleFileType.COMPONENT);
        }

        List<FileAttribute> resultingFileAttributes = getFileAttributes(kieSession);
@@ -139,13 +166,13 @@ public class ComponentDroolsExecutionService {
    }


-    private static boolean hasComponentMappingServiceGlobal(KieSession kieSession) {
+    private static boolean hasGlobalWithName(KieSession kieSession, String globalName) {

        return kieSession.getKieBase().getKiePackages()
                .stream()
                .flatMap(kiePackage -> kiePackage.getGlobalVariables()
                        .stream())
-                .anyMatch(global -> global.getName().equals(COMPONENT_MAPPING_SERVICE_GLOBAL));
+                .anyMatch(global -> global.getName().equals(globalName));
    }


@@ -45,7 +45,7 @@ public class DroolsValidationService {
    private final KieContainerCreationService kieContainerCreationService;
    private final DeprecatedElementsFinder deprecatedElementsFinder;
    private static final Pattern allowedImportsPattern = Pattern.compile(
-            "^(?:import\\s+static\\s+)?(?:import\\s+)?(?:com\\.knecon\\.fforesight|com\\.iqser\\.red|java\\.util)\\..*;$");
+            "^(?:import\\s+static\\s+)?(?:import\\s+)?(?:com\\.knecon\\.fforesight|com\\.iqser\\.red|java\\.util|java\\.text)\\..*;$");
    public static final String LINEBREAK_MATCHER = "\\R";


@@ -73,7 +73,7 @@ public class DroolsValidationService {

        DroolsValidation customValidation = ruleFileBluePrint.getDroolsValidation();

-        addSyntaxDeprecatedWarnings(ruleFileType, ruleFileBluePrint, customValidation);
+        addSyntaxDeprecatedWarnings(ruleFileBluePrint, customValidation);

        addSyntaxErrorMessages(ruleFileType, ruleFileBluePrint, customValidation);

@@ -85,7 +85,7 @@ public class DroolsValidationService {
    }


-    private void addSyntaxDeprecatedWarnings(RuleFileType ruleFileType, RuleFileBluePrint ruleFileBluePrint, DroolsValidation customValidation) {
+    private void addSyntaxDeprecatedWarnings(RuleFileBluePrint ruleFileBluePrint, DroolsValidation customValidation) {

        // find deprecated elements in the ruleFileBluePrint
        DroolsSyntaxDeprecatedWarnings warningMessageForImports = getWarningsForDeprecatedImports(ruleFileBluePrint);
@@ -93,25 +93,6 @@ public class DroolsValidationService {
            customValidation.getDeprecatedWarnings().add(warningMessageForImports);
        }
        customValidation.getDeprecatedWarnings().addAll(getWarningsForDeprecatedRules(ruleFileBluePrint));
-
-        if (ruleFileType.equals(RuleFileType.COMPONENT)) {
-            if (!ruleFileBluePrint.getGlobals().contains(ComponentDroolsExecutionService.COMPONENT_MAPPING_SERVICE_GLOBAL)) {
-                customValidation.getDeprecatedWarnings().add(buildComponentMappingServiceMissingMessage(ruleFileBluePrint));
-            }
-
-        }
-    }
-
-
-    private static DroolsSyntaxDeprecatedWarnings buildComponentMappingServiceMissingMessage(RuleFileBluePrint ruleFileBluePrint) {
-
-        return DroolsSyntaxDeprecatedWarnings.builder()
-                .message("global ComponentMappingService "
-                         + ComponentDroolsExecutionService.COMPONENT_MAPPING_SERVICE_GLOBAL
-                         + "\n is missing from the rules, consider adding it, as it will be required in future versions!")
-                .line(ruleFileBluePrint.getGlobalsLine())
-                .column(0)
-                .build();
    }


@@ -283,7 +264,9 @@ public class DroolsValidationService {

    private DroolsBlacklistErrorMessage checkAndGetBlackListedMessages(SearchImplementation blacklistedKeywordSearchImplementation, String stringToCheck, int lineIndexStart) {

-        String sanitizedRuleText = StringUtils.deleteWhitespace(stringToCheck);
+        String nonWhitespaceRuleText = StringUtils.deleteWhitespace(stringToCheck);
+        String sanitizedRuleText = nonWhitespaceRuleText.replaceAll("\"(\\\\.|[^\"\\\\])*\"|'(\\\\.|[^'\\\\])*'", "");
+
        List<SearchImplementation.MatchPosition> matches = blacklistedKeywordSearchImplementation.getMatches(sanitizedRuleText);

        if (!matches.isEmpty()) {
@@ -1,5 +1,7 @@
 package com.iqser.red.service.redaction.v1.server.service.drools;

+import static com.iqser.red.service.redaction.v1.server.service.drools.ComponentDroolsExecutionService.RULES_LOGGER_GLOBAL;
+
 import java.util.Collections;
 import java.util.HashSet;
 import java.util.LinkedList;
@@ -21,6 +23,10 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileTyp
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.BaseAnnotation;
 import com.iqser.red.service.redaction.v1.server.RedactionServiceSettings;
+import com.iqser.red.service.redaction.v1.server.logger.Context;
+import com.iqser.red.service.redaction.v1.server.logger.ObjectTrackingEventListener;
+import com.iqser.red.service.redaction.v1.server.logger.RulesLogger;
+import com.iqser.red.service.redaction.v1.server.logger.TrackingAgendaEventListener;
 import com.iqser.red.service.redaction.v1.server.model.NerEntities;
 import com.iqser.red.service.redaction.v1.server.model.dictionary.Dictionary;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
@@ -28,6 +34,7 @@ import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNo
 import com.iqser.red.service.redaction.v1.server.service.ManualChangesApplicationService;
 import com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService;
 import com.iqser.red.service.redaction.v1.server.service.document.EntityEnrichmentService;
+import com.iqser.red.service.redaction.v1.server.service.websocket.WebSocketService;
 import com.iqser.red.service.redaction.v1.server.utils.exception.DroolsTimeoutException;

 import io.micrometer.core.annotation.Timed;
@@ -48,6 +55,7 @@ public class EntityDroolsExecutionService {
    ObservationRegistry observationRegistry;
    ManualChangesApplicationService manualChangesApplicationService;
    RedactionServiceSettings settings;
+    WebSocketService webSocketService;


    @Timed("redactmanager_executeRules")
@@ -57,7 +65,8 @@ public class EntityDroolsExecutionService {
                                            Dictionary dictionary,
                                            List<FileAttribute> fileAttributes,
                                            ManualRedactions manualRedactions,
-                                            NerEntities nerEntities) {
+                                            NerEntities nerEntities,
+                                            Context context) {

        return executeRules(kieContainer,
                            document,
@@ -66,7 +75,8 @@ public class EntityDroolsExecutionService {
                            dictionary,
                            fileAttributes,
                            manualRedactions,
-                            nerEntities);
+                            nerEntities,
+                            context);
    }


@@ -78,7 +88,8 @@ public class EntityDroolsExecutionService {
                                            Dictionary dictionary,
                                            List<FileAttribute> fileAttributes,
                                            ManualRedactions manualRedactions,
-                                            NerEntities nerEntities) {
+                                            NerEntities nerEntities,
+                                            Context context) {

        addNumberOfPagesAndSectionsToAnalyseToTrace(document.getNumberOfPages(), sectionsToAnalyze.size());

@@ -88,12 +99,23 @@ public class EntityDroolsExecutionService {
                .count() ? Collections.emptySet() : buildSet(sectionsToAnalyze, document);

        EntityCreationService entityCreationService = new EntityCreationService(entityEnrichmentService, kieSession, nodesInKieSession);
+        RulesLogger logger = new RulesLogger(webSocketService, context);
+        if (settings.isDroolsDebug()) {
+            logger.enableAgendaTracking();
+            logger.enableObjectTracking();
+        }
+        kieSession.addEventListener(new TrackingAgendaEventListener(logger));
+        kieSession.addEventListener(new ObjectTrackingEventListener(logger));

        kieSession.setGlobal("document", document);
        kieSession.setGlobal("entityCreationService", entityCreationService);
        kieSession.setGlobal("manualChangesApplicationService", manualChangesApplicationService);
        kieSession.setGlobal("dictionary", dictionary);

+        if (hasGlobalWithName(kieSession, RULES_LOGGER_GLOBAL)) {
+            kieSession.setGlobal(RULES_LOGGER_GLOBAL, logger);
+        }
+
        kieSession.insert(document);

        document.getEntities()
@@ -129,17 +151,20 @@ public class EntityDroolsExecutionService {
        });

        try {
-            completableFuture.orTimeout(settings.getDroolsExecutionTimeoutSecs(document.getNumberOfPages()), TimeUnit.SECONDS)
-                    .get();
+            completableFuture.get(settings.getDroolsExecutionTimeoutSecs(document.getNumberOfPages()), TimeUnit.SECONDS);
        } catch (ExecutionException e) {
+            logger.error(e, "Exception during rule execution");
            kieSession.dispose();
            if (e.getCause() instanceof TimeoutException) {
                throw new DroolsTimeoutException(e, false, RuleFileType.ENTITY);
            }
            throw new RuntimeException(e);
        } catch (InterruptedException e) {
+            logger.error(e, "Exception during rule execution");
            kieSession.dispose();
            throw new RuntimeException(e);
+        } catch (TimeoutException e) {
+            throw new DroolsTimeoutException(e, false, RuleFileType.ENTITY);
        }

        List<FileAttribute> resultingFileAttributes = getFileAttributes(kieSession);
@@ -178,4 +203,14 @@ public class EntityDroolsExecutionService {
        }
    }

+
+    private static boolean hasGlobalWithName(KieSession kieSession, String globalName) {
+
+        return kieSession.getKieBase().getKiePackages()
+                .stream()
+                .flatMap(kiePackage -> kiePackage.getGlobalVariables()
+                        .stream())
+                .anyMatch(global -> global.getName().equals(globalName));
+    }
+
 }
@@ -32,7 +32,7 @@ import lombok.experimental.UtilityClass;
 public class RuleFileParser {

    private final static Pattern ruleIdentifierInCodeFinder = Pattern.compile(
-            "\\b(?:redact|apply|skip|remove|ignore|applyWithLineBreaks|applyWithReferences|skipWithReferences)\\s*\\(\"([a-zA-Z0-9]+.\\d+.\\d+)\",.*(?:, .*)?\\)");
+            "\\b(?:redact|apply|skip|remove|ignore|applyWithLineBreaks|applyWithReferences|skipWithReferences)\\s*\\(\\s*\"([a-zA-Z0-9]+.\\d+.\\d+)\"\\s*,\\s*.*(?:\\s*,\\s*.*)\\s*?\\)");


    @SneakyThrows
@@ -78,7 +78,8 @@ public class RuleFileParser {
                                             .map(GlobalDescr::getLine)
                                             .orElse(0),
                                     allQueries,
-                                     ruleClasses, customDroolsValidation);
+                                     ruleClasses,
+                                     customDroolsValidation);
    }


@@ -0,0 +1,22 @@
+package com.iqser.red.service.redaction.v1.server.service.websocket;
+
+import org.springframework.messaging.simp.SimpMessagingTemplate;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+
+import lombok.RequiredArgsConstructor;
+import lombok.SneakyThrows;
+
+@RequiredArgsConstructor
+public class RedisPubsubReceiver {
+
+    private final SimpMessagingTemplate websocketTemplate;
+    private final ObjectMapper mapper;
+
+    @SneakyThrows
+    public void receiveMessage(String message) {
+        var data = mapper.readValue(message, WebSocketMessage.class);
+        websocketTemplate.convertAndSend(data.getTopic(), data.getMessage());
+    }
+
+}
@@ -0,0 +1,25 @@
+package com.iqser.red.service.redaction.v1.server.service.websocket;
+
+import org.springframework.data.redis.core.StringRedisTemplate;
+import org.springframework.stereotype.Service;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+
+import lombok.RequiredArgsConstructor;
+import lombok.SneakyThrows;
+
+@Service
+@RequiredArgsConstructor
+public class RedisSyncedWebSocketService {
+
+    private final StringRedisTemplate redisTemplate;
+    private final ObjectMapper mapper;
+
+    @SneakyThrows
+    public void convertAndSend(String topic, Object message) {
+        var socketData = new WebSocketMessage(topic, mapper.writeValueAsString(message));
+        String data = mapper.writeValueAsString(socketData);
+        redisTemplate.convertAndSend("redaction-service-websocket-messages", data);
+    }
+
+}
@@ -0,0 +1,17 @@
+package com.iqser.red.service.redaction.v1.server.service.websocket;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
+public class WebSocketMessage {
+
+    private String topic;
+    private String message;
+
+}
@@ -0,0 +1,25 @@
+package com.iqser.red.service.redaction.v1.server.service.websocket;
+
+import org.springframework.stereotype.Service;
+
+import com.iqser.red.service.redaction.v1.server.logger.RuleLogEvent;
+
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+@Service
+@RequiredArgsConstructor
+public class WebSocketService {
+
+    private final RedisSyncedWebSocketService template;
+
+
+    public void sendLogEvent(RuleLogEvent ruleLogEvent) {
+
+        String destination = "/topic/" + ruleLogEvent.getTenantId() + "/rule-log-events";
+        log.debug("Sending message to url {}", destination);
+        template.convertAndSend(destination, ruleLogEvent);
+    }
+
+}
@@ -0,0 +1,236 @@
+package com.iqser.red.service.redaction.v1.server.storage;
+
+import java.io.BufferedInputStream;
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.nio.file.StandardOpenOption;
+import java.util.Arrays;
+import java.util.stream.Collectors;
+
+import org.springframework.stereotype.Service;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.primitives.Floats;
+import com.google.protobuf.Message;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
+import com.iqser.red.storage.commons.service.StorageService;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPage;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPageProto;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPositionData;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPositionDataProto;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentStructure;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentStructureProto;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentTextData;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentTextDataProto;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.EntryDataProto;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.NodeTypeProto;
+import com.knecon.fforesight.tenantcommons.TenantContext;
+
+import lombok.RequiredArgsConstructor;
+import lombok.SneakyThrows;
+import lombok.extern.slf4j.Slf4j;
+
+@Service
+@Slf4j
+@RequiredArgsConstructor
+public class DocumentDataFallbackService {
+
+    private final ObjectMapper objectMapper;
+
+    private final StorageService storageService;
+
+
+    private <T> T getOldData(String dossierId, String fileId, FileType fileType, Class<T> valueType) {
+
+        String oldStorageId = RedactionStorageService.StorageIdUtils.getStorageId(dossierId, fileId, fileType.name(), ".json");
+        try (InputStream inputStream = getObject(TenantContext.getTenantId(), oldStorageId)) {
+            return objectMapper.readValue(inputStream, valueType);
+        } catch (IOException e) {
+            log.error("Could not read JSON for " + fileType.name() + ", error was: " + e);
+            return null;
+        }
+    }
+
+
+    private void storeNewData(String dossierId, String fileId, FileType fileType, Message protoObject) {
+
+        String newStorageId = RedactionStorageService.StorageIdUtils.getStorageId(dossierId, fileId, fileType);
+        storageService.storeProtoObject(TenantContext.getTenantId(), newStorageId, protoObject);
+    }
+
+
+    public DocumentStructureProto.DocumentStructure readOldDocumentStructureAndConvert(String dossierId, String fileId) {
+
+        DocumentStructure oldDocumentStructure = getOldData(dossierId, fileId, FileType.DOCUMENT_STRUCTURE, DocumentStructure.class);
+        if (oldDocumentStructure == null) {
+            return null;
+        }
+        DocumentStructureProto.DocumentStructure documentStructure = convertDocumentStructure(oldDocumentStructure);
+        //storeNewData(dossierId, fileId, FileType.DOCUMENT_STRUCTURE, documentStructure);
+        return documentStructure;
+    }
+
+
+    public DocumentPageProto.AllDocumentPages readOldDocumentPagesAndConvert(String dossierId, String fileId) {
+
+        DocumentPage[] oldDocumentPages = getOldData(dossierId, fileId, FileType.DOCUMENT_PAGES, DocumentPage[].class);
+        if (oldDocumentPages == null) {
+            return null;
+        }
+        DocumentPageProto.AllDocumentPages allDocumentPages = convertAllDocumentPages(oldDocumentPages);
+        //storeNewData(dossierId, fileId, FileType.DOCUMENT_PAGES, allDocumentPages);
+        return allDocumentPages;
+    }
+
+
+    public DocumentPositionDataProto.AllDocumentPositionData readOldDocumentPositionDataAndConvert(String dossierId, String fileId) {
+
+        DocumentPositionData[] oldDocumentPositionData = getOldData(dossierId, fileId, FileType.DOCUMENT_POSITION, DocumentPositionData[].class);
+        if (oldDocumentPositionData == null) {
+            return null;
+        }
+        DocumentPositionDataProto.AllDocumentPositionData allDocumentPositionData = convertAllDocumentPositionData(oldDocumentPositionData);
+        //storeNewData(dossierId, fileId, FileType.DOCUMENT_POSITION, allDocumentPositionData);
+        return allDocumentPositionData;
+    }
+
+
+    public DocumentTextDataProto.AllDocumentTextData readOldDocumentTextDataAndConvert(String dossierId, String fileId) {
+
+        DocumentTextData[] oldDocumentTextData = getOldData(dossierId, fileId, FileType.DOCUMENT_TEXT, DocumentTextData[].class);
+        if (oldDocumentTextData == null) {
+            return null;
+        }
+        DocumentTextDataProto.AllDocumentTextData allDocumentTextData = convertAllDocumentTextData(oldDocumentTextData);
+        //storeNewData(dossierId, fileId, FileType.DOCUMENT_TEXT, allDocumentTextData);
+        return allDocumentTextData;
+    }
+
+
+    private static EntryDataProto.EntryData convertEntryData(DocumentStructure.EntryData oldEntryData) {
+
+        EntryDataProto.EntryData.Builder builder = EntryDataProto.EntryData.newBuilder();
+
+        builder.setType(NodeTypeProto.NodeType.valueOf(oldEntryData.getType().name()));
+        builder.addAllTreeId(Arrays.stream(oldEntryData.getTreeId()).boxed()
+                                     .collect(Collectors.toList()));
+        builder.addAllAtomicBlockIds(Arrays.asList(oldEntryData.getAtomicBlockIds()));
+        builder.addAllPageNumbers(Arrays.asList(oldEntryData.getPageNumbers()));
+
+        builder.putAllProperties(oldEntryData.getProperties());
+
+        if (oldEntryData.getChildren() != null) {
+            oldEntryData.getChildren()
+                    .forEach(child -> builder.addChildren(convertEntryData(child)));
+        }
+
+        return builder.build();
+    }
+
+
+    private static DocumentStructureProto.DocumentStructure convertDocumentStructure(DocumentStructure oldStructure) {
+
+        DocumentStructureProto.DocumentStructure.Builder newBuilder = DocumentStructureProto.DocumentStructure.newBuilder();
+
+        if (oldStructure.getRoot() != null) {
+            newBuilder.setRoot(convertEntryData(oldStructure.getRoot()));
+        }
+
+        return newBuilder.build();
+    }
+
+
+    private static DocumentPageProto.DocumentPage convertDocumentPage(DocumentPage oldPage) {
+
+        return DocumentPageProto.DocumentPage.newBuilder()
+                .setNumber(oldPage.getNumber())
+                .setHeight(oldPage.getHeight())
+                .setWidth(oldPage.getWidth())
+                .setRotation(oldPage.getRotation())
+                .build();
+    }
+
+
+    private static DocumentPageProto.AllDocumentPages convertAllDocumentPages(DocumentPage[] oldPages) {
+
+        DocumentPageProto.AllDocumentPages.Builder allPagesBuilder = DocumentPageProto.AllDocumentPages.newBuilder();
+
+        for (DocumentPage oldPage : oldPages) {
+            DocumentPageProto.DocumentPage newPage = convertDocumentPage(oldPage);
+            allPagesBuilder.addDocumentPages(newPage);
+        }
+
+        return allPagesBuilder.build();
+    }
+
+
+    private static DocumentPositionDataProto.DocumentPositionData convertDocumentPositionData(DocumentPositionData oldData) {
+
+        DocumentPositionDataProto.DocumentPositionData.Builder builder = DocumentPositionDataProto.DocumentPositionData.newBuilder()
+                .setId(oldData.getId())
+                .addAllStringIdxToPositionIdx(Arrays.stream(oldData.getStringIdxToPositionIdx()).boxed()
+                                                      .collect(Collectors.toList()));
+
+        for (float[] pos : oldData.getPositions()) {
+            DocumentPositionDataProto.DocumentPositionData.Position position = DocumentPositionDataProto.DocumentPositionData.Position.newBuilder()
+                    .addAllValue(Floats.asList(pos))
+                    .build();
+            builder.addPositions(position);
+        }
+
+        return builder.build();
+    }
+
+
+    private static DocumentPositionDataProto.AllDocumentPositionData convertAllDocumentPositionData(DocumentPositionData[] oldDataList) {
+
+        DocumentPositionDataProto.AllDocumentPositionData.Builder allDataBuilder = DocumentPositionDataProto.AllDocumentPositionData.newBuilder();
+
+        for (DocumentPositionData oldData : oldDataList) {
+            allDataBuilder.addDocumentPositionData(convertDocumentPositionData(oldData));
+        }
+
+        return allDataBuilder.build();
+    }
+
+
+    private static DocumentTextDataProto.DocumentTextData convertDocumentTextData(DocumentTextData oldData) {
+
+        DocumentTextDataProto.DocumentTextData.Builder builder = DocumentTextDataProto.DocumentTextData.newBuilder()
+                .setId(oldData.getId())
+                .setPage(oldData.getPage())
+                .setSearchText(oldData.getSearchText())
+                .setNumberOnPage(oldData.getNumberOnPage())
+                .setStart(oldData.getStart())
+                .setEnd(oldData.getEnd())
+                .addAllLineBreaks(Arrays.stream(oldData.getLineBreaks()).boxed()
+                                          .collect(Collectors.toList()));
+
+        return builder.build();
+    }
+
+
+    private static DocumentTextDataProto.AllDocumentTextData convertAllDocumentTextData(DocumentTextData[] oldDataList) {
+
+        DocumentTextDataProto.AllDocumentTextData.Builder allDataBuilder = DocumentTextDataProto.AllDocumentTextData.newBuilder();
+
+        for (DocumentTextData oldData : oldDataList) {
+            allDataBuilder.addDocumentTextData(convertDocumentTextData(oldData));
+        }
+
+        return allDataBuilder.build();
+    }
+
+
+    @SneakyThrows
+    private InputStream getObject(String tenantId, String storageId) {
+
+        File tempFile = File.createTempFile("temp", ".data");
+        storageService.downloadTo(tenantId, storageId, tempFile);
+        return new BufferedInputStream(Files.newInputStream(Paths.get(tempFile.getPath()), StandardOpenOption.DELETE_ON_CLOSE));
+    }
+
+}
@@ -1,33 +1,47 @@
 package com.iqser.red.service.redaction.v1.server.storage;

+import static com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPositionDataProto.AllDocumentPositionData;
+import static org.springframework.boot.autoconfigure.task.TaskExecutionAutoConfiguration.APPLICATION_TASK_EXECUTOR_BEAN_NAME;
+
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.InputStream;
 import java.util.Collection;
 import java.util.List;
 import java.util.Set;
+import java.util.concurrent.CompletableFuture;
+import java.util.function.Supplier;
 import java.util.stream.Collectors;

+import org.springframework.beans.factory.annotation.Qualifier;
 import org.springframework.cache.annotation.Cacheable;
+import org.springframework.core.task.TaskExecutor;
 import org.springframework.stereotype.Service;

 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentLog;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedLegalBases;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedactions;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedactionsPerPage;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog;
+import com.iqser.red.service.persistence.service.v1.api.shared.mongo.service.ComponentLogMongoService;
 import com.iqser.red.service.persistence.service.v1.api.shared.mongo.service.EntityLogMongoService;
+import com.iqser.red.service.redaction.v1.server.RedactionServiceSettings;
 import com.iqser.red.service.redaction.v1.server.client.model.NerEntitiesModel;
 import com.iqser.red.service.redaction.v1.server.model.document.DocumentData;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
 import com.iqser.red.service.redaction.v1.server.utils.exception.NotFoundException;
+import com.iqser.red.storage.commons.exception.StorageException;
 import com.iqser.red.storage.commons.exception.StorageObjectDoesNotExist;
 import com.iqser.red.storage.commons.service.StorageService;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPage;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPositionData;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentStructure;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentTextData;
+import com.knecon.fforesight.llm.service.LlmNerEntities;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPageProto;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPageProto.AllDocumentPages;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPositionDataProto;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentStructureProto.DocumentStructure;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentStructureWrapper;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentTextDataProto;
+import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentTextDataProto.AllDocumentTextData;
 import com.knecon.fforesight.tenantcommons.TenantContext;

 import io.micrometer.core.annotation.Timed;
@@ -38,12 +52,33 @@ import lombok.extern.slf4j.Slf4j;

@Slf4j
@Service
-@RequiredArgsConstructor
 public class RedactionStorageService {

+    private final RedactionServiceSettings settings;
    private final StorageService storageService;
-
    private final EntityLogMongoService entityLogMongoService;
+    private final DocumentDataFallbackService documentDataFallbackService;
+
+    private final ComponentLogMongoService componentLogMongoService;
+
+    private final TaskExecutor taskExecutor;
+
+
+    public RedactionStorageService(RedactionServiceSettings settings,
+                                   StorageService storageService,
+                                   EntityLogMongoService entityLogMongoService,
+                                   ComponentLogMongoService componentLogMongoService,
+                                   DocumentDataFallbackService documentDataFallbackService,
+                                   @Qualifier(APPLICATION_TASK_EXECUTOR_BEAN_NAME) TaskExecutor taskExecutor) {
+
+        this.settings = settings;
+
+        this.storageService = storageService;
+        this.entityLogMongoService = entityLogMongoService;
+        this.componentLogMongoService = componentLogMongoService;
+        this.documentDataFallbackService = documentDataFallbackService;
+        this.taskExecutor = taskExecutor;
+    }


    @SneakyThrows
@@ -102,6 +137,13 @@ public class RedactionStorageService {
    }


+
+    @SneakyThrows
+    public void saveComponentLog(String dossierId, String fileId, ComponentLog componentLog) {
+
+        componentLogMongoService.saveComponentLog(dossierId, fileId, componentLog);
+    }
+
    @SneakyThrows
    public void updateEntityLogEntries(String dossierId, String fileId, List<EntityLogEntry> entityLogEntries) {

@@ -147,24 +189,17 @@ public class RedactionStorageService {
    }


-    @Deprecated(forRemoval = true)
-    @Timed("redactmanager_getRedactionLog")
-    public RedactionLog getRedactionLog(String dossierId, String fileId) {
+    @Timed("redactmanager_getImportedLegalBases")
+    public ImportedLegalBases getImportedLegalBases(String dossierId, String fileId) {

        try {
-            RedactionLog redactionLog = storageService.readJSONObject(TenantContext.getTenantId(),
-                                                                      StorageIdUtils.getStorageId(dossierId, fileId, FileType.REDACTION_LOG),
-                                                                      RedactionLog.class);
-            redactionLog.setRedactionLogEntry(redactionLog.getRedactionLogEntry()
-                                                      .stream()
-                                                      .filter(entry -> !(entry.getPositions() == null || entry.getPositions().isEmpty()))
-                                                      .collect(Collectors.toList()));
-            return redactionLog;
+            return storageService.readJSONObject(TenantContext.getTenantId(),
+                                                 StorageIdUtils.getStorageId(dossierId, fileId, FileType.IMPORTED_LEGAL_BASES),
+                                                 ImportedLegalBases.class);
        } catch (StorageObjectDoesNotExist e) {
-            log.debug("RedactionLog not available.");
-            return null;
+            log.debug("Imported legal bases not available.");
+            return new ImportedLegalBases();
        }
-
    }


@@ -201,6 +236,17 @@ public class RedactionStorageService {
    }


+    public ComponentLog getComponentLog(String dossierId, String fileId) {
+
+        try {
+            return componentLogMongoService.findComponentLogByDossierIdAndFileId(dossierId, fileId)
+                    .orElseThrow(() -> new StorageObjectDoesNotExist(""));
+        } catch (StorageObjectDoesNotExist e) {
+            throw new NotFoundException("Component is not available.");
+        }
+    }
+
+
    public Set<Integer> findIdsOfSectionsToReanalyse(String dossierId, String fileId, Collection<String> entryIds) {

        return entityLogMongoService.findFirstContainingNodeIdForEachEntry(dossierId, fileId, entryIds);
@@ -213,7 +259,6 @@ public class RedactionStorageService {
    }


-
    // !Warning! before activating redis cache you need to set
    // -Dio.netty.noPreferDirect=true -XX:MaxDirectMemorySize=1000M
    // Jvm args to the largest document data size we want to process. for 4443 pages file that was 500mb.
@@ -224,27 +269,97 @@ public class RedactionStorageService {
    public DocumentData getDocumentData(String dossierId, String fileId) {

        try {
+            CompletableFuture<DocumentStructure> documentStructureFuture = fetchDocumentStructure(dossierId, fileId);
+            CompletableFuture<AllDocumentTextData> documentTextDataFuture = fetchDocumentTextData(dossierId, fileId);
+            CompletableFuture<AllDocumentPositionData> documentPositionDataFuture = fetchDocumentPositionData(dossierId, fileId);
+            CompletableFuture<AllDocumentPages> documentPagesFuture = fetchDocumentPages(dossierId, fileId);
+
+            CompletableFuture.allOf(documentStructureFuture, documentTextDataFuture, documentPositionDataFuture, documentPagesFuture).join();
+
            return DocumentData.builder()
-                    .documentStructure(storageService.readJSONObject(TenantContext.getTenantId(),
-                                                                     StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_STRUCTURE),
-                                                                     DocumentStructure.class))
-                    .documentTextData(storageService.readJSONObject(TenantContext.getTenantId(),
-                                                                    StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_TEXT),
-                                                                    DocumentTextData[].class))
-                    .documentPositionData(storageService.readJSONObject(TenantContext.getTenantId(),
-                                                                        StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_POSITION),
-                                                                        DocumentPositionData[].class))
-                    .documentPages(storageService.readJSONObject(TenantContext.getTenantId(),
-                                                                 StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_PAGES),
-                                                                 DocumentPage[].class))
+                    .documentStructureWrapper(new DocumentStructureWrapper(documentStructureFuture.get()))
+                    .documentTextData(documentTextDataFuture.get())
+                    .documentPositionData(documentPositionDataFuture.get())
+                    .documentPages(documentPagesFuture.get())
                    .build();
        } catch (StorageObjectDoesNotExist e) {
            log.debug("DocumentData not available.");
            return null;
+        } catch (Exception e) {
+            log.error("An error occurred while fetching document data", e);
+            throw new RuntimeException(e);
        }
    }


+    private CompletableFuture<DocumentStructure> fetchDocumentStructure(String dossierId, String fileId) {
+
+        Supplier<DocumentStructure> documentStructureSupplier = () -> {
+            DocumentStructure documentStructure = null;
+            String storageId = StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_STRUCTURE);
+            if (storageService.objectExists(TenantContext.getTenantId(), storageId)) {
+                documentStructure = storageService.readProtoObject(TenantContext.getTenantId(), storageId, DocumentStructure.parser());
+            }
+            if (documentStructure == null && settings.isProtobufJsonFallback()) {
+                documentStructure = documentDataFallbackService.readOldDocumentStructureAndConvert(dossierId, fileId);
+            }
+            return documentStructure;
+        };
+        return CompletableFuture.supplyAsync(documentStructureSupplier, taskExecutor);
+    }
+
+
+    private CompletableFuture<DocumentTextDataProto.AllDocumentTextData> fetchDocumentTextData(String dossierId, String fileId) {
+
+        Supplier<DocumentTextDataProto.AllDocumentTextData> documentTextDataSupplier = () -> {
+            DocumentTextDataProto.AllDocumentTextData documentTextData = null;
+            String storageId = StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_TEXT);
+            if (storageService.objectExists(TenantContext.getTenantId(), storageId)) {
+                documentTextData = storageService.readProtoObject(TenantContext.getTenantId(), storageId, DocumentTextDataProto.AllDocumentTextData.parser());
+            }
+            if (documentTextData == null && settings.isProtobufJsonFallback()) {
+                documentTextData = documentDataFallbackService.readOldDocumentTextDataAndConvert(dossierId, fileId);
+            }
+            return documentTextData;
+        };
+        return CompletableFuture.supplyAsync(documentTextDataSupplier, taskExecutor);
+    }
+
+
+    private CompletableFuture<DocumentPositionDataProto.AllDocumentPositionData> fetchDocumentPositionData(String dossierId, String fileId) {
+
+        Supplier<DocumentPositionDataProto.AllDocumentPositionData> documentPositionDataSupplier = () -> {
+            DocumentPositionDataProto.AllDocumentPositionData documentPositionData = null;
+            String storageId = StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_POSITION);
+            if (storageService.objectExists(TenantContext.getTenantId(), storageId)) {
+                documentPositionData = storageService.readProtoObject(TenantContext.getTenantId(), storageId, DocumentPositionDataProto.AllDocumentPositionData.parser());
+            }
+            if (documentPositionData == null && settings.isProtobufJsonFallback()) {
+                documentPositionData = documentDataFallbackService.readOldDocumentPositionDataAndConvert(dossierId, fileId);
+            }
+            return documentPositionData;
+        };
+        return CompletableFuture.supplyAsync(documentPositionDataSupplier, taskExecutor);
+    }
+
+
+    private CompletableFuture<DocumentPageProto.AllDocumentPages> fetchDocumentPages(String dossierId, String fileId) {
+
+        Supplier<DocumentPageProto.AllDocumentPages> documentPagesSupplier = () -> {
+            DocumentPageProto.AllDocumentPages documentPages = null;
+            String storageId = StorageIdUtils.getStorageId(dossierId, fileId, FileType.DOCUMENT_PAGES);
+            if (storageService.objectExists(TenantContext.getTenantId(), storageId)) {
+                documentPages = storageService.readProtoObject(TenantContext.getTenantId(), storageId, AllDocumentPages.parser());
+            }
+            if (documentPages == null && settings.isProtobufJsonFallback()) {
+                documentPages = documentDataFallbackService.readOldDocumentPagesAndConvert(dossierId, fileId);
+            }
+            return documentPages;
+        };
+        return CompletableFuture.supplyAsync(documentPagesSupplier, taskExecutor);
+    }
+
+
    @Timed("redactmanager_getNerEntities")
    public NerEntitiesModel getNerEntities(String dossierId, String fileId) {

@@ -256,12 +371,19 @@ public class RedactionStorageService {
    }


-    public ComponentLog getComponentLog(String dossierId, String fileId) {
+    public LlmNerEntities getLlmNerEntities(String dossierId, String fileId) {
+
+        return storageService.readJSONObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.LLM_NER_ENTITIES), LlmNerEntities.class);
+    }
+
+
+    @Timed("redactmanager_getAzureNerEntities")
+    public NerEntitiesModel getAzureNerEntities(String dossierId, String fileId) {

        try {
-            return storageService.readJSONObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.COMPONENT_LOG), ComponentLog.class);
+            return storageService.readJSONObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.AZURE_NER_ENTITIES), NerEntitiesModel.class);
        } catch (StorageObjectDoesNotExist e) {
-            throw new NotFoundException("Component Log is not available.");
+            throw new NotFoundException("NER Entities are not available.");
        }
    }

@@ -288,6 +410,12 @@ public class RedactionStorageService {
            return dossierId + "/" + fileId + "." + fileType.name() + fileType.getExtension();
        }

+
+        public static String getStorageId(String dossierId, String fileId, String fileName, String fileExtension) {
+
+            return dossierId + "/" + fileId + "." + fileName + fileExtension;
+        }
+
    }

 }
@@ -1,50 +1,64 @@
 package com.iqser.red.service.redaction.v1.server.utils;

-import java.io.BufferedReader;
-import java.io.IOException;
-import java.io.InputStreamReader;
 import java.text.DateFormat;
 import java.text.SimpleDateFormat;
 import java.time.LocalDate;
 import java.time.ZoneId;
 import java.time.format.DateTimeFormatter;
-import java.time.format.DateTimeFormatterBuilder;
 import java.time.format.DateTimeParseException;
-import java.time.format.ResolverStyle;
-import java.time.temporal.ChronoField;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Date;
 import java.util.List;
 import java.util.Locale;
-import java.util.Objects;
 import java.util.Optional;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;

-import lombok.AccessLevel;
-import lombok.experimental.FieldDefaults;
-import lombok.experimental.UtilityClass;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.DateFormatPatternErrorMessage;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.utils.DateTimeFormatterProvider;
+
 import lombok.extern.slf4j.Slf4j;

@Slf4j
-@UtilityClass
-@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
 public class DateConverter {

-    private static DateTimeFormatter DATE_TIME_FORMATTER;
+    private final DateTimeFormatter dateTimeFormatter;
    private static final List<Locale> LOCALES = Arrays.asList(Locale.UK, Locale.US);
-    private static int BASE_YEAR = 1950; // base year 1950 means, that "yy" will be interpreted in range 1950-2049
+
+
+    public DateConverter() {
+
+        List<DateFormatPatternErrorMessage> errors = new ArrayList<>();
+        this.dateTimeFormatter = DateTimeFormatterProvider.createFormatterFromResource("/dateFormats.txt", errors);
+        if (!errors.isEmpty()) {
+            throw new RuntimeException("Errors occurred while loading date formats: " + String.join(", ",
+                                                                                                    errors.stream()
+                                                                                                            .map(DateFormatPatternErrorMessage::getMessage)
+                                                                                                            .toList()));
+        }
+    }
+
+
+    public DateConverter(String dateFormats) {
+
+        List<DateFormatPatternErrorMessage> errors = new ArrayList<>();
+        this.dateTimeFormatter = DateTimeFormatterProvider.createFormatterFromInput(dateFormats, errors);
+        if (!errors.isEmpty()) {
+            throw new RuntimeException("Errors occurred while loading date formats: " + String.join(", ",
+                                                                                                    errors.stream()
+                                                                                                            .map(DateFormatPatternErrorMessage::getMessage)
+                                                                                                            .toList()));
+        }
+    }


    public Optional<Date> parseDate(String dateAsString) {

-        DateTimeFormatter formatter = getDateTimeFormatter();
        String cleanDate = dateAsString.trim();
        cleanDate = removeTrailingDot(cleanDate);

        for (Locale locale : LOCALES) {
            try {
-                return convertToDate(locale, cleanDate, formatter);
+                return convertToDate(locale, cleanDate, this.dateTimeFormatter);
            } catch (DateTimeParseException e) {
                try {
                    Optional<String> extractedDate = DateExtractorNatty.extractDate(cleanDate);
@@ -53,7 +67,7 @@ public class DateConverter {
                        return Optional.empty();
                    } else {
                        cleanDate = extractedDate.get();
-                        return convertToDate(locale, cleanDate, formatter);
+                        return convertToDate(locale, cleanDate, this.dateTimeFormatter);
                    }
                } catch (DateTimeParseException exception) {
                    log.debug("Failed to parse date: {} with locale: {}", cleanDate, locale);
@@ -63,7 +77,6 @@ public class DateConverter {

        log.warn("Failed to parse date: {}", cleanDate);
        return Optional.empty();
-
    }


@@ -82,72 +95,12 @@ public class DateConverter {
    }


-    private DateTimeFormatter getDateTimeFormatter() {
-
-        if (DATE_TIME_FORMATTER == null) {
-            DATE_TIME_FORMATTER = createFormatterFromResource();
-        }
-        return DATE_TIME_FORMATTER;
-    }
-
-
-    private DateTimeFormatter createFormatterFromResource() {
-
-        DateTimeFormatterBuilder builder = new DateTimeFormatterBuilder();
-        builder.parseCaseInsensitive();
-        try (BufferedReader reader = new BufferedReader(new InputStreamReader(Objects.requireNonNull(DateConverter.class.getResourceAsStream("/date_formats.txt"))))) {
-            String line;
-            while ((line = reader.readLine()) != null) {
-                String pattern = line.trim();
-                if (!pattern.isEmpty()) {
-                    if (hasTwoDigitsForYear(pattern)) {
-                        builder.appendOptional(setBaseYear(pattern));
-                    } else {
-                        builder.appendOptional(DateTimeFormatter.ofPattern(pattern, Locale.UK));
-                    }
-                }
-            }
-        } catch (IOException e) {
-            throw new RuntimeException("Error reading date format file: " + e.getMessage(), e);
-        }
-        return builder.toFormatter().withResolverStyle(ResolverStyle.SMART).withLocale(Locale.UK);
-    }
-
-
-    private boolean hasTwoDigitsForYear(String input) {
-        // Regex to match any string with exactly two 'y' characters
-        Pattern pattern = Pattern.compile("^[^y]*(y[^y]*){2}$");
-        Matcher matcher = pattern.matcher(input);
-
-        return matcher.matches();
-
-    }
-
-
-    private DateTimeFormatter setBaseYear(String pattern) {
-
-        DateTimeFormatterBuilder builder = new DateTimeFormatterBuilder();
-        if (pattern.startsWith("yy")) {
-            String editedPattern = pattern.substring(2);
-            builder.appendValueReduced(ChronoField.YEAR_OF_ERA, 2, 2, BASE_YEAR).appendPattern(editedPattern).toFormatter();
-        } else if (pattern.endsWith("yy")) {
-            String editedPattern = pattern.substring(0, pattern.length() - 2);
-            builder.appendPattern(editedPattern).appendValueReduced(ChronoField.YEAR_OF_ERA, 2, 2, BASE_YEAR).toFormatter();
-        } else {
-            throw new RuntimeException("Date format not supported: " + pattern);
-        }
-        return builder.toFormatter();
-    }
-
-
    private String removeTrailingDot(String dateAsString) {

-        String str = dateAsString;
-        if (str != null && !str.isEmpty() && str.charAt(str.length() - 1) == '.') {
-            str = str.substring(0, str.length() - 1);
+        if (dateAsString != null && !dateAsString.isEmpty() && dateAsString.charAt(dateAsString.length() - 1) == '.') {
+            return dateAsString.substring(0, dateAsString.length() - 1);
        }
-
-        return str;
+        return dateAsString;
    }

 }
@@ -0,0 +1,45 @@
+package com.iqser.red.service.redaction.v1.server.utils;
+
+import java.lang.reflect.Field;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Objects;
+
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.PropertyChange;
+
+import lombok.experimental.UtilityClass;
+import lombok.extern.slf4j.Slf4j;
+
+@UtilityClass
+@Slf4j
+@SuppressWarnings("PMD") // Needed in order to be able to do setAccessible(true) for field comparison
+public class EntityLogEntryDiffChecker {
+
+    public List<PropertyChange> compareEntityLogEntries(EntityLogEntry previousEntityLogEntry, EntityLogEntry currentEntityLogEntry) {
+
+        List<PropertyChange> changes = new ArrayList<>();
+        Field[] fields = EntityLogEntry.class.getDeclaredFields();
+
+        for (Field field : fields) {
+            field.setAccessible(true);
+            try {
+                Object oldValue = field.get(previousEntityLogEntry);
+                Object newValue = field.get(currentEntityLogEntry);
+
+                if (!Objects.equals(oldValue, newValue)) {
+                    changes.add(PropertyChange.builder()
+                                        .property(field.getName())
+                                        .oldValue(oldValue != null ? oldValue.toString() : "")
+                                        .newValue(newValue != null ? newValue.toString() : "")
+                                        .build());
+                }
+            } catch (IllegalAccessException e) {
+                log.warn("Failed to access field: {}", field.getName());
+            }
+        }
+
+        return changes;
+    }
+
+}
@@ -0,0 +1,54 @@
+package com.iqser.red.service.redaction.v1.server.utils;
+
+import java.time.Duration;
+import java.time.OffsetDateTime;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Objects;
+
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualChange;
+
+import lombok.experimental.UtilityClass;
+import lombok.extern.slf4j.Slf4j;
+
+@UtilityClass
+@Slf4j
+public class ManualChangesUtils {
+
+    // there is a very slight variation of time when comparing the manual changes request Date, under 1ms.
+    public static final Duration TOLERANCE = Duration.ofMillis(1);
+
+
+    public List<ManualChange> mergeManualChanges(List<ManualChange> existingChanges, List<ManualChange> allChanges) {
+
+        List<ManualChange> mergedChanges = new ArrayList<>(existingChanges);
+
+        for (ManualChange manualChange : allChanges) {
+
+            var existingChangeOpt = mergedChanges.stream()
+                    .filter(existingChange -> areChangesEquivalent(existingChange, manualChange))
+                    .findFirst();
+
+            if (existingChangeOpt.isEmpty()) {
+                mergedChanges.add(manualChange);
+            }
+        }
+
+        return mergedChanges;
+    }
+
+
+    private boolean areChangesEquivalent(ManualChange change1, ManualChange change2) {
+
+        return change1.getManualRedactionType() == change2.getManualRedactionType() && areDatesEquivalent(change1.getRequestedDate(), change2.getRequestedDate()) && Objects.equals(
+                change1.getUserId(),
+                change2.getUserId());
+    }
+
+
+    private boolean areDatesEquivalent(OffsetDateTime date1, OffsetDateTime date2) {
+
+        return Duration.between(date1, date2).abs().compareTo(TOLERANCE) <= 0;
+    }
+
+}
@@ -1,55 +0,0 @@
-package com.iqser.red.service.redaction.v1.server.utils;
-
-import java.util.Collections;
-import java.util.LinkedList;
-import java.util.Set;
-import java.util.function.BiConsumer;
-import java.util.function.BinaryOperator;
-import java.util.function.Function;
-import java.util.function.Supplier;
-import java.util.stream.Collector;
-
-import com.google.common.base.Functions;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.migration.MigratedIds;
-import com.iqser.red.service.redaction.v1.server.model.MigrationEntity;
-
-public class MigratedIdsCollector implements Collector<MigrationEntity, MigratedIds, MigratedIds> {
-
-    @Override
-    public Supplier<MigratedIds> supplier() {
-
-        return () -> new MigratedIds(new LinkedList<>(), Collections.emptyList());
-    }
-
-
-    @Override
-    public BiConsumer<MigratedIds, MigrationEntity> accumulator() {
-
-        return (migratedIds, migrationEntity) -> migratedIds.addMapping(migrationEntity.getOldId(), migrationEntity.getNewId());
-    }
-
-
-    @Override
-    public BinaryOperator<MigratedIds> combiner() {
-
-        return (migratedIds, migratedIds2) -> {
-            migratedIds.getMappings().addAll(migratedIds2.getMappings());
-            return migratedIds;
-        };
-    }
-
-
-    @Override
-    public Function<MigratedIds, MigratedIds> finisher() {
-
-        return Functions.identity();
-    }
-
-
-    @Override
-    public Set<Characteristics> characteristics() {
-
-        return Set.of(Characteristics.IDENTITY_FINISH, Characteristics.CONCURRENT, Characteristics.UNORDERED);
-    }
-
-}
@@ -3,12 +3,17 @@ package com.iqser.red.service.redaction.v1.server.utils;
 import static java.lang.String.format;

 import java.awt.geom.Rectangle2D;
+import java.awt.geom.RectangularShape;
 import java.util.LinkedList;
 import java.util.List;
+import java.util.function.Predicate;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 import java.util.stream.IntStream;

+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
 import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
@@ -18,6 +23,9 @@ import lombok.experimental.UtilityClass;
@UtilityClass
 public class RedactionSearchUtility {

+    private static final Logger log = LoggerFactory.getLogger(RedactionSearchUtility.class);
+
+
    /**
     * Checks if any part of a CharSequence matches a given regex pattern.
     *
@@ -154,14 +162,8 @@ public class RedactionSearchUtility {
     */
    public static TextRange findTextRangesOfAllLinesInYRange(double maxY, double minY, TextBlock textBlock) {

-        List<TextRange> lineBoundaries = IntStream.range(0, textBlock.numberOfLines()).boxed()
-                .map(textBlock::getLineTextRange)
-                .filter(lineBoundary -> isWithinYRange(maxY, minY, textBlock, lineBoundary))
-                .toList();
-        if (lineBoundaries.isEmpty()) {
-            return new TextRange(textBlock.getTextRange().start(), textBlock.getTextRange().start());
-        }
-        return TextRange.merge(lineBoundaries);
+        Predicate<TextRange> isWithinYRange = lineBoundary -> isWithinYRange(maxY, minY, textBlock, lineBoundary);
+        return filterLineBoundaries(textBlock, isWithinYRange);
    }


@@ -172,6 +174,49 @@ public class RedactionSearchUtility {
    }


+    /**
+     * Identifies all lines within a text block that have roughly the same vertical coordinates.
+     *
+     * @param maxY      The maximum Y-coordinate of the vertical range.
+     * @param minY      The minimum Y-coordinate of the vertical range.
+     * @param textBlock The text block containing the lines to be checked.
+     * @return A {@link TextRange} encompassing all lines within the specified Y-coordinate range.
+     */
+    public static TextRange findTextRangesOfAllLinesWithCloseYCoordinates(Double maxY, Double minY, TextBlock textBlock) {
+
+        double averageLineHeight = IntStream.range(0, textBlock.numberOfLines()).boxed()
+                .map(textBlock::getLineTextRange)
+                .flatMap((TextRange stringTextRange) -> textBlock.getPositions(stringTextRange)
+                        .stream())
+                .map(RectangularShape::getHeight)
+                .mapToDouble(Double::doubleValue).average()
+                .orElse(0);
+        Predicate<TextRange> hasCloseYRange = lineBoundary -> areYCoordinatesClose(maxY, minY, textBlock, lineBoundary, averageLineHeight);
+
+        return filterLineBoundaries(textBlock, hasCloseYRange);
+    }
+
+
+    private static boolean areYCoordinatesClose(Double maxY, Double minY, TextBlock textBlock, TextRange lineTextRange, double averageLineHeight) {
+
+        Rectangle2D lineBBox = RectangleTransformations.rectangle2DBBox(textBlock.getPositions(lineTextRange));
+        return Math.abs(lineBBox.getMinY() - minY) <= averageLineHeight && Math.abs(maxY - lineBBox.getMaxY()) <= averageLineHeight;
+    }
+
+
+    private static TextRange filterLineBoundaries(TextBlock textBlock, Predicate<TextRange> textRangePredicate) {
+
+        List<TextRange> lineBoundaries = IntStream.range(0, textBlock.numberOfLines()).boxed()
+                .map(textBlock::getLineTextRange)
+                .filter(textRangePredicate)
+                .toList();
+        if (lineBoundaries.isEmpty()) {
+            return new TextRange(textBlock.getTextRange().start(), textBlock.getTextRange().start());
+        }
+        return TextRange.merge(lineBoundaries);
+    }
+
+
    /**
     * Finds TextRanges matching a regex pattern within a TextBlock.
     *
@@ -264,8 +309,12 @@ public class RedactionSearchUtility {

        Matcher matcher = pattern.matcher(textBlock.subSequence(textBlock.getTextRange()));
        List<TextRange> boundaries = new LinkedList<>();
-        while (matcher.find()) {
-            boundaries.add(new TextRange(matcher.start(group) + textBlock.getTextRange().start(), matcher.end(group) + textBlock.getTextRange().start()));
+        try {
+            while (matcher.find()) {
+                boundaries.add(new TextRange(matcher.start(group) + textBlock.getTextRange().start(), matcher.end(group) + textBlock.getTextRange().start()));
+            }
+        }catch (StackOverflowError stackOverflowError){
+            log.warn("Stackoverflow error for pattern {} in text: {}", pattern.pattern(), textBlock);
        }
        return boundaries;
    }
@@ -273,11 +322,16 @@ public class RedactionSearchUtility {

    private static List<TextRange> getTextRangesByPatternWithLineBreaks(TextBlock textBlock, int group, Pattern pattern) {

+
        String searchTextWithLineBreaks = textBlock.searchTextWithLineBreaks();
        Matcher matcher = pattern.matcher(searchTextWithLineBreaks);
        List<TextRange> boundaries = new LinkedList<>();
-        while (matcher.find()) {
-            boundaries.add(new TextRange(matcher.start(group) + textBlock.getTextRange().start(), matcher.end(group) + textBlock.getTextRange().start()));
+        try {
+            while (matcher.find()) {
+                boundaries.add(new TextRange(matcher.start(group) + textBlock.getTextRange().start(), matcher.end(group) + textBlock.getTextRange().start()));
+            }
+        }catch (StackOverflowError stackOverflowError){
+            log.warn("Stackoverflow error for pattern {} in text with linebreaks: {}", pattern.pattern(), searchTextWithLineBreaks);
        }
        return boundaries;
    }
@@ -1,31 +1,47 @@
 package com.iqser.red.service.redaction.v1.server.utils;

+import java.util.regex.Pattern;
+
 import lombok.experimental.UtilityClass;

@UtilityClass
 public final class TextNormalizationUtilities {

-    /**
-     * Revert hyphenation due to line breaks.
-     *
-     * @param text Text to be processed.
-     * @return Text without line-break hyphenation.
-     */
-    public static String removeHyphenLineBreaks(String text) {
+    public static final Pattern hyphenLineBreaks = Pattern.compile("[-~‐‒⁻−﹣゠⁓‑\\u00AD][\\r\\n]+");
+    public static final Pattern linebreaks = Pattern.compile("[\\r\\n]+");
+    public static final Pattern doubleWhitespaces = Pattern.compile("\\s{2,}");
+    public static final Pattern WHITESPACE_REMOVAL = Pattern.compile("\\s+");

-        return text.replaceAll("([^\\s\\d\\-]{2,500})[\\-\\u00AD]\\R", "$1");
+
+    public String cleanString(String value) {
+
+        String noHyphenLinebreaks = removeHyphenLinebreaks(value);
+        String noLinebreaks = removeLinebreaks(noHyphenLinebreaks);
+        return removeMultipleWhitespaces(noLinebreaks);
    }


-    public static String removeLineBreaks(String text) {
+    public String removeHyphenLinebreaks(String value) {

-        return text.replaceAll("\n", " ");
+        return hyphenLineBreaks.matcher(value).replaceAll("");
    }


-    public static String removeRepeatingWhitespaces(String text) {
+    private String removeMultipleWhitespaces(String value) {

-        return text.replaceAll(" {2}", " ");
+        return doubleWhitespaces.matcher(value).replaceAll(" ");
+    }
+
+
+    private String removeLinebreaks(String value) {
+
+        return linebreaks.matcher(value).replaceAll(" ");
+    }
+
+
+    public String removeAllWhitespaces(String value) {
+
+        return WHITESPACE_REMOVAL.matcher(value).replaceAll("");
    }

 }
@@ -10,6 +10,7 @@ storage:
  key: minioadmin
  secret: minioadmin

+cors.enabled: true

 redaction-service:
  enableImageClassification: false
@@ -16,6 +16,17 @@ project.version: 1.0-SNAPSHOT
 server:
  port: 8080

+lifecycle:
+  base-package: com.iqser.red.service.redaction
+
+fforesight:
+  keycloak:
+    enabled: true
+    ignored-endpoints: [ '/redaction-gateway-v1', '/actuator/health/**',"/api/rules-logging/rulesocket","/api/rules-logging/rulesocket/**", '/redaction-gateway-v1/async/download/with-ott/**',
+                       '/internal-api/**', '/redaction-gateway-v1/docs/swagger-ui', '/rules/test',
+                       '/redaction-gateway-v1/docs/**','/redaction-gateway-v1/docs',
+                       '/api', '/api/','/api/docs/**','/api/docs','/api/docs/swagger-ui' ]
+
 spring:
  application:
    name: redaction-service
@@ -14,6 +14,8 @@ dd MMMM yyyy
 d MMMM yyyy
 MMMM dd, yyyy
 MMMM d, yyyy
+MMMM, d yyyy
+MMMM d,yyyy
 dd.MM.yyyy
 d.MM.yyyy
 yyyy/MM/dd
@@ -28,6 +30,7 @@ dd['.'] MMM yyyy
 d['.'] MMM yyyy
 dd['th']['st']['nd']['rd'] 'of' MMMM, yyyy
 d['th']['st']['nd']['rd'] 'of' MMMM, yyyy
+d['st']['nd']['rd']['th'] MMMM yyyy
 MMMM dd['th']['st']['nd']['rd'], yyyy
 MMMM d['th']['st']['nd']['rd'], yyyy
 yyyy, MMMM dd
@@ -72,3 +75,5 @@ dd.MM.yy
 d.MM.yy
 dd MMM. yyyy
 d MMM. yyyy
+d-MMMM-yyyy
+dd-MMMM-yyyy
--- a/Show More
+++ b/Show More
Author	SHA1	Message	Date
Kilian Schuettler	1db515acf0	extended-entity-finding: increase threshold	2024-10-22 15:09:38 +02:00
Kilian Schuettler	f9e7703063	wip: prototype extended EntityFindingUtility	2024-10-22 15:08:51 +02:00
Kilian Schüttler	62e26add99	Merge branch 'RED-10194' into 'master' RED-10194: add component mapping related classes to javadoc Closes RED-10194 See merge request redactmanager/redaction-service!542	2024-10-16 12:04:36 +02:00
Kilian Schuettler	1293484295	RED-10194: add component mapping related classes to javadoc	2024-10-16 11:38:59 +02:00
Kilian Schüttler	de0027dd6d	Merge branch 'feature/RED-10125-fp' into 'master' RED-10125: fix Normalizer in drools Closes RED-10125 See merge request redactmanager/redaction-service!541	2024-10-15 10:59:35 +02:00
Kilian Schüttler	d73ec58d8a	RED-10125: fix Normalizer in drools	2024-10-15 10:59:34 +02:00
Maverick Studer	9e032cf1a8	Merge branch 'RED-10104' into 'master' RED-10104: Add rectangle functionality to bulk-local/add endpoint Closes RED-10104 See merge request redactmanager/redaction-service!538	2024-10-14 17:19:41 +02:00
Maverick Studer	c2fd73efe7	RED-10104: Add rectangle functionality to bulk-local/add endpoint	2024-10-14 17:19:41 +02:00
Kilian Schüttler	db405278bc	Merge branch 'feature/RM-184' into 'master' RM-184 && RM-172: update SectionIdentifier javadoc, make RulesLogger and... Closes RM-184 See merge request redactmanager/redaction-service!535	2024-10-10 10:35:12 +02:00
Kilian Schüttler	11a9f2f8aa	RM-184 && RM-172: update SectionIdentifier javadoc, make RulesLogger and...	2024-10-10 10:35:12 +02:00
Maverick Studer	0aea884da2	Merge branch 'RED-9123' into 'master' RED-9123: Improve performance of re-analysis (Spike) Closes RED-9123 See merge request redactmanager/redaction-service!450	2024-10-08 09:24:19 +02:00
Maverick Studer	d32c56e101	RED-9123: Improve performance of re-analysis (Spike)	2024-10-08 09:24:18 +02:00
Maverick Studer	92d03de194	Merge branch 'hotfix-npe-component-log-fp' into 'master' Hotfix for NPE when creating component log See merge request redactmanager/redaction-service!534	2024-10-07 18:01:16 +02:00
maverickstuder	6a58bf6d44	Hotfix for NPE when creating component log	2024-10-07 16:36:53 +02:00
Dominique Eifländer	daed4e07ef	Merge branch 'RED-10131-master' into 'master' RED-10131: Catch stackoverflow error of unoptimized regexes Closes RED-10131 See merge request redactmanager/redaction-service!532	2024-10-07 13:16:24 +02:00
Dominique Eifländer	eceba3a37a	RED-10131: Catch stackoverflow error of unoptimized regexes	2024-10-07 13:00:00 +02:00
Maverick Studer	4b647b23a3	Merge branch 'RED-10146' into 'master' RED-10146: Include defined components in component log Closes RED-10146 See merge request redactmanager/redaction-service!529	2024-10-07 12:26:05 +02:00
Maverick Studer	dd1b7cf72f	RED-10146: Include defined components in component log	2024-10-07 12:26:04 +02:00
Maverick Studer	725e6c1e14	Merge branch 'RED-9933' into 'master' RED-9933: updated date formats Closes RED-9933 See merge request redactmanager/redaction-service!527	2024-09-30 11:33:03 +02:00
maverickstuder	ab49e5d296	RED-9933: updated date formats	2024-09-30 09:55:24 +02:00
Maverick Studer	48d14cff8f	Merge branch 'RED-9947-fix2' into 'master' RED-9947 && RED-10104 See merge request redactmanager/redaction-service!526	2024-09-27 11:05:26 +02:00
Maverick Studer	dbe8e08bba	RED-9947 && RED-10104	2024-09-27 11:05:26 +02:00
Maverick Studer	98456ceb6d	Merge branch 'RED-9933' into 'master' RED-9933: DocuMine DateFormat config in dossier templates Closes RED-9933 See merge request redactmanager/redaction-service!523	2024-09-25 12:21:39 +02:00
Maverick Studer	a7effce48e	RED-9933: DocuMine DateFormat config in dossier templates	2024-09-25 12:21:38 +02:00
Maverick Studer	d068160a9d	Merge branch 'RED-9933' into 'master' RED-9933: DocuMine DateFormat config in dossier templates Closes RED-9933 See merge request redactmanager/redaction-service!522	2024-09-24 11:37:51 +02:00
Maverick Studer	f37e49e8bb	RED-9933: DocuMine DateFormat config in dossier templates	2024-09-24 11:37:50 +02:00
Maverick Studer	1f3cf8d529	Merge branch 'feature/RED-9348' into 'master' RED-9348: move component log to mongodb Closes RED-9348 See merge request redactmanager/redaction-service!519	2024-09-23 16:05:03 +02:00
Maverick Studer	b61b89bc5b	RED-9348: move component log to mongodb	2024-09-23 16:05:03 +02:00
Maverick Studer	8aa31a18af	Merge branch 'feature/RED-9010' into 'master' RED-9010: remove redaction log Closes RED-9010 See merge request redactmanager/redaction-service!520	2024-09-19 11:34:34 +02:00
maverickstuder	b2a837bc54	RED-9010: remove redaction log	2024-09-18 12:29:38 +02:00
Maverick Studer	f93e59e29e	Merge branch 'RED-9524' into 'master' RED-9524: File processing does not annotate images Closes RED-9524 See merge request redactmanager/redaction-service!507	2024-09-18 10:47:10 +02:00
Maverick Studer	b8612d9b95	RED-9524: File processing does not annotate images	2024-09-18 10:47:10 +02:00
Kilian Schüttler	1683745dc7	Merge branch 'hotfix' into 'master' RED-9975: extend SectionIdentifier to alphanumeric See merge request redactmanager/redaction-service!515	2024-09-11 13:38:35 +02:00
Kilian Schüttler	ec50eca15b	RED-9975: extend SectionIdentifier to alphanumeric	2024-09-11 13:38:35 +02:00
Maverick Studer	3ff541fee6	Merge branch 'RED-9859' into 'master' RED-9859: Redactions found by et. al. rule not skipped with published information Closes RED-9859 See merge request redactmanager/redaction-service!512	2024-09-10 14:24:09 +02:00
Maverick Studer	07b9b8bf8a	RED-9859: Redactions found by et. al. rule not skipped with published information	2024-09-10 14:24:09 +02:00
Andrei Isvoran	3c165070ee	Merge branch 'RED-9986-DM12' into 'master' RED-9986 - Add component rules path to be scanned for Javadoc generation Closes RED-9986 See merge request redactmanager/redaction-service!510	2024-09-09 10:13:57 +02:00
Andrei Isvoran	ecd57e17a2	RED-9986 - Add component rules path to be scanned for Javadoc generation	2024-09-09 10:13:57 +02:00
Kilian Schüttler	9fc065518d	Merge branch 'RED-9548' into 'master' RED-9548 - Actions results in wrong results for deadlocked annotation Closes RED-9548 See merge request redactmanager/redaction-service!509	2024-09-06 14:43:57 +02:00
Corina Olariu	3c3c029cf4	RED-9548 - Actions results in wrong results for deadlocked annotation	2024-09-06 14:43:57 +02:00
Kilian Schüttler	d767966056	Merge branch 'RED-9728' into 'master' RED-9728: remove False Positives from table methods Closes RED-9728 See merge request redactmanager/redaction-service!508	2024-09-06 11:07:49 +02:00
Kilian Schüttler	bcbd4587f1	RED-9728: remove False Positives from table methods	2024-09-06 11:07:48 +02:00
Maverick Studer	03e321a824	Merge branch 'RED-9947' into 'master' RED-9947: search term occurrences implementation for local bulk Closes RED-9947 See merge request redactmanager/redaction-service!504	2024-09-04 11:51:39 +02:00
Maverick Studer	316b4c1d02	RED-9947: search term occurrences implementation for local bulk	2024-09-04 11:51:39 +02:00
Kilian Schüttler	3e0f2254ed	Merge branch 'RED-9964-fp' into 'master' RED-9964: fix errors with images Closes RED-9964 See merge request redactmanager/redaction-service!506	2024-09-04 09:30:34 +02:00
Kilian Schüttler	cd2bda15aa	RED-9964: fix errors with images	2024-09-04 09:30:33 +02:00
Kilian Schüttler	895bc56590	Merge branch 'RED-9964-fp' into 'master' RED-9964: refactor getMainBody() and getMainBodyTextBlock() in Page Closes RED-9964 See merge request redactmanager/redaction-service!501	2024-09-02 16:51:13 +02:00
Kilian Schüttler	292869c502	RED-9964: refactor getMainBody() and getMainBodyTextBlock() in Page	2024-09-02 16:51:12 +02:00
Kilian Schüttler	f3cdf46008	Merge branch 'entitylog-migration' into 'master' entitylog-mapping: add test to map entitylogs to existing DocumentData See merge request redactmanager/redaction-service!503	2024-09-02 16:24:11 +02:00
Kilian Schüttler	795f8fd31b	entitylog-mapping: add test to map entitylogs to existing DocumentData	2024-09-02 16:24:11 +02:00
Kevin Tumma	403e2f4153	Update .gitlab-ci.yml file	2024-08-30 11:22:52 +02:00
Kevin Tumma	5470157468	Update .gitlab-ci.yml file	2024-08-30 10:56:49 +02:00
Kevin Tumma	80aaeea8dd	Update .gitlab-ci.yml file	2024-08-30 10:35:20 +02:00
Kevin Tumma	5a5f14127b	Update .gitlab-ci.yml file	2024-08-30 10:23:27 +02:00
Kevin Tumma	bc49cc6e8d	Update .gitlab-ci.yml file	2024-08-30 09:52:09 +02:00
Maverick Studer	59103d3075	Merge branch 'tenants-retry' into 'master' Tenants retry logic and queue renames See merge request redactmanager/redaction-service!500	2024-08-29 16:12:38 +02:00
Maverick Studer	6cf17ef4f3	Tenants retry logic and queue renames	2024-08-29 16:12:37 +02:00
Maverick Studer	40d832fc3a	Merge branch 'REVERT-RED-7327' into 'master' Revert "RED-7327 - Change to group annotation" Closes RED-7327 See merge request redactmanager/redaction-service!499	2024-08-29 10:18:49 +02:00
Maverick Studer	b9240dec68	Revert "RED-7327 - Change to group annotation"	2024-08-29 10:18:48 +02:00
Kevin Tumma	ea7137b242	Update .gitlab-ci.yml file	2024-08-28 11:09:09 +02:00
Kevin Tumma	8375783ce8	Update .gitlab-ci.yml file	2024-08-28 10:55:28 +02:00
Kilian Schüttler	ebee9f04bd	Merge branch 'SPIKE-LLM_NER' into 'master' Spike: LLM NER See merge request redactmanager/redaction-service!498	2024-08-27 18:00:59 +02:00
Kilian Schüttler	4dc0a1fbdc	Spike: LLM NER	2024-08-27 18:00:59 +02:00
Maverick Studer	5ebe82b7ce	Merge branch 'RED-9331' into 'master' RED-9331: Explore possibilities for fair upload / analysis processing per tenant Closes RED-9331 See merge request redactmanager/redaction-service!463	2024-08-27 13:22:19 +02:00
Maverick Studer	d1c2d43ffb	RED-9331: Explore possibilities for fair upload / analysis processing per tenant	2024-08-27 13:22:18 +02:00
Maverick Studer	004f6cb5f9	Merge branch 'AZURE_NER_FP' into 'master' Update layout parser version for azure ner service See merge request redactmanager/redaction-service!497	2024-08-27 10:13:58 +02:00
Maverick Studer	4fc24fdfe3	Update layout parser version for azure ner service	2024-08-27 10:13:58 +02:00
Maverick Studer	9d668f9be1	Merge branch 'AZURE_NER_FP' into 'master' RED-9918: Azure entity recognition (Spike) See merge request redactmanager/redaction-service!496	2024-08-26 14:34:45 +02:00
Maverick Studer	4184333506	RED-9918: Azure entity recognition (Spike)	2024-08-26 14:34:44 +02:00
Maverick Studer	062d29c2ea	Merge branch 'RED-9865' into 'master' RED-9865: fix for case 2 Closes RED-9865 See merge request redactmanager/redaction-service!493	2024-08-23 17:01:14 +02:00
Maverick Studer	dafecc35b6	RED-9865: fix for case 2	2024-08-23 17:01:13 +02:00
Andrei Isvoran	5327591d54	Merge branch 'RED-8694-javadoc' into 'master' RED-8694 - Add more javadoc Closes RED-8694 See merge request redactmanager/redaction-service!492	2024-08-23 10:26:39 +02:00
Andrei Isvoran	581e367b6e	RED-8694 - Add more javadoc	2024-08-23 09:56:59 +03:00
Andrei Isvoran	b58985fa8a	Merge branch 'RED-7327-fixes' into 'master' RED-7327 - Change to group annotation Closes RED-7327 See merge request redactmanager/redaction-service!491	2024-08-22 12:09:07 +02:00
Andrei Isvoran	cdfa1afcd0	RED-7327 - Change to group annotation	2024-08-22 12:09:07 +02:00
Andrei Isvoran	78990b5555	Merge branch 'RED-7327' into 'master' RED-7327 - Add group redactions Closes RED-7327 See merge request redactmanager/redaction-service!486	2024-08-20 10:32:09 +02:00
Andrei Isvoran	2dada39717	RED-7327 - Add group redactions	2024-08-20 10:32:08 +02:00
Dominique Eifländer	66647e45b0	Merge branch 'RED-9837-master' into 'master' RED-9837: Fixed not working timeout with endless loop in drools then block Closes RED-9837 See merge request redactmanager/redaction-service!489	2024-08-19 14:08:25 +02:00
Dominique Eifländer	2ae8b0fbce	RED-9837: Fixed not working timeout with endless loop in drools then block	2024-08-19 13:18:00 +02:00
Dominique Eifländer	00409584ee	Merge branch 'RED-9760-master-rules' into 'master' RED-9760: Do not check blacklisted keywords in Strings Closes RED-9760 See merge request redactmanager/redaction-service!487	2024-08-13 11:57:37 +02:00
Dominique Eifländer	8fc1f77688	RED-9760: Do not check blacklisted keywords in Strings	2024-08-13 11:12:36 +02:00
Maverick Studer	594acb4e82	Merge branch 'RED-9782-fix' into 'master' RED-9782: Automated Analysis should be disabled when uploading a document that... Closes RED-9782 See merge request redactmanager/redaction-service!485	2024-08-12 18:41:01 +02:00
Maverick Studer	f9c77a3695	RED-9782: Automated Analysis should be disabled when uploading a document that...	2024-08-12 18:41:00 +02:00
Kilian Schüttler	30cd60f702	Merge branch 'hotfix-UOE' into 'master' hotfix: UOE in ComponentDroolsExecutionService See merge request redactmanager/redaction-service!483	2024-08-12 15:59:09 +02:00
Kilian Schuettler	6f361f0deb	hotfix: UOE in ComponentDroolsExecutionService	2024-08-12 15:22:13 +02:00
Kilian Schüttler	f629448e4b	Merge branch 'RED-9869' into 'master' RED-9869: allow java.text and find ruleIdentifiers with whitespaces/linebreaks Closes RED-9869 See merge request redactmanager/redaction-service!479	2024-08-12 12:49:47 +02:00
Kilian Schuettler	1b67ea1068	RED-9869: allow java.text and find ruleIdentifiers with whitespaces/linebreaks	2024-08-12 12:34:13 +02:00
Maverick Studer	9dbf5c479f	Merge branch 'hotfixes-dm-release-fp' into 'master' Hotfixes dm release fp See merge request redactmanager/redaction-service!478	2024-08-12 09:49:07 +02:00
Maverick Studer	c760733e41	Hotfixes dm release fp	2024-08-12 09:49:07 +02:00
Yannik Hampe	99182fba23	Merge branch 'RED-5624-migration' into 'master' RED-5625: hotfix migration Closes RED-5624 See merge request redactmanager/redaction-service!476	2024-08-09 14:08:51 +02:00
Yannik Hampe	ac5a7a73b1	RED-5625: hotfix migration	2024-08-09 14:08:51 +02:00
Maverick Studer	9d093addaf	Merge branch 'RED-9857' into 'master' RED-9857: Add new date format Closes RED-9857 See merge request redactmanager/redaction-service!474	2024-08-09 10:30:24 +02:00
Maverick Studer	56cabd54a2	RED-9857: Add new date format	2024-08-09 10:30:24 +02:00
Christoph Schabert	7edf00f014	Update .gitlab-ci.yml file	2024-08-08 14:00:44 +02:00
Christoph Schabert	3a5072d7af	Update .gitlab-ci.yml file	2024-08-08 13:26:08 +02:00
Christoph Schabert	02abe895a1	Update .gitlab-ci.yml file	2024-08-08 13:25:41 +02:00
Christoph Schabert	c29329e3df	Update .gitlab-ci.yml file	2024-08-08 12:54:24 +02:00
Christoph Schabert	b8cf0ab005	Update .gitlab-ci.yml file	2024-08-08 12:46:20 +02:00
Christoph Schabert	5a4d60eb03	Update .gitlab-ci.yml file	2024-08-08 12:15:10 +02:00
Christoph Schabert	7bc76d9306	Update .gitlab-ci.yml file	2024-08-08 12:03:36 +02:00
Christoph Schabert	c351a71831	Update .gitlab-ci.yml file	2024-08-08 11:58:03 +02:00
Christoph Schabert	1d135a26fa	Update .gitlab-ci.yml file	2024-08-08 11:44:53 +02:00
Christoph Schabert	5e8dc747bb	Update .gitlab-ci.yml file	2024-08-08 10:11:32 +02:00
Maverick Studer	5ff3ebc6cb	Merge branch 'RED-9782-fp' into 'master' RED-9782: Automated Analysis should be disabled when uploading a document that... Closes RED-9782 See merge request redactmanager/redaction-service!472	2024-08-07 12:26:01 +02:00
Maverick Studer	4aac5fb71a	RED-9782: Automated Analysis should be disabled when uploading a document that...	2024-08-07 12:26:00 +02:00
Maverick Studer	ae8b82245f	Merge branch 'RED-9782-master' into 'master' RED-9782: Added analysis function that only imports imported redactions Closes RED-9782 See merge request redactmanager/redaction-service!470	2024-08-07 10:34:29 +02:00
Dominique Eifländer	5abe73cae2	RED-9782: Added analysis function that only imports imported redactions	2024-08-07 09:44:27 +02:00
Kilian Schüttler	b6d1edacba	Merge branch 'annotationMode' into 'master' annotationMode: ignore IDs of manual adds in annotationMode See merge request redactmanager/redaction-service!465	2024-07-26 14:53:44 +02:00
Kilian Schuettler	80bc8242d0	annotationMode: ignore IDs of manual adds in annotationMode	2024-07-25 17:01:41 +02:00
Andrei Isvoran	2ab536dfee	Merge branch 'RED-9140-fp' into 'master' RED-9140 - Properly distinct between REMOVED and IGNORED Change type Closes RED-9140 See merge request redactmanager/redaction-service!464	2024-07-23 15:58:26 +02:00
Andrei Isvoran	e09cd59256	RED-9140 - Properly distinct between REMOVED and IGNORED Change type	2024-07-23 15:58:25 +02:00
Maverick Studer	3fd98c67f3	Merge branch 'RED-9246-2' into 'master' RED-9246: Resize on dossier level for dossier-template-level entry becomes skipped Closes RED-9246 See merge request redactmanager/redaction-service!458	2024-07-12 10:30:58 +02:00
Maverick Studer	1800a89b39	RED-9246: Resize on dossier level for dossier-template-level entry becomes skipped	2024-07-12 10:30:58 +02:00
Andrei Isvoran	229756d185	Merge branch 'RED-9496-graceful-shutdown' into 'master' RED-9496 - Implement graceful shutdown Closes RED-9496 See merge request redactmanager/redaction-service!456	2024-07-04 13:41:19 +02:00
Andrei Isvoran	a7d9a37be9	RED-9496 - Implement graceful shutdown	2024-07-04 14:01:40 +03:00
Dominique Eifländer	71b2dda7cd	Merge branch 'RED-9500' into 'master' RED-9500: Use Redis pub/sub to syncronize websocket topic messages accross multiple pods Closes RED-9500 See merge request redactmanager/redaction-service!455	2024-07-02 15:53:51 +02:00
Dominique Eifländer	fe7b43c32a	RED-9500: Use Redis pub/sub to syncronize websocket topic messages accross multiple pods	2024-07-02 15:38:30 +02:00
Maverick Studer	09e84cdb75	Merge branch 'RED-9123' into 'master' RED-9123: Improve performance of re-analysis (Spike) Closes RED-9123 See merge request redactmanager/redaction-service!454	2024-07-02 12:52:24 +02:00
Maverick Studer	776a24306f	RED-9123: Improve performance of re-analysis (Spike)	2024-07-02 12:52:24 +02:00
Maverick Studer	3e75f3e5e7	Merge branch 'RED-9123' into 'master' RED-9123: Improve performance of re-analysis (Spike) Closes RED-9123 See merge request redactmanager/redaction-service!450	2024-07-01 12:07:11 +02:00
Maverick Studer	1c81212a2d	RED-9123: Improve performance of re-analysis (Spike)	2024-07-01 12:07:11 +02:00
Corina Olariu	c279f54295	Merge branch 'RED-9466-fp' into 'master' RED-9466 - Adding annotation removes all AI based recommendations until forced re-analysis Closes RED-9466 See merge request redactmanager/redaction-service!453	2024-06-28 15:49:25 +02:00
Corina Olariu	5f52df1ebe	RED-9466 - Adding annotation removes all AI based recommendations until forced re-analysis	2024-06-28 15:49:25 +02:00
Andrei Isvoran	1b701a67c0	Merge branch 'RED-9349-context' into 'master' RED-9349 - Remove context from rules and check before setting the global logger Closes RED-9349 See merge request redactmanager/redaction-service!451	2024-06-28 08:29:39 +02:00
Andrei Isvoran	5cb81e7f8b	RED-9349 - Remove context from rules and check before setting the global logger	2024-06-28 08:29:39 +02:00
Andrei Isvoran	c869aab13b	Merge branch 'RED-9140' into 'master' RED-9140 - Add more information to changes Closes RED-9140 See merge request redactmanager/redaction-service!440	2024-06-27 13:49:27 +02:00
Andrei Isvoran	a58bcedccf	RED-9140 - Add more information to changes	2024-06-27 13:49:26 +02:00
Andrei Isvoran	9e45864b35	Merge branch 'RED-9349-cors' into 'master' RED-9349 - Enable cors on dev Closes RED-9349 See merge request redactmanager/redaction-service!449	2024-06-27 09:51:08 +02:00
Kilian Schüttler	67657e91b5	Merge branch 'RED-9375' into 'master' RED-9375: use storageId for cache names everywhere, such that name may be updated by a user Closes RED-9375 See merge request redactmanager/redaction-service!448	2024-06-27 09:42:24 +02:00
Kilian Schüttler	c1a2e9dee2	RED-9375: use storageId for cache names everywhere, such that name may be updated by a user	2024-06-27 09:42:24 +02:00
Andrei Isvoran	0a5cc0140f	RED-9349 - Enable cors on dev	2024-06-27 10:35:10 +03:00
Andrei Isvoran	e9fdedc684	Merge branch 'RED-9349-ignore-test-rule' into 'master' RED-9349 - Ignore rules endpoint Closes RED-9349 See merge request redactmanager/redaction-service!447	2024-06-26 14:37:53 +02:00
Andrei Isvoran	d9a55638e7	RED-9349 - Ignore rules endpoint	2024-06-26 14:37:53 +02:00
Andrei Isvoran	990c09cfad	Merge branch 'RED-9349-rule-log' into 'master' RED-9349 - Add possibility to log stuff in the rules Closes RED-9349 See merge request redactmanager/redaction-service!446	2024-06-26 10:17:58 +02:00
Andrei Isvoran	f3ae68a8da	RED-9349 - Add possibility to log stuff in the rules	2024-06-26 10:17:58 +02:00