Merge branch 'RED-10639-bp' into 'release/4.348.x'

RED-10639: RM-224: ERROR state for document after re-upload same document with... See merge request redactmanager/redaction-service!578
RED-10639: RM-224: ERROR state for document after re-upload same document with...
2024-12-12 12:57:04 +01:00 · 2024-12-12 12:57:04 +01:00 · 2024-12-10 09:20:28 +01:00 · 2024-12-09 15:09:21 +01:00 · 2024-11-05 14:34:44 +01:00 · 2024-11-05 14:34:43 +01:00
74 changed files with 28496 additions and 1101 deletions
@@ -4,7 +4,7 @@ plugins {
 }

 description = "redaction-service-api-v1"
-val persistenceServiceVersion = "2.439.0"
+val persistenceServiceVersion = "2.465.83"

 dependencies {
    implementation("org.springframework:spring-web:6.0.12")
@@ -12,11 +12,11 @@ plugins {
 description = "redaction-service-server-v1"


-val layoutParserVersion = "0.131.0"
+val layoutParserVersion = "0.142.6"
 val jacksonVersion = "2.15.2"
 val droolsVersion = "9.44.0.Final"
 val pdfBoxVersion = "3.0.0"
-val persistenceServiceVersion = "2.439.0"
+val persistenceServiceVersion = "2.465.83"
 val springBootStarterVersion = "3.1.5"
 val springCloudVersion = "4.0.4"
 val testContainersVersion = "1.19.7"
@@ -43,6 +43,7 @@ dependencies {
    implementation("com.iqser.red.commons:storage-commons:2.45.0")
    implementation("com.knecon.fforesight:tenant-commons:0.24.0")
    implementation("com.knecon.fforesight:tracing-commons:0.5.0")
+    implementation("com.knecon.fforesight:lifecycle-commons:0.6.0")

    implementation("com.fasterxml.jackson.module:jackson-module-afterburner:${jacksonVersion}")
    implementation("com.fasterxml.jackson.datatype:jackson-datatype-jsr310:${jacksonVersion}")
@@ -103,7 +104,11 @@ tasks.test {
 tasks.named<BootBuildImage>("bootBuildImage") {

    environment.put("BPE_DELIM_JAVA_TOOL_OPTIONS", " ")
-    environment.put("BPE_APPEND_JAVA_TOOL_OPTIONS", "-XX:MaxMetaspaceSize=1g -Dfile.encoding=UTF-8 -Dkie.repository.project.cache.size=50 -Dkie.repository.project.versions.cache.size=5")
+    environment.put(
+        "BPE_APPEND_JAVA_TOOL_OPTIONS",
+        "-XX:MaxMetaspaceSize=1g -Dfile.encoding=UTF-8 -Dkie.repository.project.cache.size=50 -Dkie.repository.project.versions.cache.size=5"
+    )
+    environment.put("BPE_DEFAULT_LANG", "en_US.utf8") // java.text.Normalizer does not care for file.encoding

    imageName.set("nexus.knecon.com:5001/red/${project.name}")// must build image with same name always, otherwise the builder will not know which image to use as cache. DO NOT CHANGE!
    if (project.hasProperty("buildbootDockerHostNetwork")) {
@@ -127,18 +132,19 @@ tasks.named<BootBuildImage>("bootBuildImage") {
    }
 }

-fun parseDroolsImports(droolsFilePath: String): List<String> {
-
+fun parseDroolsImports(vararg droolsFilePaths: String): List<String> {
    val imports = mutableListOf<String>()
    val importPattern = Regex("^import\\s+(com\\.iqser\\.red\\.service\\.redaction\\.v1\\.[\\w.]+);")
    val desiredPrefix = "com.iqser.red.service.redaction.v1"

-    File(droolsFilePath).forEachLine { line ->
-        importPattern.find(line)?.let { matchResult ->
-            val importPath = matchResult.groupValues[1].trim()
-            if (importPath.startsWith(desiredPrefix)) {
-                val formattedPath = importPath.replace('.', '/')
-                imports.add("$formattedPath.java")
+    droolsFilePaths.forEach { filePath ->
+        File(filePath).forEachLine { line ->
+            importPattern.find(line)?.let { matchResult ->
+                val importPath = matchResult.groupValues[1].trim()
+                if (importPath.startsWith(desiredPrefix)) {
+                    val formattedPath = importPath.replace('.', '/')
+                    imports.add("$formattedPath.java")
+                }
            }
        }
    }
@@ -146,7 +152,11 @@ fun parseDroolsImports(droolsFilePath: String): List<String> {
    return imports
 }

-val droolsImports = parseDroolsImports("redaction-service-v1/redaction-service-server-v1/src/main/resources/drools/all_rules_documine.drl")
+// Combine imports from both drools files
+val droolsImports = parseDroolsImports(
+    "redaction-service-v1/redaction-service-server-v1/src/main/resources/drools/all_rules_documine.drl",
+    "redaction-service-v1/redaction-service-server-v1/src/main/resources/drools/base_component_rules.drl"
+)

 tasks.register("generateJavaDoc", Javadoc::class) {

@@ -13,6 +13,7 @@ import org.springframework.boot.context.properties.EnableConfigurationProperties
 import org.springframework.cache.annotation.EnableCaching;
 import org.springframework.cloud.openfeign.EnableFeignClients;
 import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.EnableAspectJAutoProxy;
 import org.springframework.context.annotation.Import;
 import org.springframework.data.mongodb.repository.config.EnableMongoRepositories;

@@ -20,6 +21,7 @@ import com.iqser.red.service.dictionarymerge.commons.DictionaryMergeService;
 import com.iqser.red.service.persistence.service.v1.api.shared.mongo.SharedMongoAutoConfiguration;
 import com.iqser.red.service.redaction.v1.server.client.RulesClient;
 import com.iqser.red.storage.commons.StorageAutoConfiguration;
+import com.knecon.fforesight.lifecyclecommons.LifecycleAutoconfiguration;
 import com.knecon.fforesight.mongo.database.commons.MongoDatabaseCommonsAutoConfiguration;
 import com.knecon.fforesight.mongo.database.commons.liquibase.EnableMongoLiquibase;
 import com.knecon.fforesight.tenantcommons.MultiTenancyAutoConfiguration;
@@ -32,13 +34,14 @@ import lombok.extern.slf4j.Slf4j;

@Slf4j
@EnableCaching
-@ImportAutoConfiguration({MultiTenancyAutoConfiguration.class, SharedMongoAutoConfiguration.class})
+@ImportAutoConfiguration({MultiTenancyAutoConfiguration.class, SharedMongoAutoConfiguration.class, LifecycleAutoconfiguration.class})
@Import({MetricsConfiguration.class, StorageAutoConfiguration.class, MongoDatabaseCommonsAutoConfiguration.class})
@EnableFeignClients(basePackageClasses = RulesClient.class)
@EnableConfigurationProperties(RedactionServiceSettings.class)
@EnableMongoRepositories(basePackages = "com.iqser.red.service.persistence")
@EnableMongoLiquibase
@SpringBootApplication(exclude = {SecurityAutoConfiguration.class, ManagementWebSecurityAutoConfiguration.class, DataSourceAutoConfiguration.class, LiquibaseAutoConfiguration.class, MongoAutoConfiguration.class, MongoDataAutoConfiguration.class})
+@EnableAspectJAutoProxy
 public class Application {

    public static void main(String[] args) {
@@ -28,10 +28,18 @@ public class RedactionServiceSettings {

    private int dictionaryCacheExpireAfterAccessDays = 3;

-    private int droolsExecutionTimeoutSecs = 300;
+    private int droolsExecutionTimeoutSecs = 600;

    private boolean ruleExecutionSecured = true;

    private boolean annotationMode;

+    private boolean droolsDebug;
+
+
+    public int getDroolsExecutionTimeoutSecs(int numberOfPages) {
+
+        return (int) Math.max(getDroolsExecutionTimeoutSecs(), getDroolsExecutionTimeoutSecs() * ((float) numberOfPages / 1000));
+    }
+
 }
@@ -11,7 +11,7 @@ import lombok.NoArgsConstructor;
@NoArgsConstructor
 public class EntityRecognitionSection {

-    private int sectionNumber;
+    private String sectionNumber;
    private String text;

 }
@@ -13,6 +13,6 @@ import lombok.NoArgsConstructor;
@AllArgsConstructor
 public class NerEntitiesModel {

-    private Map<Integer, List<EntityRecognitionEntity>> data = new HashMap<>();
+    private Map<String, List<EntityRecognitionEntity>> data = new HashMap<>();

 }
@@ -21,6 +21,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.BaseAnnotation;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
@@ -119,9 +120,14 @@ public class RedactionLogToEntityLogMigrationService {
                .filter(MigrationEntity::needsManualEntry)
                .map(MigrationEntity::buildManualRedactionEntry)
                .toList();
-
        idsToMigrateInDb.setManualRedactionEntriesToAdd(manualRedactionEntriesToAdd);

+        List<String> manualForceRedactionIdsToDelete = entitiesToMigrate.stream()
+                .filter(MigrationEntity::needsForceDeletion)
+                .map(MigrationEntity::getNewId)
+                .toList();
+        idsToMigrateInDb.setForceRedactionIdsToDelete(manualForceRedactionIdsToDelete);
+
        return new MigratedEntityLog(idsToMigrateInDb, entityLog);
    }

@@ -14,6 +14,8 @@ import java.util.Optional;
 import java.util.Set;
 import java.util.stream.Collectors;

+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Change;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ChangeType;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryType;
@@ -21,6 +23,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualChangeFactory;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.BaseAnnotation;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
@@ -214,6 +217,17 @@ public final class MigrationEntity {
            entityLogEntry.setValue("Imported Redaction");
        }

+        if(entityLogEntry.getChanges() != null
+           && !entityLogEntry.getChanges().isEmpty()
+           && entityLogEntry.getChanges().stream().map(Change::getType).toList().get(entityLogEntry.getChanges().size() - 1).equals(ChangeType.REMOVED)) {
+            entityLogEntry.setState(EntryState.REMOVED);
+            if (!entityLogEntry.getManualChanges().isEmpty()) {
+                entityLogEntry.getManualChanges()
+                        .removeIf(manualChange -> manualChange.getManualRedactionType()
+                                .equals(com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualRedactionType.FORCE));
+            }
+        }
+
        return entityLogEntry;
    }

@@ -458,4 +472,10 @@ public final class MigrationEntity {
                       .anyMatch(mc -> mc instanceof ManualResizeRedaction && !((ManualResizeRedaction) mc).getUpdateDictionary()) && !(migratedEntity instanceof Image);
    }

+    public boolean needsForceDeletion() {
+
+        return manualChanges.stream()
+                       .anyMatch(mc -> mc instanceof ManualForceRedaction) && this.precursorEntity != null && this.precursorEntity.removed();
+    }
+
 }
@@ -120,7 +120,8 @@ public class PrecursorEntity implements IEntity {
        EntityType entityType = getEntityType(entryType);
        String value = Optional.ofNullable(importedRedaction.getValue())
                .orElse("");
-        return PrecursorEntity.builder()
+
+        PrecursorEntityBuilder precursorEntityBuilder = PrecursorEntity.builder()
                .id(importedRedaction.getId())
                .value(value)
                .entityPosition(rectangleWithPages)
@@ -130,14 +131,21 @@ public class PrecursorEntity implements IEntity {
                                    .orElse(""))
                .type(Optional.ofNullable(importedRedaction.getType())
                              .orElse(IMPORTED_REDACTION_TYPE))
-                .section(importedRedaction.getManualOverwriteSection())
+                .section(Optional.ofNullable(importedRedaction.getSection())
+                                 .orElse(""))
                .entityType(entityType)
                .isDictionaryEntry(false)
                .isDossierDictionaryEntry(false)
+                .manualOverwrite(new ManualChangeOverwrite(entityType))
                .rectangle(value.isBlank() || entryType.equals(EntryType.IMAGE) || entryType.equals(EntryType.IMAGE_HINT) || entryType.equals(EntryType.AREA))
-                .manualOverwrite(new ManualChangeOverwrite(entityType, importedRedaction.getManualOverwriteSection()))
-                .engines(Set.of(Engine.IMPORTED))
-                .build();
+                .engines(Set.of(Engine.IMPORTED));
+
+        if (importedRedaction.getManualOverwriteSection() != null && !importedRedaction.getManualOverwriteSection().isEmpty()) {
+            precursorEntityBuilder.section(importedRedaction.getManualOverwriteSection())
+                    .manualOverwrite(new ManualChangeOverwrite(entityType, importedRedaction.getManualOverwriteSection()));
+        }
+
+        return precursorEntityBuilder.build();
    }


@@ -172,6 +180,22 @@ public class PrecursorEntity implements IEntity {
    }


+    /**
+     * @return true when this entity is of EntityType ENTITY or HINT
+     */
+    public boolean validEntityType() {
+
+        return entityType.equals(EntityType.ENTITY) || entityType.equals(EntityType.HINT);
+    }
+
+
+    @Override
+    public boolean valid() {
+
+        return active() && validEntityType();
+    }
+
+
    private static EntityType getEntityType(EntryType entryType) {

        switch (entryType) {
@@ -76,7 +76,7 @@ public class ComponentMapping implements Queryable {

        // find all elements which fulfill query
        List<String[]> result = new LinkedList<>();
-        while (data.get(idx)[0].equals(queryParameters.query())) {
+        while (idx < data.size() && data.get(idx)[0].equals(queryParameters.query())) {
            result.add(data.get(idx));
            idx++;
        }
@@ -11,6 +11,7 @@ import java.util.stream.Stream;
 import org.ahocorasick.trie.Trie;

 import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
+import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;

 import lombok.Data;

@@ -104,6 +105,12 @@ public class SearchImplementation {
    }


+    public Stream<TextRange> getBoundaries(TextBlock textBlock) {
+
+        return getBoundaries(textBlock, textBlock.getTextRange());
+    }
+
+
    public Stream<TextRange> getBoundaries(CharSequence text, TextRange region) {

        if (this.values.isEmpty()) {
@@ -288,8 +288,8 @@ public class DocumentTree {
        if (treeId.isEmpty()) {
            return root;
        }
-        Entry entry = root.children.get(treeId.get(0));
-        for (int id : treeId.subList(1, treeId.size())) {
+        Entry entry = root;
+        for (int id : treeId) {
            entry = entry.children.get(id);
        }
        return entry;
@@ -52,6 +52,17 @@ public interface IEntity {
    String type();


+    /**
+     * An Entity is valid, when it active and not a false recommendation, a false positive or a dictionary removal.
+     *
+     * @return true, if the entity is valid, false otherwise/
+     */
+    default boolean valid() {
+
+        return active();
+    }
+
+
    /**
     * Calculates the length of the entity's value.
     *
@@ -278,6 +278,21 @@ public class TextEntity implements IEntity {
    }


+    /**
+     * @return true when this entity is of EntityType ENTITY or HINT
+     */
+    public boolean validEntityType() {
+
+        return entityType.equals(EntityType.ENTITY) || entityType.equals(EntityType.HINT);
+    }
+
+
+    public boolean valid() {
+
+        return active() && validEntityType();
+    }
+
+
    @Override
    public String value() {

@@ -71,6 +71,11 @@ public class Headline extends AbstractSemanticNode {
    }


+    /**
+     * Extracts the SectionIdentifier from the text of this headline.
+     *
+     * @return The SectionIdentifier, with which the headline starts.
+     */
    @Override
    public SectionIdentifier getSectionIdentifier() {

@@ -3,8 +3,10 @@ package com.iqser.red.service.redaction.v1.server.model.document.nodes;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
+import java.util.stream.Stream;

 import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
+import com.iqser.red.service.redaction.v1.server.model.document.textblock.AtomicTextBlock;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlockCollector;

@@ -35,7 +37,7 @@ public class Page {
    Integer width;
    Integer rotation;

-    List<SemanticNode> mainBody;
+    List<AtomicTextBlock> textBlocksOnPage;
    Header header;
    Footer footer;

@@ -53,17 +55,67 @@ public class Page {
     */
    public TextBlock getMainBodyTextBlock() {

-        return mainBody.stream()
-                .filter(SemanticNode::isLeaf)
-                .map(SemanticNode::getTextBlock)
+        return textBlocksOnPage.stream()
+                .filter(atb -> !atb.isEmpty())
                .collect(new TextBlockCollector());
    }


+    /**
+     * Retrieves the highest SemanticNodes, which appear only on this page. It is achieved by traversing the DocumentTree up, until a SemanticNode's direct parent is no longer exclusively on this page.
+     *
+     * @return A list which contains the highes SemanticNodes, which appear only on this page.
+     */
+    public List<SemanticNode> getMainBody() {
+
+        return textBlocksOnPage.stream()
+                .map(AtomicTextBlock::getParent)
+                .map(this::getHighestParentOnlyOnPage)
+                .distinct()
+                .toList();
+    }
+
+
+    /**
+     * Retrieves the highest SemanticNodes which are present on the page. There might be multiples, as two or more Main Sections start on a page.
+     * This is achieved by traversing up the document tree and returning all SemanticNodes whose direct parent is the Document
+     *
+     * @return A list of the highest SemanticNodes present on this page
+     */
+    public Stream<SemanticNode> streamHighestSemanticNodesOnPage() {
+
+        return textBlocksOnPage.stream()
+                .map(AtomicTextBlock::getParent)
+                .map(this::getHighestSemanticNodeOnPage)
+                .distinct();
+    }
+
+
+    private SemanticNode getHighestParentOnlyOnPage(SemanticNode node) {
+
+        SemanticNode currentNode = node;
+        while (currentNode.hasParent() && currentNode.getParent().onlyOnPage(this)) {
+            currentNode = currentNode.getParent();
+        }
+        return currentNode;
+    }
+
+
+    private SemanticNode getHighestSemanticNodeOnPage(SemanticNode node) {
+
+        SemanticNode currentNode = node;
+        while (currentNode.hasParent() //
+               && !currentNode.getParent().getType().equals(NodeType.DOCUMENT)) {
+            currentNode = currentNode.getParent();
+        }
+        return currentNode;
+    }
+
+
    @Override
    public String toString() {

-        return String.valueOf(number);
+        return "Page: " + number;
    }

 }
@@ -1,10 +1,12 @@
 package com.iqser.red.service.redaction.v1.server.model.document.nodes;

 import java.awt.geom.Rectangle2D;
+import java.util.ArrayList;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.stream.Collectors;

 import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.DocumentTree;
@@ -33,7 +35,6 @@ import lombok.extern.slf4j.Slf4j;
@EqualsAndHashCode(onlyExplicitlyIncluded = true, callSuper = true)
 public class Section extends AbstractSemanticNode {

-
    @Override
    public NodeType getType() {

@@ -53,6 +54,11 @@ public class Section extends AbstractSemanticNode {
    }


+    /**
+     * Returns the SectionIdentifier from the headline obtained by the getHeadline() method.
+     *
+     * @return the SectionIdentifier of the associated Headline
+     */
    @Override
    public SectionIdentifier getSectionIdentifier() {

@@ -60,7 +66,6 @@ public class Section extends AbstractSemanticNode {
    }


-
    @Override
    public String toString() {

@@ -85,7 +90,14 @@ public class Section extends AbstractSemanticNode {
     */
    public boolean anyHeadlineContainsString(String value) {

-        return streamAllSubNodesOfType(NodeType.HEADLINE).anyMatch(h -> h.containsString(value));
+        boolean found = streamAllSubNodesOfType(NodeType.HEADLINE).anyMatch(h -> h.containsString(value)) || getHeadline().containsString(value);
+        if (!found) {
+            List<Headline> previousHeadlines = new ArrayList<>();
+            headlinesByPreviousSibling(this, previousHeadlines);
+            return previousHeadlines.stream()
+                    .anyMatch(headline -> headline.containsString(value));
+        }
+        return true;
    }


@@ -97,8 +109,37 @@ public class Section extends AbstractSemanticNode {
     */
    public boolean anyHeadlineContainsStringIgnoreCase(String value) {

-        return streamAllSubNodesOfType(NodeType.HEADLINE).anyMatch(h -> h.containsStringIgnoreCase(value));
+        boolean found = streamAllSubNodesOfType(NodeType.HEADLINE).anyMatch(h -> h.containsStringIgnoreCase(value)) || getHeadline().containsStringIgnoreCase(value);
+        if (!found) {
+            List<Headline> previousHeadlines = new ArrayList<>();
+            headlinesByPreviousSibling(this, previousHeadlines);
+            return previousHeadlines.stream()
+                    .anyMatch(headline -> headline.containsStringIgnoreCase(value));
+        }
+        return true;
    }


+    private void headlinesByPreviousSibling(SemanticNode section, List<Headline> found) {
+
+        if (section.getPreviousSibling()
+                    .isPresent() && section.getPreviousSibling()
+                    .get() instanceof Section previousSection) {
+
+            var subnodes = previousSection.streamAllSubNodes()
+                    .toList();
+            if (subnodes.size() == 1 && subnodes.get(0) instanceof Headline previousHeadline) {
+                found.add(previousHeadline);
+                headlinesByPreviousSibling(previousSection, found);
+            }
+        }
+
+        if (section.getPreviousSibling()
+                    .isPresent() && section.getPreviousSibling()
+                    .get() instanceof Headline previousHeadline) {
+            found.add(previousHeadline);
+            headlinesByPreviousSibling(previousHeadline, found);
+        }
+    }
+
 }
@@ -3,29 +3,36 @@ package com.iqser.red.service.redaction.v1.server.model.document.nodes;
 import java.util.Collections;
 import java.util.LinkedList;
 import java.util.List;
+import java.util.Locale;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;

 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
+import lombok.Getter;
 import lombok.experimental.FieldDefaults;

 /**
- * Represents a unique identifier for a section within a document.
+ * Represents the textual identifier sometimes present in a Headline. For example, given the headline 3.1 Results, the section identifier is 3.1.
+ * Keep in mind, this identifier must not be unique in a single document, as there might be multiple headlines starting with the same textual identifier.
 */
@AllArgsConstructor
@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
 public class SectionIdentifier {

-    static Pattern numericalIdentifierPattern = Pattern.compile("^[\\s]?(\\d+)[\\s.,;]?(\\d+)?[\\s.,;]?(\\d+)?[\\s.,;]?(\\d+)?");
+    public static Pattern numericalIdentifierPattern = Pattern.compile("^[\\s]?(\\d+)[\\s.,;]?(\\d+)?[\\s.,;]?(\\d+)?[\\s.,;]?(\\d+)?");
+    public static Pattern alphanumericIdentifierPattern = Pattern.compile("^[\\s]?[A-Za-z][\\s.,;]?(\\d+)[\\s.,;]?(\\d+)?[\\s.,;]?(\\d+)?[\\s.,;]?(\\d+)?[\\s.,;]?");

-    private enum Format {
+    protected enum Format {
        EMPTY,
        NUMERICAL,
+        ALPHANUMERIC,
        DOCUMENT
    }

+    @Getter
    Format format;
+    @Getter
    String identifierString;
    List<Integer> identifiers;
    boolean asChild;
@@ -47,6 +54,10 @@ public class SectionIdentifier {
        if (numericalIdentifierMatcher.find()) {
            return buildNumericalSectionIdentifier(headline, numericalIdentifierMatcher);
        }
+        Matcher alphanumericIdentifierMatcher = alphanumericIdentifierPattern.matcher(headline);
+        if (alphanumericIdentifierMatcher.find()) {
+            return buildAlphanumericSectionIdentifier(headline, alphanumericIdentifierMatcher);
+        }
        // more formats here
        return SectionIdentifier.empty();
    }
@@ -105,8 +116,34 @@ public class SectionIdentifier {
    }


+    private static SectionIdentifier buildAlphanumericSectionIdentifier(String headline, Matcher alphanumericIdentifierMatcher) {
+
+        String identifierString = headline.substring(alphanumericIdentifierMatcher.start(), alphanumericIdentifierMatcher.end());
+
+        String alphanumericIdentifier = alphanumericIdentifierMatcher.group(0).substring(0, 1).toUpperCase(Locale.ENGLISH);
+        int mappedCharacterValue = alphanumericIdentifier.charAt(0) - 'A' + 1;
+        List<Integer> identifiers = new LinkedList<>();
+        identifiers.add(mappedCharacterValue);
+
+        for (int i = 1; i <= 3; i++) {
+            String numericalIdentifier = alphanumericIdentifierMatcher.group(i);
+            if (numericalIdentifier == null || numericalIdentifier.equals("0") || numericalIdentifier.isEmpty() || numericalIdentifier.isBlank()) {
+                break;
+            }
+            identifiers.add(Integer.parseInt(numericalIdentifier.trim()));
+        }
+
+        return new SectionIdentifier(Format.ALPHANUMERIC,
+                                     // Changed format to reflect alphanumeric
+                                     identifierString,
+                                     identifiers.stream()
+                                             .toList(),
+                                     false);
+    }
+
+
    /**
-     * Determines if the current section is the parent of the given section.
+     * Determines if the current SectionIdentifier is the parent of the given SectionIdentifier.
     *
     * @param sectionIdentifier The section identifier to compare against.
     * @return true if the current section is the parent of the given section, false otherwise.
@@ -155,4 +192,30 @@ public class SectionIdentifier {
        return identifierString;
    }

+
+    /**
+     * @return true, when no identifier could be found
+     */
+    public boolean isEmpty() {
+
+        return this.format.equals(Format.EMPTY);
+    }
+
+
+    /**
+     * The level of a SectionIdentifier corresponds with the count of identifiers. E.g. 1.1 is level 2, and 1. is level 1
+     *
+     * @return the level of the Headline
+     */
+    public int level() {
+
+        return identifiers.size();
+    }
+
+
+    protected List<Integer> getIdentifiers() {
+
+        return identifiers;
+    }
+
 }
@@ -16,6 +16,7 @@ import java.util.stream.Stream;

 import com.iqser.red.service.redaction.v1.server.model.document.DocumentTree;
 import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
+import com.iqser.red.service.redaction.v1.server.model.document.entity.IEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.AtomicTextBlock;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
@@ -59,6 +60,20 @@ public interface SemanticNode {
    Set<TextEntity> getEntities();


+    /**
+     * A view of the Entity Set of this SemanticNode including only the active (APPLIED or SKIPPED) Entities which are of a valid type (ENTITY or HINT).
+     * This is used for all functions, which check for the existence of an Entity, such as hasEntityOfType().
+     *
+     * @return Set of valid TextEntities
+     */
+    default Stream<TextEntity> streamValidEntities() {
+
+        return getEntities().stream()
+                .filter(IEntity::active)
+                .filter(TextEntity::validEntityType);
+    }
+
+
    /**
     * Each AtomicTextBlock is assigned a page, so to get the pages this node appears on, it collects the PageNodes from each AtomicTextBlock belonging to this node's TextBlock.
     *
@@ -149,7 +164,7 @@ public interface SemanticNode {


    /**
-     * Returns a SectionIdentifier, such that it acts as a child of the first Headline associated with this SemanticNode.
+     * Returns the SectionIdentifier as a child of the SectionIdentifier returned by the getHeadline() method.
     *
     * @return The SectionIdentifier from the first Headline.
     */
@@ -259,9 +274,7 @@ public interface SemanticNode {
     */
    default boolean hasEntitiesOfType(String type) {

-        return getEntities().stream()
-                .filter(TextEntity::active)
-                .anyMatch(redactionEntity -> redactionEntity.type().equals(type));
+        return streamValidEntities().anyMatch(redactionEntity -> redactionEntity.type().equals(type));
    }


@@ -274,10 +287,8 @@ public interface SemanticNode {
     */
    default boolean hasEntitiesOfAnyType(String... types) {

-        return getEntities().stream()
-                .filter(TextEntity::active)
-                .anyMatch(redactionEntity -> Arrays.stream(types)
-                        .anyMatch(type -> redactionEntity.type().equals(type)));
+        return streamValidEntities().anyMatch(redactionEntity -> Arrays.stream(types)
+                .anyMatch(type -> redactionEntity.type().equals(type)));
    }


@@ -290,9 +301,7 @@ public interface SemanticNode {
     */
    default boolean hasEntitiesOfAllTypes(String... types) {

-        return getEntities().stream()
-                .filter(TextEntity::active)
-                .map(TextEntity::type)
+        return streamValidEntities().map(TextEntity::type)
                .collect(Collectors.toUnmodifiableSet())
                .containsAll(Arrays.stream(types)
                                     .toList());
@@ -301,31 +310,28 @@ public interface SemanticNode {

    /**
     * Returns a List of Entities in this SemanticNode which are of the provided type such as "CBI_author".
-     * Ignores Entity with ignored == true or removed == true.
+     * Ignores Entity which are not active or of a removal type ignored == true or removed == true.
     *
     * @param type string representing the type of entities to return
     * @return List of RedactionEntities of any the type
     */
    default List<TextEntity> getEntitiesOfType(String type) {

-        return getEntities().stream()
-                .filter(TextEntity::active)
-                .filter(redactionEntity -> redactionEntity.type().equals(type))
+        return streamValidEntities().filter(redactionEntity -> redactionEntity.type().equals(type))
                .toList();
    }


    /**
     * Returns a List of Entities in this SemanticNode which have any of the provided types such as "CBI_author".
-     * Ignores Entity with ignored == true or removed == true.
+     * Ignores Entity that are not valid.
     *
     * @param types A list of strings representing the types of entities to return
     * @return List of RedactionEntities of any provided type
     */
    default List<TextEntity> getEntitiesOfType(List<String> types) {

-        return getEntities().stream()
-                .filter(TextEntity::active)
+        return streamValidEntities()//
                .filter(redactionEntity -> redactionEntity.isAnyType(types))
                .toList();
    }
@@ -333,15 +339,14 @@ public interface SemanticNode {

    /**
     * Returns a List of Entities in this SemanticNode which have any of the provided types.
-     * Ignores Entity with the ignored flag set to true or the removed flag set to true.
+     * Ignores Entity that are not valid.
     *
     * @param types A list of strings representing the types of entities to return
     * @return List of RedactionEntities that match any of the provided types
     */
    default List<TextEntity> getEntitiesOfType(String... types) {

-        return getEntities().stream()
-                .filter(TextEntity::active)
+        return streamValidEntities()//
                .filter(redactionEntity -> redactionEntity.isAnyType(Arrays.stream(types)
                                                                             .toList()))
                .toList();
@@ -445,7 +450,7 @@ public interface SemanticNode {
     */
    default boolean containsStringIgnoreCase(String string) {

-        return getTextBlock().getSearchText().toLowerCase(Locale.ROOT).contains(string.toLowerCase(Locale.ROOT));
+        return getTextBlock().getSearchTextLowerCase().contains(string.toLowerCase(Locale.ROOT));
    }


@@ -756,13 +761,12 @@ public interface SemanticNode {


    /**
-     * TODO: this produces unwanted results for sections spanning multiple columns.
     * Computes the Union of the bounding boxes of all children recursively.
     *
     * @return The union of the BoundingBoxes of all children
     */
    private Map<Page, Rectangle2D> getBBoxFromChildren() {
-
+        //TODO: this produces unwanted results for sections spanning multiple columns.
        Map<Page, Rectangle2D> bBoxPerPage = new HashMap<>();
        List<Map<Page, Rectangle2D>> childrenBBoxes = streamChildren().map(SemanticNode::getBBox)
                .toList();
@@ -808,4 +812,17 @@ public interface SemanticNode {
        streamChildren().forEach(childNode -> childNode.accept(visitor));
    }

+
+    /**
+     * Checks wether this SemanticNode appears on a single page only, and if that page is the provided one.
+     *
+     * @param page the page to check
+     * @return true, when SemanticNode is on a single page only and the page is the provided page. Otherwise, false.
+     */
+    default boolean onlyOnPage(Page page) {
+
+        Set<Page> pages = getPages();
+        return pages.size() == 1 && pages.contains(page);
+    }
+
 }
@@ -2,7 +2,6 @@ package com.iqser.red.service.redaction.v1.server.model.document.nodes;

 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
-import lombok.Builder;
 import lombok.Data;
 import lombok.EqualsAndHashCode;
 import lombok.experimental.FieldDefaults;
@@ -39,6 +38,11 @@ public class SuperSection extends AbstractSemanticNode {
    }


+    /**
+     * Returns the SectionIdentifier from the headline obtained by the getHeadline() method.
+     *
+     * @return the SectionIdentifier of the associated Headline
+     */
    @Override
    public SectionIdentifier getSectionIdentifier() {

@@ -46,7 +50,6 @@ public class SuperSection extends AbstractSemanticNode {
    }


-
    @Override
    public String toString() {

@@ -86,5 +89,4 @@ public class SuperSection extends AbstractSemanticNode {
        return streamAllSubNodesOfType(NodeType.HEADLINE).anyMatch(h -> h.containsStringIgnoreCase(value));
    }

-
 }
@@ -74,8 +74,7 @@ public class Table implements SemanticNode {
        return IntStream.range(0, numberOfRows).boxed()
                .filter(row -> rowContainsStringsIgnoreCase(row, strings))
                .flatMap(this::streamRow)
-                .map(TableCell::getEntities)
-                .flatMap(Collection::stream);
+                .flatMap(TableCell::streamValidEntities);
    }


@@ -135,11 +134,11 @@ public class Table implements SemanticNode {


    /**
-     * Streams all entities in this table, that appear in a row, which contains at least one entity with any of the provided types.
+     * Streams all entities in this table, that appear in a row, which contains at least one valid entity with any of the provided types.
     * Ignores Entity with ignored == true or removed == true.
     *
     * @param types type strings to check whether a row contains an entity like them
-     * @return Stream of all entities in this table, that appear in a row, which contains at least one entity with any of the provided types.
+     * @return Stream of all entities in this table, that appear in a row, which contains at least one valid entity with any of the provided types.
     */
    public Stream<TextEntity> streamEntitiesWhereRowContainsEntitiesOfType(List<String> types) {

@@ -192,30 +191,26 @@ public class Table implements SemanticNode {


    /**
-     * Streams all Entities in the given row.
+     * Streams all valid Entities in the given row.
     *
     * @param rowNumber the row number to look for
     * @return stream of TextEntities occurring in row
     */
    public Stream<TextEntity> streamTextEntitiesInRow(int rowNumber) {

-        return streamRow(rowNumber).map(TableCell::getEntities)
-                .flatMap(Collection::stream)
-                .filter(TextEntity::active);
+        return streamRow(rowNumber).flatMap(TableCell::streamValidEntities);
    }


    /**
-     * Streams all Entities in the given col.
+     * Streams all valid Entities in the given col.
     *
     * @param colNumber the column number to look for
     * @return stream of TextEntities occurring in row
     */
    public Stream<TextEntity> streamTextEntitiesInCol(int colNumber) {

-        return streamCol(colNumber).map(TableCell::getEntities)
-                .flatMap(Collection::stream)
-                .filter(TextEntity::active);
+        return streamCol(colNumber).flatMap(TableCell::streamValidEntities);
    }


@@ -269,6 +264,7 @@ public class Table implements SemanticNode {

        return streamHeaders().filter(tableCellNode -> tableCellNode.getTextBlock().getSearchText().contains(header))
                .map(TableCell::getCol)
+                .distinct()
                .flatMap(this::streamCol)
                .filter(tableCellNode -> !tableCellNode.isHeader());
    }
@@ -2,20 +2,13 @@ package com.iqser.red.service.redaction.v1.server.model.document.nodes;

 import java.awt.geom.Rectangle2D;
 import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
 import java.util.Map;
-import java.util.Set;

-import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
-import com.iqser.red.service.redaction.v1.server.model.document.DocumentTree;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlockCollector;
-import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;

 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
-import lombok.Builder;
 import lombok.Data;
 import lombok.EqualsAndHashCode;
 import lombok.experimental.FieldDefaults;
@@ -42,7 +35,6 @@ public class TableCell extends AbstractSemanticNode {
    TextBlock textBlock;


-
    @Override
    public Map<Page, Rectangle2D> getBBox() {

@@ -3,6 +3,7 @@ package com.iqser.red.service.redaction.v1.server.model.document.textblock;
 import static java.lang.String.format;

 import java.awt.geom.Rectangle2D;
+import java.lang.ref.SoftReference;
 import java.text.BreakIterator;
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -43,9 +44,11 @@ public class AtomicTextBlock implements TextBlock {
    //string coordinates
    TextRange textRange;
    String searchText;
-    List<String> words;
    List<Integer> lineBreaks;

+    SoftReference<String> searchTextLowerCaseCache;
+    SoftReference<List<String>> wordsCache;
+
    //position coordinates
    List<Integer> stringIdxToPositionIdx;
    @Getter
@@ -121,8 +124,31 @@ public class AtomicTextBlock implements TextBlock {
    }


+    @Override
+    public String getSearchTextLowerCase() {
+
+        String text = null;
+        if (searchTextLowerCaseCache != null) {
+            text = searchTextLowerCaseCache.get();
+        }
+
+        if (text == null) {
+            text = getSearchText().toLowerCase(Locale.ENGLISH);
+            searchTextLowerCaseCache = new SoftReference<>(text);
+        }
+
+        return text;
+    }
+
+
    public List<String> getWords() {

+        List<String> words = null;
+
+        if (wordsCache != null) {
+            words = wordsCache.get();
+        }
+
        if (words == null) {
            words = new ArrayList<>();
            BreakIterator iterator = BreakIterator.getWordInstance(Locale.ENGLISH);
@@ -131,6 +157,7 @@ public class AtomicTextBlock implements TextBlock {
            for (int end = iterator.next(); end != BreakIterator.DONE; start = end, end = iterator.next()) {
                words.add(searchText.substring(start, end));
            }
+            wordsCache = new SoftReference<>(words);
        }
        return words;
    }
@@ -3,11 +3,13 @@ package com.iqser.red.service.redaction.v1.server.model.document.textblock;
 import static java.lang.String.format;

 import java.awt.geom.Rectangle2D;
+import java.lang.ref.SoftReference;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.LinkedList;
 import java.util.List;
-import java.util.Collection;
+import java.util.Locale;
 import java.util.Map;
 import java.util.stream.Stream;

@@ -25,6 +27,7 @@ public class ConcatenatedTextBlock implements TextBlock {
    List<AtomicTextBlock> atomicTextBlocks;
    String searchText;
    TextRange textRange;
+    SoftReference<String> searchTextLowerCaseCache;


    public static ConcatenatedTextBlock empty() {
@@ -100,6 +103,23 @@ public class ConcatenatedTextBlock implements TextBlock {
    }


+    @Override
+    public String getSearchTextLowerCase() {
+
+        String text = null;
+        if (searchTextLowerCaseCache != null) {
+            text = searchTextLowerCaseCache.get();
+        }
+
+        if (text == null) {
+            text = getSearchText().toLowerCase(Locale.ENGLISH);
+            searchTextLowerCaseCache = new SoftReference<>(text);
+        }
+
+        return text;
+    }
+
+
    @Override
    public List<String> getWords() {

@@ -19,6 +19,9 @@ public interface TextBlock extends CharSequence {
    String getSearchText();


+    String getSearchTextLowerCase();
+
+
    List<String> getWords();


@@ -20,6 +20,7 @@ import org.springframework.stereotype.Service;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeResult;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.ErrorCode;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileErrorInfo;
 import com.iqser.red.service.redaction.v1.server.client.FileStatusProcessingUpdateClient;
 import com.iqser.red.service.redaction.v1.server.client.RulesClient;
@@ -110,7 +111,14 @@ public class RedactionMessageReceiver {
                    log.info("-------------------------------------------------------------------------------------------------");
                    shouldRespond = false;
                    break;
-
+                case IMPORTED_REDACTIONS_ONLY:
+                    log.info("------------------------------Imported Redactions Analysis Only------------------------------------------");
+                    log.info("Starting Imported Redactions Analysis Only for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
+                    log.debug(analyzeRequest.getManualRedactions().toString());
+                    result = analyzeService.analyzeImportedRedactionsOnly(analyzeRequest);
+                    log.info("Successful Imported Redactions Analysis Only dossier {} file {}", analyzeRequest.getDossierId(), analyzeRequest.getFileId());
+                    log.info("-------------------------------------------------------------------------------------------------");
+                    break;
                default:
                    throw new IllegalArgumentException("Unknown MessageType: " + analyzeRequest.getMessageType());
            }
@@ -132,11 +140,20 @@ public class RedactionMessageReceiver {

    private void sendAnalysisFailed(AnalyzeRequest analyzeRequest, boolean priority, Exception e) {

-        log.warn("Failed to process analyze request: {}", analyzeRequest, e);
-        var timestamp = OffsetDateTime.now().truncatedTo(ChronoUnit.MILLIS);
+        log.error("Failed to process analyze request: {}", analyzeRequest, e);
+        ErrorCode errorCode = null;
+
+        if(e instanceof DroolsTimeoutException dre){
+            if (!dre.isReported()){
+                errorCode = ErrorCode.RULES_EXECUTION_TIMEOUT;
+            } else {
+                errorCode = ErrorCode.LOCKED_RULES;
+            }
+        }
+
        fileStatusProcessingUpdateClient.analysisFailed(analyzeRequest.getDossierId(),
                                                        analyzeRequest.getFileId(),
-                                                        new FileErrorInfo(e.getMessage(), priority ? REDACTION_PRIORITY_QUEUE : REDACTION_QUEUE, "redaction-service", timestamp));
+                                                        new FileErrorInfo(e.getMessage(), priority ? REDACTION_PRIORITY_QUEUE : REDACTION_QUEUE, "redaction-service", errorCode));
    }


@@ -23,8 +23,10 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileTyp
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentLog;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogChanges;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedLegalBases;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedactions;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.mapper.ImportedLegalBasisMapper;
 import com.iqser.red.service.redaction.v1.server.RedactionServiceSettings;
 import com.iqser.red.service.redaction.v1.server.client.model.NerEntitiesModel;
 import com.iqser.red.service.redaction.v1.server.model.KieWrapper;
@@ -76,6 +78,7 @@ public class AnalyzeService {
    ImportedRedactionEntryService importedRedactionEntryService;
    ObservedStorageService observedStorageService;
    FunctionTimerValues redactmanagerAnalyzePagewiseValues;
+    ImportedLegalBasisMapper importedLegalBasisMapper = ImportedLegalBasisMapper.INSTANCE;


    @Timed("redactmanager_reanalyze")
@@ -128,7 +131,7 @@ public class AnalyzeService {
                                    document,
                                    document.getNumberOfPages(),
                                    true,
-                                    Collections.emptySet());
+                                    new HashSet<>());
        }

        KieWrapper kieWrapperEntityRules = kieContainerCreationService.getLatestKieContainer(analyzeRequest.getDossierTemplateId(), RuleFileType.ENTITY);
@@ -152,7 +155,7 @@ public class AnalyzeService {
        log.info("Finished Dictionary Search for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());

        // we could add the imported redactions similar to the manual redactions here as well for additional processing
-        List<FileAttribute> allFileAttributes = entityDroolsExecutionService.executeRules(kieWrapperEntityRules.container(),
+        List<FileAttribute> allFileAttributes = entityDroolsExecutionService.executeRules(analyzeRequest.getFileId(), kieWrapperEntityRules.container(),
                                                                                          document,
                                                                                          sectionsToReAnalyse,
                                                                                          dictionary,
@@ -218,7 +221,7 @@ public class AnalyzeService {
        log.info("Finished Dictionary Search for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());

        // we could add the imported redactions similar to the manual redactions here as well for additional processing
-        List<FileAttribute> allFileAttributes = entityDroolsExecutionService.executeRules(kieWrapperEntityRules.container(),
+        List<FileAttribute> allFileAttributes = entityDroolsExecutionService.executeRules(analyzeRequest.getFileId(), kieWrapperEntityRules.container(),
                                                                                          document,
                                                                                          dictionary,
                                                                                          analyzeRequest.getFileAttributes(),
@@ -245,6 +248,39 @@ public class AnalyzeService {
    }


+    @Timed("redactmanager_analyzeImportedRedactionsOnly")
+    @Observed(name = "AnalyzeService", contextualName = "analyzeImportedRedactionsOnly")
+    public AnalyzeResult analyzeImportedRedactionsOnly(AnalyzeRequest analyzeRequest) {
+
+        long startTime = System.currentTimeMillis();
+
+        dictionaryService.updateDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId());
+
+        Document document = DocumentGraphMapper.toDocumentGraph(observedStorageService.getDocumentData(analyzeRequest.getDossierId(), analyzeRequest.getFileId()));
+        log.info("Loaded Document Graph for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
+
+        ImportedRedactions importedRedactions = redactionStorageService.getImportedRedactions(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
+        log.info("Loaded Imported Redactions for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
+
+        ImportedLegalBases importedLegalBases = redactionStorageService.getImportedLegalBases(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
+        log.info("Loaded Imported Legal Bases for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
+
+        var notFoundImportedEntries = importedRedactionEntryService.addImportedEntriesAndReturnNotFoundEntries(analyzeRequest, importedRedactions, document);
+
+        EntityLogChanges entityLogChanges = entityLogCreatorService.createInitialEntityLog(analyzeRequest, document, notFoundImportedEntries, new DictionaryVersion(0, 0), 0);
+
+        entityLogChanges.getEntityLog()
+                .setLegalBasis(importedLegalBases.getImportedLegalBases()
+                                       .stream()
+                                       .map(importedLegalBasisMapper::toEntityLogLegalBasis)
+                                       .toList());
+
+        notFoundImportedEntitiesService.processEntityLog(entityLogChanges.getEntityLog(), analyzeRequest, notFoundImportedEntries);
+
+        return finalizeAnalysis(analyzeRequest, startTime, KieWrapper.empty(), entityLogChanges, document, document.getNumberOfPages(), false, new HashSet<>());
+    }
+
+
    private AnalyzeResult finalizeAnalysis(AnalyzeRequest analyzeRequest,
                                           long startTime,
                                           KieWrapper kieWrapperComponentRules,
@@ -314,7 +350,7 @@ public class AnalyzeService {
        // We need the latest EntityLog entries for components rules execution
        entityLog.setEntityLogEntry(redactionStorageService.getEntityLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId()).getEntityLogEntry());

-        List<Component> components = componentDroolsExecutionService.executeRules(kieWrapperComponentRules.container(),
+        List<Component> components = componentDroolsExecutionService.executeRules(analyzeRequest.getFileId(), kieWrapperComponentRules.container(),
                                                                                  entityLog,
                                                                                  document,
                                                                                  addedFileAttributes,
@@ -322,7 +358,10 @@ public class AnalyzeService {

        log.info("Finished component rule execution for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());

-        ComponentLog componentLog = componentLogCreatorService.buildComponentLog(analyzeRequest.getAnalysisNumber(), components, kieWrapperComponentRules.rulesVersion());
+        ComponentLog componentLog = componentLogCreatorService.buildComponentLog(analyzeRequest.getAnalysisNumber(),
+                                                                                 components,
+                                                                                 kieWrapperComponentRules.rulesVersion(),
+                                                                                 analyzeRequest.getComponentDefinitions());

        redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.COMPONENT_LOG, componentLog);

@@ -367,7 +406,7 @@ public class AnalyzeService {

        return new NerEntitiesModel(nerEntitiesModel.getData().entrySet()
                                            .stream() //
-                                            .filter(entry -> sectionsToReanalyseIds.contains(entry.getKey())) //
+                                            .filter(entry -> sectionsToReanalyseIds.contains(getSuperSectionID(entry.getKey()))) //
                                            .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)));
    }

@@ -383,4 +422,11 @@ public class AnalyzeService {
        return nerEntities;
    }

+
+    private static Integer getSuperSectionID(String section) {
+
+        return NerEntitiesAdapter.sectionNumberToTreeId(section)
+                .get(0);
+    }
+
 }
@@ -58,7 +58,7 @@ public class DictionarySearchService {

        Set<Engine> engines = isDossierDictionaryEntry ? Set.of(Engine.DOSSIER_DICTIONARY) : Set.of(Engine.DICTIONARY);
        EntityCreationService entityCreationService = new EntityCreationService(entityEnrichmentService);
-        searchImplementation.getBoundaries(node.getTextBlock(), node.getTextRange())
+        searchImplementation.getBoundaries(node.getTextBlock())
                .filter(boundary -> entityCreationService.isValidEntityTextRange(node.getTextBlock(), boundary))
                .forEach(bounds -> entityCreationService.byTextRangeWithEngine(bounds, type, entityType, node, engines)
                        .ifPresent(entity -> {
@@ -74,6 +74,12 @@ public class DictionaryService {
    }


+    public void clearTenantDictionaryCache() {
+
+        tenantDictionaryCache.invalidateAll();
+    }
+
+
    @SneakyThrows
    @Observed(name = "DictionaryService", contextualName = "update-dictionary")
    @Timed("redactmanager_updateDictionary")
@@ -223,7 +223,6 @@ public class EntityLogCreatorService {

        String type = precursorEntity.getManualOverwrite().getType()
                .orElse(precursorEntity.getType());
-        boolean isHint = isHint(precursorEntity.getEntityType());
        return EntityLogEntry.builder()
                .id(precursorEntity.getId())
                .reason(precursorEntity.buildReasonWithManualChangeDescriptions())
@@ -341,12 +340,6 @@ public class EntityLogCreatorService {
    }


-    private boolean isHint(EntityType entityType) {
-
-        return entityType.equals(EntityType.HINT);
-    }
-
-
    public static EntryState buildEntryState(IEntity entity) {

        if (entity.applied() && entity.active()) {
@@ -13,7 +13,8 @@ import org.springframework.amqp.rabbit.core.RabbitTemplate;
 import org.springframework.stereotype.Service;

 import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.BaseAnnotation;
@@ -61,7 +62,9 @@ public class UnprocessedChangesService {

        dictionaryService.updateDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId());

-        EntityLog previousEntityLog = redactionStorageService.getEntityLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
+        List<String> annotationIds = analyzeRequest.getManualRedactions().buildAll().stream().map(BaseAnnotation::getAnnotationId).toList();
+
+        List<EntityLogEntry> entityLogEntries = redactionStorageService.getEntityLogEntriesById(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), annotationIds);
        Document document = DocumentGraphMapper.toDocumentGraph(observedStorageService.getDocumentData(analyzeRequest.getDossierId(), analyzeRequest.getFileId()));

        Set<String> allAnnotationIds = analyzeRequest.getManualRedactions().getEntriesToAdd()
@@ -77,9 +80,10 @@ public class UnprocessedChangesService {
        List<ManualResizeRedaction> manualResizeRedactions = analyzeRequest.getManualRedactions().getResizeRedactions()
                .stream()
                .toList();
-        List<PrecursorEntity> manualEntitiesToBeResized = previousEntityLog.getEntityLogEntry()
+        List<PrecursorEntity> manualEntitiesToBeResized = entityLogEntries
                .stream()
                .filter(entityLogEntry -> resizeIds.contains(entityLogEntry.getId()))
+                .filter(entityLogEntry -> entityLogEntry.getEngines().contains(Engine.MANUAL))
                .toList()
                .stream()
                .map(PrecursorEntity::fromEntityLogEntry)
@@ -1,9 +1,15 @@
 package com.iqser.red.service.redaction.v1.server.service.components;

 import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
+import java.util.Locale;
 import java.util.Map;
+import java.util.Set;
+import java.util.function.Function;
+import java.util.stream.Collectors;

 import org.springframework.stereotype.Service;

@@ -12,6 +18,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentLogEntry;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentLogEntryValue;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.component.ComponentDefinition;
 import com.iqser.red.service.redaction.v1.server.model.component.Component;
 import com.iqser.red.service.redaction.v1.server.model.component.Entity;
 import com.iqser.red.service.redaction.v1.server.service.document.ComponentComparator;
@@ -20,7 +27,10 @@ import com.iqser.red.service.redaction.v1.server.service.document.EntityComparat
@Service
 public class ComponentLogCreatorService {

-    public ComponentLog buildComponentLog(int analysisNumber, List<Component> components, long componentRulesVersion) {
+    public ComponentLog buildComponentLog(int analysisNumber,
+                                          List<Component> components,
+                                          long componentRulesVersion,
+                                          List<ComponentDefinition> componentDefinitions) {

        Map<String, List<ComponentLogEntryValue>> map = new HashMap<>();
        components.stream()
@@ -29,11 +39,23 @@ public class ComponentLogCreatorService {
                    ComponentLogEntryValue componentLogEntryValue = buildComponentLogEntry(component);
                    map.computeIfAbsent(component.getName(), k -> new ArrayList<>()).add(componentLogEntryValue);
                });
-        List<ComponentLogEntry> componentLogComponents = map.entrySet()
+        List<ComponentLogEntry> componentLogEntries = map.entrySet()
                .stream()
                .map(entry -> new ComponentLogEntry(entry.getKey(), entry.getValue(), false))
-                .toList();
-        return new ComponentLog(analysisNumber, componentRulesVersion, componentLogComponents);
+                .collect(Collectors.toList());
+
+        Set<String> existingComponentNames = map.keySet()
+                .stream()
+                .map(s -> s.toLowerCase(Locale.ROOT))
+                .collect(Collectors.toSet());
+        for (ComponentDefinition componentDefinition : componentDefinitions) {
+            String technicalName = componentDefinition.getTechnicalName().toLowerCase(Locale.ROOT);
+            if (!existingComponentNames.contains(technicalName)) {
+                componentLogEntries.add(new ComponentLogEntry(componentDefinition.getDisplayName(), new ArrayList<>(), false));
+            }
+        }
+
+        return new ComponentLog(analysisNumber, componentRulesVersion, componentLogEntries);
    }


@@ -41,6 +63,7 @@ public class ComponentLogCreatorService {

        return ComponentLogEntryValue.builder()
                .value(component.getValue())
+                .originalValue(component.getValue())
                .componentRuleId(component.getMatchedRule().toString())
                .valueDescription(component.getValueDescription())
                .componentLogEntityReferences(toComponentEntityReferences(component.getReferences()
@@ -40,7 +40,8 @@ public class ComponentMappingFileSystemCache {
    public File getComponentMappingFile(ComponentMappingMetadata metadata) {

        Path mappingFile = getMappingFileFromMetadata(metadata);
-        Path mappingFileMetaDataFile = mappingFile.resolveSibling(metadata.getName() + METADATA_SUFFIX);
+        Path mappingFileMetaDataFile = getMappingMetadataFileFromMetadata(metadata);
+
        synchronized (ComponentMappingFileSystemCache.class) {

            if (fileExistsAndUpToDate(metadata, mappingFile, mappingFileMetaDataFile)) {
@@ -52,7 +53,7 @@ public class ComponentMappingFileSystemCache {
            InputStreamResource inputStreamResource = storageService.getObject(TenantContext.getTenantId(), metadata.getStorageId());

            Files.write(mappingFile, inputStreamResource.getContentAsByteArray(), StandardOpenOption.WRITE, StandardOpenOption.TRUNCATE_EXISTING, StandardOpenOption.CREATE);
-            
+
            mapper.writeValue(mappingFileMetaDataFile.toFile(), metadata);

            return mappingFile.toFile();
@@ -87,6 +88,13 @@ public class ComponentMappingFileSystemCache {
    }


+    private Path getMappingMetadataFileFromMetadata(ComponentMappingMetadata metadata) {
+
+        Path tenantStem = mappingFileDir.resolve(TenantContext.getTenantId());
+        return tenantStem.resolve(metadata.getStorageId() + METADATA_SUFFIX);
+    }
+
+
    private boolean fileExistsAndUpToDate(ComponentMappingMetadata metadata, Path mappingFile, Path mappingFileMetaDataFile) {

        if (mappingFile.toFile().exists() && mappingFile.toFile().isFile() && mappingFileMetaDataFile.toFile().exists() && mappingFileMetaDataFile.toFile().isFile()) {
@@ -4,6 +4,7 @@ import java.io.File;
 import java.io.FileReader;
 import java.io.Reader;
 import java.nio.charset.Charset;
+import java.time.Duration;
 import java.util.List;

 import org.springframework.stereotype.Service;
@@ -36,7 +37,7 @@ public class ComponentMappingMemoryCache {
    public ComponentMappingMemoryCache(ComponentMappingFileSystemCache componentMappingFileSystemCache) {

        this.fileSystemCache = componentMappingFileSystemCache;
-        cache = CacheBuilder.newBuilder().maximumWeight(MAX_NUMBER_OF_LINES).weigher(COMPONENT_MAPPING_WEIGHER).build();
+        cache = CacheBuilder.newBuilder().maximumWeight(MAX_NUMBER_OF_LINES).weigher(COMPONENT_MAPPING_WEIGHER).expireAfterAccess(Duration.ofDays(1)).build();
    }


@@ -1,6 +1,5 @@
 package com.iqser.red.service.redaction.v1.server.service.document;

-
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
@@ -64,7 +63,7 @@ public class DocumentGraphMapper {
        for (DocumentStructure.EntryData entryData : entries) {

            List<Page> pages = Arrays.stream(entryData.getPageNumbers())
-                    .map(pageNumber -> getPage(pageNumber, context))
+                    .map(context::getPage)
                    .toList();

            SemanticNode node = switch (entryData.getType()) {
@@ -83,6 +82,15 @@ public class DocumentGraphMapper {
            if (entryData.getAtomicBlockIds().length > 0) {
                TextBlock textBlock = toTextBlock(entryData.getAtomicBlockIds(), context, node);
                node.setLeafTextBlock(textBlock);
+
+                switch (entryData.getType()) {
+                    case HEADER -> pages.forEach(page -> page.setHeader((Header) node));
+                    case FOOTER -> pages.forEach(page -> page.setFooter((Footer) node));
+                    case IMAGE -> pages.forEach(page -> page.getImages().add((Image) node));
+                    default -> textBlock.getAtomicTextBlocks()
+                            .forEach(atb -> atb.getPage().getTextBlocksOnPage().add(atb));
+                }
+
            }
            List<Integer> treeId = Arrays.stream(entryData.getTreeId()).boxed()
                    .toList();
@@ -94,13 +102,8 @@ public class DocumentGraphMapper {
            }
            node.setTreeId(treeId);

-            switch (entryData.getType()) {
-                case HEADER -> pages.forEach(page -> page.setHeader((Header) node));
-                case FOOTER -> pages.forEach(page -> page.setFooter((Footer) node));
-                default -> pages.forEach(page -> page.getMainBody().add(node));
-            }
-
            newEntries.add(DocumentTree.Entry.builder().treeId(treeId).children(buildEntries(entryData.getChildren(), context)).node(node).build());
+
        }
        return newEntries;
    }
@@ -115,7 +118,7 @@ public class DocumentGraphMapper {
    private Image buildImage(Context context, Map<String, String> properties, Long[] pageNumbers) {

        assert pageNumbers.length == 1;
-        Page page = getPage(pageNumbers[0], context);
+        Page page = context.getPage(pageNumbers[0]);
        var builder = Image.builder();
        PropertiesMapper.parseImageProperties(properties, builder);
        return builder.documentTree(context.documentTree).page(page).build();
@@ -161,6 +164,7 @@ public class DocumentGraphMapper {
        return SuperSection.builder().documentTree(context.documentTree).build();
    }

+
    private Paragraph buildParagraph(Context context, Map<String, String> properties) {

        if (PropertiesMapper.isDuplicateParagraph(properties)) {
@@ -189,21 +193,13 @@ public class DocumentGraphMapper {
        return AtomicTextBlock.fromAtomicTextBlockData(context.documentTextData.get(Math.toIntExact(atomicTextBlockId)),
                                                       context.documentPositionData.get(Math.toIntExact(atomicTextBlockId)),
                                                       parent,
-                                                       getPage(context.documentTextData.get(Math.toIntExact(atomicTextBlockId)).getPage(), context));
+                                                       context.getPage(context.documentTextData.get(Math.toIntExact(atomicTextBlockId)).getPage()));
    }


    private Page buildPage(DocumentPage p) {

-        return Page.builder().rotation(p.getRotation()).height(p.getHeight()).width(p.getWidth()).number(p.getNumber()).mainBody(new LinkedList<>()).build();
-    }
-
-
-    private Page getPage(Long pageIndex, Context context) {
-
-        Page page = context.pageData.get(Math.toIntExact(pageIndex) - 1);
-        assert page.getNumber() == Math.toIntExact(pageIndex);
-        return page;
+        return Page.builder().rotation(p.getRotation()).height(p.getHeight()).width(p.getWidth()).number(p.getNumber()).textBlocksOnPage(new LinkedList<>()).build();
    }


@@ -226,6 +222,14 @@ public class DocumentGraphMapper {

        }

+
+        private Page getPage(Long pageIndex) {
+
+            Page page = pageData.get(Math.toIntExact(pageIndex) - 1);
+            assert page.getNumber() == Math.toIntExact(pageIndex);
+            return page;
+        }
+
    }

 }
@@ -10,6 +10,7 @@ import static com.iqser.red.service.redaction.v1.server.service.document.EntityC
 import static com.iqser.red.service.redaction.v1.server.utils.SeparatorUtils.boundaryIsSurroundedBySeparators;

 import java.util.Collection;
+import java.util.Collections;
 import java.util.Comparator;
 import java.util.LinkedList;
 import java.util.List;
@@ -49,12 +50,22 @@ public class EntityCreationService {

    private final EntityEnrichmentService entityEnrichmentService;
    private final KieSession kieSession;
+    private final Set<SemanticNode> nodesInKieSession; // empty set means all nodes are in kieSession


    public EntityCreationService(EntityEnrichmentService entityEnrichmentService) {

        this.entityEnrichmentService = entityEnrichmentService;
        this.kieSession = null;
+        this.nodesInKieSession = Collections.emptySet();
+    }
+
+
+    public EntityCreationService(EntityEnrichmentService entityEnrichmentService, KieSession kieSession) {
+
+        this.entityEnrichmentService = entityEnrichmentService;
+        this.kieSession = kieSession;
+        this.nodesInKieSession = Collections.emptySet();
    }


@@ -464,7 +475,7 @@ public class EntityCreationService {
     */
    public Stream<TextEntity> bySearchImplementation(SearchImplementation searchImplementation, String type, EntityType entityType, SemanticNode node) {

-        return searchImplementation.getBoundaries(node.getTextBlock(), node.getTextRange())
+        return searchImplementation.getBoundaries(node.getTextBlock())
                .filter(boundary -> isValidEntityTextRange(node.getTextBlock(), boundary))
                .map(bounds -> byTextRange(bounds, type, entityType, node))
                .filter(Optional::isPresent)
@@ -485,7 +496,7 @@ public class EntityCreationService {

        TextBlock textBlock = node.getTextBlock();
        SearchImplementation searchImplementation = new SearchImplementation(strings, false);
-        return searchImplementation.getBoundaries(textBlock, node.getTextRange())
+        return searchImplementation.getBoundaries(textBlock)
                .map(boundary -> toLineAfterTextRange(textBlock, boundary))
                .filter(boundary -> isValidEntityTextRange(textBlock, boundary))
                .map(boundary -> byTextRange(boundary, type, entityType, node))
@@ -507,7 +518,7 @@ public class EntityCreationService {

        TextBlock textBlock = node.getTextBlock();
        SearchImplementation searchImplementation = new SearchImplementation(strings, true);
-        return searchImplementation.getBoundaries(textBlock, node.getTextRange())
+        return searchImplementation.getBoundaries(textBlock)
                .map(boundary -> toLineAfterTextRange(textBlock, boundary))
                .filter(boundary -> isValidEntityTextRange(textBlock, boundary))
                .map(boundary -> byTextRange(boundary, type, entityType, node))
@@ -603,21 +614,21 @@ public class EntityCreationService {
    /**
     * Looks across the remaining table row to the right of the provided TableCell if any line intersects the y coordinates of the found text.
     *
-     * @param TextRanges a list of textRanges
+     * @param textRanges a list of textRanges
     * @param tableCell  the table cell
     * @param type       the type
     * @param entityType the entity type
     * @param tableNode  the table node
     * @return a stream of RedactionEntities
     */
-    private Stream<TextEntity> lineAfterBoundariesAcrossColumns(List<TextRange> TextRanges, TableCell tableCell, String type, EntityType entityType, Table tableNode) {
+    private Stream<TextEntity> lineAfterBoundariesAcrossColumns(List<TextRange> textRanges, TableCell tableCell, String type, EntityType entityType, Table tableNode) {

-        return TextRanges.stream()
+        return textRanges.stream()
                .map(boundary -> RectangleTransformations.rectangle2DBBox(tableCell.getTextBlock().getPositions(boundary)))
                .map(bBox -> Pair.of(bBox.getMaxY(), bBox.getMinY()))
                .map(maxMinPair -> tableNode.streamRow(tableCell.getRow())
                        .filter(nextTableCell -> nextTableCell.getCol() > tableCell.getCol())
-                        .map(nextTableCell -> RedactionSearchUtility.findTextRangesOfAllLinesInYRange(maxMinPair.getLeft(), maxMinPair.getRight(), nextTableCell.getTextBlock()))
+                        .map(nextTableCell -> RedactionSearchUtility.findTextRangesOfAllLinesWithCloseYCoordinates(maxMinPair.getLeft(), maxMinPair.getRight(), nextTableCell.getTextBlock()))
                        .map(b -> b.trim(tableNode.getTextBlock()))
                        .filter(boundary -> isValidEntityTextRange(tableNode.getTextBlock(), boundary))
                        .map(boundary -> byTextRange(boundary, type, entityType, tableNode))
@@ -989,16 +1000,19 @@ public class EntityCreationService {
                return Optional.empty(); // Entity has been recategorized and should not be created at all.
            }
            TextEntity existingEntity = optionalTextEntity.get();
-            if (existingEntity.getTextRange().equals(textRange)) {
+            if (existingEntity.getTextRange().equals(trimmedTextRange)) {
                return optionalTextEntity; // exactly the same entity, return directly
            }
            if (!existingEntity.resized()) {
-                addDuplicateEntityToGraph(existingEntity, textRange, node);
+                addDuplicateEntityToGraph(existingEntity, trimmedTextRange, node);
                return optionalTextEntity; // If Entity has not been resized, insert as duplicate at appropriate position
            }
            return Optional.empty(); // Entity has been resized, if there are duplicates they should be treated there
        }
-        addEntityToGraph(entity, node.getDocumentTree());
+        boolean added = addEntityToGraph(entity, node.getDocumentTree());
+        if (!added) {
+            return Optional.empty();
+        }
        entity.addEngines(engines);
        insertToKieSession(entity);
        return Optional.of(entity);
@@ -1295,15 +1309,23 @@ public class EntityCreationService {
    }


-    private void addEntityToGraph(TextEntity entity, DocumentTree documentTree) {
+    private boolean addEntityToGraph(TextEntity entity, DocumentTree documentTree) {

        documentTree.getRoot().getNode().addThisToEntityIfIntersects(entity);

+        if (!nodesInKieSession.isEmpty() && entity.getIntersectingNodes()
+                .stream()
+                .anyMatch(node -> !nodesInKieSession.contains(node))) {
+            entity.removeFromGraph();
+            return false;
+        }
+
        TextBlock textBlock = entity.getDeepestFullyContainingNode().getTextBlock();
        entityEnrichmentService.enrichEntity(entity, textBlock);

        addToPages(entity);
        addEntityToNodeEntitySets(entity);
+        return true;
    }

 }
@@ -11,13 +11,18 @@ import java.util.stream.Collectors;

 import org.springframework.stereotype.Service;

+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedactions;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.BaseAnnotation;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;
+import com.iqser.red.service.redaction.v1.server.RedactionServiceSettings;
 import com.iqser.red.service.redaction.v1.server.model.PrecursorEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
+import com.iqser.red.service.redaction.v1.server.model.document.nodes.Page;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
 import com.iqser.red.service.redaction.v1.server.service.DictionaryService;

@@ -35,11 +40,17 @@ public class EntityFromPrecursorCreationService {
    static double MATCH_THRESHOLD = 10; // Is compared to the average sum of distances in pdf coordinates for each corner of the bounding box of the entities
    EntityFindingUtility entityFindingUtility;
    DictionaryService dictionaryService;
+    RedactionServiceSettings settings;


-    public List<PrecursorEntity> createEntitiesIfFoundAndReturnNotFoundEntries(ManualRedactions manualRedactions, SemanticNode node, String dossierTemplateId) {
+    public List<PrecursorEntity> createEntitiesIfFoundAndReturnNotFoundEntries(ManualRedactions manualRedactions, Document document, String dossierTemplateId) {

+        Set<Integer> pageNumbers = document.getPages()
+                .stream()
+                .map(Page::getNumber)
+                .collect(Collectors.toSet());
        Set<IdRemoval> idRemovals = manualRedactions.getIdsToRemove();
+
        List<PrecursorEntity> manualEntities = manualRedactions.getEntriesToAdd()
                .stream()
                .filter(BaseAnnotation::isLocal)
@@ -48,6 +59,10 @@ public class EntityFromPrecursorCreationService {
                        .filter(idRemoval -> idRemoval.getRequestDate().isAfter(manualRedactionEntry.getRequestDate()))
                        .findAny()//
                        .isEmpty())
+                .filter(manualRedactionEntry -> manualRedactionEntry.getPositions()
+                        .stream()
+                        .map(Rectangle::getPage)
+                        .allMatch(pageNumbers::contains))
                .map(manualRedactionEntry -> //
                             PrecursorEntity.fromManualRedactionEntry(manualRedactionEntry, dictionaryService.isHint(manualRedactionEntry.getType(), dossierTemplateId)))
                .peek(manualEntity -> {
@@ -58,7 +73,7 @@ public class EntityFromPrecursorCreationService {
                    }
                })
                .toList();
-        return toTextEntity(manualEntities, node);
+        return toTextEntity(manualEntities, document);
    }


@@ -92,7 +107,7 @@ public class EntityFromPrecursorCreationService {
                notFoundEntities.add(precursorEntity);
                continue;
            }
-            createCorrectEntity(precursorEntity, optionalClosestEntity.get());
+            createCorrectEntity(precursorEntity, optionalClosestEntity.get(), settings.isAnnotationMode());
        }

        tempEntitiesByValue.values()
@@ -125,12 +140,16 @@ public class EntityFromPrecursorCreationService {
                                                         precursorEntity.getEntityType(),
                                                         closestEntity.getDeepestFullyContainingNode());
        } else {
-            correctEntity = TextEntity.initialEntityNode(closestEntity.getTextRange(),
-                                                         precursorEntity.type(),
-                                                         precursorEntity.getEntityType(),
-                                                         precursorEntity.getId(),
-                                                         precursorEntity.getManualOverwrite().getSection()
-                                                                 .orElse(null));
+            String section = precursorEntity.getManualOverwrite().getSection()
+                    .orElse(null);
+            if ((section == null || section.isBlank())
+                && precursorEntity.getSection() != null
+                && !precursorEntity.getSection().isBlank()
+                && precursorEntity.getEngines().contains(Engine.IMPORTED)) {
+                section = precursorEntity.getSection();
+            }
+
+            correctEntity = TextEntity.initialEntityNode(closestEntity.getTextRange(), precursorEntity.type(), precursorEntity.getEntityType(), precursorEntity.getId(), section);
        }
        correctEntity.setDeepestFullyContainingNode(closestEntity.getDeepestFullyContainingNode());
        correctEntity.setIntersectingNodes(new ArrayList<>(closestEntity.getIntersectingNodes()));
@@ -1,10 +1,13 @@
 package com.iqser.red.service.redaction.v1.server.service.document;

+import java.util.Arrays;
 import java.util.Collection;
 import java.util.Comparator;
 import java.util.LinkedList;
 import java.util.List;
+import java.util.Map;
 import java.util.Set;
+import java.util.stream.Collectors;
 import java.util.stream.Stream;

 import com.iqser.red.service.redaction.v1.server.client.model.EntityRecognitionEntity;
@@ -14,7 +17,6 @@ import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.NodeType;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
-import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;

 import lombok.AccessLevel;
 import lombok.experimental.FieldDefaults;
@@ -44,11 +46,10 @@ public class NerEntitiesAdapter {
     */
    public NerEntities toNerEntities(NerEntitiesModel nerEntitiesModel, Document document) {

-        return new NerEntities(addOffsetsAndFlatten(getStringStartOffsetsForMainSectionsHeadersFooters(document),
-                                                    nerEntitiesModel).map(nerEntityModel -> new NerEntities.NerEntity(nerEntityModel.getValue(),
-                                                                                                                      new TextRange(nerEntityModel.getStartOffset(),
-                                                                                                                                    nerEntityModel.getEndOffset()),
-                                                                                                                      nerEntityModel.getType()))
+        return new NerEntities(addOffsetsAndFlatten(getStringStartOffsetsForMainSectionsHeadersFooters(document), nerEntitiesModel).map(nerEntityModel -> new NerEntities.NerEntity(
+                        nerEntityModel.getValue(),
+                        new TextRange(nerEntityModel.getStartOffset(), nerEntityModel.getEndOffset()),
+                        nerEntityModel.getType()))
                                       .toList());
    }

@@ -161,11 +162,12 @@ public class NerEntitiesAdapter {
    }


-    private static Stream<EntityRecognitionEntity> addOffsetsAndFlatten(List<Integer> stringOffsetsForMainSectionsHeadersFooters, NerEntitiesModel nerEntitiesModel) {
+    private static Stream<EntityRecognitionEntity> addOffsetsAndFlatten(Map<List<Integer>, Integer> stringOffsetsForMainSectionsHeadersFooters, NerEntitiesModel nerEntitiesModel) {

        nerEntitiesModel.getData()
                .forEach((sectionNumber, listOfNerEntities) -> listOfNerEntities.forEach(entityRecognitionEntity -> {
-                    int newStartOffset = entityRecognitionEntity.getStartOffset() + stringOffsetsForMainSectionsHeadersFooters.get(sectionNumber);
+                    int newStartOffset = entityRecognitionEntity.getStartOffset() + stringOffsetsForMainSectionsHeadersFooters.getOrDefault(sectionNumberToTreeId(sectionNumber),
+                                                                                                                                            0);
                    entityRecognitionEntity.setStartOffset(newStartOffset);
                    entityRecognitionEntity.setEndOffset(newStartOffset + entityRecognitionEntity.getValue().length());
                }));
@@ -175,14 +177,19 @@ public class NerEntitiesAdapter {
    }


-    private static List<Integer> getStringStartOffsetsForMainSectionsHeadersFooters(Document document) {
+    public static List<Integer> sectionNumberToTreeId(String sectionNumber) {
+
+        return Arrays.stream(sectionNumber.split("\\."))
+                .map(Integer::parseInt)
+                .collect(Collectors.toList());
+    }
+
+
+    private static Map<List<Integer>, Integer> getStringStartOffsetsForMainSectionsHeadersFooters(Document document) {

        return document.streamAllSubNodes()
-                .filter(child -> (child.getType().equals(NodeType.FOOTER) ||child.getType().equals(NodeType.HEADER) ||child.getType().equals(NodeType.SECTION)))
-                .map(SemanticNode::getTextBlock)
-                .map(TextBlock::getTextRange)
-                .map(TextRange::start)
-                .toList();
+                .filter(child -> child.getType().equals(NodeType.FOOTER) || child.getType().equals(NodeType.HEADER) || child.getType().equals(NodeType.SECTION))
+                .collect(Collectors.toMap(SemanticNode::getTreeId, child -> child.getTextBlock().getTextRange().start()));
    }

 }
@@ -11,7 +11,6 @@ import java.util.stream.Stream;
 import org.springframework.stereotype.Service;

 import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedaction;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedactions;
@@ -27,7 +26,6 @@ import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryIncr
 import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryIncrementValue;
 import com.iqser.red.service.redaction.v1.server.model.dictionary.SearchImplementation;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
-import com.iqser.red.service.redaction.v1.server.model.document.nodes.NodeType;

 import io.micrometer.core.annotation.Timed;
 import lombok.AccessLevel;
@@ -51,7 +49,6 @@ public class SectionFinderService {
        long start = System.currentTimeMillis();
        Set<Integer> sectionsToReanalyse = new HashSet<>();

-
        var dictionaryIncrementsSearch = new SearchImplementation(dictionaryIncrement.getValues()
                                                                          .stream()
                                                                          .map(DictionaryIncrementValue::getValue)
@@ -82,9 +79,7 @@ public class SectionFinderService {
        return document.getPages()
                .stream()
                .filter(page -> relevantPagesForReanalysis.contains(page.getNumber()))
-                .flatMap(page -> Stream.concat(page.getMainBody()
-                                                       .stream()
-                                                       .filter(node -> node.getType().equals(NodeType.SECTION)), Stream.of(page.getHeader(), page.getFooter())))
+                .flatMap(page -> Stream.concat(page.streamHighestSemanticNodesOnPage(), Stream.of(page.getHeader(), page.getFooter())))
                .map(node -> node.getTreeId()
                        .get(0))
                .toList();
@@ -49,7 +49,7 @@ public class ComponentDroolsExecutionService {
    ComponentMappingMemoryCache componentMappingMemoryCache;


-    public List<Component> executeRules(KieContainer kieContainer,
+    public List<Component> executeRules(String fileId, KieContainer kieContainer,
                                        EntityLog entityLog,
                                        Document document,
                                        Set<FileAttribute> fileAttributes,
@@ -61,21 +61,22 @@ public class ComponentDroolsExecutionService {

        kieSession.setGlobal("componentCreationService", componentCreationService);

-        if (hasComponentMappingServiceGlobal(kieSession)) {
+        if (hasGlobalWithName(kieSession, COMPONENT_MAPPING_SERVICE_GLOBAL)) {
            kieSession.setGlobal(COMPONENT_MAPPING_SERVICE_GLOBAL, componentMappingService);
        }

        entityLog.getEntityLogEntry()
                .stream()
                .filter(this::isApplied)
-                .filter(entityLogEntry -> entityLogEntry.getState().equals(EntryState.APPLIED))
                .flatMap(entry -> {
                    List<Entity> entities = new ArrayList<>();
                    entities.add(Entity.fromEntityLogEntry(entry, document, entry.getStartOffset(), entry.getEndOffset()));
                    if (entry.getDuplicatedTextRanges() != null && !entry.getDuplicatedTextRanges().isEmpty()) {
-                        entry.getDuplicatedTextRanges().forEach(duplicatedTextRange -> {
-                            entities.add(Entity.fromEntityLogEntry(entry, document, duplicatedTextRange.getStart(), duplicatedTextRange.getEnd()));
-                        });
+                        entry.getDuplicatedTextRanges()
+                                .forEach(duplicatedTextRange -> entities.add(Entity.fromEntityLogEntry(entry,
+                                                                                                       document,
+                                                                                                       duplicatedTextRange.getStart(),
+                                                                                                       duplicatedTextRange.getEnd())));
                    }
                    return entities.stream();
                })
@@ -94,17 +95,18 @@ public class ComponentDroolsExecutionService {
        });

        try {
-            completableFuture.orTimeout(settings.getDroolsExecutionTimeoutSecs(), TimeUnit.SECONDS)
-                    .get();
+            completableFuture.get(settings.getDroolsExecutionTimeoutSecs(document.getNumberOfPages()), TimeUnit.SECONDS);
        } catch (ExecutionException e) {
            kieSession.dispose();
            if (e.getCause() instanceof TimeoutException) {
-                throw new DroolsTimeoutException(e, false, RuleFileType.COMPONENT);
+                throw new DroolsTimeoutException(String.format("The file %s caused a timeout", fileId), e, false, RuleFileType.COMPONENT);
            }
            throw new RuntimeException(e);
        } catch (InterruptedException e) {
            kieSession.dispose();
            throw new RuntimeException(e);
+        } catch (TimeoutException e) {
+            throw new DroolsTimeoutException(String.format("The file %s caused a timeout", fileId), e, false, RuleFileType.COMPONENT);
        }

        List<FileAttribute> resultingFileAttributes = getFileAttributes(kieSession);
@@ -124,7 +126,7 @@ public class ComponentDroolsExecutionService {
    private static void addOrUpdate(Set<FileAttribute> fileAttributes, List<FileAttribute> resultingFileAttributes) {

        for (FileAttribute resultingFileAttribute : resultingFileAttributes) {
-            fileAttributes.remove(resultingFileAttribute);
+            fileAttributes.removeIf(fa -> fa.getLabel().equals(resultingFileAttribute.getLabel()));
            fileAttributes.add(resultingFileAttribute);
        }
    }
@@ -139,13 +141,13 @@ public class ComponentDroolsExecutionService {
    }


-    private static boolean hasComponentMappingServiceGlobal(KieSession kieSession) {
+    private static boolean hasGlobalWithName(KieSession kieSession, String globalName) {

        return kieSession.getKieBase().getKiePackages()
                .stream()
                .flatMap(kiePackage -> kiePackage.getGlobalVariables()
                        .stream())
-                .anyMatch(global -> global.getName().equals(COMPONENT_MAPPING_SERVICE_GLOBAL));
+                .anyMatch(global -> global.getName().equals(globalName));
    }


@@ -45,7 +45,7 @@ public class DroolsValidationService {
    private final KieContainerCreationService kieContainerCreationService;
    private final DeprecatedElementsFinder deprecatedElementsFinder;
    private static final Pattern allowedImportsPattern = Pattern.compile(
-            "^(?:import\\s+static\\s+)?(?:import\\s+)?(?:com\\.knecon\\.fforesight|com\\.iqser\\.red|java\\.util)\\..*;$");
+            "^(?:import\\s+static\\s+)?(?:import\\s+)?(?:com\\.knecon\\.fforesight|com\\.iqser\\.red|java\\.util|java\\.text)\\..*;$");
    public static final String LINEBREAK_MATCHER = "\\R";


@@ -73,7 +73,7 @@ public class DroolsValidationService {

        DroolsValidation customValidation = ruleFileBluePrint.getDroolsValidation();

-        addSyntaxDeprecatedWarnings(ruleFileType, ruleFileBluePrint, customValidation);
+        addSyntaxDeprecatedWarnings(ruleFileBluePrint, customValidation);

        addSyntaxErrorMessages(ruleFileType, ruleFileBluePrint, customValidation);

@@ -85,7 +85,7 @@ public class DroolsValidationService {
    }


-    private void addSyntaxDeprecatedWarnings(RuleFileType ruleFileType, RuleFileBluePrint ruleFileBluePrint, DroolsValidation customValidation) {
+    private void addSyntaxDeprecatedWarnings(RuleFileBluePrint ruleFileBluePrint, DroolsValidation customValidation) {

        // find deprecated elements in the ruleFileBluePrint
        DroolsSyntaxDeprecatedWarnings warningMessageForImports = getWarningsForDeprecatedImports(ruleFileBluePrint);
@@ -93,25 +93,6 @@ public class DroolsValidationService {
            customValidation.getDeprecatedWarnings().add(warningMessageForImports);
        }
        customValidation.getDeprecatedWarnings().addAll(getWarningsForDeprecatedRules(ruleFileBluePrint));
-
-        if (ruleFileType.equals(RuleFileType.COMPONENT)) {
-            if (!ruleFileBluePrint.getGlobals().contains(ComponentDroolsExecutionService.COMPONENT_MAPPING_SERVICE_GLOBAL)) {
-                customValidation.getDeprecatedWarnings().add(buildComponentMappingServiceMissingMessage(ruleFileBluePrint));
-            }
-
-        }
-    }
-
-
-    private static DroolsSyntaxDeprecatedWarnings buildComponentMappingServiceMissingMessage(RuleFileBluePrint ruleFileBluePrint) {
-
-        return DroolsSyntaxDeprecatedWarnings.builder()
-                .message("global ComponentMappingService "
-                         + ComponentDroolsExecutionService.COMPONENT_MAPPING_SERVICE_GLOBAL
-                         + "\n is missing from the rules, consider adding it, as it will be required in future versions!")
-                .line(ruleFileBluePrint.getGlobalsLine())
-                .column(0)
-                .build();
    }


@@ -251,22 +232,23 @@ public class DroolsValidationService {
        if (!blacklistedKeywords.isEmpty()) {
            SearchImplementation blacklistedKeywordSearchImplementation = new SearchImplementation(blacklistedKeywords, false);

+            // check also the imports
+            DroolsBlacklistErrorMessage blacklistErrorMessage = checkAndGetBlackListedMessages(blacklistedKeywordSearchImplementation,
+                                                                                               ruleFileBluePrint.getImports(),
+                                                                                               ruleFileBluePrint.getImportLine());
+            if (blacklistErrorMessage != null) {
+                blacklistErrorMessages.add(blacklistErrorMessage);
+            }
+
+            // check the rules
            for (RuleClass ruleClass : ruleFileBluePrint.getRuleClasses()) {
                for (RuleUnit ruleUnit : ruleClass.ruleUnits()) {
                    for (BasicRule basicRule : ruleUnit.rules()) {
-                        String sanitizedRuleText = StringUtils.deleteWhitespace(basicRule.getCode());
-                        List<SearchImplementation.MatchPosition> matches = blacklistedKeywordSearchImplementation.getMatches(sanitizedRuleText);
-
-                        if (!matches.isEmpty()) {
-                            List<String> foundBlacklistedKeywords = matches.stream()
-                                    .map(m -> sanitizedRuleText.substring(m.startIndex(), m.endIndex()))
-                                    .distinct()
-                                    .toList();
-                            blacklistErrorMessages.add(DroolsBlacklistErrorMessage.builder()
-                                                               .line(basicRule.getLine())
-                                                               .column(0)
-                                                               .blacklistedKeywords(foundBlacklistedKeywords)
-                                                               .build());
+                        DroolsBlacklistErrorMessage ruleBlacklistErrorMessage = checkAndGetBlackListedMessages(blacklistedKeywordSearchImplementation,
+                                                                                                               basicRule.getCode(),
+                                                                                                               basicRule.getLine());
+                        if (ruleBlacklistErrorMessage != null) {
+                            blacklistErrorMessages.add(ruleBlacklistErrorMessage);
                        }
                    }
                }
@@ -280,6 +262,24 @@ public class DroolsValidationService {
    }


+    private DroolsBlacklistErrorMessage checkAndGetBlackListedMessages(SearchImplementation blacklistedKeywordSearchImplementation, String stringToCheck, int lineIndexStart) {
+
+        String nonWhitespaceRuleText = StringUtils.deleteWhitespace(stringToCheck);
+        String sanitizedRuleText = nonWhitespaceRuleText.replaceAll("\"(\\\\.|[^\"\\\\])*\"|'(\\\\.|[^'\\\\])*'", "");
+
+        List<SearchImplementation.MatchPosition> matches = blacklistedKeywordSearchImplementation.getMatches(sanitizedRuleText);
+
+        if (!matches.isEmpty()) {
+            List<String> foundBlacklistedKeywords = matches.stream()
+                    .map(m -> sanitizedRuleText.substring(m.startIndex(), m.endIndex()))
+                    .distinct()
+                    .toList();
+            return DroolsBlacklistErrorMessage.builder().line(lineIndexStart).column(0).blacklistedKeywords(foundBlacklistedKeywords).build();
+        }
+        return null;
+    }
+
+
    private List<String> parseBlacklistFile(String blacklistFileString) {

        return Stream.of(blacklistFileString.split(LINEBREAK_MATCHER))
@@ -1,7 +1,10 @@
 package com.iqser.red.service.redaction.v1.server.service.drools;

+import java.util.Collections;
+import java.util.HashSet;
 import java.util.LinkedList;
 import java.util.List;
+import java.util.Set;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.TimeUnit;
@@ -49,14 +52,14 @@ public class EntityDroolsExecutionService {

    @Timed("redactmanager_executeRules")
    @Observed(name = "EntityDroolsExecutionService", contextualName = "execute-entity-rules")
-    public List<FileAttribute> executeRules(KieContainer kieContainer,
+    public List<FileAttribute> executeRules(String fileId, KieContainer kieContainer,
                                            Document document,
                                            Dictionary dictionary,
                                            List<FileAttribute> fileAttributes,
                                            ManualRedactions manualRedactions,
                                            NerEntities nerEntities) {

-        return executeRules(kieContainer,
+        return executeRules(fileId, kieContainer,
                            document,
                            document.streamChildren()
                                    .toList(),
@@ -69,7 +72,7 @@ public class EntityDroolsExecutionService {

    @Timed("redactmanager_executeRules")
    @Observed(name = "EntityDroolsExecutionService", contextualName = "execute-entity-rules")
-    public List<FileAttribute> executeRules(KieContainer kieContainer,
+    public List<FileAttribute> executeRules(String fileId, KieContainer kieContainer,
                                            Document document,
                                            List<SemanticNode> sectionsToAnalyze,
                                            Dictionary dictionary,
@@ -80,7 +83,11 @@ public class EntityDroolsExecutionService {
        addNumberOfPagesAndSectionsToAnalyseToTrace(document.getNumberOfPages(), sectionsToAnalyze.size());

        KieSession kieSession = kieContainer.newKieSession();
-        EntityCreationService entityCreationService = new EntityCreationService(entityEnrichmentService, kieSession);
+
+        Set<SemanticNode> nodesInKieSession = sectionsToAnalyze.size() == document.streamAllSubNodes()
+                .count() ? Collections.emptySet() : buildSet(sectionsToAnalyze, document);
+
+        EntityCreationService entityCreationService = new EntityCreationService(entityEnrichmentService, kieSession, nodesInKieSession);

        kieSession.setGlobal("document", document);
        kieSession.setGlobal("entityCreationService", entityCreationService);
@@ -122,17 +129,18 @@ public class EntityDroolsExecutionService {
        });

        try {
-            completableFuture.orTimeout(settings.getDroolsExecutionTimeoutSecs(), TimeUnit.SECONDS)
-                    .get();
+            completableFuture.get(settings.getDroolsExecutionTimeoutSecs(document.getNumberOfPages()), TimeUnit.SECONDS);
        } catch (ExecutionException e) {
            kieSession.dispose();
            if (e.getCause() instanceof TimeoutException) {
-                throw new DroolsTimeoutException(e, false, RuleFileType.ENTITY);
+                throw new DroolsTimeoutException(String.format("The file %s caused a timeout",fileId), e, false, RuleFileType.ENTITY);
            }
            throw new RuntimeException(e);
        } catch (InterruptedException e) {
            kieSession.dispose();
            throw new RuntimeException(e);
+        } catch (TimeoutException e) {
+            throw new DroolsTimeoutException(String.format("The file %s caused a timeout",fileId), e, false, RuleFileType.ENTITY);
        }

        List<FileAttribute> resultingFileAttributes = getFileAttributes(kieSession);
@@ -141,6 +149,17 @@ public class EntityDroolsExecutionService {
    }


+    private static Set<SemanticNode> buildSet(List<SemanticNode> sectionsToAnalyze, Document document) {
+
+        Set<SemanticNode> nodes = new HashSet<>(sectionsToAnalyze);
+        sectionsToAnalyze.stream()
+                .flatMap(SemanticNode::streamAllSubNodes)
+                .forEach(nodes::add);
+        nodes.add(document);
+        return nodes;
+    }
+
+
    public List<FileAttribute> getFileAttributes(KieSession kieSession) {

        List<FileAttribute> fileAttributes = new LinkedList<>();
@@ -32,7 +32,7 @@ import lombok.experimental.UtilityClass;
 public class RuleFileParser {

    private final static Pattern ruleIdentifierInCodeFinder = Pattern.compile(
-            "\\b(?:redact|apply|skip|remove|ignore|applyWithLineBreaks|applyWithReferences|skipWithReferences)\\s*\\(\"([a-zA-Z0-9]+.\\d+.\\d+)\",.*(?:, .*)?\\)");
+            "\\b(?:redact|apply|skip|remove|ignore|applyWithLineBreaks|applyWithReferences|skipWithReferences)\\s*\\(\\s*\"([a-zA-Z0-9]+.\\d+.\\d+)\"\\s*,\\s*.*(?:\\s*,\\s*.*)\\s*?\\)");


    @SneakyThrows
@@ -78,7 +78,8 @@ public class RuleFileParser {
                                             .map(GlobalDescr::getLine)
                                             .orElse(0),
                                     allQueries,
-                                     ruleClasses, customDroolsValidation);
+                                     ruleClasses,
+                                     customDroolsValidation);
    }


@@ -9,10 +9,12 @@ import java.util.Set;
 import java.util.stream.Collectors;

 import org.springframework.cache.annotation.Cacheable;
+import org.springframework.context.annotation.Import;
 import org.springframework.stereotype.Service;

 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentLog;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedLegalBases;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedactions;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedactionsPerPage;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
@@ -109,6 +111,12 @@ public class RedactionStorageService {
    }


+    public List<EntityLogEntry> getEntityLogEntriesById(String dossierId, String fileId, List<String> ids) {
+
+        return entityLogMongoService.findEntityLogEntriesByIds(dossierId, fileId, ids);
+    }
+
+
    @Timed("redactmanager_getImportedRedactions")
    public ImportedRedactions getImportedRedactions(String dossierId, String fileId) {

@@ -141,6 +149,21 @@ public class RedactionStorageService {
    }


+
+    @Timed("redactmanager_getImportedLegalBases")
+    public ImportedLegalBases getImportedLegalBases(String dossierId, String fileId) {
+
+        try {
+            return storageService.readJSONObject(TenantContext.getTenantId(),
+                                                                             StorageIdUtils.getStorageId(dossierId, fileId, FileType.IMPORTED_LEGAL_BASES),
+                                                                             ImportedLegalBases.class);
+        } catch (StorageObjectDoesNotExist e) {
+            log.debug("Imported legal bases not available.");
+            return new ImportedLegalBases();
+        }
+    }
+
+
    @Deprecated(forRemoval = true)
    @Timed("redactmanager_getRedactionLog")
    public RedactionLog getRedactionLog(String dossierId, String fileId) {
@@ -11,12 +11,15 @@ import java.time.format.DateTimeFormatter;
 import java.time.format.DateTimeFormatterBuilder;
 import java.time.format.DateTimeParseException;
 import java.time.format.ResolverStyle;
+import java.time.temporal.ChronoField;
 import java.util.Arrays;
 import java.util.Date;
 import java.util.List;
 import java.util.Locale;
 import java.util.Objects;
 import java.util.Optional;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;

 import lombok.AccessLevel;
 import lombok.experimental.FieldDefaults;
@@ -30,6 +33,7 @@ public class DateConverter {

    private static DateTimeFormatter DATE_TIME_FORMATTER;
    private static final List<Locale> LOCALES = Arrays.asList(Locale.UK, Locale.US);
+    private static int BASE_YEAR = 1950; // base year 1950 means, that "yy" will be interpreted in range 1950-2049


    public Optional<Date> parseDate(String dateAsString) {
@@ -96,7 +100,11 @@ public class DateConverter {
            while ((line = reader.readLine()) != null) {
                String pattern = line.trim();
                if (!pattern.isEmpty()) {
-                    builder.appendOptional(DateTimeFormatter.ofPattern(pattern, Locale.UK));
+                    if (hasTwoDigitsForYear(pattern)) {
+                        builder.appendOptional(setBaseYear(pattern));
+                    } else {
+                        builder.appendOptional(DateTimeFormatter.ofPattern(pattern, Locale.UK));
+                    }
                }
            }
        } catch (IOException e) {
@@ -106,6 +114,32 @@ public class DateConverter {
    }


+    private boolean hasTwoDigitsForYear(String input) {
+        // Regex to match any string with exactly two 'y' characters
+        Pattern pattern = Pattern.compile("^[^y]*(y[^y]*){2}$");
+        Matcher matcher = pattern.matcher(input);
+
+        return matcher.matches();
+
+    }
+
+
+    private DateTimeFormatter setBaseYear(String pattern) {
+
+        DateTimeFormatterBuilder builder = new DateTimeFormatterBuilder();
+        if (pattern.startsWith("yy")) {
+            String editedPattern = pattern.substring(2);
+            builder.appendValueReduced(ChronoField.YEAR_OF_ERA, 2, 2, BASE_YEAR).appendPattern(editedPattern).toFormatter();
+        } else if (pattern.endsWith("yy")) {
+            String editedPattern = pattern.substring(0, pattern.length() - 2);
+            builder.appendPattern(editedPattern).appendValueReduced(ChronoField.YEAR_OF_ERA, 2, 2, BASE_YEAR).toFormatter();
+        } else {
+            throw new RuntimeException("Date format not supported: " + pattern);
+        }
+        return builder.toFormatter();
+    }
+
+
    private String removeTrailingDot(String dateAsString) {

        String str = dateAsString;
@@ -18,7 +18,7 @@ public class MigratedIdsCollector implements Collector<MigrationEntity, Migrated
    @Override
    public Supplier<MigratedIds> supplier() {

-        return () -> new MigratedIds(new LinkedList<>(), Collections.emptyList());
+        return () -> new MigratedIds(new LinkedList<>(), Collections.emptyList(), Collections.emptyList());
    }


@@ -3,12 +3,17 @@ package com.iqser.red.service.redaction.v1.server.utils;
 import static java.lang.String.format;

 import java.awt.geom.Rectangle2D;
+import java.awt.geom.RectangularShape;
 import java.util.LinkedList;
 import java.util.List;
+import java.util.function.Predicate;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 import java.util.stream.IntStream;

+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
 import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
@@ -18,6 +23,9 @@ import lombok.experimental.UtilityClass;
@UtilityClass
 public class RedactionSearchUtility {

+    private static final Logger log = LoggerFactory.getLogger(RedactionSearchUtility.class);
+
+
    /**
     * Checks if any part of a CharSequence matches a given regex pattern.
     *
@@ -154,14 +162,8 @@ public class RedactionSearchUtility {
     */
    public static TextRange findTextRangesOfAllLinesInYRange(double maxY, double minY, TextBlock textBlock) {

-        List<TextRange> lineBoundaries = IntStream.range(0, textBlock.numberOfLines()).boxed()
-                .map(textBlock::getLineTextRange)
-                .filter(lineBoundary -> isWithinYRange(maxY, minY, textBlock, lineBoundary))
-                .toList();
-        if (lineBoundaries.isEmpty()) {
-            return new TextRange(textBlock.getTextRange().start(), textBlock.getTextRange().start());
-        }
-        return TextRange.merge(lineBoundaries);
+        Predicate<TextRange> isWithinYRange = lineBoundary -> isWithinYRange(maxY, minY, textBlock, lineBoundary);
+        return filterLineBoundaries(textBlock, isWithinYRange);
    }


@@ -172,6 +174,49 @@ public class RedactionSearchUtility {
    }


+    /**
+     * Identifies all lines within a text block that have roughly the same vertical coordinates.
+     *
+     * @param maxY      The maximum Y-coordinate of the vertical range.
+     * @param minY      The minimum Y-coordinate of the vertical range.
+     * @param textBlock The text block containing the lines to be checked.
+     * @return A {@link TextRange} encompassing all lines within the specified Y-coordinate range.
+     */
+    public static TextRange findTextRangesOfAllLinesWithCloseYCoordinates(Double maxY, Double minY, TextBlock textBlock) {
+
+        double averageLineHeight = IntStream.range(0, textBlock.numberOfLines()).boxed()
+                .map(textBlock::getLineTextRange)
+                .flatMap((TextRange stringTextRange) -> textBlock.getPositions(stringTextRange)
+                        .stream())
+                .map(RectangularShape::getHeight)
+                .mapToDouble(Double::doubleValue).average()
+                .orElse(0);
+        Predicate<TextRange> hasCloseYRange = lineBoundary -> areYCoordinatesClose(maxY, minY, textBlock, lineBoundary, averageLineHeight);
+
+        return filterLineBoundaries(textBlock, hasCloseYRange);
+    }
+
+
+    private static boolean areYCoordinatesClose(Double maxY, Double minY, TextBlock textBlock, TextRange lineTextRange, double averageLineHeight) {
+
+        Rectangle2D lineBBox = RectangleTransformations.rectangle2DBBox(textBlock.getPositions(lineTextRange));
+        return Math.abs(lineBBox.getMinY() - minY) <= averageLineHeight && Math.abs(maxY - lineBBox.getMaxY()) <= averageLineHeight;
+    }
+
+
+    private static TextRange filterLineBoundaries(TextBlock textBlock, Predicate<TextRange> textRangePredicate) {
+
+        List<TextRange> lineBoundaries = IntStream.range(0, textBlock.numberOfLines()).boxed()
+                .map(textBlock::getLineTextRange)
+                .filter(textRangePredicate)
+                .toList();
+        if (lineBoundaries.isEmpty()) {
+            return new TextRange(textBlock.getTextRange().start(), textBlock.getTextRange().start());
+        }
+        return TextRange.merge(lineBoundaries);
+    }
+
+
    /**
     * Finds TextRanges matching a regex pattern within a TextBlock.
     *
@@ -264,8 +309,12 @@ public class RedactionSearchUtility {

        Matcher matcher = pattern.matcher(textBlock.subSequence(textBlock.getTextRange()));
        List<TextRange> boundaries = new LinkedList<>();
-        while (matcher.find()) {
-            boundaries.add(new TextRange(matcher.start(group) + textBlock.getTextRange().start(), matcher.end(group) + textBlock.getTextRange().start()));
+        try {
+            while (matcher.find()) {
+                boundaries.add(new TextRange(matcher.start(group) + textBlock.getTextRange().start(), matcher.end(group) + textBlock.getTextRange().start()));
+            }
+        }catch (StackOverflowError stackOverflowError){
+            log.warn("Stackoverflow error for pattern {} in text: {}", pattern.pattern(), textBlock);
        }
        return boundaries;
    }
@@ -273,11 +322,16 @@ public class RedactionSearchUtility {

    private static List<TextRange> getTextRangesByPatternWithLineBreaks(TextBlock textBlock, int group, Pattern pattern) {

+
        String searchTextWithLineBreaks = textBlock.searchTextWithLineBreaks();
        Matcher matcher = pattern.matcher(searchTextWithLineBreaks);
        List<TextRange> boundaries = new LinkedList<>();
-        while (matcher.find()) {
-            boundaries.add(new TextRange(matcher.start(group) + textBlock.getTextRange().start(), matcher.end(group) + textBlock.getTextRange().start()));
+        try {
+            while (matcher.find()) {
+                boundaries.add(new TextRange(matcher.start(group) + textBlock.getTextRange().start(), matcher.end(group) + textBlock.getTextRange().start()));
+            }
+        }catch (StackOverflowError stackOverflowError){
+            log.warn("Stackoverflow error for pattern {} in text with linebreaks: {}", pattern.pattern(), searchTextWithLineBreaks);
        }
        return boundaries;
    }
@@ -22,6 +22,15 @@ public class DroolsTimeoutException extends RuntimeException {
    }


+    public DroolsTimeoutException(String message, Throwable cause, boolean reported, RuleFileType ruleFileType) {
+
+        super(message, cause);
+        this.reported = reported;
+        this.ruleFileType = ruleFileType;
+    }
+
+
+
    public DroolsTimeoutException(boolean reported, RuleFileType ruleFileType) {

        super(DROOLS_TIMEOUT_MESSAGE);
@@ -16,6 +16,9 @@ project.version: 1.0-SNAPSHOT
 server:
  port: 8080

+lifecycle:
+  base-package: com.iqser.red.service.redaction
+
 spring:
  application:
    name: redaction-service
@@ -1,40 +1,79 @@
+yyyy MMM dd
+yyyy MMM d
+yyyy MM dd
+yyyy MM d
 dd-MMM-yyyy
+d-MMM-yyyy
 dd MMM yyyy
+d MMM yyyy
 dd MMM yy
 d MMM yy
 dd MM yyyy
+d MM yyyy
 dd MMMM yyyy
-MMMM dd, yyyy
-dd.MM.yyyy
-yyyy/MM/dd
-yyyy-MM-dd
-dd-MM-yyyy
-MMMM d, yyyy
 d MMMM yyyy
+MMMM dd, yyyy
+MMMM d, yyyy
+MMMM, d yyyy
+MMMM d,yyyy
+dd.MM.yyyy
+d.MM.yyyy
+yyyy/MM/dd
+yyyy/MM/d
+yyyy-MM-dd
+yyyy-MM-d
+dd-MM-yyyy
+d-MM-yyyy
+MMM dd, yyyy
 MMM d, yyyy
+dd['.'] MMM yyyy
 d['.'] MMM yyyy
-d-MMM-yyyy
+dd['th']['st']['nd']['rd'] 'of' MMMM, yyyy
 d['th']['st']['nd']['rd'] 'of' MMMM, yyyy
+d['st']['nd']['rd']['th'] MMMM yyyy
+MMMM dd['th']['st']['nd']['rd'], yyyy
 MMMM d['th']['st']['nd']['rd'], yyyy
+yyyy, MMMM dd
 yyyy, MMMM d
 yyyy.MM.dd
-yyyyMMdd
+yyyy.MM.d
 dd-MM-yy
+d-MM-yy
 dd/MM/yy
+d/MM/yy
+MMMM dd, yy
 MMMM d, yy
+dd MMMM, yy
 d MMMM, yy
+dd['th']['st']['nd']['rd'] MMM yyyy
 d['th']['st']['nd']['rd'] MMM yyyy
+MMM dd['th']['st']['nd']['rd'], yy
 MMM d['th']['st']['nd']['rd'], yy
 yyyy-MMM-dd
+yyyy-MMM-d
 MMM-dd-yyyy
-dd-MM-yyyy
-yyyy, MMMM dd
+MMM-d-yyyy
 dd.MMM.yyyy
+d.MMM.yyyy
 dd.MMMM.yyyy
+d.MMMM.yyyy
 dd.MMM.yy
+d.MMM.yy
 dd.MMMM.yy
+d.MMMM.yy
 dd.MMM-yyyy
+d.MMM-yyyy
 dd.MMMM-yyyy
+d.MMMM-yyyy
 dd. MMM yy
+d. MMM yy
+dd['th']['st']['nd']['rd'] MMMM yy
 d['th']['st']['nd']['rd'] MMMM yy
-d['th']['st']['nd']['rd'] MMMM yyyy
+dd['th']['st']['nd']['rd'] MMMM yyyy
+d['th']['st']['nd']['rd'] MMMM yyyy
+dd.MM.yy
+d.MM.yy
+dd MMM. yyyy
+d MMM. yyyy
+d-MMMM-yyyy
+dd-MMMM-yyyy
@@ -3,6 +3,7 @@ Runtime.
 Thread.
 Class.

+TenantContext
 ProcessBuilder
 SecurityManager
 ClassLoader
@@ -13,6 +14,4 @@ java.io.Object

 java.net

-java.lang
-
 java.util.zip
@@ -13,5 +13,5 @@
    <root level="INFO">
        <appender-ref ref="${logType}"/>
    </root>
-
+    <logger name="org.drools.mvel" level="ERROR"/>
 </configuration>
@@ -100,6 +100,8 @@ public abstract class AbstractRedactionIntegrationTest {
    protected static final String ROTATE_SIMPLE_INDICATOR = "RotateSimple";

    protected final static String TEST_DOSSIER_TEMPLATE_ID = "123";
+    protected final static String TEST_DOSSIER_ID = "123";
+    protected final static String TEST_FILE_ID = "123";
    public static final String IMPORTED_REDACTION_TYPE_ID = IMPORTED_REDACTION_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID;
    public static final String DOSSIER_REDACTIONS_TYPE_ID = DOSSIER_REDACTIONS_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID;
    public static final String ROTATE_SIMPLE_TYPE_ID = ROTATE_SIMPLE_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID;
@@ -112,12 +114,14 @@ public abstract class AbstractRedactionIntegrationTest {
    public static final String PII_TYPE_ID = DICTIONARY_PII + ":" + TEST_DOSSIER_TEMPLATE_ID;
    public static final String TEST_METHOD_TYPE_ID = TEST_METHOD_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID;
    public static final String PUBLISHED_INFORMATION_TYPE_ID = PUBLISHED_INFORMATION_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID;
+    public static final String DOSSIER_PUBLISHED_INFORMATION_TYPE_ID = PUBLISHED_INFORMATION_TYPE_ID + ":" + TEST_DOSSIER_ID;
    public static final String MUST_REDACT_TYPE_ID = MUST_REDACT_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID;
    public static final String HINT_ONLY_TYPE_ID = HINT_ONLY_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID;
    public static final String REDACTION_TYPE_ID = REDACTION_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID;
    public static final String NO_REDACTION_TYPE_ID = NO_REDACTION_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID;
    public static final String SPONSOR_TYPE_ID = DICTIONARY_SPONSOR + ":" + TEST_DOSSIER_TEMPLATE_ID;
    public static final String AUTHOR_TYPE_ID = DICTIONARY_AUTHOR + ":" + TEST_DOSSIER_TEMPLATE_ID;
+    public static final String DOSSIER_AUTHOR_TYPE_ID = AUTHOR_TYPE_ID + ":" + TEST_DOSSIER_ID;
    public static final String ADDRESS_TYPE_ID = DICTIONARY_ADDRESS + ":" + TEST_DOSSIER_TEMPLATE_ID;
    public static final String VERTEBRATE_TYPE_ID = VERTEBRATE_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID;

@@ -181,8 +185,6 @@ public abstract class AbstractRedactionIntegrationTest {
    protected final Map<String, Long> reanlysisVersions = new HashMap<>();
    protected final Set<String> deleted = new HashSet<>();

-    protected final static String TEST_DOSSIER_ID = "123";
-    protected final static String TEST_FILE_ID = "123";

    @MockBean
    protected RulesClient rulesClient;
@@ -249,6 +251,10 @@ public abstract class AbstractRedactionIntegrationTest {
                                                                                                                                                 true));
        when(dictionaryClient.getDictionaryForType(IMPORTED_REDACTION_TYPE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(IMPORTED_REDACTION_INDICATOR,
                                                                                                                                                 true));
+        when(dictionaryClient.getDictionaryForType(DOSSIER_PUBLISHED_INFORMATION_TYPE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(
+                PUBLISHED_INFORMATION_INDICATOR,
+                true));
+        when(dictionaryClient.getDictionaryForType(DOSSIER_AUTHOR_TYPE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(DICTIONARY_AUTHOR, true));

    }

@@ -346,6 +352,8 @@ public abstract class AbstractRedactionIntegrationTest {
                                .map(this::cleanDictionaryEntry)
                                .collect(Collectors.toSet()));
        dossierDictionary.put(IMPORTED_REDACTION_INDICATOR, new ArrayList<>());
+        dossierDictionary.put(PUBLISHED_INFORMATION_INDICATOR, new ArrayList<>());
+        dossierDictionary.put(DICTIONARY_AUTHOR, new ArrayList<>());

        falsePositive.computeIfAbsent(DICTIONARY_PII, v -> new ArrayList<>())
                .addAll(ResourceLoader.load("dictionaries/PII_false_positive.txt")
@@ -9,6 +9,7 @@ import static org.mockito.Mockito.when;

 import java.io.File;
 import java.io.FileInputStream;
+import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.Arrays;
@@ -17,11 +18,13 @@ import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 import java.util.Optional;
 import java.util.Set;
 import java.util.UUID;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.stream.Collectors;
+import java.util.stream.Stream;
 import java.util.zip.GZIPInputStream;

 import org.junit.jupiter.api.BeforeEach;
@@ -56,7 +59,6 @@ import com.iqser.red.service.redaction.v1.server.service.AnalyzeService;
 import com.iqser.red.service.redaction.v1.server.service.DictionaryService;
 import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
 import com.iqser.red.service.redaction.v1.server.testcontainers.MongoDBTestContainer;
-import com.iqser.red.service.redaction.v1.server.utils.exception.NotFoundException;
 import com.iqser.red.storage.commons.service.StorageService;
 import com.knecon.fforesight.mongo.database.commons.liquibase.TenantMongoLiquibaseExecutor;
 import com.knecon.fforesight.mongo.database.commons.service.MongoConnectionProvider;
@@ -80,7 +82,25 @@ import lombok.extern.slf4j.Slf4j;
 * This way you can recreate what is happening on the stack almost exactly.
 */ public class AnalysisEnd2EndTest {

-    Path dossierTemplateToUse = Path.of("/home/kschuettler/iqser/fforesight/dossier-templates-v2/dev/LayoutParsingDatasetEvaluation"); // Add your dossier-template here
+    // These files will be uploaded if they are present in the folder
+    public static final Set<FileType> ENDINGS_TO_UPLOAD = Set.of(FileType.ORIGIN,
+                                                                 FileType.DOCUMENT_PAGES,
+                                                                 FileType.DOCUMENT_POSITION,
+                                                                 FileType.DOCUMENT_STRUCTURE,
+                                                                 FileType.DOCUMENT_TEXT,
+                                                                 FileType.IMAGE_INFO,
+                                                                 FileType.NER_ENTITIES,
+                                                                 FileType.TABLES,
+                                                                 FileType.IMPORTED_REDACTIONS);
+
+    // These files must be present in the folder or the test will skip the file
+    public static final Set<FileType> REQUIRED_FILES = Set.of(FileType.ORIGIN,
+                                                              FileType.DOCUMENT_PAGES,
+                                                              FileType.DOCUMENT_POSITION,
+                                                              FileType.DOCUMENT_STRUCTURE,
+                                                              FileType.DOCUMENT_TEXT);
+
+    Path dossierTemplateToUse = Path.of("/home/kschuettler/Downloads/New Folder/DOSSIER_TEMPLATE"); // Add your dossier-template here
    ObjectMapper mapper = ObjectMapperFactory.create();
    final String TENANT_ID = "tenant";

@@ -121,7 +141,7 @@ import lombok.extern.slf4j.Slf4j;
    @SneakyThrows
    public void runAnalysisEnd2End() {

-        String folder = "/home/kschuettler/Dokumente/analysisend2end/file1"; // Should contain all files from minio directly, still zipped. Can contain multiple files.
+        String folder = "/home/kschuettler/Downloads/New Folder/436e4a2a-0ba3-4d3c-9944-c355f5c1cca2"; // Should contain all files from minio directly, still zipped. Can contain multiple files.

        Path absoluteFolderPath;
        if (folder.startsWith("files")) { // if it starts with "files" it is most likely in the resources folder, else it should be an absolute path
@@ -133,11 +153,16 @@ import lombok.extern.slf4j.Slf4j;

        log.info("Starting end2end analyses for all distinct filenames in folder: {}", folder);
        List<AnalyzeRequest> analyzeRequests = prepareStorageForFolder(absoluteFolderPath);
-        log.info("Found {} distinct fileIds", analyzeRequests.size());
+        log.info("Found {} distinct fileIds with all required files", analyzeRequests.size());
        for (int i = 0; i < analyzeRequests.size(); i++) {
+            long start = System.currentTimeMillis();
            AnalyzeRequest analyzeRequest = analyzeRequests.get(i);
+            log.info("----------------------------------------------------------------------------------");
            log.info("{}/{}: Starting analysis for file {}", i + 1, analyzeRequests.size(), analyzeRequest.getFileId());
            analyzeService.analyze(analyzeRequest);
+            log.info("{}/{}: Finished analysis for file {} in {} ms", i + 1, analyzeRequests.size(), analyzeRequest.getFileId(), System.currentTimeMillis() - start);
+            log.info("----------------------------------------------------------------------------------");
+            log.info("");
        }
    }

@@ -188,22 +213,36 @@ import lombok.extern.slf4j.Slf4j;
    @SneakyThrows
    private List<AnalyzeRequest> prepareStorageForFolder(Path folder) {

-        return Files.list(folder)
-                .map(this::parseFileId)
-                .distinct()
+        return findOriginFiles(folder).stream()
                .map(fileId -> prepareStorageForFile(fileId, folder))
+                .filter(Optional::isPresent)
+                .map(Optional::get)
                .toList();
    }


-    private String parseFileId(Path path) {
+    private Set<String> findOriginFiles(Path folder) throws IOException {

-        return path.getFileName().toString().split("\\.")[0];
+        return Files.walk(folder)
+                .map(this::parseFileName)
+                .filter(Objects::nonNull)
+                .collect(Collectors.toSet());
+    }
+
+
+    private String parseFileName(Path path) {
+
+        String suffix = ".ORIGIN.pdf";
+        if (!path.getFileName().toString().endsWith(suffix)) {
+            return null;
+        }
+
+        return path.getFileName().toString().replace(suffix, "");
    }


    @SneakyThrows
-    private AnalyzeRequest prepareStorageForFile(String fileId, Path folder) {
+    private Optional<AnalyzeRequest> prepareStorageForFile(String fileName, Path folder) {

        AnalyzeRequest request = new AnalyzeRequest();
        request.setDossierId(UUID.randomUUID().toString());
@@ -211,53 +250,65 @@ import lombok.extern.slf4j.Slf4j;
        request.setDossierTemplateId(testDossierTemplate.id);
        request.setAnalysisNumber(-1);

-        Path manualRedactionFile = folder.resolve(fileId + ".MANUAL_REDACTIONS.json");
+        Path manualRedactionFile = folder.resolve(fileName + ".MANUAL_REDACTIONS.json");
        if (Files.exists(manualRedactionFile)) {
-            request.setManualRedactions(mapper.readValue(manualRedactionFile.toFile(), ManualRedactions.class));
+            request.setManualRedactions(parseManualRedactions(manualRedactionFile));
        } else {
            request.setManualRedactions(new ManualRedactions());
        }

-        Set<FileType> endingsToUpload = Set.of("ORIGIN",
-                                               "DOCUMENT_PAGES",
-                                               "DOCUMENT_POSITION",
-                                               "DOCUMENT_STRUCTURE",
-                                               "DOCUMENT_TEXT",
-                                               "IMAGE_INFO",
-                                               "NER_ENTITIES",
-                                               "TABLES",
-                                               "IMPORTED_REDACTIONS")
-                .stream()
-                .map(FileType::valueOf)
-                .collect(Collectors.toSet());
-
-        Set<FileType> uploadedFileTypes = Files.walk(folder)
-                .filter(path -> path.toFile().isFile())
-                .filter(path -> parseFileTypeFromPath(path).map(endingsToUpload::contains)
-                        .orElse(false))
-                .map(filePath -> uploadFile(filePath, request))
-                .filter(Optional::isPresent)
-                .map(Optional::get)
+        Set<FileType> uploadedFileTypes = findFilesToUpload(fileName, folder, ENDINGS_TO_UPLOAD).map(filePath -> uploadFile(filePath, request))
+                .map(FileToUpload::fileType)
                .collect(Collectors.toUnmodifiableSet());

-        Set<FileType> missingFileTypes = Sets.difference(endingsToUpload, uploadedFileTypes);
+        Set<FileType> missingFileTypes = Sets.difference(REQUIRED_FILES, uploadedFileTypes);
+
        if (!missingFileTypes.isEmpty()) {
            log.error("Folder {} is missing files of type {}",
                      folder.toFile(),
                      missingFileTypes.stream()
                              .map(Enum::toString)
                              .collect(Collectors.joining(", ")));
-            throw new NotFoundException("Not all required file types are present.");
+            return Optional.empty();
        }
-        return request;
+        return Optional.of(request);
    }


-    private static Optional<FileType> parseFileTypeFromPath(Path path) {
+    private static Stream<FileToUpload> findFilesToUpload(String fileName, Path folder, Set<FileType> endingsToUpload) throws IOException {
+
+        return Files.walk(folder)
+                .filter(path -> path.toFile().isFile())
+                .map(path -> parseFileTypeFromPath(path, fileName, endingsToUpload))
+                .filter(Optional::isPresent)
+                .map(Optional::get);
+    }
+
+
+    private ManualRedactions parseManualRedactions(Path manualRedactionFile) {

-        String fileType = path.getFileName().toString().split("\\.")[1];
        try {
-            return Optional.of(FileType.valueOf(fileType));
+            return mapper.readValue(manualRedactionFile.toFile(), ManualRedactions.class);
+        } catch (IOException e) {
+            log.error("Could not parse manual redactions");
+            return new ManualRedactions();
+        }
+    }
+
+
+    private static Optional<FileToUpload> parseFileTypeFromPath(Path path, String fileName, Set<FileType> endingsToUpload) {
+
+        if (!path.getFileName().toString().startsWith(fileName)) {
+            return Optional.empty();
+        }
+
+        try {
+            String fileTypeString = path.getFileName().toString().split("\\.")[1];
+            FileType fileType = FileType.valueOf(fileTypeString);
+            if (!endingsToUpload.contains(fileType)) {
+                return Optional.empty();
+            }
+            return Optional.of(new FileToUpload(path, fileType));
        } catch (IllegalArgumentException e) {
            return Optional.empty();
        }
@@ -265,21 +316,26 @@ import lombok.extern.slf4j.Slf4j;


    @SneakyThrows
-    private Optional<FileType> uploadFile(Path path, AnalyzeRequest request) {
+    private FileToUpload uploadFile(FileToUpload fileToUpload, AnalyzeRequest request) {

-        Optional<FileType> fileType = parseFileTypeFromPath(path);
-        if (fileType.isEmpty()) {
-            return Optional.empty();
+        if (fileToUpload.path().getFileName().toString().endsWith(".gz")) {
+            try (var fis = new FileInputStream(fileToUpload.path().toFile()); var in = new GZIPInputStream(fis);) {
+                storageService.storeObject(TENANT_ID,
+                                           RedactionStorageService.StorageIdUtils.getStorageId(request.getDossierId(), request.getFileId(), fileToUpload.fileType()),
+                                           in);
+            }
+        } else {
+            try (var in = new FileInputStream(fileToUpload.path().toFile())) {
+                storageService.storeObject(TENANT_ID,
+                                           RedactionStorageService.StorageIdUtils.getStorageId(request.getDossierId(), request.getFileId(), fileToUpload.fileType()),
+                                           in);
+            }
        }
-        try (var fis = new FileInputStream(path.toFile()); var in = new GZIPInputStream(fis);) {
-            storageService.storeObject(TENANT_ID, RedactionStorageService.StorageIdUtils.getStorageId(request.getDossierId(), request.getFileId(), fileType.get()), in);
-
-        }
-        return fileType;
+        return fileToUpload;
    }


-    private class TestDossierTemplate {
+    public class TestDossierTemplate {

        String id;
        Dictionary testDictionary;
@@ -379,4 +435,8 @@ import lombok.extern.slf4j.Slf4j;

    }

+    private record FileToUpload(Path path, FileType fileType) {
+
+    }
+
 }
@@ -8,6 +8,7 @@ import static org.mockito.Mockito.when;
 import java.io.FileOutputStream;
 import java.io.IOException;
 import java.time.OffsetDateTime;
+import java.util.ArrayList;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
@@ -165,12 +166,31 @@ public class DocumineFloraTest extends AbstractRedactionIntegrationTest {
                .get();
        assertNotNull(experimentalDates);

+        List<String> expectedDates = new ArrayList<>();
+        expectedDates.add("10/01/2022");
+        expectedDates.add("01/01/2022");
+        expectedDates.add("08/09/2024");
+        expectedDates.add("03/01/2022");
+        expectedDates.add("03/08/1992");
+        expectedDates.add("13/08/1992");
+        expectedDates.add("27/02/1992");
+        expectedDates.add("27/10/1989");
+        expectedDates.add("07/10/1989");
+        expectedDates.add("21/08/1998");
+        expectedDates.add("02/08/1998");
+        expectedDates.add("01/05/1988");
+        expectedDates.add("02/06/2003");
+        expectedDates.add("03/09/2005");
+        expectedDates.add("06/09/2005");
+        expectedDates.add("17/08/2005");
+        expectedDates.add("22/08/2035");
+
        String dates = experimentalDates.getComponentValues()
                .get(0).getValue();
        String[] dateArray = dates.split(", ");
        boolean allEqual = true;
        for (String date : dateArray) {
-            if (!"10/01/2022".equals(date) && !"08/09/2024".equals(date)) {
+            if (!expectedDates.contains(date)) {
                allEqual = false;
                break;
            }
@@ -107,7 +107,7 @@ public class MigrationIntegrationTest extends BuildDocumentIntegrationTest {
    @SneakyThrows
    public void testSave() {

-        MigratedIds ids = new MigratedIds(new LinkedList<>(), null);
+        MigratedIds ids = new MigratedIds(new LinkedList<>(), null, null);
        ids.addMapping("123", "321");
        ids.addMapping("123", "321");
        ids.addMapping("123", "321");
@@ -7,13 +7,16 @@ import static org.mockito.Mockito.when;
 import java.io.FileOutputStream;
 import java.io.IOException;
 import java.time.OffsetDateTime;
+import java.util.ArrayList;
 import java.util.List;
 import java.util.Set;
+import java.util.UUID;
 import java.util.stream.Stream;

 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.extension.ExtendWith;
+import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
 import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration;
 import org.springframework.boot.test.context.SpringBootTest;
@@ -23,23 +26,30 @@ import org.springframework.context.annotation.Configuration;
 import org.springframework.context.annotation.FilterType;
 import org.springframework.context.annotation.Import;
 import org.springframework.context.annotation.Primary;
+import org.springframework.core.io.ClassPathResource;
 import org.springframework.test.context.junit.jupiter.SpringExtension;

 import com.iqser.red.commons.jackson.ObjectMapperFactory;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeResult;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryType;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type;
 import com.iqser.red.service.redaction.v1.server.annotate.AnnotateRequest;
 import com.iqser.red.service.redaction.v1.server.annotate.AnnotateResponse;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.OsUtils;
+import com.iqser.red.service.redaction.v1.server.service.DictionaryService;
+import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
 import com.iqser.red.storage.commons.StorageAutoConfiguration;
 import com.iqser.red.storage.commons.service.StorageService;
 import com.iqser.red.storage.commons.utils.FileSystemBackedStorageService;
@@ -47,6 +57,8 @@ import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsi
 import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingServiceProcessorConfiguration;
 import com.knecon.fforesight.tenantcommons.TenantContext;

+import lombok.SneakyThrows;
+
@ExtendWith(SpringExtension.class)
@SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
@Import(RedactionIntegrationTest.RedactionIntegrationTestConfiguration.class)
@@ -54,6 +66,9 @@ public class RedactionAcceptanceTest extends AbstractRedactionIntegrationTest {

    private static final String RULES = loadFromClassPath("drools/acceptance_rules.drl");

+    @Autowired
+    DictionaryService dictionaryService;
+
    @Configuration
    @EnableAutoConfiguration(exclude = {RabbitAutoConfiguration.class})
    @Import(LayoutParsingServiceProcessorConfiguration.class)
@@ -87,14 +102,37 @@ public class RedactionAcceptanceTest extends AbstractRedactionIntegrationTest {

        when(dictionaryClient.getVersion(TEST_DOSSIER_ID)).thenReturn(0L);
        when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, true)).thenReturn(List.of(Type.builder()
-                                                                                                       .id(DOSSIER_REDACTIONS_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID)
+                                                                                                       .id(DOSSIER_REDACTIONS_TYPE_ID)
                                                                                                       .type(DOSSIER_REDACTIONS_INDICATOR)
-                                                                                                       .dossierTemplateId(TEST_DOSSIER_ID)
+                                                                                                       .dossierTemplateId(TEST_DOSSIER_TEMPLATE_ID)
+                                                                                                       .dossierId(TEST_DOSSIER_ID)
                                                                                                       .hexColor("#ffe187")
                                                                                                       .isHint(hintTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
                                                                                                       .isCaseInsensitive(caseInSensitiveMap.get(DOSSIER_REDACTIONS_INDICATOR))
                                                                                                       .isRecommendation(recommendationTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
                                                                                                       .rank(rankTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
+                                                                                                       .build(),
+                                                                                               Type.builder()
+                                                                                                       .id(DOSSIER_AUTHOR_TYPE_ID)
+                                                                                                       .type(DICTIONARY_AUTHOR)
+                                                                                                       .dossierTemplateId(TEST_DOSSIER_TEMPLATE_ID)
+                                                                                                       .dossierId(TEST_DOSSIER_ID)
+                                                                                                       .hexColor("#ffe184")
+                                                                                                       .isHint(hintTypeMap.get(DICTIONARY_AUTHOR))
+                                                                                                       .isCaseInsensitive(caseInSensitiveMap.get(DICTIONARY_AUTHOR))
+                                                                                                       .isRecommendation(recommendationTypeMap.get(DICTIONARY_AUTHOR))
+                                                                                                       .rank(rankTypeMap.get(DICTIONARY_AUTHOR))
+                                                                                                       .build(),
+                                                                                               Type.builder()
+                                                                                                       .id(DOSSIER_PUBLISHED_INFORMATION_TYPE_ID)
+                                                                                                       .type(PUBLISHED_INFORMATION_INDICATOR)
+                                                                                                       .dossierTemplateId(TEST_DOSSIER_TEMPLATE_ID)
+                                                                                                       .dossierId(TEST_DOSSIER_ID)
+                                                                                                       .hexColor("#ffe180")
+                                                                                                       .isHint(hintTypeMap.get(PUBLISHED_INFORMATION_INDICATOR))
+                                                                                                       .isCaseInsensitive(caseInSensitiveMap.get(PUBLISHED_INFORMATION_INDICATOR))
+                                                                                                       .isRecommendation(recommendationTypeMap.get(PUBLISHED_INFORMATION_INDICATOR))
+                                                                                                       .rank(rankTypeMap.get(PUBLISHED_INFORMATION_INDICATOR))
                                                                                                       .build()));

        mockDictionaryCalls(null);
@@ -128,6 +166,8 @@ public class RedactionAcceptanceTest extends AbstractRedactionIntegrationTest {

        String EFSA_SANITISATION_RULES = loadFromClassPath("drools/efsa_sanitisation.drl");
        when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.ENTITY)).thenReturn(JSONPrimitive.of(EFSA_SANITISATION_RULES));
+        dossierDictionary.put(PUBLISHED_INFORMATION_INDICATOR, new ArrayList<>());
+        dictionaryService.clearTenantDictionaryCache();
        AnalyzeRequest request = uploadFileToStorage("files/syngenta/CustomerFiles/SYNGENTA_EFSA_sanitisation_GFL_v1_moreSections.pdf");
        System.out.println("Start Full integration test");
        analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request);
@@ -138,10 +178,10 @@ public class RedactionAcceptanceTest extends AbstractRedactionIntegrationTest {

        var publishedInformationEntry1 = findEntityByTypeAndValue(entityLog, "published_information", "Oxford University Press").findFirst()
                .orElseThrow();
-        assertThat(publishedInformationEntry1.getSection().startsWith("Paragraph:"));
+        assertThat(publishedInformationEntry1.getSection()).startsWith("Paragraph:");
        var asyaLyon1 = findEntityByTypeAndValueAndSectionNumber(entityLog, "CBI_author", "Asya Lyon", publishedInformationEntry1.getContainingNodeId()).findFirst()
                .orElseThrow();
-        assertThat(publishedInformationEntry1.getSection().startsWith("Paragraph:"));
+        assertThat(asyaLyon1.getSection()).startsWith("Paragraph:");
        assertEquals(EntryState.SKIPPED, asyaLyon1.getState());

        var idRemoval = buildIdRemoval(publishedInformationEntry1.getId());
@@ -175,7 +215,7 @@ public class RedactionAcceptanceTest extends AbstractRedactionIntegrationTest {

        String EFSA_SANITISATION_RULES = loadFromClassPath("drools/efsa_sanitisation.drl");
        when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.ENTITY)).thenReturn(JSONPrimitive.of(EFSA_SANITISATION_RULES));
-        AnalyzeRequest request = uploadFileToStorage("files/syngenta/CustomerFiles/SYNGENTA_EFSA_sanitisation_GFL_v1_moreSections.pdf");
+        AnalyzeRequest request = uploadFileToStorage("files/new/SYNGENTA_EFSA_sanitisation_GFL_v1 3.pdf");
        System.out.println("Start Full integration test");
        analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request);
        System.out.println("Finished structure analysis");
@@ -192,10 +232,10 @@ public class RedactionAcceptanceTest extends AbstractRedactionIntegrationTest {

        var publishedInformationEntry1 = findEntityByTypeAndValue(entityLog, "published_information", "Oxford University Press").findFirst()
                .orElseThrow();
-        assertThat(publishedInformationEntry1.getSection().startsWith("Paragraph:"));
+        assertThat(publishedInformationEntry1.getSection()).startsWith("Paragraph:");
        var asyaLyon1 = findEntityByTypeAndValueAndSectionNumber(entityLog, "CBI_author", "Asya Lyon", publishedInformationEntry1.getContainingNodeId()).findFirst()
                .orElseThrow();
-        assertThat(publishedInformationEntry1.getSection().startsWith("Paragraph:"));
+        assertThat(publishedInformationEntry1.getSection()).startsWith("Paragraph:");
        assertEquals(EntryState.SKIPPED, asyaLyon1.getState());

        var idRemoval = buildIdRemoval(publishedInformationEntry1.getId());
@@ -220,7 +260,78 @@ public class RedactionAcceptanceTest extends AbstractRedactionIntegrationTest {
        try (FileOutputStream fileOutputStream = new FileOutputStream(outputFileName)) {
            fileOutputStream.write(annotateResponse.getDocument());
        }
+        dictionary.get(PUBLISHED_INFORMATION_INDICATOR).remove("Press");
+        assertThat(dictionary.get(PUBLISHED_INFORMATION_INDICATOR).contains("Press")).isFalse();
+    }

+
+    @Test
+    public void testPublishedInformationRemovalAtDossierLevel() throws IOException {
+
+        String EFSA_SANITISATION_RULES = loadFromClassPath("drools/efsa_sanitisation.drl");
+        when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.ENTITY)).thenReturn(JSONPrimitive.of(EFSA_SANITISATION_RULES));
+        AnalyzeRequest request = uploadFileToStorage("files/new/SYNGENTA_EFSA_sanitisation_GFL_v1 3.pdf");
+        System.out.println("Start Full integration test");
+        analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request);
+        System.out.println("Finished structure analysis");
+        mockDictionaryCalls(0L);
+        AnalyzeResult result = analyzeService.analyze(request);
+        System.out.println("Finished analysis");
+        // add dossier template published information  and the cbi_author
+        var publishedInfoValue = "great job";
+        var cbiAuthorValue = "Kara Hunt";
+        dictionary.get(PUBLISHED_INFORMATION_INDICATOR).add(publishedInfoValue);
+        dictionary.get(DICTIONARY_AUTHOR).add(cbiAuthorValue);
+        reanlysisVersions.put(publishedInfoValue, 3L);
+        reanlysisVersions.put(cbiAuthorValue, 3L);
+        when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(3L);
+        mockDictionaryCalls(2L);
+
+        request.setAnalysisNumber(3);
+        analyzeService.reanalyze(request);
+
+        EntityLog entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
+        var publishedInformationEntry = findEntityByTypeAndValue(entityLog, "published_information", publishedInfoValue).findFirst()
+                .orElseThrow();
+        var cbiAuthor = findEntityByTypeAndValueAndSectionNumber(entityLog, "CBI_author", cbiAuthorValue, publishedInformationEntry.getContainingNodeId()).findFirst()
+                .orElseThrow();
+        assertEquals(EntryState.SKIPPED, cbiAuthor.getState());
+
+        // remove at dossier level
+        var idRemoval = buildIdRemoval(publishedInformationEntry.getId());
+        idRemoval.setRemoveFromDictionary(true);
+
+        var manualRedactions = ManualRedactions.builder().idsToRemove(Set.of(idRemoval)).build();
+        request.setManualRedactions(manualRedactions);
+
+        deleted.add(publishedInfoValue);
+        reanlysisVersions.put(publishedInfoValue, 4L);
+
+        when(dictionaryClient.getVersionForDossier(TEST_DOSSIER_ID)).thenReturn(3L);
+        dossierDictionary.get(PUBLISHED_INFORMATION_INDICATOR).add(publishedInfoValue);
+
+        request.setAnalysisNumber(3);
+        mockDictionaryCalls(3L);
+        analyzeService.reanalyze(request);
+
+        entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
+
+        publishedInformationEntry = findEntityByTypeAndValue(entityLog, "published_information", publishedInfoValue).findFirst()
+                .orElseThrow();
+        cbiAuthor = findEntityByTypeAndValueAndSectionNumber(entityLog, "CBI_author", cbiAuthorValue, publishedInformationEntry.getContainingNodeId()).findFirst()
+                .orElseThrow();
+
+        assertEquals(EntryState.APPLIED, cbiAuthor.getState());
+
+        // cleanup changes
+        deleted.remove(publishedInfoValue);
+        assertThat(deleted.contains(publishedInfoValue)).isFalse();
+        dossierDictionary.put(PUBLISHED_INFORMATION_INDICATOR, new ArrayList<>());
+        assertThat(dossierDictionary.get(PUBLISHED_INFORMATION_INDICATOR).size()).isEqualTo(0);
+        dictionary.get(DICTIONARY_AUTHOR).remove(cbiAuthorValue);
+        assertThat(dictionary.get(DICTIONARY_AUTHOR).contains(cbiAuthorValue)).isFalse();
+        dictionary.get(PUBLISHED_INFORMATION_INDICATOR).remove(publishedInfoValue);
+        assertThat(dictionary.get(PUBLISHED_INFORMATION_INDICATOR).contains(publishedInfoValue)).isFalse();
    }


@@ -275,6 +386,75 @@ public class RedactionAcceptanceTest extends AbstractRedactionIntegrationTest {
    }


+    @Test
+    @SneakyThrows
+    void testNerEntitiesAfterReanalysis() {
+
+        String EFSA_SANITISATION_RULES = loadFromClassPath("drools/efsa_sanitisation.drl");
+        when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.ENTITY)).thenReturn(JSONPrimitive.of(EFSA_SANITISATION_RULES));
+
+        ClassPathResource responseJson = new ClassPathResource("files/new/SYNGENTA_EFSA_sanitisation_GFL_v1 3.NER_ENTITIES.json");
+        storageService.storeObject(TenantContext.getTenantId(),
+                                   RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.NER_ENTITIES),
+                                   responseJson.getInputStream());
+
+        String pdfFile = "files/new/SYNGENTA_EFSA_sanitisation_GFL_v1 3.pdf";
+
+        AnalyzeRequest request = uploadFileToStorage(pdfFile);
+
+        analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request);
+        request.setAnalysisNumber(1);
+        dossierDictionary.put(DICTIONARY_AUTHOR, new ArrayList<>());
+        mockDictionaryCalls(0L);
+
+        analyzeService.analyze(request);
+
+        var entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
+
+        String nerValue = "Osip S.";
+        var nerEntity = findEntityByTypeAndValue(entityLog, DICTIONARY_AUTHOR, nerValue).findFirst()
+                .orElseThrow();
+        assertThat(nerEntity.getEngines()).contains(Engine.NER);
+
+        String dictionaryAddValue = "cooperation";
+        ManualRedactionEntry manualRedactionEntry = ManualRedactionEntry.builder()
+                .value(dictionaryAddValue)
+                .type(DICTIONARY_AUTHOR)
+                .user("user")
+                .addToDossierDictionary(true)
+                .positions(List.of(Rectangle.builder().topLeftX(180.748f).topLeftY(546.564f).width(56.592f).height(15.408f).page(1).build()))
+                .type("dossier_redaction")
+                .fileId(TEST_FILE_ID)
+                .requestDate(OffsetDateTime.now())
+                .annotationId(UUID.randomUUID().toString())
+                .build();
+        request.setManualRedactions(ManualRedactions.builder().entriesToAdd(Set.of(manualRedactionEntry)).build());
+
+        request.setAnalysisNumber(2);
+        dossierDictionary.get(DICTIONARY_AUTHOR).add(dictionaryAddValue);
+        reanlysisVersions.put(dictionaryAddValue, 2L);
+        when(dictionaryClient.getVersionForDossier(TEST_DOSSIER_ID)).thenReturn(2L);
+        mockDictionaryCalls(1L);
+
+        AnalyzeResult reanalyzeResult = analyzeService.reanalyze(request);
+
+        entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
+
+        EntityLogEntry entityLogEntryAdded = entityLog.getEntityLogEntry()
+                .stream()
+                .filter(entityLogEntry -> entityLogEntry.getValue().equals(dictionaryAddValue))
+                .findFirst()
+                .get();
+        assertEquals(EntryState.APPLIED, entityLogEntryAdded.getState());
+
+        nerEntity = findEntityByTypeAndValue(entityLog, DICTIONARY_AUTHOR, nerValue).findFirst()
+                .orElseThrow();
+        assertThat(nerEntity.getEngines()).contains(Engine.NER);
+        dossierDictionary.get(DICTIONARY_AUTHOR).remove(dictionaryAddValue);
+
+    }
+
+
    private static IdRemoval buildIdRemoval(String id) {

        return IdRemoval.builder().annotationId(id).user("user").requestDate(OffsetDateTime.now()).fileId(TEST_FILE_ID).build();
@@ -119,7 +119,7 @@ public class RedactionIntegrationTest extends RulesIntegrationTest {
        when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L);
        when(dictionaryClient.getAllTypesForDossierTemplate(TEST_DOSSIER_TEMPLATE_ID, true)).thenReturn(getTypeResponse());

-        when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L);
+        when(dictionaryClient.getVersionForDossier(TEST_DOSSIER_ID)).thenReturn(0L);
        when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, true)).thenReturn(List.of(Type.builder()
                                                                                                       .id(DOSSIER_REDACTIONS_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID)
                                                                                                       .type(DOSSIER_REDACTIONS_INDICATOR)
@@ -1254,8 +1254,6 @@ public class RedactionIntegrationTest extends RulesIntegrationTest {


    @Test
-    @Disabled
-    // todo: fix me in RED-9257
    public void signaturesAreRedactionAfterReanalyse() throws IOException {

        AnalyzeRequest request = uploadFileToStorage("files/new/SYNGENTA_EFSA_sanitisation_GFL_v1 3.pdf");
@@ -1297,8 +1295,6 @@ public class RedactionIntegrationTest extends RulesIntegrationTest {


    @Test
-    @Disabled
-    // todo: fix me in RED-9257
    public void entityIsAppliedAfterRecategorize() throws IOException {

        AnalyzeRequest request = uploadFileToStorage("files/new/SYNGENTA_EFSA_sanitisation_GFL_v1 (1).pdf");
@@ -43,7 +43,17 @@ public class DateConverterTest {
                                                         "28 March 2018 (animal 1 - 5000 mg/kg bw)",
                                                         "28 March 2018 (animal1 - 5000 mg/kg bw)",
                                                         "28 August 2018 (animal 1)",
-                                                         "31 August 2018 (animal 1)");
+                                                         "31 August 2018 (animal 1)",
+                                                         "October, 27 1989",
+                                                         "October, 7 1989",
+                                                         "August 21,1998",
+                                                         "August 2,1998",
+                                                         "1st May 1988",
+                                                         "2nd June 2003",
+                                                         "3rd September 2005",
+                                                         "6th September 2005",
+                                                         "17th August 2005",
+                                                         "22nd August 2035");

        for (String dateStr : goldenStandardDates) {
            Optional<Date> parsedDate = DateConverter.parseDate(dateStr);
@@ -308,7 +308,7 @@ public class DocumentIEntityInsertionIntegrationTest extends BuildDocumentIntegr
        assertEquals("2-[(2-(1-hydroxy-ethyl)-6methyl-phenyl-amino]propan-1-ol (", textEntity.getTextBefore());
        assertEquals(" of metabolite of", textEntity.getTextAfter());
        assertEquals(searchTerm, textEntity.getValue());
-        assertEquals(7, textEntity.getIntersectingNodes().size());
+        assertEquals(8, textEntity.getIntersectingNodes().size());
        assertEquals("Table 2.7-1: List of substances and metabolites and related structural formula ",
                     textEntity.getDeepestFullyContainingNode().getHeadline().getTextBlock().getSearchText());
        assertTrue(textEntity.getPages()
@@ -154,7 +154,7 @@ public class DocumentPerformanceIntegrationTest extends BuildDocumentIntegration
        System.out.printf("Inserting entities into the graph took %d ms\n", System.currentTimeMillis() - graphInsertionStart);

        long droolsStart = System.currentTimeMillis();
-        List<FileAttribute> fileAttributes = entityDroolsExecutionService.executeRules(kieContainer,
+        List<FileAttribute> fileAttributes = entityDroolsExecutionService.executeRules(TEST_FILE_ID, kieContainer,
                                                                                       document,
                                                                                       dictionary,
                                                                                       Collections.emptyList(),
@@ -361,7 +361,6 @@ class DroolsValidationServiceTest {
    }


-
    @Test
    @SneakyThrows
    void testRulesWithBlacklistedKeyword() {
@@ -373,28 +372,97 @@ class DroolsValidationServiceTest {

        String rulesString = new String(rulesFile.getInputStream().readAllBytes());

-        String evilRule = """
+        String globalStart = "\nglobal Document document";
+        String importTenantContext = "import com.knecon.fforesight.tenantcommons.TenantContext;\n";
+        int indexGlobalStart = rulesString.indexOf(globalStart);
+        String rulesString1 = rulesString.substring(0, indexGlobalStart) + importTenantContext + rulesString.substring(indexGlobalStart);
+
+        String evilRulePart1 = """
                
                //------------------------------------ All the evil rules ------------------------------------
-
+                
                // Rule unit: EV.1
                rule "EV.1.0: Remove duplicate FileAttributes but also do very evil things"
                    salience 999
                    when
                        $fileAttribute: FileAttribute($label: label, $value: value)
                        $duplicate: FileAttribute(this != $fileAttribute, label == $label, value == $value)
+                """;
+        String evilRulePart2 = """
                    then
                        retract($duplicate);
                        System.exit(0);
                    end
                """;
-        rulesString = rulesString + evilRule;
+        String usingTenantContext = """
+                $tenantId: TenantContext.getTenantId()
+                """;
+        String usingStaticTenantContext = """
+                $tenantId: getTenantId()
+                """;
+        rulesString1 = rulesString1 + evilRulePart1 + usingTenantContext + evilRulePart2;

-        DroolsValidation droolsValidation = droolsValidationService.testRules(new RuleValidationModel(RuleFileType.ENTITY.name(), rulesString));
+        DroolsValidation droolsValidation = droolsValidationService.testRules(new RuleValidationModel(RuleFileType.ENTITY.name(), rulesString1));
        droolsValidation.getBlacklistErrorMessages()
                .forEach(System.out::println);
        assertFalse(droolsValidation.isCompiled());
-        assertEquals(droolsValidation.getBlacklistErrorMessages().size(), 1);
+        assertEquals(2, droolsValidation.getBlacklistErrorMessages().size());
+        assertEquals(1,
+                     droolsValidation.getBlacklistErrorMessages()
+                             .get(0).getBlacklistedKeywords().size());
+        assertTrue(droolsValidation.getBlacklistErrorMessages()
+                           .get(0).getBlacklistedKeywords().contains("TenantContext"));
+        assertEquals(2,
+                     droolsValidation.getBlacklistErrorMessages()
+                             .get(1).getBlacklistedKeywords().size());
+        assertTrue(droolsValidation.getBlacklistErrorMessages()
+                           .get(1).getBlacklistedKeywords().contains("TenantContext"));
+
+        String importStaticTenantContext = "\nimport static com.knecon.fforesight.tenantcommons.TenantContext.getTenantId;";
+        String importStaticStart = "\nimport static java.lang.String.format;";
+        int indexStaticImportStart = rulesString.indexOf(importStaticStart);
+        String rulesStringWithStaticImport = rulesString.substring(0, indexStaticImportStart) + importStaticTenantContext + rulesString.substring(indexStaticImportStart);
+
+        rulesStringWithStaticImport = rulesStringWithStaticImport + evilRulePart1 + usingStaticTenantContext + evilRulePart2;
+
+        DroolsValidation droolsValidation2 = droolsValidationService.testRules(new RuleValidationModel(RuleFileType.ENTITY.name(), rulesStringWithStaticImport));
+        droolsValidation2.getBlacklistErrorMessages()
+                .forEach(System.out::println);
+        assertFalse(droolsValidation2.isCompiled());
+        assertEquals(2, droolsValidation2.getBlacklistErrorMessages().size());
+        assertEquals(1,
+                     droolsValidation2.getBlacklistErrorMessages()
+                             .get(0).getBlacklistedKeywords().size());
+        assertTrue(droolsValidation2.getBlacklistErrorMessages()
+                           .get(0).getBlacklistedKeywords().contains("TenantContext"));
+        assertEquals(1,
+                     droolsValidation2.getBlacklistErrorMessages()
+                             .get(1).getBlacklistedKeywords().size());
+        assertTrue(droolsValidation2.getBlacklistErrorMessages()
+                           .get(1).getBlacklistedKeywords().contains("System."));
+    }
+
+
+    @Test
+    void assertRuleIdentifierDoesNotMatch() {
+
+        String ruleString = RuleManagementResources.getBaseRuleFileString() + """
+                rule "CBI.0.0: Redact CBI Authors (non vertebrate Study)"
+                    when
+                        not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
+                        $entity: TextEntity(type() == "CBI_author", dictionaryEntry)
+                    then
+                        $entity.redact(
+                        "CBI.1.0", 
+                        "Author found", 
+                        "Article 39(e)(3) of Regulation (EC) No 178/2002"
+                        );
+                    end
+                """;
+
+        RuleFileBluePrint ruleFileBluePrint = RuleFileParser.buildBluePrintFromRulesString(ruleString);
+
+        assertFalse(ruleFileBluePrint.getDroolsValidation().isCompiled());
    }

 }
@@ -1066,6 +1066,8 @@ rule "AI.0.0: Add all NER Entities of type CBI_author"
        nerEntities: NerEntities(hasEntitiesOfType("CBI_author"))
    then
        nerEntities.streamEntitiesOfType("CBI_author")
+            .filter(entity -> entity.value().length() > 3)
+            .filter(entity -> entity.value().length() < 40)
            .forEach(nerEntity -> entityCreationService.optionalByNerEntity(nerEntity, EntityType.RECOMMENDATION, document));
    end

@@ -1696,6 +1696,8 @@ rule "AI.0.0: Add all NER Entities of type CBI_author"
        nerEntities: NerEntities(hasEntitiesOfType("CBI_author"))
    then
        nerEntities.streamEntitiesOfType("CBI_author")
+            .filter(entity -> entity.value().length() > 3)
+            .filter(entity -> entity.value().length() < 40)
            .forEach(nerEntity -> entityCreationService.optionalByNerEntity(nerEntity, EntityType.RECOMMENDATION, document));
    end

@@ -625,6 +625,8 @@ rule "AI.0.0: Add all NER Entities of type CBI_author"
        nerEntities: NerEntities(hasEntitiesOfType("CBI_author"))
    then
        nerEntities.streamEntitiesOfType("CBI_author")
+            .filter(entity -> entity.value().length() > 3)
+            .filter(entity -> entity.value().length() < 40)
            .forEach(nerEntity -> entityCreationService.optionalByNerEntity(nerEntity, EntityType.RECOMMENDATION, document));
    end

@@ -1154,6 +1154,8 @@ rule "AI.0.0: Add all NER Entities of type CBI_author"
        nerEntities: NerEntities(hasEntitiesOfType("CBI_author"))
    then
        nerEntities.streamEntitiesOfType("CBI_author")
+            .filter(entity -> entity.value().length() > 3)
+            .filter(entity -> entity.value().length() < 40)
            .forEach(nerEntity -> entityCreationService.optionalByNerEntity(nerEntity, EntityType.RECOMMENDATION, document));
    end

@@ -100,6 +100,8 @@ rule "AI.0.0: Add all NER Entities of type CBI_author"
        nerEntities: NerEntities(hasEntitiesOfType("CBI_author"))
    then
        nerEntities.streamEntitiesOfType("CBI_author")
+            .filter(entity -> entity.value().length() > 3)
+            .filter(entity -> entity.value().length() < 40)
            .forEach(nerEntity -> entityCreationService.optionalByNerEntity(nerEntity, EntityType.RECOMMENDATION, document));
    end

@@ -14,4 +14,7 @@
        <appender-ref ref="${logType}"/>
    </root>

+    <logger name="org.apache.fontbox.ttf" level="ERROR"/>
+    <logger name="org.drools.mvel" level="ERROR"/>
+
 </configuration>
@@ -1692,6 +1692,8 @@ rule "AI.0.0: Add all NER Entities of type CBI_author"
        nerEntities: NerEntities(hasEntitiesOfType("CBI_author"))
    then
        nerEntities.streamEntitiesOfType("CBI_author")
+            .filter(entity -> entity.value().length() > 3)
+            .filter(entity -> entity.value().length() < 40)
            .forEach(nerEntity -> entityCreationService.optionalByNerEntity(nerEntity, EntityType.RECOMMENDATION, document));
    end

@@ -25,9 +25,10 @@ public class RuleFileMigrationTest {

    // Put your redaction service drools paths and dossier-templates paths both RM and DM here
    static final List<String> ruleFileDirs = List.of(
-            "/Users/maverickstuder/Documents/RedactManager/redaction-service/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools",
-            "/Users/maverickstuder/Documents/DocuMine/dossier-templates-v2/",
-            "/Users/maverickstuder/Documents/RedactManager/dossier-templates-v2/");
+            "/home/kschuettler/iqser/redaction/redaction-service/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools",
+            "/home/kschuettler/iqser/redaction/dossier-templates-v2",
+            "/home/kschuettler/iqser/fforesight/dossier-templates-v2",
+            "/home/kschuettler/iqser/business-logic");


    @Test
Author	SHA1	Message	Date
Maverick Studer	19892bde17	Merge branch 'RED-10639-bp' into 'release/4.348.x' RED-10639: RM-224: ERROR state for document after re-upload same document with... See merge request redactmanager/redaction-service!578	2024-12-12 12:57:04 +01:00
Maverick Studer	0851e6ddd9	RED-10639: RM-224: ERROR state for document after re-upload same document with...	2024-12-12 12:57:04 +01:00
Dominique Eifländer	29cab89def	Merge branch 'RED-10644-4.2' into 'release/4.348.x' RED-10644: Fixed dublicated entries with whitespace at the end See merge request redactmanager/redaction-service!574	2024-12-10 09:20:28 +01:00
Dominique Eifländer	4f36677933	RED-10644: Fixed dublicated entries with whitespace at the end	2024-12-09 15:09:21 +01:00
Timo Bejan	5ef3699648	Merge branch 'RED-10353-4.2' into 'release/4.348.x' RED-10353: Increase rules timeout, added different error message and error... See merge request redactmanager/redaction-service!549	2024-11-05 14:34:44 +01:00
Dominique Eifländer	9ee4ae2757	RED-10353: Increase rules timeout, added different error message and error...	2024-11-05 14:34:43 +01:00
Kilian Schüttler	3de0bcf99c	Merge branch 'feature/RED-10125' into 'release/4.348.x' RED-10125: fix Normalizer in drools See merge request redactmanager/redaction-service!540	2024-10-15 10:47:18 +02:00
Kilian Schuettler	28c243dd3d	RED-10125: fix Normalizer in drools	2024-10-15 10:40:59 +02:00
Kilian Schüttler	4a8e62919b	Merge branch 'feature/RM-184-bp' into 'release/4.348.x' RM-184 && RM-172: update SectionIdentifier javadoc, make... See merge request redactmanager/redaction-service!536	2024-10-10 10:21:10 +02:00
Kilian Schüttler	9e3d202a91	RM-184 && RM-172: update SectionIdentifier javadoc, make...	2024-10-10 10:21:10 +02:00
Maverick Studer	dc48145452	Merge branch 'hotfix-npe-component-log' into 'release/4.348.x' Hotfix for NPE when creating component log See merge request redactmanager/redaction-service!533	2024-10-07 16:50:07 +02:00
maverickstuder	1d2abf18a2	Hotfix for NPE when creating component log	2024-10-07 16:35:27 +02:00
Dominique Eifländer	d631abcc39	Merge branch 'RED-10131-4.2' into 'release/4.348.x' RED-10131: Catch stackoverflow error of unoptimized regexes See merge request redactmanager/redaction-service!531	2024-10-07 13:12:49 +02:00
Dominique Eifländer	c0a191b139	RED-10131: Catch stackoverflow error of unoptimized regexes	2024-10-07 12:56:27 +02:00
Maverick Studer	333481e1c9	Merge branch 'RED-10146-bp' into 'release/4.348.x' RED-10146: Include defined components in component log See merge request redactmanager/redaction-service!530	2024-10-07 12:26:00 +02:00
Maverick Studer	1cff5666b5	RED-10146: Include defined components in component log	2024-10-07 12:26:00 +02:00
Dominique Eifländer	c7bd8c8a8d	Merge branch 'RED-10130' into 'release/4.348.x' RED-9728: remove False Positives from table methods See merge request redactmanager/redaction-service!528	2024-10-01 15:45:51 +02:00
Kilian Schüttler	1bdb9a2ef1	RED-9728: remove False Positives from table methods	2024-10-01 15:12:11 +02:00
Kilian Schüttler	ddca2ffc01	Merge branch 'hotfix-bp' into 'release/4.348.x' RED-9975: extend SectionIdentifier to alphanumeric See merge request redactmanager/redaction-service!514	2024-09-11 13:38:22 +02:00
Kilian Schüttler	fc30e05b7c	RED-9975: extend SectionIdentifier to alphanumeric	2024-09-11 13:38:22 +02:00
Andrei Isvoran	3ea73aa859	Merge branch 'RED-9986-bp' into 'release/4.348.x' RED-9986 - Add component rules path to be scanned for Javadoc generation See merge request redactmanager/redaction-service!511	2024-09-09 12:44:01 +02:00
Andrei Isvoran	179ac6d9ad	RED-9986 - Add component rules path to be scanned for Javadoc generation	2024-09-09 11:33:31 +03:00
Kilian Schüttler	bdc6ab7e96	Merge branch 'RED-9964' into 'release/4.348.x' RED-9964: fix errors with images See merge request redactmanager/redaction-service!505	2024-09-04 09:16:11 +02:00
Kilian Schüttler	e959d60ec0	RED-9964: fix errors with images	2024-09-04 09:16:11 +02:00
Kilian Schüttler	6b6d06d24e	Merge branch 'RED-9964' into 'release/4.348.x' RED-9964: refactor getMainBody() and getMainBodyTextBlock() in Page See merge request redactmanager/redaction-service!502	2024-09-02 16:51:05 +02:00
Kilian Schüttler	8ac0657795	RED-9964: refactor getMainBody() and getMainBodyTextBlock() in Page	2024-09-02 16:51:04 +02:00
Maverick Studer	dad17bb504	Merge branch 'RED-9865-bp2' into 'release/4.348.x' RED-9865: fix for case 2 See merge request redactmanager/redaction-service!495	2024-08-23 17:01:21 +02:00
Maverick Studer	f445b7fe69	RED-9865: fix for case 2	2024-08-23 17:01:21 +02:00
Dominique Eifländer	0692cc90e4	Merge branch 'RED-9837-4.1' into 'release/4.348.x' RED-9837: Fixed not working timeout with endless loop in drools then block See merge request redactmanager/redaction-service!490	2024-08-19 14:08:31 +02:00
Dominique Eifländer	ab114b0920	RED-9837: Fixed not working timeout with endless loop in drools then block	2024-08-19 13:20:04 +02:00
Dominique Eifländer	7396c04314	Merge branch 'RED-9760-4.1-rules' into 'release/4.348.x' RED-9760: Do not check blacklisted keywords in Strings See merge request redactmanager/redaction-service!488	2024-08-13 12:01:49 +02:00
Dominique Eifländer	305cd8f5ac	RED-9760: Do not check blacklisted keywords in Strings	2024-08-13 11:11:25 +02:00
Maverick Studer	b4ecbde89e	Merge branch 'RED-9782-fix-bp' into 'release/4.348.x' RED-9782: Automated Analysis should be disabled when uploading a document that... See merge request redactmanager/redaction-service!484	2024-08-12 18:40:56 +02:00
Maverick Studer	7c31d4f70b	RED-9782: Automated Analysis should be disabled when uploading a document that...	2024-08-12 18:40:56 +02:00
Kilian Schüttler	f08654a082	Merge branch 'hotfix' into 'release/4.348.x' Fix UOE in ComponentDroolsExecutionService See merge request redactmanager/redaction-service!482	2024-08-12 15:59:14 +02:00
Kilian Schüttler	2cf7f7c7b2	Fix UOE in ComponentDroolsExecutionService	2024-08-12 15:59:14 +02:00
Kilian Schüttler	a51f10b9d1	Merge branch 'RED-9869-bp' into 'release/4.348.x' RED-9869: allow java.text and find ruleIdentifiers with whitespaces/linebreaks See merge request redactmanager/redaction-service!480	2024-08-12 15:19:40 +02:00
Kilian Schuettler	8c36035655	RED-9869: allow java.text and find ruleIdentifiers with whitespaces/linebreaks	2024-08-12 12:39:42 +02:00
Maverick Studer	67bb4fe7f9	Merge branch 'hotfixes-dm-release' into 'release/4.348.x' Hotfixes dm release See merge request redactmanager/redaction-service!477	2024-08-09 16:52:40 +02:00
Maverick Studer	2a9101306c	Hotfixes dm release	2024-08-09 16:52:40 +02:00
Maverick Studer	6ecac11df5	Merge branch 'RED-9857-bp' into 'release/4.348.x' RED-9857: Add new date format See merge request redactmanager/redaction-service!475	2024-08-09 10:30:23 +02:00
Maverick Studer	e663fd2f2a	RED-9857: Add new date format	2024-08-09 10:30:22 +02:00
Dominique Eifländer	0ef4087b36	Merge branch 'RED-9760-anyheadline-4.1' into 'release/4.348.x' RED-9760: Changed anyHeadlineContains to act like in the previous version See merge request redactmanager/redaction-service!473	2024-08-07 15:17:33 +02:00
Dominique Eifländer	bf3ae1606b	RED-9760: Changed anyHeadlineContains to act like in the previous version	2024-08-07 14:56:16 +02:00
Maverick Studer	43620f7b52	Merge branch 'RED-9782' into 'release/4.348.x' RED-9782: Automated Analysis should be disabled when uploading a document that... See merge request redactmanager/redaction-service!471	2024-08-07 12:26:03 +02:00
Maverick Studer	92fc003576	RED-9782: Automated Analysis should be disabled when uploading a document that...	2024-08-07 12:26:02 +02:00
Dominique Eifländer	ed02a83289	Merge branch 'RED-9782-4.1' into 'release/4.348.x' Resolve RED-9782 "4.1" See merge request redactmanager/redaction-service!469	2024-08-02 14:40:19 +02:00
Dominique Eifländer	78f5aaa54e	Resolve RED-9782 "4.1"	2024-08-02 14:40:19 +02:00
Andrei Isvoran	acb5b4c308	Merge branch 'RED-9770' into 'release/4.348.x' RED-9770 - Extend date converter See merge request redactmanager/redaction-service!467	2024-07-30 10:29:48 +02:00
Andrei Isvoran	61ee1c12ca	RED-9770 - Extend date converter	2024-07-30 10:56:58 +03:00
Kilian Schüttler	abec7ae6bf	Merge branch 'annotationMode-bp' into 'release/4.348.x' annotationMode: ignore IDs of manual adds in annotationMode See merge request redactmanager/redaction-service!466	2024-07-26 14:53:41 +02:00
Kilian Schuettler	afeddb4d91	annotationMode: ignore IDs of manual adds in annotationMode	2024-07-26 14:03:28 +02:00
Dominique Eifländer	359c237943	Merge branch 'RED-9658-mongo-4.1' into 'release/4.348.x' RED-9658: Fixed wrong mongo database name See merge request redactmanager/redaction-service!461	2024-07-17 11:02:22 +02:00
Dominique Eifländer	9789943f45	RED-9658: Fixed wrong mongo database name	2024-07-17 10:44:30 +02:00
Andrei Isvoran	f096aab156	Merge branch 'RED-9667' into 'release/4.348.x' RED-9667 - Extend convert dates See merge request redactmanager/redaction-service!459	2024-07-16 16:02:20 +02:00
Andrei Isvoran	156b102e87	RED-9667 - Extend convert dates	2024-07-16 16:02:20 +02:00
Andrei Isvoran	180728721a	Merge branch 'RED-9496-graceful-shutdown-bp' into 'release/4.348.x' RED-9496 - Implement graceful shutdown See merge request redactmanager/redaction-service!457	2024-07-04 13:56:45 +02:00
Andrei Isvoran	fb9d1042ac	RED-9496 - Implement graceful shutdown	2024-07-04 14:03:43 +03:00
Corina Olariu	046b4b29b9	Merge branch 'RED-9466-bp' into 'release/4.348.x' RED-9466 - Adding annotation removes all AI based recommendations until forced re-analysis See merge request redactmanager/redaction-service!452	2024-06-28 15:31:16 +02:00
Corina Olariu	dce797ef8e	RED-9466 - Adding annotation removes all AI based recommendations until forced re-analysis	2024-06-28 15:31:16 +02:00
Kilian Schüttler	8b8dab2a18	RED-9375: use storageId for cache names everywhere, such that name may be updated by a user (cherry picked from commit `c1a2e9dee2`)	2024-06-27 16:13:24 +02:00
Kilian Schüttler	e26e0e34f5	Merge branch 'hotfix' into 'master' hotfix: forward port dynamic timeout for rule execution See merge request redactmanager/redaction-service!445	2024-06-25 13:52:34 +02:00
Kilian Schuettler	7f94090dac	hotfix: forward port dynamic timeout for rule execution	2024-06-25 13:39:18 +02:00
Corina Olariu	ea88ea339d	Merge branch 'RED-9350-2' into 'master' RED-9350- Prohibit access to tenant context in rule execution Closes RED-9350 See merge request redactmanager/redaction-service!444	2024-06-24 12:37:18 +02:00
Corina Olariu	74a10cd77f	RED-9350- Prohibit access to tenant context in rule execution	2024-06-24 12:37:18 +02:00
Kilian Schüttler	0a31dc6c07	Merge branch 'RED-9361' into 'master' RED-9361: update rule AI.0.0 Closes RED-9361 See merge request redactmanager/redaction-service!436	2024-06-24 10:14:35 +02:00
Kilian Schüttler	21f68a0f1c	RED-9361: update rule AI.0.0	2024-06-24 10:14:35 +02:00
Ali Oezyetimoglu	abf0f6b6ea	Merge branch 'RED-9413' into 'master' RED-9413: fixed date issue Closes RED-9413 See merge request redactmanager/redaction-service!442	2024-06-21 09:30:59 +02:00
Ali Oezyetimoglu	fcd09a928c	RED-9413: fixed date issue	2024-06-21 09:30:59 +02:00
Andrei Isvoran	9ef104bbae	Merge branch 'RED-9378' into 'master' RED-9378 - Don't create UnprocessedManualEntities for non-manual entities Closes RED-9378 See merge request redactmanager/redaction-service!441	2024-06-21 07:57:04 +02:00
Andrei Isvoran	badbc08c2e	RED-9378 - Don't create UnprocessedManualEntities for non-manual entities	2024-06-20 17:26:04 +03:00
Maverick Studer	ebb9b7f4c8	Merge branch 'RED-9374' into 'master' RED-9374: Ner Entities are at wrong locations Closes RED-9374 See merge request redactmanager/redaction-service!438	2024-06-19 12:15:50 +02:00
Maverick Studer	7918db25a9	RED-9374: Ner Entities are at wrong locations	2024-06-19 12:15:50 +02:00
Corina Olariu	36c7a61265	Merge branch 'RED-9350' into 'master' RED-9350- Prohibit access to tenant context in rule execution Closes RED-9350 See merge request redactmanager/redaction-service!437	2024-06-18 12:42:59 +02:00
Corina Olariu	9d7c1a6157	RED-9350- Prohibit access to tenant context in rule execution	2024-06-18 12:42:59 +02:00
Kilian Schüttler	f8cfc4bd1a	Merge branch 'hotfix' into 'master' hotfix: fix mapping query error See merge request redactmanager/redaction-service!435	2024-06-17 17:50:22 +02:00
Kilian Schüttler	7fb84c42e0	hotfix: fix mapping query error	2024-06-17 17:50:22 +02:00
Dominique Eifländer	9f171e515b	Merge branch 'RED-9274-master' into 'master' RED-9274: Set state REMOVED if last change is REMOVED in migration Closes RED-9274 See merge request redactmanager/redaction-service!434	2024-06-17 13:21:53 +02:00
Dominique Eifländer	5cb7510aac	RED-9274: Set state REMOVED if last change is REMOVED in migration	2024-06-17 13:06:31 +02:00
Ali Oezyetimoglu	32a43202c9	Merge branch 'fix' into 'master' RED-9322 & RED-9036: See merge request redactmanager/redaction-service!432	2024-06-17 09:51:15 +02:00
Ali Oezyetimoglu	3f17fb6bf5	RED-9322 & RED-9036:	2024-06-17 09:51:15 +02:00
Kilian Schüttler	3ef9db74df	Merge branch 'hotfix' into 'master' hotfix: fix annotationMode See merge request redactmanager/redaction-service!430	2024-06-14 16:37:41 +02:00
Kilian Schuettler	499215db02	hotfix: fix annotationMode	2024-06-14 16:23:50 +02:00
Dominique Eifländer	d4251e7b4e	Merge branch 'gotfixExc-master' into 'master' hotfix: show stacktrace when exceptions occurs See merge request redactmanager/redaction-service!429	2024-06-14 15:42:33 +02:00
Dominique Eifländer	edd2254dfe	hotfix: show stacktrace when exceptions occurs	2024-06-14 15:27:13 +02:00
Ali Oezyetimoglu	6c36188689	Merge branch 'RED-9322_RED-9036' into 'master' RED-9322 & RED-9036: Closes RED-9322 See merge request redactmanager/redaction-service!427	2024-06-14 13:48:47 +02:00
Ali Oezyetimoglu	6c8095e968	RED-9322 & RED-9036:	2024-06-14 13:48:47 +02:00
Corina Olariu	c2acb90080	Merge branch 'RED-9187-junit' into 'master' RED-9187 - Annotation does not switch back to redacted if published... Closes RED-9187 See merge request redactmanager/redaction-service!425	2024-06-13 08:36:57 +02:00
Corina Olariu	18966be460	RED-9187 - Annotation does not switch back to redacted if published...	2024-06-13 08:36:57 +02:00
Maverick Studer	98e9bf1d63	Merge branch 'RED-9257' into 'master' RED-9257: intersectingNodes update crashes kieSession Closes RED-9257 See merge request redactmanager/redaction-service!419	2024-06-10 16:24:33 +02:00
Kilian Schüttler	82b0f1e6c2	RED-9257: intersectingNodes update crashes kieSession	2024-06-10 16:24:33 +02:00
Ali Oezyetimoglu	d042e5b38c	Merge branch 'RED-8339' into 'master' RED-8339: readded deprecated originalValue as it is currently used by FE until... Closes RED-8339 See merge request redactmanager/redaction-service!424	2024-06-10 14:27:07 +02:00
Ali Oezyetimoglu	2c97b7f04e	RED-8339: readded deprecated originalValue as it is currently used by FE until...	2024-06-10 14:27:07 +02:00
Maverick Studer	6e9548dca4	Merge branch 'RED-9317-fp' into 'master' RED-9317: Added missing date format dd.MM.yy Closes RED-9317 See merge request redactmanager/redaction-service!423	2024-06-07 15:24:34 +02:00
Dominique Eifländer	570ecc5797	RED-9317: Added missing date format dd.MM.yy	2024-06-07 15:10:38 +02:00
Kilian Schüttler	58416173fa	Merge branch 'RED-7074' into 'master' RED-7074: increase layoutparser version Closes RED-7074 See merge request redactmanager/redaction-service!421	2024-06-07 10:20:13 +02:00
Kilian Schüttler	eec42aca66	RED-7074: increase layoutparser version	2024-06-07 10:20:13 +02:00