RED-9843: Delete entitylog when new layout analyis is needed for documine

RED-9837: Fixed not working timeout with endless loop in drools then block

See merge request redactmanager/redaction-service!490

redaction-service-v1/redaction-service-api-v1/build.gradle.kts

@@ -4,7 +4,7 @@ plugins {
 }
 
 description = "redaction-service-api-v1"
-val persistenceServiceVersion = "2.439.0"
+val persistenceServiceVersion = "2.465.41"
 
 dependencies {
     implementation("org.springframework:spring-web:6.0.12")

redaction-service-v1/redaction-service-server-v1/build.gradle.kts

@@ -12,11 +12,11 @@ plugins {
 description = "redaction-service-server-v1"
 
 
-val layoutParserVersion = "0.141.0"
+val layoutParserVersion = "0.142.6"
 val jacksonVersion = "2.15.2"
 val droolsVersion = "9.44.0.Final"
 val pdfBoxVersion = "3.0.0"
-val persistenceServiceVersion = "2.444.0"
+val persistenceServiceVersion = "2.465.41"
 val springBootStarterVersion = "3.1.5"
 val springCloudVersion = "4.0.4"
 val testContainersVersion = "1.19.7"
@@ -43,6 +43,7 @@ dependencies {
     implementation("com.iqser.red.commons:storage-commons:2.45.0")
     implementation("com.knecon.fforesight:tenant-commons:0.24.0")
     implementation("com.knecon.fforesight:tracing-commons:0.5.0")
+    implementation("com.knecon.fforesight:lifecycle-commons:0.6.0")
 
     implementation("com.fasterxml.jackson.module:jackson-module-afterburner:${jacksonVersion}")
     implementation("com.fasterxml.jackson.datatype:jackson-datatype-jsr310:${jacksonVersion}")

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/Application.java

@@ -13,6 +13,7 @@ import org.springframework.boot.context.properties.EnableConfigurationProperties
 import org.springframework.cache.annotation.EnableCaching;
 import org.springframework.cloud.openfeign.EnableFeignClients;
 import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.EnableAspectJAutoProxy;
 import org.springframework.context.annotation.Import;
 import org.springframework.data.mongodb.repository.config.EnableMongoRepositories;
 
@@ -20,6 +21,7 @@ import com.iqser.red.service.dictionarymerge.commons.DictionaryMergeService;
 import com.iqser.red.service.persistence.service.v1.api.shared.mongo.SharedMongoAutoConfiguration;
 import com.iqser.red.service.redaction.v1.server.client.RulesClient;
 import com.iqser.red.storage.commons.StorageAutoConfiguration;
+import com.knecon.fforesight.lifecyclecommons.LifecycleAutoconfiguration;
 import com.knecon.fforesight.mongo.database.commons.MongoDatabaseCommonsAutoConfiguration;
 import com.knecon.fforesight.mongo.database.commons.liquibase.EnableMongoLiquibase;
 import com.knecon.fforesight.tenantcommons.MultiTenancyAutoConfiguration;
@@ -32,13 +34,14 @@ import lombok.extern.slf4j.Slf4j;
 
 @Slf4j
 @EnableCaching
-@ImportAutoConfiguration({MultiTenancyAutoConfiguration.class, SharedMongoAutoConfiguration.class})
+@ImportAutoConfiguration({MultiTenancyAutoConfiguration.class, SharedMongoAutoConfiguration.class, LifecycleAutoconfiguration.class})
 @Import({MetricsConfiguration.class, StorageAutoConfiguration.class, MongoDatabaseCommonsAutoConfiguration.class})
 @EnableFeignClients(basePackageClasses = RulesClient.class)
 @EnableConfigurationProperties(RedactionServiceSettings.class)
 @EnableMongoRepositories(basePackages = "com.iqser.red.service.persistence")
 @EnableMongoLiquibase
 @SpringBootApplication(exclude = {SecurityAutoConfiguration.class, ManagementWebSecurityAutoConfiguration.class, DataSourceAutoConfiguration.class, LiquibaseAutoConfiguration.class, MongoAutoConfiguration.class, MongoDataAutoConfiguration.class})
+@EnableAspectJAutoProxy
 public class Application {
 
     public static void main(String[] args) {

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/PrecursorEntity.java

@@ -120,7 +120,8 @@ public class PrecursorEntity implements IEntity {
         EntityType entityType = getEntityType(entryType);
         String value = Optional.ofNullable(importedRedaction.getValue())
                 .orElse("");
-        return PrecursorEntity.builder()
+
+        PrecursorEntityBuilder precursorEntityBuilder = PrecursorEntity.builder()
                 .id(importedRedaction.getId())
                 .value(value)
                 .entityPosition(rectangleWithPages)
@@ -130,14 +131,21 @@ public class PrecursorEntity implements IEntity {
                                     .orElse(""))
                 .type(Optional.ofNullable(importedRedaction.getType())
                               .orElse(IMPORTED_REDACTION_TYPE))
-                .section(importedRedaction.getManualOverwriteSection())
+                .section(Optional.ofNullable(importedRedaction.getSection())
+                                 .orElse(""))
                 .entityType(entityType)
                 .isDictionaryEntry(false)
                 .isDossierDictionaryEntry(false)
+                .manualOverwrite(new ManualChangeOverwrite(entityType))
                 .rectangle(value.isBlank() || entryType.equals(EntryType.IMAGE) || entryType.equals(EntryType.IMAGE_HINT) || entryType.equals(EntryType.AREA))
-                .manualOverwrite(new ManualChangeOverwrite(entityType, importedRedaction.getManualOverwriteSection()))
-                .engines(Set.of(Engine.IMPORTED))
-                .build();
+                .engines(Set.of(Engine.IMPORTED));
+
+        if (importedRedaction.getManualOverwriteSection() != null && !importedRedaction.getManualOverwriteSection().isEmpty()) {
+            precursorEntityBuilder.section(importedRedaction.getManualOverwriteSection())
+                    .manualOverwrite(new ManualChangeOverwrite(entityType, importedRedaction.getManualOverwriteSection()));
+        }
+
+        return precursorEntityBuilder.build();
     }
 
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/component/Entity.java

@@ -83,7 +83,7 @@ public class Entity {
                 .section(e.getSection())
                 .color(e.getColor())
                 .positions(e.getPositions())
-                .containingNode(document.getDocumentTree().getEntryById(e.getContainingNodeId()).getNode())
+//                .containingNode(document.getDocumentTree().getEntryById(e.getContainingNodeId()).getNode())
                 .textBefore(e.getTextBefore())
                 .textAfter(e.getTextAfter())
                 .startOffset(startOffset)

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/DocumentTree.java

@@ -288,8 +288,8 @@ public class DocumentTree {
         if (treeId.isEmpty()) {
             return root;
         }
-        Entry entry = root.children.get(treeId.get(0));
-        for (int id : treeId.subList(1, treeId.size())) {
+        Entry entry = root;
+        for (int id : treeId) {
             entry = entry.children.get(id);
         }
         return entry;

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Section.java

@@ -1,10 +1,12 @@
 package com.iqser.red.service.redaction.v1.server.model.document.nodes;
 
 import java.awt.geom.Rectangle2D;
+import java.util.ArrayList;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.stream.Collectors;
 
 import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.DocumentTree;
@@ -33,7 +35,6 @@ import lombok.extern.slf4j.Slf4j;
 @EqualsAndHashCode(onlyExplicitlyIncluded = true, callSuper = true)
 public class Section extends AbstractSemanticNode {
 
-
     @Override
     public NodeType getType() {
 
@@ -60,7 +61,6 @@ public class Section extends AbstractSemanticNode {
     }
 
 
-
     @Override
     public String toString() {
 
@@ -85,7 +85,14 @@ public class Section extends AbstractSemanticNode {
      */
     public boolean anyHeadlineContainsString(String value) {
 
-        return streamAllSubNodesOfType(NodeType.HEADLINE).anyMatch(h -> h.containsString(value));
+        boolean found = streamAllSubNodesOfType(NodeType.HEADLINE).anyMatch(h -> h.containsString(value)) || getHeadline().containsString(value);
+        if (!found) {
+            List<Headline> previousHeadlines = new ArrayList<>();
+            headlinesByPreviousSibling(this, previousHeadlines);
+            return previousHeadlines.stream()
+                    .anyMatch(headline -> headline.containsString(value));
+        }
+        return true;
     }
 
 
@@ -97,8 +104,37 @@ public class Section extends AbstractSemanticNode {
      */
     public boolean anyHeadlineContainsStringIgnoreCase(String value) {
 
-        return streamAllSubNodesOfType(NodeType.HEADLINE).anyMatch(h -> h.containsStringIgnoreCase(value));
+        boolean found = streamAllSubNodesOfType(NodeType.HEADLINE).anyMatch(h -> h.containsStringIgnoreCase(value)) || getHeadline().containsStringIgnoreCase(value);
+        if (!found) {
+            List<Headline> previousHeadlines = new ArrayList<>();
+            headlinesByPreviousSibling(this, previousHeadlines);
+            return previousHeadlines.stream()
+                    .anyMatch(headline -> headline.containsStringIgnoreCase(value));
+        }
+        return true;
     }
 
 
+    private void headlinesByPreviousSibling(SemanticNode section, List<Headline> found) {
+
+        if (section.getPreviousSibling()
+                    .isPresent() && section.getPreviousSibling()
+                    .get() instanceof Section previousSection) {
+
+            var subnodes = previousSection.streamAllSubNodes()
+                    .toList();
+            if (subnodes.size() == 1 && subnodes.get(0) instanceof Headline previousHeadline) {
+                found.add(previousHeadline);
+                headlinesByPreviousSibling(previousSection, found);
+            }
+        }
+
+        if (section.getPreviousSibling()
+                    .isPresent() && section.getPreviousSibling()
+                    .get() instanceof Headline previousHeadline) {
+            found.add(previousHeadline);
+            headlinesByPreviousSibling(previousHeadline, found);
+        }
+    }
+
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/queue/RedactionMessageReceiver.java

@@ -110,7 +110,14 @@ public class RedactionMessageReceiver {
                     log.info("-------------------------------------------------------------------------------------------------");
                     shouldRespond = false;
                     break;
-
+                case IMPORTED_REDACTIONS_ONLY:
+                    log.info("------------------------------Imported Redactions Analysis Only------------------------------------------");
+                    log.info("Starting Imported Redactions Analysis Only for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
+                    log.debug(analyzeRequest.getManualRedactions().toString());
+                    result = analyzeService.analyzeImportedRedactionsOnly(analyzeRequest);
+                    log.info("Successful Imported Redactions Analysis Only dossier {} file {}", analyzeRequest.getDossierId(), analyzeRequest.getFileId());
+                    log.info("-------------------------------------------------------------------------------------------------");
+                    break;
                 default:
                     throw new IllegalArgumentException("Unknown MessageType: " + analyzeRequest.getMessageType());
             }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalyzeService.java

@@ -23,8 +23,10 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileTyp
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentLog;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogChanges;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedLegalBases;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedactions;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.mapper.ImportedLegalBasisMapper;
 import com.iqser.red.service.redaction.v1.server.RedactionServiceSettings;
 import com.iqser.red.service.redaction.v1.server.client.model.NerEntitiesModel;
 import com.iqser.red.service.redaction.v1.server.model.KieWrapper;
@@ -76,6 +78,7 @@ public class AnalyzeService {
     ImportedRedactionEntryService importedRedactionEntryService;
     ObservedStorageService observedStorageService;
     FunctionTimerValues redactmanagerAnalyzePagewiseValues;
+    ImportedLegalBasisMapper importedLegalBasisMapper = ImportedLegalBasisMapper.INSTANCE;
 
 
     @Timed("redactmanager_reanalyze")
@@ -128,7 +131,7 @@ public class AnalyzeService {
                                     document,
                                     document.getNumberOfPages(),
                                     true,
-                                    Collections.emptySet());
+                                    new HashSet<>());
         }
 
         KieWrapper kieWrapperEntityRules = kieContainerCreationService.getLatestKieContainer(analyzeRequest.getDossierTemplateId(), RuleFileType.ENTITY);
@@ -245,6 +248,39 @@ public class AnalyzeService {
     }
 
 
+    @Timed("redactmanager_analyzeImportedRedactionsOnly")
+    @Observed(name = "AnalyzeService", contextualName = "analyzeImportedRedactionsOnly")
+    public AnalyzeResult analyzeImportedRedactionsOnly(AnalyzeRequest analyzeRequest) {
+
+        long startTime = System.currentTimeMillis();
+
+        dictionaryService.updateDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId());
+
+        Document document = DocumentGraphMapper.toDocumentGraph(observedStorageService.getDocumentData(analyzeRequest.getDossierId(), analyzeRequest.getFileId()));
+        log.info("Loaded Document Graph for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
+
+        ImportedRedactions importedRedactions = redactionStorageService.getImportedRedactions(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
+        log.info("Loaded Imported Redactions for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
+
+        ImportedLegalBases importedLegalBases = redactionStorageService.getImportedLegalBases(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
+        log.info("Loaded Imported Legal Bases for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
+
+        var notFoundImportedEntries = importedRedactionEntryService.addImportedEntriesAndReturnNotFoundEntries(analyzeRequest, importedRedactions, document);
+
+        EntityLogChanges entityLogChanges = entityLogCreatorService.createInitialEntityLog(analyzeRequest, document, notFoundImportedEntries, new DictionaryVersion(0, 0), 0);
+
+        entityLogChanges.getEntityLog()
+                .setLegalBasis(importedLegalBases.getImportedLegalBases()
+                                       .stream()
+                                       .map(importedLegalBasisMapper::toEntityLogLegalBasis)
+                                       .toList());
+
+        notFoundImportedEntitiesService.processEntityLog(entityLogChanges.getEntityLog(), analyzeRequest, notFoundImportedEntries);
+
+        return finalizeAnalysis(analyzeRequest, startTime, KieWrapper.empty(), entityLogChanges, document, document.getNumberOfPages(), false, new HashSet<>());
+    }
+
+
     private AnalyzeResult finalizeAnalysis(AnalyzeRequest analyzeRequest,
                                            long startTime,
                                            KieWrapper kieWrapperComponentRules,
@@ -367,7 +403,7 @@ public class AnalyzeService {
 
         return new NerEntitiesModel(nerEntitiesModel.getData().entrySet()
                                             .stream() //
-                                            .filter(entry -> sectionsToReanalyseIds.contains(entry.getKey())) //
+                                            .filter(entry -> sectionsToReanalyseIds.contains(getSuperSectionID(entry.getKey()))) //
                                             .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)));
     }
 
@@ -383,4 +419,11 @@ public class AnalyzeService {
         return nerEntities;
     }
 
+
+    private static Integer getSuperSectionID(String section) {
+
+        return NerEntitiesAdapter.sectionNumberToTreeId(section)
+                .get(0);
+    }
+
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java

@@ -223,7 +223,6 @@ public class EntityLogCreatorService {
 
         String type = precursorEntity.getManualOverwrite().getType()
                 .orElse(precursorEntity.getType());
-        boolean isHint = isHint(precursorEntity.getEntityType());
         return EntityLogEntry.builder()
                 .id(precursorEntity.getId())
                 .reason(precursorEntity.buildReasonWithManualChangeDescriptions())

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/components/ComponentMappingFileSystemCache.java

@@ -40,7 +40,8 @@ public class ComponentMappingFileSystemCache {
     public File getComponentMappingFile(ComponentMappingMetadata metadata) {
 
         Path mappingFile = getMappingFileFromMetadata(metadata);
-        Path mappingFileMetaDataFile = mappingFile.resolveSibling(metadata.getName() + METADATA_SUFFIX);
+        Path mappingFileMetaDataFile = getMappingMetadataFileFromMetadata(metadata);
+
         synchronized (ComponentMappingFileSystemCache.class) {
 
             if (fileExistsAndUpToDate(metadata, mappingFile, mappingFileMetaDataFile)) {
@@ -87,6 +88,13 @@ public class ComponentMappingFileSystemCache {
     }
 
 
+    private Path getMappingMetadataFileFromMetadata(ComponentMappingMetadata metadata) {
+
+        Path tenantStem = mappingFileDir.resolve(TenantContext.getTenantId());
+        return tenantStem.resolve(metadata.getStorageId() + METADATA_SUFFIX);
+    }
+
+
     private boolean fileExistsAndUpToDate(ComponentMappingMetadata metadata, Path mappingFile, Path mappingFileMetaDataFile) {
 
         if (mappingFile.toFile().exists() && mappingFile.toFile().isFile() && mappingFileMetaDataFile.toFile().exists() && mappingFileMetaDataFile.toFile().isFile()) {

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/components/ComponentMappingMemoryCache.java

@@ -4,6 +4,7 @@ import java.io.File;
 import java.io.FileReader;
 import java.io.Reader;
 import java.nio.charset.Charset;
+import java.time.Duration;
 import java.util.List;
 
 import org.springframework.stereotype.Service;
@@ -36,7 +37,7 @@ public class ComponentMappingMemoryCache {
     public ComponentMappingMemoryCache(ComponentMappingFileSystemCache componentMappingFileSystemCache) {
 
         this.fileSystemCache = componentMappingFileSystemCache;
-        cache = CacheBuilder.newBuilder().maximumWeight(MAX_NUMBER_OF_LINES).weigher(COMPONENT_MAPPING_WEIGHER).build();
+        cache = CacheBuilder.newBuilder().maximumWeight(MAX_NUMBER_OF_LINES).weigher(COMPONENT_MAPPING_WEIGHER).expireAfterAccess(Duration.ofDays(1)).build();
     }
 
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityCreationService.java

@@ -614,21 +614,21 @@ public class EntityCreationService {
     /**
      * Looks across the remaining table row to the right of the provided TableCell if any line intersects the y coordinates of the found text.
      *
-     * @param TextRanges a list of textRanges
+     * @param textRanges a list of textRanges
      * @param tableCell  the table cell
      * @param type       the type
      * @param entityType the entity type
      * @param tableNode  the table node
      * @return a stream of RedactionEntities
      */
-    private Stream<TextEntity> lineAfterBoundariesAcrossColumns(List<TextRange> TextRanges, TableCell tableCell, String type, EntityType entityType, Table tableNode) {
+    private Stream<TextEntity> lineAfterBoundariesAcrossColumns(List<TextRange> textRanges, TableCell tableCell, String type, EntityType entityType, Table tableNode) {
 
-        return TextRanges.stream()
+        return textRanges.stream()
                 .map(boundary -> RectangleTransformations.rectangle2DBBox(tableCell.getTextBlock().getPositions(boundary)))
                 .map(bBox -> Pair.of(bBox.getMaxY(), bBox.getMinY()))
                 .map(maxMinPair -> tableNode.streamRow(tableCell.getRow())
                         .filter(nextTableCell -> nextTableCell.getCol() > tableCell.getCol())
-                        .map(nextTableCell -> RedactionSearchUtility.findTextRangesOfAllLinesInYRange(maxMinPair.getLeft(), maxMinPair.getRight(), nextTableCell.getTextBlock()))
+                        .map(nextTableCell -> RedactionSearchUtility.findTextRangesOfAllLinesWithCloseYCoordinates(maxMinPair.getLeft(), maxMinPair.getRight(), nextTableCell.getTextBlock()))
                         .map(b -> b.trim(tableNode.getTextBlock()))
                         .filter(boundary -> isValidEntityTextRange(tableNode.getTextBlock(), boundary))
                         .map(boundary -> byTextRange(boundary, type, entityType, tableNode))

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityFromPrecursorCreationService.java

@@ -11,10 +11,12 @@ import java.util.stream.Collectors;
 
 import org.springframework.stereotype.Service;
 
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedactions;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.BaseAnnotation;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;
+import com.iqser.red.service.redaction.v1.server.RedactionServiceSettings;
 import com.iqser.red.service.redaction.v1.server.model.PrecursorEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
@@ -35,6 +37,7 @@ public class EntityFromPrecursorCreationService {
     static double MATCH_THRESHOLD = 10; // Is compared to the average sum of distances in pdf coordinates for each corner of the bounding box of the entities
     EntityFindingUtility entityFindingUtility;
     DictionaryService dictionaryService;
+    RedactionServiceSettings settings;
 
 
     public List<PrecursorEntity> createEntitiesIfFoundAndReturnNotFoundEntries(ManualRedactions manualRedactions, SemanticNode node, String dossierTemplateId) {
@@ -92,7 +95,7 @@ public class EntityFromPrecursorCreationService {
                 notFoundEntities.add(precursorEntity);
                 continue;
             }
-            createCorrectEntity(precursorEntity, optionalClosestEntity.get());
+            createCorrectEntity(precursorEntity, optionalClosestEntity.get(), settings.isAnnotationMode());
         }
 
         tempEntitiesByValue.values()
@@ -125,12 +128,16 @@ public class EntityFromPrecursorCreationService {
                                                          precursorEntity.getEntityType(),
                                                          closestEntity.getDeepestFullyContainingNode());
         } else {
-            correctEntity = TextEntity.initialEntityNode(closestEntity.getTextRange(),
-                                                         precursorEntity.type(),
-                                                         precursorEntity.getEntityType(),
-                                                         precursorEntity.getId(),
-                                                         precursorEntity.getManualOverwrite().getSection()
-                                                                 .orElse(null));
+            String section = precursorEntity.getManualOverwrite().getSection()
+                    .orElse(null);
+            if ((section == null || section.isBlank())
+                && precursorEntity.getSection() != null
+                && !precursorEntity.getSection().isBlank()
+                && precursorEntity.getEngines().contains(Engine.IMPORTED)) {
+                section = precursorEntity.getSection();
+            }
+
+            correctEntity = TextEntity.initialEntityNode(closestEntity.getTextRange(), precursorEntity.type(), precursorEntity.getEntityType(), precursorEntity.getId(), section);
         }
         correctEntity.setDeepestFullyContainingNode(closestEntity.getDeepestFullyContainingNode());
         correctEntity.setIntersectingNodes(new ArrayList<>(closestEntity.getIntersectingNodes()));

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/NerEntitiesAdapter.java

@@ -177,7 +177,7 @@ public class NerEntitiesAdapter {
     }
 
 
-    private static List<Integer> sectionNumberToTreeId(String sectionNumber) {
+    public static List<Integer> sectionNumberToTreeId(String sectionNumber) {
 
         return Arrays.stream(sectionNumber.split("\\."))
                 .map(Integer::parseInt)

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/ComponentDroolsExecutionService.java

@@ -73,9 +73,10 @@ public class ComponentDroolsExecutionService {
                     entities.add(Entity.fromEntityLogEntry(entry, document, entry.getStartOffset(), entry.getEndOffset()));
                     if (entry.getDuplicatedTextRanges() != null && !entry.getDuplicatedTextRanges().isEmpty()) {
                         entry.getDuplicatedTextRanges()
-                                .forEach(duplicatedTextRange -> {
-                                    entities.add(Entity.fromEntityLogEntry(entry, document, duplicatedTextRange.getStart(), duplicatedTextRange.getEnd()));
-                                });
+                                .forEach(duplicatedTextRange -> entities.add(Entity.fromEntityLogEntry(entry,
+                                                                                                       document,
+                                                                                                       duplicatedTextRange.getStart(),
+                                                                                                       duplicatedTextRange.getEnd())));
                     }
                     return entities.stream();
                 })
@@ -94,8 +95,7 @@ public class ComponentDroolsExecutionService {
         });
 
         try {
-            completableFuture.orTimeout(settings.getDroolsExecutionTimeoutSecs(document.getNumberOfPages()), TimeUnit.SECONDS)
-                    .get();
+            completableFuture.get(settings.getDroolsExecutionTimeoutSecs(document.getNumberOfPages()), TimeUnit.SECONDS);
         } catch (ExecutionException e) {
             kieSession.dispose();
             if (e.getCause() instanceof TimeoutException) {
@@ -105,6 +105,8 @@ public class ComponentDroolsExecutionService {
         } catch (InterruptedException e) {
             kieSession.dispose();
             throw new RuntimeException(e);
+        } catch (TimeoutException e) {
+            throw new DroolsTimeoutException(e, false, RuleFileType.COMPONENT);
         }
 
         List<FileAttribute> resultingFileAttributes = getFileAttributes(kieSession);

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/DroolsValidationService.java

@@ -45,7 +45,7 @@ public class DroolsValidationService {
     private final KieContainerCreationService kieContainerCreationService;
     private final DeprecatedElementsFinder deprecatedElementsFinder;
     private static final Pattern allowedImportsPattern = Pattern.compile(
-            "^(?:import\\s+static\\s+)?(?:import\\s+)?(?:com\\.knecon\\.fforesight|com\\.iqser\\.red|java\\.util)\\..*;$");
+            "^(?:import\\s+static\\s+)?(?:import\\s+)?(?:com\\.knecon\\.fforesight|com\\.iqser\\.red|java\\.util|java\\.text)\\..*;$");
     public static final String LINEBREAK_MATCHER = "\\R";
 
 
@@ -283,7 +283,9 @@ public class DroolsValidationService {
 
     private DroolsBlacklistErrorMessage checkAndGetBlackListedMessages(SearchImplementation blacklistedKeywordSearchImplementation, String stringToCheck, int lineIndexStart) {
 
-        String sanitizedRuleText = StringUtils.deleteWhitespace(stringToCheck);
+        String nonWhitespaceRuleText = StringUtils.deleteWhitespace(stringToCheck);
+        String sanitizedRuleText= nonWhitespaceRuleText.replaceAll("\"(\\\\.|[^\"\\\\])*\"|'(\\\\.|[^'\\\\])*'" ,"");
+
         List<SearchImplementation.MatchPosition> matches = blacklistedKeywordSearchImplementation.getMatches(sanitizedRuleText);
 
         if (!matches.isEmpty()) {

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/EntityDroolsExecutionService.java

@@ -129,8 +129,7 @@ public class EntityDroolsExecutionService {
         });
 
         try {
-            completableFuture.orTimeout(settings.getDroolsExecutionTimeoutSecs(document.getNumberOfPages()), TimeUnit.SECONDS)
-                    .get();
+            completableFuture.get(settings.getDroolsExecutionTimeoutSecs(document.getNumberOfPages()), TimeUnit.SECONDS);
         } catch (ExecutionException e) {
             kieSession.dispose();
             if (e.getCause() instanceof TimeoutException) {
@@ -140,6 +139,8 @@ public class EntityDroolsExecutionService {
         } catch (InterruptedException e) {
             kieSession.dispose();
             throw new RuntimeException(e);
+        } catch (TimeoutException e) {
+            throw new DroolsTimeoutException(e, false, RuleFileType.ENTITY);
         }
 
         List<FileAttribute> resultingFileAttributes = getFileAttributes(kieSession);

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/RuleFileParser.java

@@ -32,7 +32,7 @@ import lombok.experimental.UtilityClass;
 public class RuleFileParser {
 
     private final static Pattern ruleIdentifierInCodeFinder = Pattern.compile(
-            "\\b(?:redact|apply|skip|remove|ignore|applyWithLineBreaks|applyWithReferences|skipWithReferences)\\s*\\(\"([a-zA-Z0-9]+.\\d+.\\d+)\",.*(?:, .*)?\\)");
+            "\\b(?:redact|apply|skip|remove|ignore|applyWithLineBreaks|applyWithReferences|skipWithReferences)\\s*\\(\\s*\"([a-zA-Z0-9]+.\\d+.\\d+)\"\\s*,\\s*.*(?:\\s*,\\s*.*)\\s*?\\)");
 
 
     @SneakyThrows
@@ -78,7 +78,8 @@ public class RuleFileParser {
                                              .map(GlobalDescr::getLine)
                                              .orElse(0),
                                      allQueries,
-                                     ruleClasses, customDroolsValidation);
+                                     ruleClasses,
+                                     customDroolsValidation);
     }
 
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/RedactionStorageService.java

@@ -9,10 +9,12 @@ import java.util.Set;
 import java.util.stream.Collectors;
 
 import org.springframework.cache.annotation.Cacheable;
+import org.springframework.context.annotation.Import;
 import org.springframework.stereotype.Service;
 
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.componentlog.ComponentLog;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedLegalBases;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedactions;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.imported.ImportedRedactionsPerPage;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
@@ -147,6 +149,21 @@ public class RedactionStorageService {
     }
 
 
+
+    @Timed("redactmanager_getImportedLegalBases")
+    public ImportedLegalBases getImportedLegalBases(String dossierId, String fileId) {
+
+        try {
+            return storageService.readJSONObject(TenantContext.getTenantId(),
+                                                                             StorageIdUtils.getStorageId(dossierId, fileId, FileType.IMPORTED_LEGAL_BASES),
+                                                                             ImportedLegalBases.class);
+        } catch (StorageObjectDoesNotExist e) {
+            log.debug("Imported legal bases not available.");
+            return new ImportedLegalBases();
+        }
+    }
+
+
     @Deprecated(forRemoval = true)
     @Timed("redactmanager_getRedactionLog")
     public RedactionLog getRedactionLog(String dossierId, String fileId) {

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/utils/RedactionSearchUtility.java

@@ -3,8 +3,10 @@ package com.iqser.red.service.redaction.v1.server.utils;
 import static java.lang.String.format;
 
 import java.awt.geom.Rectangle2D;
+import java.awt.geom.RectangularShape;
 import java.util.LinkedList;
 import java.util.List;
+import java.util.function.Predicate;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 import java.util.stream.IntStream;
@@ -154,14 +156,8 @@ public class RedactionSearchUtility {
      */
     public static TextRange findTextRangesOfAllLinesInYRange(double maxY, double minY, TextBlock textBlock) {
 
-        List<TextRange> lineBoundaries = IntStream.range(0, textBlock.numberOfLines()).boxed()
-                .map(textBlock::getLineTextRange)
-                .filter(lineBoundary -> isWithinYRange(maxY, minY, textBlock, lineBoundary))
-                .toList();
-        if (lineBoundaries.isEmpty()) {
-            return new TextRange(textBlock.getTextRange().start(), textBlock.getTextRange().start());
-        }
-        return TextRange.merge(lineBoundaries);
+        Predicate<TextRange> isWithinYRange = lineBoundary -> isWithinYRange(maxY, minY, textBlock, lineBoundary);
+        return filterLineBoundaries(textBlock, isWithinYRange);
     }
 
 
@@ -172,6 +168,49 @@ public class RedactionSearchUtility {
     }
 
 
+    /**
+     * Identifies all lines within a text block that have roughly the same vertical coordinates.
+     *
+     * @param maxY      The maximum Y-coordinate of the vertical range.
+     * @param minY      The minimum Y-coordinate of the vertical range.
+     * @param textBlock The text block containing the lines to be checked.
+     * @return A {@link TextRange} encompassing all lines within the specified Y-coordinate range.
+     */
+    public static TextRange findTextRangesOfAllLinesWithCloseYCoordinates(Double maxY, Double minY, TextBlock textBlock) {
+
+        double averageLineHeight = IntStream.range(0, textBlock.numberOfLines()).boxed()
+                .map(textBlock::getLineTextRange)
+                .flatMap((TextRange stringTextRange) -> textBlock.getPositions(stringTextRange)
+                        .stream())
+                .map(RectangularShape::getHeight)
+                .mapToDouble(Double::doubleValue).average()
+                .orElse(0);
+        Predicate<TextRange> hasCloseYRange = lineBoundary -> areYCoordinatesClose(maxY, minY, textBlock, lineBoundary, averageLineHeight);
+
+        return filterLineBoundaries(textBlock, hasCloseYRange);
+    }
+
+
+    private static boolean areYCoordinatesClose(Double maxY, Double minY, TextBlock textBlock, TextRange lineTextRange, double averageLineHeight) {
+
+        Rectangle2D lineBBox = RectangleTransformations.rectangle2DBBox(textBlock.getPositions(lineTextRange));
+        return Math.abs(lineBBox.getMinY() - minY) <= averageLineHeight && Math.abs(maxY - lineBBox.getMaxY()) <= averageLineHeight;
+    }
+
+
+    private static TextRange filterLineBoundaries(TextBlock textBlock, Predicate<TextRange> textRangePredicate) {
+
+        List<TextRange> lineBoundaries = IntStream.range(0, textBlock.numberOfLines()).boxed()
+                .map(textBlock::getLineTextRange)
+                .filter(textRangePredicate)
+                .toList();
+        if (lineBoundaries.isEmpty()) {
+            return new TextRange(textBlock.getTextRange().start(), textBlock.getTextRange().start());
+        }
+        return TextRange.merge(lineBoundaries);
+    }
+
+
     /**
      * Finds TextRanges matching a regex pattern within a TextBlock.
      *

redaction-service-v1/redaction-service-server-v1/src/main/resources/application.yml

@@ -16,6 +16,9 @@ project.version: 1.0-SNAPSHOT
 server:
   port: 8080
 
+lifecycle:
+  base-package: com.iqser.red.service.redaction
+
 spring:
   application:
     name: redaction-service

redaction-service-v1/redaction-service-server-v1/src/main/resources/date_formats.txt

@@ -14,6 +14,8 @@ dd MMMM yyyy
 d MMMM yyyy
 MMMM dd, yyyy
 MMMM d, yyyy
+MMMM, d yyyy
+MMMM d,yyyy
 dd.MM.yyyy
 d.MM.yyyy
 yyyy/MM/dd
@@ -28,6 +30,7 @@ dd['.'] MMM yyyy
 d['.'] MMM yyyy
 dd['th']['st']['nd']['rd'] 'of' MMMM, yyyy
 d['th']['st']['nd']['rd'] 'of' MMMM, yyyy
+d['st']['nd']['rd']['th'] MMMM yyyy
 MMMM dd['th']['st']['nd']['rd'], yyyy
 MMMM d['th']['st']['nd']['rd'], yyyy
 yyyy, MMMM dd
@@ -72,3 +75,5 @@ dd.MM.yy
 d.MM.yy
 dd MMM. yyyy
 d MMM. yyyy
+d-MMMM-yyyy
+dd-MMMM-yyyy

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AbstractRedactionIntegrationTest.java

@@ -114,13 +114,14 @@ public abstract class AbstractRedactionIntegrationTest {
     public static final String PII_TYPE_ID = DICTIONARY_PII + ":" + TEST_DOSSIER_TEMPLATE_ID;
     public static final String TEST_METHOD_TYPE_ID = TEST_METHOD_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID;
     public static final String PUBLISHED_INFORMATION_TYPE_ID = PUBLISHED_INFORMATION_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID;
-    public static final String DOSSIER_PUBLISHED_INFORMATION_TYPE_ID = PUBLISHED_INFORMATION_TYPE_ID+ ":" + TEST_DOSSIER_ID;
+    public static final String DOSSIER_PUBLISHED_INFORMATION_TYPE_ID = PUBLISHED_INFORMATION_TYPE_ID + ":" + TEST_DOSSIER_ID;
     public static final String MUST_REDACT_TYPE_ID = MUST_REDACT_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID;
     public static final String HINT_ONLY_TYPE_ID = HINT_ONLY_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID;
     public static final String REDACTION_TYPE_ID = REDACTION_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID;
     public static final String NO_REDACTION_TYPE_ID = NO_REDACTION_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID;
     public static final String SPONSOR_TYPE_ID = DICTIONARY_SPONSOR + ":" + TEST_DOSSIER_TEMPLATE_ID;
     public static final String AUTHOR_TYPE_ID = DICTIONARY_AUTHOR + ":" + TEST_DOSSIER_TEMPLATE_ID;
+    public static final String DOSSIER_AUTHOR_TYPE_ID = AUTHOR_TYPE_ID + ":" + TEST_DOSSIER_ID;
     public static final String ADDRESS_TYPE_ID = DICTIONARY_ADDRESS + ":" + TEST_DOSSIER_TEMPLATE_ID;
     public static final String VERTEBRATE_TYPE_ID = VERTEBRATE_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID;
 
@@ -250,8 +251,10 @@ public abstract class AbstractRedactionIntegrationTest {
                                                                                                                                                  true));
         when(dictionaryClient.getDictionaryForType(IMPORTED_REDACTION_TYPE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(IMPORTED_REDACTION_INDICATOR,
                                                                                                                                                  true));
-        when(dictionaryClient.getDictionaryForType(DOSSIER_PUBLISHED_INFORMATION_TYPE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(PUBLISHED_INFORMATION_INDICATOR,
+        when(dictionaryClient.getDictionaryForType(DOSSIER_PUBLISHED_INFORMATION_TYPE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(
+                PUBLISHED_INFORMATION_INDICATOR,
                 true));
+        when(dictionaryClient.getDictionaryForType(DOSSIER_AUTHOR_TYPE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(DICTIONARY_AUTHOR, true));
 
     }
 
@@ -350,6 +353,7 @@ public abstract class AbstractRedactionIntegrationTest {
                                 .collect(Collectors.toSet()));
         dossierDictionary.put(IMPORTED_REDACTION_INDICATOR, new ArrayList<>());
         dossierDictionary.put(PUBLISHED_INFORMATION_INDICATOR, new ArrayList<>());
+        dossierDictionary.put(DICTIONARY_AUTHOR, new ArrayList<>());
 
         falsePositive.computeIfAbsent(DICTIONARY_PII, v -> new ArrayList<>())
                 .addAll(ResourceLoader.load("dictionaries/PII_false_positive.txt")

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DocumineFloraTest.java

@@ -174,6 +174,16 @@ public class DocumineFloraTest extends AbstractRedactionIntegrationTest {
         expectedDates.add("03/08/1992");
         expectedDates.add("13/08/1992");
         expectedDates.add("27/02/1992");
+        expectedDates.add("27/10/1989");
+        expectedDates.add("07/10/1989");
+        expectedDates.add("21/08/1998");
+        expectedDates.add("02/08/1998");
+        expectedDates.add("01/05/1988");
+        expectedDates.add("02/06/2003");
+        expectedDates.add("03/09/2005");
+        expectedDates.add("06/09/2005");
+        expectedDates.add("17/08/2005");
+        expectedDates.add("22/08/2035");
 
         String dates = experimentalDates.getComponentValues()
                 .get(0).getValue();

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionAcceptanceTest.java

@@ -10,6 +10,7 @@ import java.time.OffsetDateTime;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Set;
+import java.util.UUID;
 import java.util.stream.Stream;
 
 import org.junit.jupiter.api.BeforeEach;
@@ -25,24 +26,30 @@ import org.springframework.context.annotation.Configuration;
 import org.springframework.context.annotation.FilterType;
 import org.springframework.context.annotation.Import;
 import org.springframework.context.annotation.Primary;
+import org.springframework.core.io.ClassPathResource;
 import org.springframework.test.context.junit.jupiter.SpringExtension;
 
 import com.iqser.red.commons.jackson.ObjectMapperFactory;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeResult;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryType;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type;
 import com.iqser.red.service.redaction.v1.server.annotate.AnnotateRequest;
 import com.iqser.red.service.redaction.v1.server.annotate.AnnotateResponse;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.OsUtils;
 import com.iqser.red.service.redaction.v1.server.service.DictionaryService;
+import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
 import com.iqser.red.storage.commons.StorageAutoConfiguration;
 import com.iqser.red.storage.commons.service.StorageService;
 import com.iqser.red.storage.commons.utils.FileSystemBackedStorageService;
@@ -50,6 +57,8 @@ import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsi
 import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingServiceProcessorConfiguration;
 import com.knecon.fforesight.tenantcommons.TenantContext;
 
+import lombok.SneakyThrows;
+
 @ExtendWith(SpringExtension.class)
 @SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
 @Import(RedactionIntegrationTest.RedactionIntegrationTestConfiguration.class)
@@ -103,6 +112,17 @@ public class RedactionAcceptanceTest extends AbstractRedactionIntegrationTest {
                                                                                                        .isRecommendation(recommendationTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
                                                                                                        .rank(rankTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
                                                                                                        .build(),
+                                                                                               Type.builder()
+                                                                                                       .id(DOSSIER_AUTHOR_TYPE_ID)
+                                                                                                       .type(DICTIONARY_AUTHOR)
+                                                                                                       .dossierTemplateId(TEST_DOSSIER_TEMPLATE_ID)
+                                                                                                       .dossierId(TEST_DOSSIER_ID)
+                                                                                                       .hexColor("#ffe184")
+                                                                                                       .isHint(hintTypeMap.get(DICTIONARY_AUTHOR))
+                                                                                                       .isCaseInsensitive(caseInSensitiveMap.get(DICTIONARY_AUTHOR))
+                                                                                                       .isRecommendation(recommendationTypeMap.get(DICTIONARY_AUTHOR))
+                                                                                                       .rank(rankTypeMap.get(DICTIONARY_AUTHOR))
+                                                                                                       .build(),
                                                                                                Type.builder()
                                                                                                        .id(DOSSIER_PUBLISHED_INFORMATION_TYPE_ID)
                                                                                                        .type(PUBLISHED_INFORMATION_INDICATOR)
@@ -158,10 +178,10 @@ public class RedactionAcceptanceTest extends AbstractRedactionIntegrationTest {
 
         var publishedInformationEntry1 = findEntityByTypeAndValue(entityLog, "published_information", "Oxford University Press").findFirst()
                 .orElseThrow();
-        assertThat(publishedInformationEntry1.getSection().startsWith("Paragraph:"));
+        assertThat(publishedInformationEntry1.getSection()).startsWith("Paragraph:");
         var asyaLyon1 = findEntityByTypeAndValueAndSectionNumber(entityLog, "CBI_author", "Asya Lyon", publishedInformationEntry1.getContainingNodeId()).findFirst()
                 .orElseThrow();
-        assertThat(asyaLyon1.getSection().startsWith("Paragraph:"));
+        assertThat(asyaLyon1.getSection()).startsWith("Paragraph:");
         assertEquals(EntryState.SKIPPED, asyaLyon1.getState());
 
         var idRemoval = buildIdRemoval(publishedInformationEntry1.getId());
@@ -212,10 +232,10 @@ public class RedactionAcceptanceTest extends AbstractRedactionIntegrationTest {
 
         var publishedInformationEntry1 = findEntityByTypeAndValue(entityLog, "published_information", "Oxford University Press").findFirst()
                 .orElseThrow();
-        assertThat(publishedInformationEntry1.getSection().startsWith("Paragraph:"));
+        assertThat(publishedInformationEntry1.getSection()).startsWith("Paragraph:");
         var asyaLyon1 = findEntityByTypeAndValueAndSectionNumber(entityLog, "CBI_author", "Asya Lyon", publishedInformationEntry1.getContainingNodeId()).findFirst()
                 .orElseThrow();
-        assertThat(publishedInformationEntry1.getSection().startsWith("Paragraph:"));
+        assertThat(publishedInformationEntry1.getSection()).startsWith("Paragraph:");
         assertEquals(EntryState.SKIPPED, asyaLyon1.getState());
 
         var idRemoval = buildIdRemoval(publishedInformationEntry1.getId());
@@ -244,6 +264,7 @@ public class RedactionAcceptanceTest extends AbstractRedactionIntegrationTest {
         assertThat(dictionary.get(PUBLISHED_INFORMATION_INDICATOR).contains("Press")).isFalse();
     }
 
+
     @Test
     public void testPublishedInformationRemovalAtDossierLevel() throws IOException {
 
@@ -365,6 +386,75 @@ public class RedactionAcceptanceTest extends AbstractRedactionIntegrationTest {
     }
 
 
+    @Test
+    @SneakyThrows
+    void testNerEntitiesAfterReanalysis() {
+
+        String EFSA_SANITISATION_RULES = loadFromClassPath("drools/efsa_sanitisation.drl");
+        when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.ENTITY)).thenReturn(JSONPrimitive.of(EFSA_SANITISATION_RULES));
+
+        ClassPathResource responseJson = new ClassPathResource("files/new/SYNGENTA_EFSA_sanitisation_GFL_v1 3.NER_ENTITIES.json");
+        storageService.storeObject(TenantContext.getTenantId(),
+                                   RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.NER_ENTITIES),
+                                   responseJson.getInputStream());
+
+        String pdfFile = "files/new/SYNGENTA_EFSA_sanitisation_GFL_v1 3.pdf";
+
+        AnalyzeRequest request = uploadFileToStorage(pdfFile);
+
+        analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request);
+        request.setAnalysisNumber(1);
+        dossierDictionary.put(DICTIONARY_AUTHOR, new ArrayList<>());
+        mockDictionaryCalls(0L);
+
+        analyzeService.analyze(request);
+
+        var entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
+
+        String nerValue = "Osip S.";
+        var nerEntity = findEntityByTypeAndValue(entityLog, DICTIONARY_AUTHOR, nerValue).findFirst()
+                .orElseThrow();
+        assertThat(nerEntity.getEngines()).contains(Engine.NER);
+
+        String dictionaryAddValue = "cooperation";
+        ManualRedactionEntry manualRedactionEntry = ManualRedactionEntry.builder()
+                .value(dictionaryAddValue)
+                .type(DICTIONARY_AUTHOR)
+                .user("user")
+                .addToDossierDictionary(true)
+                .positions(List.of(Rectangle.builder().topLeftX(180.748f).topLeftY(546.564f).width(56.592f).height(15.408f).page(1).build()))
+                .type("dossier_redaction")
+                .fileId(TEST_FILE_ID)
+                .requestDate(OffsetDateTime.now())
+                .annotationId(UUID.randomUUID().toString())
+                .build();
+        request.setManualRedactions(ManualRedactions.builder().entriesToAdd(Set.of(manualRedactionEntry)).build());
+
+        request.setAnalysisNumber(2);
+        dossierDictionary.get(DICTIONARY_AUTHOR).add(dictionaryAddValue);
+        reanlysisVersions.put(dictionaryAddValue, 2L);
+        when(dictionaryClient.getVersionForDossier(TEST_DOSSIER_ID)).thenReturn(2L);
+        mockDictionaryCalls(1L);
+
+        AnalyzeResult reanalyzeResult = analyzeService.reanalyze(request);
+
+        entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
+
+        EntityLogEntry entityLogEntryAdded = entityLog.getEntityLogEntry()
+                .stream()
+                .filter(entityLogEntry -> entityLogEntry.getValue().equals(dictionaryAddValue))
+                .findFirst()
+                .get();
+        assertEquals(EntryState.APPLIED, entityLogEntryAdded.getState());
+
+        nerEntity = findEntityByTypeAndValue(entityLog, DICTIONARY_AUTHOR, nerValue).findFirst()
+                .orElseThrow();
+        assertThat(nerEntity.getEngines()).contains(Engine.NER);
+        dossierDictionary.get(DICTIONARY_AUTHOR).remove(dictionaryAddValue);
+
+    }
+
+
     private static IdRemoval buildIdRemoval(String id) {
 
         return IdRemoval.builder().annotationId(id).user("user").requestDate(OffsetDateTime.now()).fileId(TEST_FILE_ID).build();

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/date/DateConverterTest.java

@@ -43,7 +43,17 @@ public class DateConverterTest {
                                                          "28 March 2018 (animal 1 - 5000 mg/kg bw)",
                                                          "28 March 2018 (animal1 - 5000 mg/kg bw)",
                                                          "28 August 2018 (animal 1)",
-                                                         "31 August 2018 (animal 1)");
+                                                         "31 August 2018 (animal 1)",
+                                                         "October, 27 1989",
+                                                         "October, 7 1989",
+                                                         "August 21,1998",
+                                                         "August 2,1998",
+                                                         "1st May 1988",
+                                                         "2nd June 2003",
+                                                         "3rd September 2005",
+                                                         "6th September 2005",
+                                                         "17th August 2005",
+                                                         "22nd August 2035");
 
         for (String dateStr : goldenStandardDates) {
             Optional<Date> parsedDate = DateConverter.parseDate(dateStr);

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/drools/files/management/services/DroolsValidationServiceTest.java

@@ -361,7 +361,6 @@ class DroolsValidationServiceTest {
     }
 
 
-
     @Test
     @SneakyThrows
     void testRulesWithBlacklistedKeyword() {
@@ -408,11 +407,13 @@ class DroolsValidationServiceTest {
                 .forEach(System.out::println);
         assertFalse(droolsValidation.isCompiled());
         assertEquals(2, droolsValidation.getBlacklistErrorMessages().size());
-        assertEquals(1, droolsValidation.getBlacklistErrorMessages()
+        assertEquals(1,
+                     droolsValidation.getBlacklistErrorMessages()
                              .get(0).getBlacklistedKeywords().size());
         assertTrue(droolsValidation.getBlacklistErrorMessages()
                            .get(0).getBlacklistedKeywords().contains("TenantContext"));
-        assertEquals(2, droolsValidation.getBlacklistErrorMessages()
+        assertEquals(2,
+                     droolsValidation.getBlacklistErrorMessages()
                              .get(1).getBlacklistedKeywords().size());
         assertTrue(droolsValidation.getBlacklistErrorMessages()
                            .get(1).getBlacklistedKeywords().contains("TenantContext"));
@@ -429,14 +430,39 @@ class DroolsValidationServiceTest {
                 .forEach(System.out::println);
         assertFalse(droolsValidation2.isCompiled());
         assertEquals(2, droolsValidation2.getBlacklistErrorMessages().size());
-        assertEquals(1, droolsValidation2.getBlacklistErrorMessages()
+        assertEquals(1,
+                     droolsValidation2.getBlacklistErrorMessages()
                              .get(0).getBlacklistedKeywords().size());
         assertTrue(droolsValidation2.getBlacklistErrorMessages()
                            .get(0).getBlacklistedKeywords().contains("TenantContext"));
-        assertEquals(1, droolsValidation2.getBlacklistErrorMessages()
+        assertEquals(1,
+                     droolsValidation2.getBlacklistErrorMessages()
                              .get(1).getBlacklistedKeywords().size());
         assertTrue(droolsValidation2.getBlacklistErrorMessages()
                            .get(1).getBlacklistedKeywords().contains("System."));
     }
 
+
+    @Test
+    void assertRuleIdentifierDoesNotMatch() {
+
+        String ruleString = RuleManagementResources.getBaseRuleFileString() + """
+                rule "CBI.0.0: Redact CBI Authors (non vertebrate Study)"
+                    when
+                        not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
+                        $entity: TextEntity(type() == "CBI_author", dictionaryEntry)
+                    then
+                        $entity.redact(
+                        "CBI.1.0", 
+                        "Author found", 
+                        "Article 39(e)(3) of Regulation (EC) No 178/2002"
+                        );
+                    end
+                """;
+
+        RuleFileBluePrint ruleFileBluePrint = RuleFileParser.buildBluePrintFromRulesString(ruleString);
+
+        assertFalse(ruleFileBluePrint.getDroolsValidation().isCompiled());
+    }
+
 }