RED-9760: Fixed not found headline in anyheadlinecontains

RED-9770 - Extend date converter

See merge request redactmanager/redaction-service!467

redaction-service-v1/redaction-service-server-v1/build.gradle.kts

@@ -12,11 +12,11 @@ plugins {
 description = "redaction-service-server-v1"
 
 
-val layoutParserVersion = "0.141.0"
+val layoutParserVersion = "0.142.6"
 val jacksonVersion = "2.15.2"
 val droolsVersion = "9.44.0.Final"
 val pdfBoxVersion = "3.0.0"
-val persistenceServiceVersion = "2.444.0"
+val persistenceServiceVersion = "2.465.25"
 val springBootStarterVersion = "3.1.5"
 val springCloudVersion = "4.0.4"
 val testContainersVersion = "1.19.7"
@@ -43,6 +43,7 @@ dependencies {
     implementation("com.iqser.red.commons:storage-commons:2.45.0")
     implementation("com.knecon.fforesight:tenant-commons:0.24.0")
     implementation("com.knecon.fforesight:tracing-commons:0.5.0")
+    implementation("com.knecon.fforesight:lifecycle-commons:0.6.0")
 
     implementation("com.fasterxml.jackson.module:jackson-module-afterburner:${jacksonVersion}")
     implementation("com.fasterxml.jackson.datatype:jackson-datatype-jsr310:${jacksonVersion}")

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/Application.java

@@ -13,6 +13,7 @@ import org.springframework.boot.context.properties.EnableConfigurationProperties
 import org.springframework.cache.annotation.EnableCaching;
 import org.springframework.cloud.openfeign.EnableFeignClients;
 import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.EnableAspectJAutoProxy;
 import org.springframework.context.annotation.Import;
 import org.springframework.data.mongodb.repository.config.EnableMongoRepositories;
 
@@ -20,6 +21,7 @@ import com.iqser.red.service.dictionarymerge.commons.DictionaryMergeService;
 import com.iqser.red.service.persistence.service.v1.api.shared.mongo.SharedMongoAutoConfiguration;
 import com.iqser.red.service.redaction.v1.server.client.RulesClient;
 import com.iqser.red.storage.commons.StorageAutoConfiguration;
+import com.knecon.fforesight.lifecyclecommons.LifecycleAutoconfiguration;
 import com.knecon.fforesight.mongo.database.commons.MongoDatabaseCommonsAutoConfiguration;
 import com.knecon.fforesight.mongo.database.commons.liquibase.EnableMongoLiquibase;
 import com.knecon.fforesight.tenantcommons.MultiTenancyAutoConfiguration;
@@ -32,13 +34,14 @@ import lombok.extern.slf4j.Slf4j;
 
 @Slf4j
 @EnableCaching
-@ImportAutoConfiguration({MultiTenancyAutoConfiguration.class, SharedMongoAutoConfiguration.class})
+@ImportAutoConfiguration({MultiTenancyAutoConfiguration.class, SharedMongoAutoConfiguration.class, LifecycleAutoconfiguration.class})
 @Import({MetricsConfiguration.class, StorageAutoConfiguration.class, MongoDatabaseCommonsAutoConfiguration.class})
 @EnableFeignClients(basePackageClasses = RulesClient.class)
 @EnableConfigurationProperties(RedactionServiceSettings.class)
 @EnableMongoRepositories(basePackages = "com.iqser.red.service.persistence")
 @EnableMongoLiquibase
 @SpringBootApplication(exclude = {SecurityAutoConfiguration.class, ManagementWebSecurityAutoConfiguration.class, DataSourceAutoConfiguration.class, LiquibaseAutoConfiguration.class, MongoAutoConfiguration.class, MongoDataAutoConfiguration.class})
+@EnableAspectJAutoProxy
 public class Application {
 
     public static void main(String[] args) {

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Section.java

@@ -85,7 +85,7 @@ public class Section extends AbstractSemanticNode {
      */
     public boolean anyHeadlineContainsString(String value) {
 
-        return streamAllSubNodesOfType(NodeType.HEADLINE).anyMatch(h -> h.containsString(value));
+        return streamAllSubNodesOfType(NodeType.HEADLINE).anyMatch(h -> h.containsString(value)) || getHeadline().containsString(value);
     }
 
 
@@ -97,7 +97,7 @@ public class Section extends AbstractSemanticNode {
      */
     public boolean anyHeadlineContainsStringIgnoreCase(String value) {
 
-        return streamAllSubNodesOfType(NodeType.HEADLINE).anyMatch(h -> h.containsStringIgnoreCase(value));
+        return streamAllSubNodesOfType(NodeType.HEADLINE).anyMatch(h -> h.containsStringIgnoreCase(value)) || getHeadline().containsStringIgnoreCase(value);
     }
 
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalyzeService.java

@@ -367,7 +367,7 @@ public class AnalyzeService {
 
         return new NerEntitiesModel(nerEntitiesModel.getData().entrySet()
                                             .stream() //
-                                            .filter(entry -> sectionsToReanalyseIds.contains(entry.getKey())) //
+                                            .filter(entry -> sectionsToReanalyseIds.contains(getSuperSectionID(entry.getKey()))) //
                                             .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)));
     }
 
@@ -383,4 +383,11 @@ public class AnalyzeService {
         return nerEntities;
     }
 
+
+    private static Integer getSuperSectionID(String section) {
+
+        return NerEntitiesAdapter.sectionNumberToTreeId(section)
+                .get(0);
+    }
+
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java

@@ -223,7 +223,6 @@ public class EntityLogCreatorService {
 
         String type = precursorEntity.getManualOverwrite().getType()
                 .orElse(precursorEntity.getType());
-        boolean isHint = isHint(precursorEntity.getEntityType());
         return EntityLogEntry.builder()
                 .id(precursorEntity.getId())
                 .reason(precursorEntity.buildReasonWithManualChangeDescriptions())

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/components/ComponentMappingFileSystemCache.java

@@ -40,7 +40,8 @@ public class ComponentMappingFileSystemCache {
     public File getComponentMappingFile(ComponentMappingMetadata metadata) {
 
         Path mappingFile = getMappingFileFromMetadata(metadata);
-        Path mappingFileMetaDataFile = mappingFile.resolveSibling(metadata.getName() + METADATA_SUFFIX);
+        Path mappingFileMetaDataFile = getMappingMetadataFileFromMetadata(metadata);
+
         synchronized (ComponentMappingFileSystemCache.class) {
 
             if (fileExistsAndUpToDate(metadata, mappingFile, mappingFileMetaDataFile)) {
@@ -52,7 +53,7 @@ public class ComponentMappingFileSystemCache {
             InputStreamResource inputStreamResource = storageService.getObject(TenantContext.getTenantId(), metadata.getStorageId());
 
             Files.write(mappingFile, inputStreamResource.getContentAsByteArray(), StandardOpenOption.WRITE, StandardOpenOption.TRUNCATE_EXISTING, StandardOpenOption.CREATE);
-            
+
             mapper.writeValue(mappingFileMetaDataFile.toFile(), metadata);
 
             return mappingFile.toFile();
@@ -87,6 +88,13 @@ public class ComponentMappingFileSystemCache {
     }
 
 
+    private Path getMappingMetadataFileFromMetadata(ComponentMappingMetadata metadata) {
+
+        Path tenantStem = mappingFileDir.resolve(TenantContext.getTenantId());
+        return tenantStem.resolve(metadata.getStorageId() + METADATA_SUFFIX);
+    }
+
+
     private boolean fileExistsAndUpToDate(ComponentMappingMetadata metadata, Path mappingFile, Path mappingFileMetaDataFile) {
 
         if (mappingFile.toFile().exists() && mappingFile.toFile().isFile() && mappingFileMetaDataFile.toFile().exists() && mappingFileMetaDataFile.toFile().isFile()) {

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/components/ComponentMappingMemoryCache.java

@@ -4,6 +4,7 @@ import java.io.File;
 import java.io.FileReader;
 import java.io.Reader;
 import java.nio.charset.Charset;
+import java.time.Duration;
 import java.util.List;
 
 import org.springframework.stereotype.Service;
@@ -36,7 +37,7 @@ public class ComponentMappingMemoryCache {
     public ComponentMappingMemoryCache(ComponentMappingFileSystemCache componentMappingFileSystemCache) {
 
         this.fileSystemCache = componentMappingFileSystemCache;
-        cache = CacheBuilder.newBuilder().maximumWeight(MAX_NUMBER_OF_LINES).weigher(COMPONENT_MAPPING_WEIGHER).build();
+        cache = CacheBuilder.newBuilder().maximumWeight(MAX_NUMBER_OF_LINES).weigher(COMPONENT_MAPPING_WEIGHER).expireAfterAccess(Duration.ofDays(1)).build();
     }
 
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityFromPrecursorCreationService.java

@@ -15,6 +15,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.BaseAnnotation;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;
+import com.iqser.red.service.redaction.v1.server.RedactionServiceSettings;
 import com.iqser.red.service.redaction.v1.server.model.PrecursorEntity;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
@@ -35,6 +36,7 @@ public class EntityFromPrecursorCreationService {
     static double MATCH_THRESHOLD = 10; // Is compared to the average sum of distances in pdf coordinates for each corner of the bounding box of the entities
     EntityFindingUtility entityFindingUtility;
     DictionaryService dictionaryService;
+    RedactionServiceSettings settings;
 
 
     public List<PrecursorEntity> createEntitiesIfFoundAndReturnNotFoundEntries(ManualRedactions manualRedactions, SemanticNode node, String dossierTemplateId) {
@@ -92,7 +94,7 @@ public class EntityFromPrecursorCreationService {
                 notFoundEntities.add(precursorEntity);
                 continue;
             }
-            createCorrectEntity(precursorEntity, optionalClosestEntity.get());
+            createCorrectEntity(precursorEntity, optionalClosestEntity.get(), settings.isAnnotationMode());
         }
 
         tempEntitiesByValue.values()
@@ -129,8 +131,7 @@ public class EntityFromPrecursorCreationService {
                                                          precursorEntity.type(),
                                                          precursorEntity.getEntityType(),
                                                          precursorEntity.getId(),
-                                                         precursorEntity.getManualOverwrite().getSection()
-                                                                 .orElse(null));
+                                                         precursorEntity.getManualOverwrite().getSection().orElse(null));
         }
         correctEntity.setDeepestFullyContainingNode(closestEntity.getDeepestFullyContainingNode());
         correctEntity.setIntersectingNodes(new ArrayList<>(closestEntity.getIntersectingNodes()));

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/NerEntitiesAdapter.java

@@ -177,7 +177,7 @@ public class NerEntitiesAdapter {
     }
 
 
-    private static List<Integer> sectionNumberToTreeId(String sectionNumber) {
+    public static List<Integer> sectionNumberToTreeId(String sectionNumber) {
 
         return Arrays.stream(sectionNumber.split("\\."))
                 .map(Integer::parseInt)

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/ComponentDroolsExecutionService.java

@@ -73,9 +73,10 @@ public class ComponentDroolsExecutionService {
                     entities.add(Entity.fromEntityLogEntry(entry, document, entry.getStartOffset(), entry.getEndOffset()));
                     if (entry.getDuplicatedTextRanges() != null && !entry.getDuplicatedTextRanges().isEmpty()) {
                         entry.getDuplicatedTextRanges()
-                                .forEach(duplicatedTextRange -> {
-                                    entities.add(Entity.fromEntityLogEntry(entry, document, duplicatedTextRange.getStart(), duplicatedTextRange.getEnd()));
-                                });
+                                .forEach(duplicatedTextRange -> entities.add(Entity.fromEntityLogEntry(entry,
+                                                                                                       document,
+                                                                                                       duplicatedTextRange.getStart(),
+                                                                                                       duplicatedTextRange.getEnd())));
                     }
                     return entities.stream();
                 })

redaction-service-v1/redaction-service-server-v1/src/main/resources/application.yml

@@ -16,6 +16,9 @@ project.version: 1.0-SNAPSHOT
 server:
   port: 8080
 
+lifecycle:
+  base-package: com.iqser.red.service.redaction
+
 spring:
   application:
     name: redaction-service

redaction-service-v1/redaction-service-server-v1/src/main/resources/date_formats.txt

@@ -14,6 +14,8 @@ dd MMMM yyyy
 d MMMM yyyy
 MMMM dd, yyyy
 MMMM d, yyyy
+MMMM, d yyyy
+MMMM d,yyyy
 dd.MM.yyyy
 d.MM.yyyy
 yyyy/MM/dd
@@ -28,6 +30,7 @@ dd['.'] MMM yyyy
 d['.'] MMM yyyy
 dd['th']['st']['nd']['rd'] 'of' MMMM, yyyy
 d['th']['st']['nd']['rd'] 'of' MMMM, yyyy
+d['st']['nd']['rd']['th'] MMMM yyyy
 MMMM dd['th']['st']['nd']['rd'], yyyy
 MMMM d['th']['st']['nd']['rd'], yyyy
 yyyy, MMMM dd

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AbstractRedactionIntegrationTest.java

@@ -114,13 +114,14 @@ public abstract class AbstractRedactionIntegrationTest {
     public static final String PII_TYPE_ID = DICTIONARY_PII + ":" + TEST_DOSSIER_TEMPLATE_ID;
     public static final String TEST_METHOD_TYPE_ID = TEST_METHOD_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID;
     public static final String PUBLISHED_INFORMATION_TYPE_ID = PUBLISHED_INFORMATION_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID;
-    public static final String DOSSIER_PUBLISHED_INFORMATION_TYPE_ID = PUBLISHED_INFORMATION_TYPE_ID+ ":" + TEST_DOSSIER_ID;
+    public static final String DOSSIER_PUBLISHED_INFORMATION_TYPE_ID = PUBLISHED_INFORMATION_TYPE_ID + ":" + TEST_DOSSIER_ID;
     public static final String MUST_REDACT_TYPE_ID = MUST_REDACT_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID;
     public static final String HINT_ONLY_TYPE_ID = HINT_ONLY_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID;
     public static final String REDACTION_TYPE_ID = REDACTION_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID;
     public static final String NO_REDACTION_TYPE_ID = NO_REDACTION_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID;
     public static final String SPONSOR_TYPE_ID = DICTIONARY_SPONSOR + ":" + TEST_DOSSIER_TEMPLATE_ID;
     public static final String AUTHOR_TYPE_ID = DICTIONARY_AUTHOR + ":" + TEST_DOSSIER_TEMPLATE_ID;
+    public static final String DOSSIER_AUTHOR_TYPE_ID = AUTHOR_TYPE_ID + ":" + TEST_DOSSIER_ID;
     public static final String ADDRESS_TYPE_ID = DICTIONARY_ADDRESS + ":" + TEST_DOSSIER_TEMPLATE_ID;
     public static final String VERTEBRATE_TYPE_ID = VERTEBRATE_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID;
 
@@ -250,8 +251,10 @@ public abstract class AbstractRedactionIntegrationTest {
                                                                                                                                                  true));
         when(dictionaryClient.getDictionaryForType(IMPORTED_REDACTION_TYPE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(IMPORTED_REDACTION_INDICATOR,
                                                                                                                                                  true));
-        when(dictionaryClient.getDictionaryForType(DOSSIER_PUBLISHED_INFORMATION_TYPE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(PUBLISHED_INFORMATION_INDICATOR,
-                                                                                                                                                 true));
+        when(dictionaryClient.getDictionaryForType(DOSSIER_PUBLISHED_INFORMATION_TYPE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(
+                PUBLISHED_INFORMATION_INDICATOR,
+                true));
+        when(dictionaryClient.getDictionaryForType(DOSSIER_AUTHOR_TYPE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(DICTIONARY_AUTHOR, true));
 
     }
 
@@ -350,6 +353,7 @@ public abstract class AbstractRedactionIntegrationTest {
                                 .collect(Collectors.toSet()));
         dossierDictionary.put(IMPORTED_REDACTION_INDICATOR, new ArrayList<>());
         dossierDictionary.put(PUBLISHED_INFORMATION_INDICATOR, new ArrayList<>());
+        dossierDictionary.put(DICTIONARY_AUTHOR, new ArrayList<>());
 
         falsePositive.computeIfAbsent(DICTIONARY_PII, v -> new ArrayList<>())
                 .addAll(ResourceLoader.load("dictionaries/PII_false_positive.txt")

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DocumineFloraTest.java

@@ -174,6 +174,16 @@ public class DocumineFloraTest extends AbstractRedactionIntegrationTest {
         expectedDates.add("03/08/1992");
         expectedDates.add("13/08/1992");
         expectedDates.add("27/02/1992");
+        expectedDates.add("27/10/1989");
+        expectedDates.add("07/10/1989");
+        expectedDates.add("21/08/1998");
+        expectedDates.add("02/08/1998");
+        expectedDates.add("01/05/1988");
+        expectedDates.add("02/06/2003");
+        expectedDates.add("03/09/2005");
+        expectedDates.add("06/09/2005");
+        expectedDates.add("17/08/2005");
+        expectedDates.add("22/08/2035");
 
         String dates = experimentalDates.getComponentValues()
                 .get(0).getValue();

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionAcceptanceTest.java

@@ -10,6 +10,7 @@ import java.time.OffsetDateTime;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Set;
+import java.util.UUID;
 import java.util.stream.Stream;
 
 import org.junit.jupiter.api.BeforeEach;
@@ -25,24 +26,30 @@ import org.springframework.context.annotation.Configuration;
 import org.springframework.context.annotation.FilterType;
 import org.springframework.context.annotation.Import;
 import org.springframework.context.annotation.Primary;
+import org.springframework.core.io.ClassPathResource;
 import org.springframework.test.context.junit.jupiter.SpringExtension;
 
 import com.iqser.red.commons.jackson.ObjectMapperFactory;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeResult;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryType;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type;
 import com.iqser.red.service.redaction.v1.server.annotate.AnnotateRequest;
 import com.iqser.red.service.redaction.v1.server.annotate.AnnotateResponse;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.OsUtils;
 import com.iqser.red.service.redaction.v1.server.service.DictionaryService;
+import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
 import com.iqser.red.storage.commons.StorageAutoConfiguration;
 import com.iqser.red.storage.commons.service.StorageService;
 import com.iqser.red.storage.commons.utils.FileSystemBackedStorageService;
@@ -50,6 +57,8 @@ import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsi
 import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingServiceProcessorConfiguration;
 import com.knecon.fforesight.tenantcommons.TenantContext;
 
+import lombok.SneakyThrows;
+
 @ExtendWith(SpringExtension.class)
 @SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
 @Import(RedactionIntegrationTest.RedactionIntegrationTestConfiguration.class)
@@ -103,6 +112,17 @@ public class RedactionAcceptanceTest extends AbstractRedactionIntegrationTest {
                                                                                                        .isRecommendation(recommendationTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
                                                                                                        .rank(rankTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
                                                                                                        .build(),
+                                                                                               Type.builder()
+                                                                                                       .id(DOSSIER_AUTHOR_TYPE_ID)
+                                                                                                       .type(DICTIONARY_AUTHOR)
+                                                                                                       .dossierTemplateId(TEST_DOSSIER_TEMPLATE_ID)
+                                                                                                       .dossierId(TEST_DOSSIER_ID)
+                                                                                                       .hexColor("#ffe184")
+                                                                                                       .isHint(hintTypeMap.get(DICTIONARY_AUTHOR))
+                                                                                                       .isCaseInsensitive(caseInSensitiveMap.get(DICTIONARY_AUTHOR))
+                                                                                                       .isRecommendation(recommendationTypeMap.get(DICTIONARY_AUTHOR))
+                                                                                                       .rank(rankTypeMap.get(DICTIONARY_AUTHOR))
+                                                                                                       .build(),
                                                                                                Type.builder()
                                                                                                        .id(DOSSIER_PUBLISHED_INFORMATION_TYPE_ID)
                                                                                                        .type(PUBLISHED_INFORMATION_INDICATOR)
@@ -158,10 +178,10 @@ public class RedactionAcceptanceTest extends AbstractRedactionIntegrationTest {
 
         var publishedInformationEntry1 = findEntityByTypeAndValue(entityLog, "published_information", "Oxford University Press").findFirst()
                 .orElseThrow();
-        assertThat(publishedInformationEntry1.getSection().startsWith("Paragraph:"));
+        assertThat(publishedInformationEntry1.getSection()).startsWith("Paragraph:");
         var asyaLyon1 = findEntityByTypeAndValueAndSectionNumber(entityLog, "CBI_author", "Asya Lyon", publishedInformationEntry1.getContainingNodeId()).findFirst()
                 .orElseThrow();
-        assertThat(asyaLyon1.getSection().startsWith("Paragraph:"));
+        assertThat(asyaLyon1.getSection()).startsWith("Paragraph:");
         assertEquals(EntryState.SKIPPED, asyaLyon1.getState());
 
         var idRemoval = buildIdRemoval(publishedInformationEntry1.getId());
@@ -212,10 +232,10 @@ public class RedactionAcceptanceTest extends AbstractRedactionIntegrationTest {
 
         var publishedInformationEntry1 = findEntityByTypeAndValue(entityLog, "published_information", "Oxford University Press").findFirst()
                 .orElseThrow();
-        assertThat(publishedInformationEntry1.getSection().startsWith("Paragraph:"));
+        assertThat(publishedInformationEntry1.getSection()).startsWith("Paragraph:");
         var asyaLyon1 = findEntityByTypeAndValueAndSectionNumber(entityLog, "CBI_author", "Asya Lyon", publishedInformationEntry1.getContainingNodeId()).findFirst()
                 .orElseThrow();
-        assertThat(publishedInformationEntry1.getSection().startsWith("Paragraph:"));
+        assertThat(publishedInformationEntry1.getSection()).startsWith("Paragraph:");
         assertEquals(EntryState.SKIPPED, asyaLyon1.getState());
 
         var idRemoval = buildIdRemoval(publishedInformationEntry1.getId());
@@ -244,6 +264,7 @@ public class RedactionAcceptanceTest extends AbstractRedactionIntegrationTest {
         assertThat(dictionary.get(PUBLISHED_INFORMATION_INDICATOR).contains("Press")).isFalse();
     }
 
+
     @Test
     public void testPublishedInformationRemovalAtDossierLevel() throws IOException {
 
@@ -365,6 +386,75 @@ public class RedactionAcceptanceTest extends AbstractRedactionIntegrationTest {
     }
 
 
+    @Test
+    @SneakyThrows
+    void testNerEntitiesAfterReanalysis() {
+
+        String EFSA_SANITISATION_RULES = loadFromClassPath("drools/efsa_sanitisation.drl");
+        when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.ENTITY)).thenReturn(JSONPrimitive.of(EFSA_SANITISATION_RULES));
+
+        ClassPathResource responseJson = new ClassPathResource("files/new/SYNGENTA_EFSA_sanitisation_GFL_v1 3.NER_ENTITIES.json");
+        storageService.storeObject(TenantContext.getTenantId(),
+                                   RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.NER_ENTITIES),
+                                   responseJson.getInputStream());
+
+        String pdfFile = "files/new/SYNGENTA_EFSA_sanitisation_GFL_v1 3.pdf";
+
+        AnalyzeRequest request = uploadFileToStorage(pdfFile);
+
+        analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request);
+        request.setAnalysisNumber(1);
+        dossierDictionary.put(DICTIONARY_AUTHOR, new ArrayList<>());
+        mockDictionaryCalls(0L);
+
+        analyzeService.analyze(request);
+
+        var entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
+
+        String nerValue = "Osip S.";
+        var nerEntity = findEntityByTypeAndValue(entityLog, DICTIONARY_AUTHOR, nerValue).findFirst()
+                .orElseThrow();
+        assertThat(nerEntity.getEngines()).contains(Engine.NER);
+
+        String dictionaryAddValue = "cooperation";
+        ManualRedactionEntry manualRedactionEntry = ManualRedactionEntry.builder()
+                .value(dictionaryAddValue)
+                .type(DICTIONARY_AUTHOR)
+                .user("user")
+                .addToDossierDictionary(true)
+                .positions(List.of(Rectangle.builder().topLeftX(180.748f).topLeftY(546.564f).width(56.592f).height(15.408f).page(1).build()))
+                .type("dossier_redaction")
+                .fileId(TEST_FILE_ID)
+                .requestDate(OffsetDateTime.now())
+                .annotationId(UUID.randomUUID().toString())
+                .build();
+        request.setManualRedactions(ManualRedactions.builder().entriesToAdd(Set.of(manualRedactionEntry)).build());
+
+        request.setAnalysisNumber(2);
+        dossierDictionary.get(DICTIONARY_AUTHOR).add(dictionaryAddValue);
+        reanlysisVersions.put(dictionaryAddValue, 2L);
+        when(dictionaryClient.getVersionForDossier(TEST_DOSSIER_ID)).thenReturn(2L);
+        mockDictionaryCalls(1L);
+
+        AnalyzeResult reanalyzeResult = analyzeService.reanalyze(request);
+
+        entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
+
+        EntityLogEntry entityLogEntryAdded = entityLog.getEntityLogEntry()
+                .stream()
+                .filter(entityLogEntry -> entityLogEntry.getValue().equals(dictionaryAddValue))
+                .findFirst()
+                .get();
+        assertEquals(EntryState.APPLIED, entityLogEntryAdded.getState());
+
+        nerEntity = findEntityByTypeAndValue(entityLog, DICTIONARY_AUTHOR, nerValue).findFirst()
+                .orElseThrow();
+        assertThat(nerEntity.getEngines()).contains(Engine.NER);
+        dossierDictionary.get(DICTIONARY_AUTHOR).remove(dictionaryAddValue);
+
+    }
+
+
     private static IdRemoval buildIdRemoval(String id) {
 
         return IdRemoval.builder().annotationId(id).user("user").requestDate(OffsetDateTime.now()).fileId(TEST_FILE_ID).build();

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/date/DateConverterTest.java

@@ -43,7 +43,17 @@ public class DateConverterTest {
                                                          "28 March 2018 (animal 1 - 5000 mg/kg bw)",
                                                          "28 March 2018 (animal1 - 5000 mg/kg bw)",
                                                          "28 August 2018 (animal 1)",
-                                                         "31 August 2018 (animal 1)");
+                                                         "31 August 2018 (animal 1)",
+                                                         "October, 27 1989",
+                                                         "October, 7 1989",
+                                                         "August 21,1998",
+                                                         "August 2,1998",
+                                                         "1st May 1988",
+                                                         "2nd June 2003",
+                                                         "3rd September 2005",
+                                                         "6th September 2005",
+                                                         "17th August 2005",
+                                                         "22nd August 2035");
 
         for (String dateStr : goldenStandardDates) {
             Optional<Date> parsedDate = DateConverter.parseDate(dateStr);