Merge branch 'master' into poc-dict-trie

# Conflicts:
#	redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionAcceptanceTest.java

redaction-service-v1/redaction-service-server-v1/build.gradle.kts

@@ -16,7 +16,7 @@ val layoutParserVersion = "0.181.0"
 val jacksonVersion = "2.15.2"
 val droolsVersion = "9.44.0.Final"
 val pdfBoxVersion = "3.0.0"
-val persistenceServiceVersion = "2.587.0"
+val persistenceServiceVersion = "2.592.0-RED10260.0"
 val llmServiceVersion = "1.11.0"
 val springBootStarterVersion = "3.1.5"
 val springCloudVersion = "4.0.4"
@@ -52,7 +52,7 @@ dependencies {
 
     implementation("com.iqser.red.commons:dictionary-merge-commons:1.5.0")
     implementation("com.iqser.red.commons:storage-commons:2.50.0")
-    implementation("com.knecon.fforesight:tenant-commons:0.30.0")
+    implementation("com.knecon.fforesight:tenant-commons:0.31.0")
     implementation("com.knecon.fforesight:keycloak-commons:0.30.0") {
         exclude(group = "com.knecon.fforesight", module = "tenant-commons")
     }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/configuraton/PriorityTenantMessagingConfigurationImpl.java

@@ -1,12 +0,0 @@
-package com.iqser.red.service.redaction.v1.server.configuraton;
-
-import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
-import org.springframework.context.annotation.Configuration;
-
-import com.knecon.fforesight.tenantcommons.queue.TenantMessagingConfiguration;
-
-@Configuration
-@ConditionalOnProperty(prefix = "redaction-service", name = "priorityMode", havingValue = "true")
-public class PriorityTenantMessagingConfigurationImpl extends TenantMessagingConfiguration {
-
-}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/configuraton/TenantMessagingConfigurationImpl.java

@@ -1,12 +0,0 @@
-package com.iqser.red.service.redaction.v1.server.configuraton;
-
-import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
-import org.springframework.context.annotation.Configuration;
-
-import com.knecon.fforesight.tenantcommons.queue.TenantMessagingConfiguration;
-
-@Configuration
-@ConditionalOnProperty(prefix = "redaction-service", name = "priorityMode", havingValue = "false")
-public class TenantMessagingConfigurationImpl extends TenantMessagingConfiguration {
-
-}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/queue/PriorityTenantExchangeMessageReceiverImpl.java

@@ -1,68 +0,0 @@
-package com.iqser.red.service.redaction.v1.server.queue;
-
-import static com.iqser.red.service.redaction.v1.model.QueueNames.*;
-import java.util.Map;
-import java.util.Set;
-
-import org.springframework.amqp.rabbit.annotation.RabbitHandler;
-import org.springframework.amqp.rabbit.annotation.RabbitListener;
-import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
-import org.springframework.boot.context.event.ApplicationReadyEvent;
-import org.springframework.context.event.EventListener;
-import org.springframework.stereotype.Service;
-
-import com.knecon.fforesight.tenantcommons.TenantProvider;
-import com.knecon.fforesight.tenantcommons.model.TenantCreatedEvent;
-import com.knecon.fforesight.tenantcommons.model.TenantQueueConfiguration;
-import com.knecon.fforesight.tenantcommons.model.TenantResponse;
-import com.knecon.fforesight.tenantcommons.queue.RabbitQueueFromExchangeService;
-import com.knecon.fforesight.tenantcommons.queue.TenantExchangeMessageReceiver;
-
-@Service
-@ConditionalOnProperty(prefix = "redaction-service", name = "priorityMode", havingValue = "true")
-public class PriorityTenantExchangeMessageReceiverImpl extends TenantExchangeMessageReceiver {
-
-    public PriorityTenantExchangeMessageReceiverImpl(RabbitQueueFromExchangeService rabbitQueueService, TenantProvider tenantProvider) {
-
-        super(rabbitQueueService, tenantProvider);
-    }
-
-
-    @Override
-    protected Set<TenantQueueConfiguration> getTenantQueueConfigs() {
-
-        return Set.of(TenantQueueConfiguration.builder()
-                              .listenerId(PriorityMessageReceiver.REDACTION_PRIORITY_REQUEST_LISTENER_ID)
-                              .exchangeName(REDACTION_PRIORITY_REQUEST_EXCHANGE)
-                              .queuePrefix(REDACTION_PRIORITY_REQUEST_QUEUE_PREFIX)
-                              .dlqName(REDACTION_DLQ)
-                              .arguments(Map.of("x-max-priority", 2))
-                              .build());
-    }
-
-
-    @EventListener(ApplicationReadyEvent.class)
-    public void onApplicationReady() {
-
-        System.out.println("application ready invoked");
-        super.initializeQueues();
-    }
-
-
-    @RabbitHandler
-    @RabbitListener(queues = "#{priorityTenantMessagingConfigurationImpl.getTenantCreatedQueueName()}")
-    public void reactToTenantCreation(TenantCreatedEvent tenantCreatedEvent) {
-
-        super.reactToTenantCreation(tenantCreatedEvent);
-    }
-
-
-    @RabbitHandler
-    @RabbitListener(queues = "#{priorityTenantMessagingConfigurationImpl.getTenantDeletedQueueName()}")
-    public void reactToTenantDeletion(TenantResponse tenantResponse) {
-
-        super.reactToTenantDeletion(tenantResponse);
-
-    }
-
-}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/queue/TenantExchangeMessageReceiverImpl.java

@@ -1,68 +0,0 @@
-package com.iqser.red.service.redaction.v1.server.queue;
-
-import static com.iqser.red.service.redaction.v1.model.QueueNames.*;
-import java.util.Map;
-import java.util.Set;
-
-import org.springframework.amqp.rabbit.annotation.RabbitHandler;
-import org.springframework.amqp.rabbit.annotation.RabbitListener;
-import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
-import org.springframework.boot.context.event.ApplicationReadyEvent;
-import org.springframework.context.event.EventListener;
-import org.springframework.stereotype.Service;
-
-import com.knecon.fforesight.tenantcommons.TenantProvider;
-import com.knecon.fforesight.tenantcommons.model.TenantCreatedEvent;
-import com.knecon.fforesight.tenantcommons.model.TenantQueueConfiguration;
-import com.knecon.fforesight.tenantcommons.model.TenantResponse;
-import com.knecon.fforesight.tenantcommons.queue.RabbitQueueFromExchangeService;
-import com.knecon.fforesight.tenantcommons.queue.TenantExchangeMessageReceiver;
-
-@Service
-@ConditionalOnProperty(prefix = "redaction-service", name = "priorityMode", havingValue = "false")
-public class TenantExchangeMessageReceiverImpl extends TenantExchangeMessageReceiver {
-
-    public TenantExchangeMessageReceiverImpl(RabbitQueueFromExchangeService rabbitQueueService, TenantProvider tenantProvider) {
-
-        super(rabbitQueueService, tenantProvider);
-    }
-
-
-    @Override
-    protected Set<TenantQueueConfiguration> getTenantQueueConfigs() {
-
-        return Set.of(TenantQueueConfiguration.builder()
-                              .listenerId(MessageReceiver.REDACTION_REQUEST_LISTENER_ID)
-                              .exchangeName(REDACTION_REQUEST_EXCHANGE)
-                              .queuePrefix(REDACTION_REQUEST_QUEUE_PREFIX)
-                              .dlqName(REDACTION_DLQ)
-                              .arguments(Map.of("x-max-priority", 2))
-                              .build());
-    }
-
-
-    @EventListener(ApplicationReadyEvent.class)
-    public void onApplicationReady() {
-
-        System.out.println("application ready invoked");
-        super.initializeQueues();
-    }
-
-
-    @RabbitHandler
-    @RabbitListener(queues = "#{tenantMessagingConfigurationImpl.getTenantCreatedQueueName()}")
-    public void reactToTenantCreation(TenantCreatedEvent tenantCreatedEvent) {
-
-        super.reactToTenantCreation(tenantCreatedEvent);
-    }
-
-
-    @RabbitHandler
-    @RabbitListener(queues = "#{tenantMessagingConfigurationImpl.getTenantDeletedQueueName()}")
-    public void reactToTenantDeletion(TenantResponse tenantResponse) {
-
-        super.reactToTenantDeletion(tenantResponse);
-
-    }
-
-}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/queue/TenantQueueProviderConfig.java

@@ -0,0 +1,49 @@
+package com.iqser.red.service.redaction.v1.server.queue;
+
+import static com.iqser.red.service.redaction.v1.model.QueueNames.REDACTION_DLQ;
+import static com.iqser.red.service.redaction.v1.model.QueueNames.REDACTION_PRIORITY_REQUEST_EXCHANGE;
+import static com.iqser.red.service.redaction.v1.model.QueueNames.REDACTION_PRIORITY_REQUEST_QUEUE_PREFIX;
+import static com.iqser.red.service.redaction.v1.model.QueueNames.REDACTION_REQUEST_EXCHANGE;
+import static com.iqser.red.service.redaction.v1.model.QueueNames.REDACTION_REQUEST_QUEUE_PREFIX;
+
+import java.util.Map;
+import java.util.Set;
+
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+
+import com.knecon.fforesight.tenantcommons.model.TenantQueueConfiguration;
+import com.knecon.fforesight.tenantcommons.model.TenantQueueProvider;
+
+@Configuration
+public class TenantQueueProviderConfig {
+
+    @Bean
+    @ConditionalOnProperty(prefix = "redaction-service", name = "priorityMode", havingValue = "true")
+    protected TenantQueueProvider getTenantPriorityQueueConfigs() {
+
+        return new TenantQueueProvider(Set.of(TenantQueueConfiguration.builder()
+                                                      .listenerId(PriorityMessageReceiver.REDACTION_PRIORITY_REQUEST_LISTENER_ID)
+                                                      .exchangeName(REDACTION_PRIORITY_REQUEST_EXCHANGE)
+                                                      .queuePrefix(REDACTION_PRIORITY_REQUEST_QUEUE_PREFIX)
+                                                      .dlqName(REDACTION_DLQ)
+                                                      .arguments(Map.of("x-max-priority", 2))
+                                                      .build()));
+    }
+
+
+    @Bean
+    @ConditionalOnProperty(prefix = "redaction-service", name = "priorityMode", havingValue = "false", matchIfMissing = true)
+    protected TenantQueueProvider getTenantQueueConfigs() {
+
+        return new TenantQueueProvider(Set.of(TenantQueueConfiguration.builder()
+                                                      .listenerId(MessageReceiver.REDACTION_REQUEST_LISTENER_ID)
+                                                      .exchangeName(REDACTION_REQUEST_EXCHANGE)
+                                                      .queuePrefix(REDACTION_REQUEST_QUEUE_PREFIX)
+                                                      .dlqName(REDACTION_DLQ)
+                                                      .arguments(Map.of("x-max-priority", 2))
+                                                      .build()));
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalysisFinalizationService.java

@@ -61,6 +61,7 @@ public class AnalysisFinalizationService {
         // as workaround for duplicate key exceptions occurring due to simultaneous analyses and reanalyses save instead of insert is used
         // also analysis numbers should be incremented in every follow-up request, so checking if the log exists is not needed
         if (!redactionStorageService.entityLogExists(analyzeRequest.getDossierId(), analyzeRequest.getFileId())) {
+            entityLog.setEntityLogEntry(entityLogChanges.getNewEntityLogEntries());
             redactionStorageService.saveEntityLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), entityLog);
 
         } else {

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityChangeLogService.java

@@ -41,7 +41,7 @@ public class EntityChangeLogService {
         var now = OffsetDateTime.now();
         if (previousEntityLogEntries.isEmpty()) {
             newEntityLogEntries.forEach(entry -> entry.getChanges().add(new Change(analysisNumber, ChangeType.ADDED, now, Collections.emptyMap())));
-            return new EntryChanges(newEntityLogEntries, new ArrayList<>());
+            return new EntryChanges(newEntityLogEntries.stream().filter(entry -> !entry.getState().equals(EntryState.REMOVED)).collect(Collectors.toList()), new ArrayList<>());
         }
 
         List<EntityLogEntry> toInsert = new ArrayList<>();
@@ -51,6 +51,9 @@ public class EntityChangeLogService {
                     .filter(entry -> entry.getId().equals(entityLogEntry.getId()))
                     .findAny();
             if (optionalPreviousEntity.isEmpty()) {
+                if (entityLogEntry.getState().equals(EntryState.REMOVED)) {
+                    continue;
+                }
                 entityLogEntry.getChanges().add(new Change(analysisNumber, ChangeType.ADDED, now, Collections.emptyMap()));
                 toInsert.add(entityLogEntry);
                 continue;

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java

@@ -180,7 +180,6 @@ public class EntityLogCreatorService {
                 .stream()
                 .filter(entity -> !entity.getValue().isEmpty())
                 .filter(EntityLogCreatorService::notFalsePositiveOrFalseRecommendationOrRemoval)
-                .filter(entity -> !entity.removed())
                 .toList();
 
         List<Image> images = document.streamAllImages()

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/components/ComponentMappingMemoryCache.java

@@ -76,7 +76,7 @@ public class ComponentMappingMemoryCache {
 
         Charset charSet = Charset.forName(metadata.getEncoding());
         List<String[]> data;
-        try (Reader fileReader = new FileReader(mappingFile, charSet); CSVReader reader = buildReader(fileReader, metadata.getDelimiter())) {
+        try (Reader fileReader = new FileReader(mappingFile, charSet); CSVReader reader = buildReader(fileReader, metadata.getDelimiter(), metadata.getQuoteChar())) {
             List<String[]> rawData = reader.readAll();
             rawData.remove(0); // remove header row
             data = rawData;
@@ -86,9 +86,9 @@ public class ComponentMappingMemoryCache {
 
 
     @SneakyThrows
-    private static CSVReader buildReader(Reader reader, char delimiter) {
+    private static CSVReader buildReader(Reader reader, char delimiter, char quoteChar) {
 
-        return new CSVReaderBuilder(reader).withCSVParser(new CSVParserBuilder().withSeparator(delimiter).build()).build();
+        return new CSVReaderBuilder(reader).withCSVParser(new CSVParserBuilder().withSeparator(delimiter).withQuoteChar(quoteChar).build()).build();
     }
 
 

redaction-service-v1/redaction-service-server-v1/src/main/resources/drools/all_rules_documine.drl

@@ -1454,12 +1454,23 @@ rule "MAN.4.1: Apply legal basis change"
 rule "X.0.0: Remove Entity contained by Entity of same type"
     salience 65
     when
-        $larger: TextEntity($type: type(), $entityType: entityType, !removed())
+        $larger: TextEntity($type: type(), $entityType: entityType, !removed(), !hasManualChanges())
-        $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges(), !removed())
+        not TextEntity(getTextRange().equals($larger.getTextRange()), type() == $type, entityType == EntityType.DICTIONARY_REMOVAL, engines contains Engine.DOSSIER_DICTIONARY, !hasManualChanges())
+        $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges())
+    then
+        $contained.remove("X.0.0", "remove Entity contained by Entity of same type");
+        retract($contained);
+    end
+
+rule "X.0.1: Remove Entity contained by Entity of same type with manual changes"
+    salience 65
+    when
+        $larger: TextEntity($type: type(), $entityType: entityType, !removed(), hasManualChanges())
+        $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges())
     then
         $contained.getIntersectingNodes().forEach(node -> update(node));
-        $contained.remove("X.0.0", "remove Entity contained by Entity of same type");
+        $contained.remove("X.0.1", "remove Entity contained by Entity of same type with manual changes");
-        update($contained);
+        retract($contained);
     end
 
 
@@ -1468,10 +1479,21 @@ rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
     salience 64
     when
         $falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active())
-        $entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges())
+        $entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.ENTITY), !hasManualChanges())
+    then
+        $entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE");
+        retract($entity)
+    end
+
+
+rule "X.2.1: Remove Entity of type HINT when contained by FALSE_POSITIVE"
+    salience 64
+    when
+        $falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active())
+        $entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.HINT), !hasManualChanges())
     then
         $entity.getIntersectingNodes().forEach(node -> update(node));
-        $entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE");
+        $entity.remove("X.2.1", "remove Entity of type ENTITY when contained by FALSE_POSITIVE");
         retract($entity)
     end
 
@@ -1593,15 +1615,15 @@ rule "X.11.0: Remove dictionary entity which intersects with a manual entity"
 //------------------------------------ Dictionary merging rules ------------------------------------
 
 // Rule unit: DICT.0
-rule "DICT.0.0: Ignore Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL"
+rule "DICT.0.0: Remove Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL"
     salience 64
     when
         $dictionaryRemoval: TextEntity($type: type(), entityType == EntityType.DICTIONARY_REMOVAL, engines contains Engine.DOSSIER_DICTIONARY)
         $entity: TextEntity(getTextRange().equals($dictionaryRemoval.getTextRange()), engines contains Engine.DICTIONARY, type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges())
-        not TextEntity(containedBy($entity), engines contains Engine.DOSSIER_DICTIONARY, type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges())
     then
         $entity.getIntersectingNodes().forEach(node -> update(node));
-        $entity.ignore("DICT.0.0", "Ignore Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL");
+        $entity.remove("DICT.0.0", "Remove Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL");
+        $entity.addEngine(Engine.DOSSIER_DICTIONARY);
     end
 
 

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AbstractRedactionIntegrationTest.java

@@ -5,6 +5,7 @@ import static com.iqser.red.service.redaction.v1.server.testcontainers.MongoDBTe
 import static com.iqser.red.service.redaction.v1.server.testcontainers.MongoDBTestContainer.MONGO_USERNAME;
 import static com.knecon.fforesight.tenantcommons.model.TenantResponse.builder;
 import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
 
 import java.io.File;
@@ -30,17 +31,27 @@ import org.junit.jupiter.api.BeforeEach;
 import org.mockito.stubbing.Answer;
 import org.springframework.amqp.rabbit.core.RabbitAdmin;
 import org.springframework.amqp.rabbit.core.RabbitTemplate;
+import org.springframework.amqp.rabbit.listener.MessageListenerContainer;
 import org.springframework.amqp.rabbit.listener.RabbitListenerEndpointRegistry;
 import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
+import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration;
 import org.springframework.boot.test.mock.mockito.MockBean;
 import org.springframework.boot.test.util.TestPropertyValues;
 import org.springframework.context.ApplicationContextInitializer;
 import org.springframework.context.ConfigurableApplicationContext;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.ComponentScan;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.context.annotation.FilterType;
+import org.springframework.context.annotation.Import;
+import org.springframework.context.annotation.Primary;
 import org.springframework.core.io.ClassPathResource;
 import org.springframework.data.redis.listener.RedisMessageListenerContainer;
 import org.springframework.test.context.ContextConfiguration;
 
 import com.fasterxml.jackson.databind.ObjectMapper;
+import com.iqser.red.commons.jackson.ObjectMapperFactory;
 import com.iqser.red.service.dictionarymerge.commons.DictionaryEntry;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
@@ -65,6 +76,7 @@ import com.iqser.red.service.redaction.v1.server.testcontainers.MongoDBTestConta
 import com.iqser.red.service.redaction.v1.server.utils.LayoutParsingRequestProvider;
 import com.iqser.red.service.redaction.v1.server.utils.ResourceLoader;
 import com.iqser.red.service.redaction.v1.server.utils.TextNormalizationUtilities;
+import com.iqser.red.storage.commons.StorageAutoConfiguration;
 import com.iqser.red.storage.commons.service.StorageService;
 import com.iqser.red.storage.commons.utils.FileSystemBackedStorageService;
 import com.knecon.fforesight.keycloakcommons.security.TenantAuthenticationManagerResolver;
@@ -73,6 +85,7 @@ import com.knecon.fforesight.mongo.database.commons.service.MongoConnectionProvi
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingFinishedEvent;
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
 import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingPipeline;
+import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingServiceProcessorConfiguration;
 import com.knecon.fforesight.tenantcommons.TenantContext;
 import com.knecon.fforesight.tenantcommons.TenantProvider;
 import com.knecon.fforesight.tenantcommons.model.MongoDBConnection;
@@ -86,6 +99,7 @@ import lombok.SneakyThrows;
 import lombok.extern.slf4j.Slf4j;
 
 @ContextConfiguration(initializers = {AbstractRedactionIntegrationTest.Initializer.class})
+@Import(AbstractRedactionIntegrationTest.TestConfiguration.class)
 public abstract class AbstractRedactionIntegrationTest {
 
     protected static final String VERTEBRATE_INDICATOR = "vertebrate";
@@ -173,9 +187,6 @@ public abstract class AbstractRedactionIntegrationTest {
     @MockBean
     protected RabbitAdmin rabbitAdmin;
 
-    @MockBean
-    protected RabbitListenerEndpointRegistry rabbitListenerEndpointRegistry;
-
     @MockBean
     protected LegalBasisClient legalBasisClient;
 
@@ -745,4 +756,30 @@ public abstract class AbstractRedactionIntegrationTest {
         }
     }
 
+
+    @Configuration
+    @EnableAutoConfiguration(exclude = {RabbitAutoConfiguration.class})
+    @Import(LayoutParsingServiceProcessorConfiguration.class)
+    @ComponentScan(excludeFilters = {@ComponentScan.Filter(type = FilterType.ASSIGNABLE_TYPE, value = StorageAutoConfiguration.class)})
+    public static class TestConfiguration {
+
+        @Bean
+        public RabbitListenerEndpointRegistry rabbitListenerEndpointRegistry() {
+
+            var mock = mock(RabbitListenerEndpointRegistry.class);
+            when(mock.getListenerContainer(any())).thenReturn(mock(MessageListenerContainer.class));
+
+            return mock;
+        }
+
+
+        @Bean
+        @Primary
+        public StorageService inMemoryStorage() {
+
+            return new FileSystemBackedStorageService(ObjectMapperFactory.create());
+        }
+
+    }
+
 }

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AnalysisEnd2EndTest.java

@@ -85,7 +85,7 @@ import lombok.extern.slf4j.Slf4j;
 @Slf4j
 @ExtendWith(SpringExtension.class)
 @SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
-@Import(RedactionIntegrationTest.RedactionIntegrationTestConfiguration.class)
+@Import(AbstractRedactionIntegrationTest.TestConfiguration.class)
 @Disabled
 /*
  * This test is meant to be used directly with a download from blob storage (e.g. minio). You need to define the dossier template you want to use by supplying an absolute path.

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AnalysisTest.java

@@ -6,42 +6,26 @@ import static org.mockito.Mockito.when;
 
 import java.io.FileOutputStream;
 import java.io.IOException;
-import java.util.List;
 
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.extension.ExtendWith;
-import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
-import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration;
 import org.springframework.boot.test.context.SpringBootTest;
-import org.springframework.context.annotation.Bean;
-import org.springframework.context.annotation.ComponentScan;
-import org.springframework.context.annotation.Configuration;
-import org.springframework.context.annotation.FilterType;
-import org.springframework.context.annotation.Import;
-import org.springframework.context.annotation.Primary;
 import org.springframework.test.context.junit.jupiter.SpringExtension;
 
-import com.iqser.red.commons.jackson.ObjectMapperFactory;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeResult;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type;
 import com.iqser.red.service.redaction.v1.server.annotate.AnnotateRequest;
 import com.iqser.red.service.redaction.v1.server.annotate.AnnotateResponse;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.OsUtils;
-import com.iqser.red.storage.commons.StorageAutoConfiguration;
-import com.iqser.red.storage.commons.service.StorageService;
-import com.iqser.red.storage.commons.utils.FileSystemBackedStorageService;
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
-import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingServiceProcessorConfiguration;
 import com.knecon.fforesight.tenantcommons.TenantContext;
 
 @ExtendWith(SpringExtension.class)
 @SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT, properties = {"application.type=DocuMine"})
-@Import(AnalysisTest.RedactionIntegrationTestConfiguration.class)
 public class AnalysisTest extends AbstractRedactionIntegrationTest {
 
     private static final String RULES = loadFromClassPath("drools/test_rules.drl");
@@ -71,22 +55,6 @@ public class AnalysisTest extends AbstractRedactionIntegrationTest {
     }
 
 
-    @Configuration
-    @EnableAutoConfiguration(exclude = {RabbitAutoConfiguration.class})
-    @Import(LayoutParsingServiceProcessorConfiguration.class)
-    @ComponentScan(excludeFilters = {@ComponentScan.Filter(type = FilterType.ASSIGNABLE_TYPE, value = StorageAutoConfiguration.class)})
-    static class RedactionIntegrationTestConfiguration {
-
-        @Bean
-        @Primary
-        public StorageService inmemoryStorage() {
-
-            return new FileSystemBackedStorageService(ObjectMapperFactory.create());
-        }
-
-    }
-
-
     @BeforeEach
     public void stubClients() {
 

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DictionaryServiceTest.java

@@ -17,21 +17,13 @@ import org.junit.jupiter.api.extension.ExtendWith;
 import org.kie.api.runtime.KieContainer;
 import org.springframework.amqp.rabbit.core.RabbitAdmin;
 import org.springframework.amqp.rabbit.core.RabbitTemplate;
-import org.springframework.amqp.rabbit.listener.RabbitListenerEndpointRegistry;
 import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
-import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration;
 import org.springframework.boot.test.context.SpringBootTest;
 import org.springframework.boot.test.mock.mockito.MockBean;
-import org.springframework.context.annotation.Bean;
-import org.springframework.context.annotation.Configuration;
-import org.springframework.context.annotation.Import;
-import org.springframework.context.annotation.Primary;
 import org.springframework.data.redis.listener.RedisMessageListenerContainer;
 import org.springframework.test.context.junit.jupiter.SpringExtension;
 
 import com.google.common.collect.Sets;
-import com.iqser.red.commons.jackson.ObjectMapperFactory;
 import com.iqser.red.service.dictionarymerge.commons.DictionaryEntry;
 import com.iqser.red.service.dictionarymerge.commons.DictionaryEntryModel;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.configuration.Colors;
@@ -40,15 +32,12 @@ import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
 import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryVersion;
 import com.iqser.red.service.redaction.v1.server.service.DictionaryService;
 import com.iqser.red.service.redaction.v1.server.service.websocket.RedisSyncedWebSocketService;
-import com.iqser.red.storage.commons.service.StorageService;
-import com.iqser.red.storage.commons.utils.FileSystemBackedStorageService;
 import com.knecon.fforesight.keycloakcommons.security.TenantAuthenticationManagerResolver;
 import com.knecon.fforesight.tenantcommons.TenantContext;
 import com.knecon.fforesight.tenantcommons.TenantsClient;
 
 @ExtendWith(SpringExtension.class)
 @SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
-@Import(RedactionIntegrationTest.RedactionIntegrationTestConfiguration.class)
 public class DictionaryServiceTest {
 
     @MockBean
@@ -57,9 +46,6 @@ public class DictionaryServiceTest {
     @MockBean
     protected RabbitAdmin rabbitAdmin;
 
-    @MockBean
-    protected RabbitListenerEndpointRegistry rabbitListenerEndpointRegistry;
-
     @MockBean
     private TenantsClient tenantsClient;
 
@@ -81,19 +67,6 @@ public class DictionaryServiceTest {
     @Autowired
     protected DictionaryService dictionaryService;
 
-    @Configuration
-    @EnableAutoConfiguration(exclude = {RabbitAutoConfiguration.class})
-    public static class RedactionIntegrationTestConfiguration {
-
-        @Bean
-        @Primary
-        public StorageService inmemoryStorage() {
-
-            return new FileSystemBackedStorageService(ObjectMapperFactory.create());
-        }
-
-    }
-
 
     @Test
     public void testDictionaryServiceConsistency() {

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DocumineFloraTest.java

@@ -3,7 +3,6 @@ package com.iqser.red.service.redaction.v1.server;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertNotNull;
 import static org.junit.jupiter.api.Assertions.assertTrue;
-import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.Mockito.when;
 
 import java.io.FileOutputStream;
@@ -19,19 +18,10 @@ import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.extension.ExtendWith;
-import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
-import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration;
 import org.springframework.boot.test.context.SpringBootTest;
-import org.springframework.context.annotation.Bean;
-import org.springframework.context.annotation.ComponentScan;
-import org.springframework.context.annotation.Configuration;
-import org.springframework.context.annotation.FilterType;
-import org.springframework.context.annotation.Import;
-import org.springframework.context.annotation.Primary;
 import org.springframework.core.io.ClassPathResource;
 import org.springframework.test.context.junit.jupiter.SpringExtension;
 
-import com.iqser.red.commons.jackson.ObjectMapperFactory;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeResult;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType;
@@ -40,21 +30,15 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type;
 import com.iqser.red.service.redaction.v1.server.annotate.AnnotateRequest;
 import com.iqser.red.service.redaction.v1.server.annotate.AnnotateResponse;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.OsUtils;
 import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
-import com.iqser.red.storage.commons.StorageAutoConfiguration;
-import com.iqser.red.storage.commons.service.StorageService;
-import com.iqser.red.storage.commons.utils.FileSystemBackedStorageService;
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
-import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingServiceProcessorConfiguration;
 import com.knecon.fforesight.tenantcommons.TenantContext;
 
 @ExtendWith(SpringExtension.class)
 @SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT, properties = {"application.type=DocuMine"})
-@Import(DocumineFloraTest.RedactionIntegrationTestConfiguration.class)
 public class DocumineFloraTest extends AbstractRedactionIntegrationTest {
 
     private static final String RULES = loadFromClassPath("drools/documine_flora.drl");
@@ -192,22 +176,6 @@ public class DocumineFloraTest extends AbstractRedactionIntegrationTest {
     }
 
 
-    @Configuration
-    @EnableAutoConfiguration(exclude = {RabbitAutoConfiguration.class})
-    @Import(LayoutParsingServiceProcessorConfiguration.class)
-    @ComponentScan(excludeFilters = {@ComponentScan.Filter(type = FilterType.ASSIGNABLE_TYPE, value = StorageAutoConfiguration.class)})
-    static class RedactionIntegrationTestConfiguration {
-
-        @Bean
-        @Primary
-        public StorageService inmemoryStorage() {
-
-            return new FileSystemBackedStorageService(ObjectMapperFactory.create());
-        }
-
-    }
-
-
     @BeforeEach
     public void stubClients() {
 
@@ -277,9 +245,7 @@ public class DocumineFloraTest extends AbstractRedactionIntegrationTest {
                                      .fileId(TEST_FILE_ID)
                                      .build());
         });
-        request.setManualRedactions(ManualRedactions.builder()
+        request.setManualRedactions(ManualRedactions.builder().idsToRemove(idRemovalSet).build());
-                                            .idsToRemove(idRemovalSet)
-                                            .build());
 
         analyzeService.reanalyze(request);
 
@@ -307,12 +273,20 @@ public class DocumineFloraTest extends AbstractRedactionIntegrationTest {
         System.out.println("Finished analysis");
 
         var componentLog = redactionStorageService.getComponentLog(TEST_DOSSIER_ID, TEST_FILE_ID);
-        var doseMortality = componentLog.getComponentLogEntries().stream().filter(componentLogEntry -> componentLogEntry.getName().equals("Dose_Mortality")).findFirst().get();
+        var doseMortality = componentLog.getComponentLogEntries()
+                .stream()
+                .filter(componentLogEntry -> componentLogEntry.getName().equals("Dose_Mortality"))
+                .findFirst()
+                .get();
 
         assertEquals(doseMortality.getValues().size(), 5);
 
         Pattern pattern = Pattern.compile("^5000, [SD]$");
-        boolean allMatch = doseMortality.getValues().stream().map(ComponentLogEntryValue::getValue).allMatch(pattern.asPredicate());
+        boolean allMatch = doseMortality.getValues()
+                .stream()
+                .map(ComponentLogEntryValue::getValue)
+                .allMatch(pattern.asPredicate());
         assertTrue(allMatch);
     }
+
 }

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/MapEntityLogToDocumentDataTest.java

@@ -24,15 +24,7 @@ import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.extension.ExtendWith;
 import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
-import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration;
 import org.springframework.boot.test.context.SpringBootTest;
-import org.springframework.context.annotation.Bean;
-import org.springframework.context.annotation.ComponentScan;
-import org.springframework.context.annotation.Configuration;
-import org.springframework.context.annotation.FilterType;
-import org.springframework.context.annotation.Import;
-import org.springframework.context.annotation.Primary;
 import org.springframework.test.context.junit.jupiter.SpringExtension;
 
 import com.fasterxml.jackson.databind.ObjectMapper;
@@ -53,15 +45,11 @@ import com.iqser.red.service.redaction.v1.server.service.DictionaryService;
 import com.iqser.red.service.redaction.v1.server.service.EntityLogCreatorService;
 import com.iqser.red.service.redaction.v1.server.service.document.DocumentGraphMapper;
 import com.iqser.red.service.redaction.v1.server.service.document.EntityFindingUtility;
-import com.iqser.red.storage.commons.StorageAutoConfiguration;
-import com.iqser.red.storage.commons.service.StorageService;
-import com.iqser.red.storage.commons.utils.FileSystemBackedStorageService;
 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPageProto;
 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentPositionDataProto;
 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentStructureProto;
 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentStructureWrapper;
 import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.DocumentTextDataProto;
-import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingServiceProcessorConfiguration;
 import com.knecon.fforesight.tenantcommons.TenantContext;
 
 import lombok.SneakyThrows;
@@ -71,7 +59,6 @@ import lombok.extern.slf4j.Slf4j;
 @Slf4j
 @ExtendWith(SpringExtension.class)
 @SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
-@Import(RedactionIntegrationTest.RedactionIntegrationTestConfiguration.class)
 public class MapEntityLogToDocumentDataTest extends AbstractRedactionIntegrationTest {
 
     public static final int MATCH_THRESHOLD = 25;
@@ -91,21 +78,6 @@ public class MapEntityLogToDocumentDataTest extends AbstractRedactionIntegration
     private AtomicLong totalNotFoundEntities;
     private AtomicLong totalRemovedDuplicates;
 
-    @Configuration
-    @EnableAutoConfiguration(exclude = {RabbitAutoConfiguration.class})
-    @Import({LayoutParsingServiceProcessorConfiguration.class})
-    @ComponentScan(excludeFilters = {@ComponentScan.Filter(type = FilterType.ASSIGNABLE_TYPE, value = StorageAutoConfiguration.class)})
-    public static class RedactionIntegrationTestConfiguration {
-
-        @Bean
-        @Primary
-        public StorageService inmemoryStorage() {
-
-            return new FileSystemBackedStorageService(ObjectMapperFactory.create());
-        }
-
-    }
-
 
     @BeforeEach
     public void stubClients() {

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionAcceptanceTest.java

@@ -2,7 +2,6 @@ package com.iqser.red.service.redaction.v1.server;
 
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.junit.jupiter.api.Assertions.assertEquals;
-import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.Mockito.when;
 
 import java.io.FileOutputStream;
@@ -18,19 +17,10 @@ import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.extension.ExtendWith;
 import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
-import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration;
 import org.springframework.boot.test.context.SpringBootTest;
-import org.springframework.context.annotation.Bean;
-import org.springframework.context.annotation.ComponentScan;
-import org.springframework.context.annotation.Configuration;
-import org.springframework.context.annotation.FilterType;
-import org.springframework.context.annotation.Import;
-import org.springframework.context.annotation.Primary;
 import org.springframework.core.io.ClassPathResource;
 import org.springframework.test.context.junit.jupiter.SpringExtension;
 
-import com.iqser.red.commons.jackson.ObjectMapperFactory;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeResult;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType;
@@ -51,18 +41,13 @@ import com.iqser.red.service.redaction.v1.server.redaction.utils.OsUtils;
 import com.iqser.red.service.redaction.v1.server.service.DictionaryCacheService;
 import com.iqser.red.service.redaction.v1.server.service.DictionaryService;
 import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
-import com.iqser.red.storage.commons.StorageAutoConfiguration;
-import com.iqser.red.storage.commons.service.StorageService;
-import com.iqser.red.storage.commons.utils.FileSystemBackedStorageService;
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
-import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingServiceProcessorConfiguration;
 import com.knecon.fforesight.tenantcommons.TenantContext;
 
 import lombok.SneakyThrows;
 
 @ExtendWith(SpringExtension.class)
 @SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
-@Import(RedactionIntegrationTest.RedactionIntegrationTestConfiguration.class)
 public class RedactionAcceptanceTest extends AbstractRedactionIntegrationTest {
 
     private static final String RULES = loadFromClassPath("drools/acceptance_rules.drl");
@@ -341,7 +326,7 @@ public class RedactionAcceptanceTest extends AbstractRedactionIntegrationTest {
         var entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
 
         var desireeEtAl = findEntityByTypeAndValue(entityLog, "CBI_author", "Desiree").filter(e -> e.getEntryType().equals(EntryType.ENTITY))
-                .filter(e -> e.getMatchedRule().startsWith("CBI.7"))
+                .filter(e -> e.getMatchedRule().startsWith("CBI.0"))
                 .findAny()
                 .orElseThrow();
         IdRemoval removal = buildIdRemoval(desireeEtAl.getId());

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java

@@ -6,8 +6,6 @@ import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.ArgumentMatchers.anyString;
-import static org.mockito.Mockito.times;
-import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
 
 import java.io.BufferedWriter;
@@ -37,21 +35,11 @@ import org.junit.jupiter.api.Order;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.TestMethodOrder;
 import org.junit.jupiter.api.extension.ExtendWith;
-import org.mockito.Mock;
-import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
-import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration;
 import org.springframework.boot.test.context.SpringBootTest;
-import org.springframework.context.annotation.Bean;
-import org.springframework.context.annotation.ComponentScan;
-import org.springframework.context.annotation.Configuration;
-import org.springframework.context.annotation.FilterType;
-import org.springframework.context.annotation.Import;
-import org.springframework.context.annotation.Primary;
 import org.springframework.core.io.ClassPathResource;
 import org.springframework.test.context.junit.jupiter.SpringExtension;
 
 import com.fasterxml.jackson.core.type.TypeReference;
-import com.iqser.red.commons.jackson.ObjectMapperFactory;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeResult;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;
@@ -83,12 +71,7 @@ import com.iqser.red.service.redaction.v1.server.model.document.nodes.Section;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.OsUtils;
 import com.iqser.red.service.redaction.v1.server.rules.RulesIntegrationTest;
 import com.iqser.red.service.redaction.v1.server.service.document.DocumentGraphMapper;
-import com.iqser.red.service.redaction.v1.server.storage.DocumentDataFallbackService;
-import com.iqser.red.storage.commons.StorageAutoConfiguration;
-import com.iqser.red.storage.commons.service.StorageService;
-import com.iqser.red.storage.commons.utils.FileSystemBackedStorageService;
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
-import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingServiceProcessorConfiguration;
 import com.knecon.fforesight.service.layoutparser.processor.utils.ProtobufUtil;
 import com.knecon.fforesight.tenantcommons.TenantContext;
 
@@ -96,29 +79,12 @@ import lombok.SneakyThrows;
 
 @ExtendWith(SpringExtension.class)
 @SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
-@Import(RedactionIntegrationTest.RedactionIntegrationTestConfiguration.class)
 @TestMethodOrder(MethodOrderer.OrderAnnotation.class)
 public class RedactionIntegrationTest extends RulesIntegrationTest {
 
     private static final String RULES = loadFromClassPath("drools/rules.drl");
 
 
-    @Configuration
-    @EnableAutoConfiguration(exclude = {RabbitAutoConfiguration.class})
-    @Import({LayoutParsingServiceProcessorConfiguration.class})
-    @ComponentScan(excludeFilters = {@ComponentScan.Filter(type = FilterType.ASSIGNABLE_TYPE, value = StorageAutoConfiguration.class)})
-    public static class RedactionIntegrationTestConfiguration {
-
-        @Bean
-        @Primary
-        public StorageService inmemoryStorage() {
-
-            return new FileSystemBackedStorageService(ObjectMapperFactory.create());
-        }
-
-    }
-
-
     @BeforeEach
     public void stubClients() {
 
@@ -279,9 +245,9 @@ public class RedactionIntegrationTest extends RulesIntegrationTest {
         analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request);
         AnalyzeResult result = analyzeService.analyze(request);
 
-        var redactionLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
+        var entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
 
-        var values = redactionLog.getEntityLogEntry()
+        var values = entityLog.getEntityLogEntry()
                 .stream()
                 .map(EntityLogEntry::getValue)
                 .collect(Collectors.toList());
@@ -1313,6 +1279,10 @@ public class RedactionIntegrationTest extends RulesIntegrationTest {
     @Test
     public void entityIsAppliedAfterRecategorize() throws IOException {
 
+        String EFSA_SANITISATION_RULES = loadFromClassPath("drools/efsa_sanitisation.drl");
+        when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.ENTITY)).thenReturn(JSONPrimitive.of(EFSA_SANITISATION_RULES));
+
+
         AnalyzeRequest request = uploadFileToStorage("files/new/SYNGENTA_EFSA_sanitisation_GFL_v1 (1).pdf");
         ClassPathResource imageServiceResponseFileResource = new ClassPathResource("files/new/SYNGENTA_EFSA_sanitisation_GFL_v1 (1).IMAGE_INFO.json");
 
@@ -1333,9 +1303,12 @@ public class RedactionIntegrationTest extends RulesIntegrationTest {
                 .findFirst()
                 .get();
 
+
+        var newId = UUID.randomUUID().toString();
         request.setManualRedactions(ManualRedactions.builder()
                                             .legalBasisChanges(Set.of(ManualLegalBasisChange.builder()
                                                                               .annotationId("3029651d0842a625f2d23f8375c23600")
+//                                                                              .annotationId(newId)
                                                                               .section("[19, 2]: Paragraph: Contact point: LexCo Contact:")
                                                                               .value("0049 331 441 551 14")
                                                                               .requestDate(OffsetDateTime.now())
@@ -1345,6 +1318,7 @@ public class RedactionIntegrationTest extends RulesIntegrationTest {
                                                                               .build()))
                                             .recategorizations(Set.of(ManualRecategorization.builder()
                                                                               .annotationId("3029651d0842a625f2d23f8375c23600")
+//                                                                              .annotationId(newId)
                                                                               .type("CBI_author")
                                                                               .legalBasis("")
                                                                               .requestDate(OffsetDateTime.now())
@@ -1360,6 +1334,7 @@ public class RedactionIntegrationTest extends RulesIntegrationTest {
         var changedAnnotation = entityLog.getEntityLogEntry()
                 .stream()
                 .filter(entityLogEntry -> entityLogEntry.getId().equals("3029651d0842a625f2d23f8375c23600"))
+//                .filter(entityLogEntry -> entityLogEntry.getId().equals(newId))
                 .findFirst()
                 .get();
 
@@ -1770,14 +1745,14 @@ public class RedactionIntegrationTest extends RulesIntegrationTest {
 
         var entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
 
-        var listCBI_0_0 = entityLog.getEntityLogEntry()
+        var listCBI_0_3 = entityLog.getEntityLogEntry()
                 .stream()
-                .filter(entityLogEntry -> entityLogEntry.getMatchedRule().startsWith("CBI.0.0"))
+                .filter(entityLogEntry -> entityLogEntry.getMatchedRule().startsWith("CBI.0.3"))
                 .toList();
 
-        var listCBI_0_1 = entityLog.getEntityLogEntry()
+        var listCBI_0_4 = entityLog.getEntityLogEntry()
                 .stream()
-                .filter(entityLogEntry -> entityLogEntry.getMatchedRule().startsWith("CBI.0.1"))
+                .filter(entityLogEntry -> entityLogEntry.getMatchedRule().startsWith("CBI.0.4"))
                 .toList();
 
         FileAttribute vertebrateNo = FileAttribute.builder().id("fileAttributeId").label("Vertebrate Study").placeholder("{fileattributes.vertebrateStudy}").value("No").build();
@@ -1789,18 +1764,18 @@ public class RedactionIntegrationTest extends RulesIntegrationTest {
 
         var entityLog2 = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
 
-        var list2CBI_0_0 = entityLog2.getEntityLogEntry()
+        var list2CBI_0_3 = entityLog2.getEntityLogEntry()
                 .stream()
-                .filter(entityLogEntry -> entityLogEntry.getMatchedRule().startsWith("CBI.0.0"))
+                .filter(entityLogEntry -> entityLogEntry.getMatchedRule().startsWith("CBI.0.3"))
                 .toList();
 
-        var list2CBI_0_1 = entityLog2.getEntityLogEntry()
+        var list2CBI_0_4 = entityLog2.getEntityLogEntry()
                 .stream()
-                .filter(entityLogEntry -> entityLogEntry.getMatchedRule().startsWith("CBI.0.1"))
+                .filter(entityLogEntry -> entityLogEntry.getMatchedRule().startsWith("CBI.0.4"))
                 .toList();
 
-        assertEquals(listCBI_0_0.size(), list2CBI_0_1.size());
+        assertEquals(listCBI_0_3.size(), list2CBI_0_4.size());
-        assertEquals(listCBI_0_1.size(), list2CBI_0_0.size());
+        assertEquals(listCBI_0_4.size(), list2CBI_0_3.size());
 
         request.setFileAttributes(List.of(vertebrateYes));
 
@@ -1810,18 +1785,18 @@ public class RedactionIntegrationTest extends RulesIntegrationTest {
 
         var entityLog3 = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
 
-        var list3CBI_0_0 = entityLog3.getEntityLogEntry()
+        var list3CBI_0_3 = entityLog3.getEntityLogEntry()
                 .stream()
-                .filter(entityLogEntry -> entityLogEntry.getMatchedRule().startsWith("CBI.0.0"))
+                .filter(entityLogEntry -> entityLogEntry.getMatchedRule().startsWith("CBI.0.3"))
                 .toList();
 
-        var list3CBI_0_1 = entityLog3.getEntityLogEntry()
+        var list3CBI_0_4 = entityLog3.getEntityLogEntry()
                 .stream()
-                .filter(entityLogEntry -> entityLogEntry.getMatchedRule().startsWith("CBI.0.1"))
+                .filter(entityLogEntry -> entityLogEntry.getMatchedRule().startsWith("CBI.0.4"))
                 .toList();
 
-        assertEquals(list3CBI_0_0.size(), list2CBI_0_1.size());
+        assertEquals(list3CBI_0_3.size(), list2CBI_0_4.size());
-        assertEquals(list3CBI_0_1.size(), list2CBI_0_0.size());
+        assertEquals(list3CBI_0_4.size(), list2CBI_0_3.size());
     }
 
 
@@ -2013,7 +1988,7 @@ public class RedactionIntegrationTest extends RulesIntegrationTest {
                              .get("reason"), "No vertebrate found -> removed by manual override");
         assertEquals(responseDavidKsenia.getChanges()
                              .get(1).getPropertyChanges()
-                             .get("matchedRule"), "CBI.3.2 -> ");
+                             .get("matchedRule"), "CBI.13.2 -> ");
         assertEquals(responseDavidKsenia.getChanges()
                              .get(1).getPropertyChanges()
                              .get("state"), "SKIPPED -> IGNORED");
@@ -2067,7 +2042,7 @@ public class RedactionIntegrationTest extends RulesIntegrationTest {
                              .get("reason"), "No vertebrate found -> Recategorized entities are applied by default., recategorized by manual override");
         assertEquals(responseDavidKsenia.getChanges()
                              .get(1).getPropertyChanges()
-                             .get("matchedRule"), "CBI.3.2 -> MAN.3.3");
+                             .get("matchedRule"), "CBI.13.2 -> MAN.3.3");
         assertEquals(responseDavidKsenia.getChanges()
                              .get(1).getPropertyChanges()
                              .get("legalBasis"), " -> new legal basis");
@@ -2205,11 +2180,13 @@ public class RedactionIntegrationTest extends RulesIntegrationTest {
                 .get();
         var entityLogEntry2 = entityLog.getEntityLogEntry()
                 .stream()
-                .filter(entityLogEntry -> entityLogEntry.getValue().equals(assessment))
+                .filter(entityLogEntry -> entityLogEntry.getId().equals("64795d65b8654503a25f7c44fce71fcb"))
                 .findFirst()
                 .get();
-        assertEquals(entityLogEntry1.getState(), EntryState.APPLIED);
+        assertEquals(entityLogEntry1.getState(), EntryState.REMOVED);
+        assertEquals(entityLogEntry1.getMatchedRule(), "DICT.0.0");
         assertEquals(entityLogEntry2.getState(), EntryState.APPLIED);
+        assertEquals(entityLogEntry2.getMatchedRule(), "CBI.0.3");
         assertThat(entityLogEntry2.getStartOffset() > entityLogEntry1.getEndOffset());
     }
 
@@ -2252,9 +2229,7 @@ public class RedactionIntegrationTest extends RulesIntegrationTest {
     @Test
     public void testFileWithImagesAndNoText() {
 
-        AnalyzeRequest request = prepareStorage("files/new/only_images.pdf",
+        AnalyzeRequest request = prepareStorage("files/new/only_images.pdf", "files/cv_service_empty_response.json", "files/only_images_file_image_response.json");
-                                                "files/cv_service_empty_response.json",
-                                                "files/only_images_file_image_response.json");
 
         analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request);
         AnalyzeResult result = analyzeService.analyze(request);

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationV2Test.java

@@ -10,30 +10,16 @@ import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.extension.ExtendWith;
-import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
-import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration;
 import org.springframework.boot.test.context.SpringBootTest;
-import org.springframework.context.annotation.Bean;
-import org.springframework.context.annotation.ComponentScan;
-import org.springframework.context.annotation.Configuration;
-import org.springframework.context.annotation.FilterType;
-import org.springframework.context.annotation.Import;
-import org.springframework.context.annotation.Primary;
 import org.springframework.test.context.junit.jupiter.SpringExtension;
 
-import com.iqser.red.commons.jackson.ObjectMapperFactory;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type;
-import com.iqser.red.storage.commons.StorageAutoConfiguration;
-import com.iqser.red.storage.commons.service.StorageService;
-import com.iqser.red.storage.commons.utils.FileSystemBackedStorageService;
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
-import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingServiceProcessorConfiguration;
 import com.knecon.fforesight.tenantcommons.TenantContext;
 
 import lombok.SneakyThrows;
@@ -41,26 +27,10 @@ import lombok.SneakyThrows;
 @Disabled // TODO: re-enable, when redaction log is removed
 @ExtendWith(SpringExtension.class)
 @SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
-@Import(RedactionIntegrationV2Test.RedactionIntegrationTestConfiguration.class)
 public class RedactionIntegrationV2Test extends AbstractRedactionIntegrationTest {
 
     private static final String RULES = loadFromClassPath("drools/rules_v2.drl");
 
-    @Configuration
-    @EnableAutoConfiguration(exclude = {RabbitAutoConfiguration.class})
-    @Import(LayoutParsingServiceProcessorConfiguration.class)
-    @ComponentScan(excludeFilters = {@ComponentScan.Filter(type = FilterType.ASSIGNABLE_TYPE, value = StorageAutoConfiguration.class)})
-    static class RedactionIntegrationTestConfiguration {
-
-        @Bean
-        @Primary
-        public StorageService inmemoryStorage() {
-
-            return new FileSystemBackedStorageService(ObjectMapperFactory.create());
-        }
-
-    }
-
 
     @BeforeEach
     public void stubClients() {

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/entity/TextEntityTest.java

@@ -16,11 +16,22 @@ public class TextEntityTest {
         PrecursorEntity entity = PrecursorEntity.builder().type("PII").entityType(EntityType.ENTITY).build();
         entity.skip("CBI.1.0", "");
         entity.skip("CBI.2.0", "");
-        entity.skip("CBI.3.0", "");
+        entity.skip("CBI.13.1", "");
-        entity.skip("CBI.4.1", "");
+        entity.skip("CBI.13.3", "");
-        entity.skip("CBI.4.0", "");
+        assertThat(entity.getMatchedRule().getRuleIdentifier().toString()).isEqualTo("CBI.13.3");
-        assertThat(entity.getMatchedRule().getRuleIdentifier().toString()).isEqualTo("CBI.4.1");
+        assertThat(entity.getMatchedRuleUnit()).isEqualTo(13);
-        assertThat(entity.getMatchedRuleUnit()).isEqualTo(4);
+    }
+
+    @Test
+    public void testMatchedRule2() {
+
+        PrecursorEntity entity = PrecursorEntity.builder().type("PII").entityType(EntityType.ENTITY).build();
+        entity.redact("PII.4.1", "", "legal");
+        entity.apply("MAN.3.0", "");
+        entity.apply("MAN.3.3", "");
+        entity.skip("CBI.13.2", "");
+        assertThat(entity.getMatchedRule().getRuleIdentifier().toString()).isEqualTo("CBI.13.2");
+        assertThat(entity.getMatchedRuleUnit()).isEqualTo(13);
     }
 
 

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/BuildDocumentIntegrationTest.java

@@ -1,52 +1,22 @@
 package com.iqser.red.service.redaction.v1.server.document.graph;
 
 import org.junit.jupiter.api.extension.ExtendWith;
-import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
-import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration;
 import org.springframework.boot.test.context.SpringBootTest;
-import org.springframework.context.annotation.Bean;
-import org.springframework.context.annotation.ComponentScan;
-import org.springframework.context.annotation.Configuration;
-import org.springframework.context.annotation.FilterType;
-import org.springframework.context.annotation.Import;
-import org.springframework.context.annotation.Primary;
 import org.springframework.test.context.junit.jupiter.SpringExtension;
 
-import com.iqser.red.commons.jackson.ObjectMapperFactory;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
 import com.iqser.red.service.redaction.v1.server.AbstractRedactionIntegrationTest;
 import com.iqser.red.service.redaction.v1.server.Application;
-import com.iqser.red.service.redaction.v1.server.service.document.DocumentGraphMapper;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
-import com.iqser.red.storage.commons.StorageAutoConfiguration;
+import com.iqser.red.service.redaction.v1.server.service.document.DocumentGraphMapper;
-import com.iqser.red.storage.commons.service.StorageService;
-import com.iqser.red.storage.commons.utils.FileSystemBackedStorageService;
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
-import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingServiceProcessorConfiguration;
 
 import lombok.SneakyThrows;
 
 @ExtendWith(SpringExtension.class)
 @SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
-@Import(BuildDocumentIntegrationTest.TestConfiguration.class)
 public class BuildDocumentIntegrationTest extends AbstractRedactionIntegrationTest {
 
-    @Configuration
-    @EnableAutoConfiguration(exclude = {RabbitAutoConfiguration.class})
-    @Import(LayoutParsingServiceProcessorConfiguration.class)
-    @ComponentScan(excludeFilters = {@ComponentScan.Filter(type = FilterType.ASSIGNABLE_TYPE, value = StorageAutoConfiguration.class)})
-    public static class TestConfiguration {
-
-        @Bean
-        @Primary
-        public StorageService inmemoryStorage() {
-
-            return new FileSystemBackedStorageService(ObjectMapperFactory.create());
-        }
-
-    }
-
-
     @SneakyThrows
     protected Document buildGraph(String filename) {
 

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentPerformanceIntegrationTest.java

@@ -16,17 +16,9 @@ import org.apache.pdfbox.Loader;
 import org.apache.pdfbox.pdmodel.PDDocument;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
-import org.kie.api.KieServices;
-import org.kie.api.builder.KieBuilder;
-import org.kie.api.builder.KieFileSystem;
-import org.kie.api.builder.KieModule;
 import org.kie.api.runtime.KieContainer;
-import org.kie.internal.io.ResourceFactory;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Qualifier;
-import org.springframework.context.annotation.Bean;
-import org.springframework.context.annotation.Configuration;
-import org.springframework.context.annotation.Import;
 import org.springframework.core.io.ClassPathResource;
 
 import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;
@@ -44,6 +36,7 @@ import com.iqser.red.service.redaction.v1.server.model.document.nodes.Page;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Section;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
 import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
+import com.iqser.red.service.redaction.v1.server.rules.RulesIntegrationTest;
 import com.iqser.red.service.redaction.v1.server.service.DictionaryService;
 import com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService;
 import com.iqser.red.service.redaction.v1.server.service.document.EntityEnrichmentService;
@@ -53,8 +46,7 @@ import com.knecon.fforesight.tenantcommons.TenantContext;
 
 import lombok.SneakyThrows;
 
-@Import(DocumentPerformanceIntegrationTest.TestConfiguration.class)
+public class DocumentPerformanceIntegrationTest extends RulesIntegrationTest {
-public class DocumentPerformanceIntegrationTest extends BuildDocumentIntegrationTest {
 
     private static final String RULES = "drools/rules.drl";
 
@@ -72,26 +64,6 @@ public class DocumentPerformanceIntegrationTest extends BuildDocumentIntegration
     @Autowired
     private KieContainer kieContainer;
 
-    @Configuration
-    @Import(BuildDocumentIntegrationTest.TestConfiguration.class)
-    public static class TestConfiguration {
-
-        @Bean
-        public KieContainer kieContainer() {
-
-            KieServices kieServices = KieServices.Factory.get();
-
-            KieFileSystem kieFileSystem = kieServices.newKieFileSystem();
-            kieFileSystem.write(ResourceFactory.newClassPathResource(RULES));
-            KieBuilder kieBuilder = kieServices.newKieBuilder(kieFileSystem);
-            kieBuilder.buildAll();
-            KieModule kieModule = kieBuilder.getKieModule();
-
-            return kieServices.newKieContainer(kieModule.getReleaseId());
-        }
-
-    }
-
 
     @BeforeEach
     public void stubClients() {

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesEnd2EndTest.java

@@ -24,18 +24,9 @@ import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.extension.ExtendWith;
 import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
-import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration;
 import org.springframework.boot.test.context.SpringBootTest;
-import org.springframework.context.annotation.Bean;
-import org.springframework.context.annotation.ComponentScan;
-import org.springframework.context.annotation.Configuration;
-import org.springframework.context.annotation.FilterType;
-import org.springframework.context.annotation.Import;
-import org.springframework.context.annotation.Primary;
 import org.springframework.test.context.junit.jupiter.SpringExtension;
 
-import com.iqser.red.commons.jackson.ObjectMapperFactory;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeResult;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType;
@@ -53,7 +44,6 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Point;
 import com.iqser.red.service.redaction.v1.server.AbstractRedactionIntegrationTest;
 import com.iqser.red.service.redaction.v1.server.Application;
@@ -65,11 +55,7 @@ import com.iqser.red.service.redaction.v1.server.redaction.utils.OsUtils;
 import com.iqser.red.service.redaction.v1.server.service.document.DocumentGraphMapper;
 import com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService;
 import com.iqser.red.service.redaction.v1.server.service.document.EntityEnrichmentService;
-import com.iqser.red.storage.commons.StorageAutoConfiguration;
-import com.iqser.red.storage.commons.service.StorageService;
-import com.iqser.red.storage.commons.utils.FileSystemBackedStorageService;
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
-import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingServiceProcessorConfiguration;
 import com.knecon.fforesight.tenantcommons.TenantContext;
 
 import lombok.SneakyThrows;
@@ -77,7 +63,6 @@ import lombok.SneakyThrows;
 @Disabled // TODO: only disabled due to OOM in gradle testRunner in GitLab, find a solution!
 @ExtendWith(SpringExtension.class)
 @SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
-@Import(ManualChangesEnd2EndTest.TestConfiguration.class)
 public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest {
 
     private static final String RULES = loadFromClassPath("drools/acceptance_rules.drl");
@@ -87,21 +72,6 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest {
 
     private EntityCreationService entityCreationService;
 
-    @Configuration
-    @EnableAutoConfiguration(exclude = {RabbitAutoConfiguration.class})
-    @Import(LayoutParsingServiceProcessorConfiguration.class)
-    @ComponentScan(excludeFilters = {@ComponentScan.Filter(type = FilterType.ASSIGNABLE_TYPE, value = StorageAutoConfiguration.class)})
-    public static class TestConfiguration {
-
-        @Bean
-        @Primary
-        public StorageService inmemoryStorage() {
-
-            return new FileSystemBackedStorageService(ObjectMapperFactory.create());
-        }
-
-    }
-
 
     @BeforeEach
     public void createServices() {
@@ -352,7 +322,7 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest {
                 .findFirst()
                 .get();
 
-        assertEquals("CBI.7.0", asyaLyon.getMatchedRule());
+        assertEquals("CBI.0.0", asyaLyon.getMatchedRule());
         assertEquals("Published Information found in section", asyaLyon.getReason());
         assertNotEquals(asyaLyon.getState(), EntryState.APPLIED);
 

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/PrecursorEntityTest.java

@@ -18,6 +18,7 @@ import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.boot.test.mock.mockito.MockBean;
 
 import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogChanges;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle;
@@ -134,15 +135,17 @@ public class PrecursorEntityTest extends BuildDocumentIntegrationTest {
         assertEquals(1, notFoundManualEntities.size());
         assertTrue(document.getEntities().isEmpty());
 
-        List<EntityLogEntry> redactionLogEntries = entityLogCreatorService.createInitialEntityLog(new AnalyzeRequest(),
+        EntityLogChanges entityLogChanges = entityLogCreatorService.createInitialEntityLog(new AnalyzeRequest(),
-                                                                                                  document,
+                                                                                           document,
-                                                                                                  notFoundManualEntities,
+                                                                                           notFoundManualEntities,
-                                                                                                  new DictionaryVersion(),
+                                                                                           new DictionaryVersion(),
-                                                                                                  0L).getEntityLog().getEntityLogEntry();
+                                                                                           0L);
+        List<EntityLogEntry> entityLogEntries = entityLogChanges.getEntityLog().getEntityLogEntry();
+
+        assertEquals(1, entityLogEntries.size());
+        assertEquals(value, entityLogEntries.get(0).getValue());
+        assertEquals(type, entityLogEntries.get(0).getType());
 
-        assertEquals(1, redactionLogEntries.size());
-        assertEquals(value, redactionLogEntries.get(0).getValue());
-        assertEquals(type, redactionLogEntries.get(0).getType());
         return new DocumentAndEntity(document, notFoundManualEntities.get(0));
     }
 

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/realdata/LiveDataIntegrationTest.java

@@ -20,19 +20,11 @@ import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.extension.ExtendWith;
 import org.springframework.amqp.rabbit.core.RabbitAdmin;
 import org.springframework.amqp.rabbit.core.RabbitTemplate;
-import org.springframework.amqp.rabbit.listener.RabbitListenerEndpointRegistry;
 import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
-import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration;
 import org.springframework.boot.test.autoconfigure.actuate.observability.AutoConfigureObservability;
 import org.springframework.boot.test.context.SpringBootTest;
 import org.springframework.boot.test.mock.mockito.MockBean;
-import org.springframework.context.annotation.Bean;
-import org.springframework.context.annotation.ComponentScan;
-import org.springframework.context.annotation.Configuration;
-import org.springframework.context.annotation.FilterType;
 import org.springframework.context.annotation.Import;
-import org.springframework.context.annotation.Primary;
 import org.springframework.core.io.ClassPathResource;
 import org.springframework.core.io.Resource;
 import org.springframework.core.io.support.ResourcePatternResolver;
@@ -41,12 +33,12 @@ import org.springframework.test.context.junit.jupiter.SpringExtension;
 
 import com.fasterxml.jackson.core.type.TypeReference;
 import com.fasterxml.jackson.databind.ObjectMapper;
-import com.iqser.red.commons.jackson.ObjectMapperFactory;
 import com.iqser.red.service.dictionarymerge.commons.DictionaryEntry;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.configuration.Colors;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type;
+import com.iqser.red.service.redaction.v1.server.AbstractRedactionIntegrationTest;
 import com.iqser.red.service.redaction.v1.server.Application;
 import com.iqser.red.service.redaction.v1.server.RedactionServiceSettings;
 import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
@@ -58,9 +50,6 @@ import com.iqser.red.service.redaction.v1.server.service.DictionaryService;
 import com.iqser.red.service.redaction.v1.server.service.websocket.RedisSyncedWebSocketService;
 import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
 import com.iqser.red.service.redaction.v1.server.utils.ExceptionProvider;
-import com.iqser.red.storage.commons.StorageAutoConfiguration;
-import com.iqser.red.storage.commons.service.StorageService;
-import com.iqser.red.storage.commons.utils.FileSystemBackedStorageService;
 import com.knecon.fforesight.keycloakcommons.security.TenantAuthenticationManagerResolver;
 import com.knecon.fforesight.tenantcommons.TenantContext;
 import com.knecon.fforesight.tenantcommons.TenantsClient;
@@ -70,7 +59,7 @@ import lombok.SneakyThrows;
 
 @ExtendWith(SpringExtension.class)
 @SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
-@Import(LiveDataIntegrationTest.RedactionIntegrationTestConfiguration.class)
+@Import(AbstractRedactionIntegrationTest.TestConfiguration.class)
 @AutoConfigureObservability
 public class LiveDataIntegrationTest {
 
@@ -88,9 +77,6 @@ public class LiveDataIntegrationTest {
     @MockBean
     protected RabbitAdmin rabbitAdmin;
 
-    @MockBean
-    protected RabbitListenerEndpointRegistry rabbitListenerEndpointRegistry;
-
     @MockBean
     private TenantsClient tenantsClient;
 
@@ -135,20 +121,6 @@ public class LiveDataIntegrationTest {
 
     private List<Type> types;
 
-    @Configuration
-    @EnableAutoConfiguration(exclude = {RabbitAutoConfiguration.class})
-    @ComponentScan(excludeFilters = {@ComponentScan.Filter(type = FilterType.ASSIGNABLE_TYPE, value = StorageAutoConfiguration.class)})
-    public static class RedactionIntegrationTestConfiguration {
-
-        @Bean
-        @Primary
-        public StorageService inmemoryStorage() {
-
-            return new FileSystemBackedStorageService(ObjectMapperFactory.create());
-        }
-
-    }
-
 
     @SneakyThrows
     @BeforeEach

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/rules/RulesIntegrationTest.java

@@ -19,6 +19,7 @@ import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;
 import org.springframework.context.annotation.Import;
 
+import com.iqser.red.service.redaction.v1.server.AbstractRedactionIntegrationTest;
 import com.iqser.red.service.redaction.v1.server.document.graph.BuildDocumentIntegrationTest;
 import com.iqser.red.service.redaction.v1.server.model.dictionary.Dictionary;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
@@ -27,6 +28,7 @@ import com.iqser.red.service.redaction.v1.server.service.document.EntityCreation
 import com.iqser.red.service.redaction.v1.server.service.document.EntityEnrichmentService;
 
 @ExtendWith(MockitoExtension.class)
+@Import(RulesIntegrationTest.TestConfiguration.class)
 public class RulesIntegrationTest extends BuildDocumentIntegrationTest {
 
     protected static final String RULES = "drools/rules.drl";
@@ -55,7 +57,7 @@ public class RulesIntegrationTest extends BuildDocumentIntegrationTest {
 
 
     @Configuration
-    @Import(BuildDocumentIntegrationTest.TestConfiguration.class)
+    @Import(AbstractRedactionIntegrationTest.TestConfiguration.class)
     public static class TestConfiguration {
 
         @Bean

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/service/document/DocumentSearchServiceTest.java

@@ -14,19 +14,10 @@ import org.junit.jupiter.api.extension.ExtendWith;
 import org.mockito.ArgumentCaptor;
 import org.mockito.Captor;
 import org.springframework.amqp.rabbit.core.RabbitTemplate;
-import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
-import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration;
 import org.springframework.boot.test.context.SpringBootTest;
 import org.springframework.boot.test.mock.mockito.SpyBean;
-import org.springframework.context.annotation.Bean;
-import org.springframework.context.annotation.ComponentScan;
-import org.springframework.context.annotation.Configuration;
-import org.springframework.context.annotation.FilterType;
-import org.springframework.context.annotation.Import;
-import org.springframework.context.annotation.Primary;
 import org.springframework.test.context.junit.jupiter.SpringExtension;
 
-import com.iqser.red.commons.jackson.ObjectMapperFactory;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.BulkLocalRequest;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.BulkLocalResponse;
@@ -36,38 +27,17 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSON
 import com.iqser.red.service.redaction.v1.model.QueueNames;
 import com.iqser.red.service.redaction.v1.server.AbstractRedactionIntegrationTest;
 import com.iqser.red.service.redaction.v1.server.Application;
-import com.iqser.red.service.redaction.v1.server.RedactionIntegrationTest;
-import com.iqser.red.storage.commons.StorageAutoConfiguration;
-import com.iqser.red.storage.commons.service.StorageService;
-import com.iqser.red.storage.commons.utils.FileSystemBackedStorageService;
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
-import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingServiceProcessorConfiguration;
 import com.knecon.fforesight.tenantcommons.TenantContext;
 
 import lombok.SneakyThrows;
 
 @ExtendWith(SpringExtension.class)
 @SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
-@Import(RedactionIntegrationTest.RedactionIntegrationTestConfiguration.class)
 public class DocumentSearchServiceTest extends AbstractRedactionIntegrationTest {
 
     private static final String RULES = loadFromClassPath("drools/acceptance_rules.drl");
 
-    @Configuration
-    @EnableAutoConfiguration(exclude = {RabbitAutoConfiguration.class})
-    @Import({LayoutParsingServiceProcessorConfiguration.class})
-    @ComponentScan(excludeFilters = {@ComponentScan.Filter(type = FilterType.ASSIGNABLE_TYPE, value = StorageAutoConfiguration.class)})
-    public static class RedactionIntegrationTestConfiguration {
-
-        @Bean
-        @Primary
-        public StorageService inmemoryStorage() {
-
-            return new FileSystemBackedStorageService(ObjectMapperFactory.create());
-        }
-
-    }
-
     @Captor
     ArgumentCaptor<BulkLocalResponse> captor;
 

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/service/document/UnprocessedChangesServiceTest.java

@@ -3,7 +3,6 @@ package com.iqser.red.service.redaction.v1.server.service.document;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertFalse;
 import static org.junit.jupiter.api.Assertions.assertTrue;
-import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.ArgumentMatchers.eq;
 import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
@@ -21,19 +20,10 @@ import org.junit.jupiter.api.extension.ExtendWith;
 import org.mockito.ArgumentCaptor;
 import org.mockito.Captor;
 import org.springframework.amqp.rabbit.core.RabbitTemplate;
-import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
-import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration;
 import org.springframework.boot.test.context.SpringBootTest;
 import org.springframework.boot.test.mock.mockito.SpyBean;
-import org.springframework.context.annotation.Bean;
-import org.springframework.context.annotation.ComponentScan;
-import org.springframework.context.annotation.Configuration;
-import org.springframework.context.annotation.FilterType;
-import org.springframework.context.annotation.Import;
-import org.springframework.context.annotation.Primary;
 import org.springframework.test.context.junit.jupiter.SpringExtension;
 
-import com.iqser.red.commons.jackson.ObjectMapperFactory;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.RuleFileType;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
@@ -41,44 +31,22 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type;
 import com.iqser.red.service.redaction.v1.model.AnalyzeResponse;
 import com.iqser.red.service.redaction.v1.model.QueueNames;
 import com.iqser.red.service.redaction.v1.model.UnprocessedManualEntity;
 import com.iqser.red.service.redaction.v1.server.AbstractRedactionIntegrationTest;
 import com.iqser.red.service.redaction.v1.server.Application;
-import com.iqser.red.service.redaction.v1.server.RedactionIntegrationTest;
-import com.iqser.red.storage.commons.StorageAutoConfiguration;
-import com.iqser.red.storage.commons.service.StorageService;
-import com.iqser.red.storage.commons.utils.FileSystemBackedStorageService;
 import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
-import com.knecon.fforesight.service.layoutparser.processor.LayoutParsingServiceProcessorConfiguration;
 import com.knecon.fforesight.tenantcommons.TenantContext;
 
 import lombok.SneakyThrows;
 
 @ExtendWith(SpringExtension.class)
 @SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
-@Import(RedactionIntegrationTest.RedactionIntegrationTestConfiguration.class)
 public class UnprocessedChangesServiceTest extends AbstractRedactionIntegrationTest {
 
     private static final String RULES = loadFromClassPath("drools/acceptance_rules.drl");
 
-    @Configuration
-    @EnableAutoConfiguration(exclude = {RabbitAutoConfiguration.class})
-    @Import({LayoutParsingServiceProcessorConfiguration.class})
-    @ComponentScan(excludeFilters = {@ComponentScan.Filter(type = FilterType.ASSIGNABLE_TYPE, value = StorageAutoConfiguration.class)})
-    public static class RedactionIntegrationTestConfiguration {
-
-        @Bean
-        @Primary
-        public StorageService inmemoryStorage() {
-
-            return new FileSystemBackedStorageService(ObjectMapperFactory.create());
-        }
-
-    }
-
     @Captor
     ArgumentCaptor<AnalyzeResponse> captor;
 

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/acceptance_rules.drl

@@ -85,20 +85,58 @@ rule "SYN.1.0: Recommend CTL/BL laboratory that start with BL or CTL"
 //------------------------------------ CBI rules ------------------------------------
 
 // Rule unit: CBI.0
-rule "CBI.0.0: Redact CBI Authors (non vertebrate Study)"
+rule "CBI.0.0: Add CBI_author with \"et al.\" RegEx"
+    agenda-group "LOCAL_DICTIONARY_ADDS"
+    when
+        $section: Section(containsString("et al."))
+    then
+        entityCreationService.byRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, 1, $section)
+            .forEach(entity -> {
+                entity.skip("CBI.0.0", "Author found by \"et al\" regex");
+                dictionary.recommendEverywhere(entity);
+            });
+    end
+
+rule "CBI.0.1: Add CBI_author with \"et al.\" RegEx (non vertebrate study)"
+    agenda-group "LOCAL_DICTIONARY_ADDS"
+    when
+        not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
+        $section: Section(containsString("et al."))
+    then
+        entityCreationService.byRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, 1, $section)
+            .forEach(entity -> {
+                entity.redact("CBI.0.1", "Author found by \"et al\" regex", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+                dictionary.recommendEverywhere(entity);
+            });
+    end
+
+rule "CBI.0.2: Add CBI_author with \"et al.\" RegEx (vertebrate study)"
+    agenda-group "LOCAL_DICTIONARY_ADDS"
+    when
+        FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
+        $section: Section(containsString("et al."))
+    then
+        entityCreationService.byRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, 1, $section)
+            .forEach(entity -> {
+                entity.redact("CBI.0.2", "Author found by \"et al\" regex", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+                dictionary.recommendEverywhere(entity);
+            });
+    end
+
+rule "CBI.0.3: Redact CBI Authors (non vertebrate Study)"
     when
         not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
         $entity: TextEntity(type() == "CBI_author", dictionaryEntry)
     then
-        $entity.redact("CBI.0.0", "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        $entity.redact("CBI.0.3", "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
     end
 
-rule "CBI.0.1: Redact CBI Authors (vertebrate Study)"
+rule "CBI.0.4: Redact CBI Authors (vertebrate Study)"
     when
         FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
         $entity: TextEntity(type() == "CBI_author", dictionaryEntry)
     then
-        $entity.redact("CBI.0.1", "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        $entity.redact("CBI.0.4", "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
     end
 
 
@@ -130,58 +168,6 @@ rule "CBI.2.0: Do not redact genitive CBI Author"
     end
 
 
-// Rule unit: CBI.7
-rule "CBI.7.0: Add CBI_author with \"et al.\" RegEx"
-    agenda-group "LOCAL_DICTIONARY_ADDS"
-    when
-        $section: Section(containsString("et al."))
-    then
-        entityCreationService.byRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, 1, $section)
-            .forEach(entity -> {
-                entity.redact("CBI.7.0", "Author found by \"et al\" regex", "Reg (EC) No 1107/2009 Art. 63 (2g)");
-                dictionary.recommendEverywhere(entity);
-            });
-    end
-
-rule "CBI.7.1: Add CBI_author with \"et al.\" RegEx (non vertebrate study)"
-    agenda-group "LOCAL_DICTIONARY_ADDS"
-    when
-        not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
-        $section: Section(containsString("et al."))
-    then
-        entityCreationService.byRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, 1, $section)
-            .forEach(entity -> {
-                entity.redact("CBI.7.1", "Author found by \"et al\" regex", "Article 39(e)(3) of Regulation (EC) No 178/2002");
-                dictionary.recommendEverywhere(entity);
-            });
-    end
-
-rule "CBI.7.2: Add CBI_author with \"et al.\" RegEx (vertebrate study)"
-    agenda-group "LOCAL_DICTIONARY_ADDS"
-    when
-        FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
-        $section: Section(containsString("et al."))
-    then
-        entityCreationService.byRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, 1, $section)
-            .forEach(entity -> {
-                entity.redact("CBI.7.2", "Author found by \"et al\" regex", "Article 39(e)(2) of Regulation (EC) No 178/2002");
-                dictionary.recommendEverywhere(entity);
-            });
-    end
-
-rule "CBI.7.3: Add CBI_author with \"et al.\" RegEx"
-    agenda-group "LOCAL_DICTIONARY_ADDS"
-    when
-        $section: Section(containsString("et al."))
-    then
-        entityCreationService.byRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, 1, $section)
-            .forEach(entity -> {
-                entity.redact("CBI.7.3", "Author found by \"et al\" regex", "Article 4(1)(b), Regulation (EC) No 1049/2001 (Personal data)");
-                dictionary.recommendEverywhere(entity);
-            });
-    end
-
-
 // Rule unit: CBI.9
 rule "CBI.9.0: Redact all cells with Header Author(s) as CBI_author (non vertebrate study)"
     agenda-group "LOCAL_DICTIONARY_ADDS"
@@ -287,7 +273,6 @@ rule "CBI.11.0: Recommend all CBI_author entities in Table with Vertebrate Study
         $table.getEntitiesOfType("CBI_author").stream().filter(IEntity::applied).forEach(entity -> dictionary.addMultipleAuthorsAsRecommendation(entity));
     end
 
-
 // Rule unit: CBI.16
 rule "CBI.16.0: Do not redact Names and Addresses if published information found in Section without tables"
     when
@@ -366,9 +351,9 @@ rule "CBI.20.0: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJEC
     when
         $section: Section(!hasTables(), containsString("PERFORMING LABORATORY:"), containsString("LABORATORY PROJECT ID:"))
     then
-        entityCreationService.betweenStrings("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", EntityType.ENTITY, $section)
+        entityCreationService.betweenStrings("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "PII", EntityType.ENTITY, $section)
          .forEach(laboratoryEntity -> {
-             laboratoryEntity.redact("CBI.20.0", "PERFORMING LABORATORY was found", "Reg (EC) No 1107/2009 Art. 63 (2g)");
+             laboratoryEntity.redact("CBI.20.0", "PERFORMING LABORATORY was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
              dictionary.recommendEverywhere(laboratoryEntity);
          });
     end
@@ -418,7 +403,7 @@ rule "CBI.23.0: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\" (no
         not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
         $document: Document(containsStringIgnoreCase("AUTHOR(S)"), containsAnyStringIgnoreCase("COMPLETION DATE", "STUDY COMPLETION DATE"))
     then
-        entityCreationService.shortestBetweenAnyStringIgnoreCase(List.of("AUTHOR(S)", "AUTHOR(S):"), List.of("COMPLETION DATE", "COMPLETION DATE:", "STUDY COMPLETION DATE", "STUDY COMPLETION DATE:"), "CBI_author", EntityType.ENTITY, $document)
+        entityCreationService.shortestBetweenAnyStringIgnoreCase(List.of("AUTHOR(S)", "AUTHOR(S):"), List.of("COMPLETION DATE", "COMPLETION DATE:", "STUDY COMPLETION DATE", "STUDY COMPLETION DATE:"), "CBI_author", EntityType.ENTITY, $document, 200)
          .forEach(authorEntity -> authorEntity.redact("CBI.23.0", "AUTHOR(S) was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
     end
 
@@ -427,7 +412,7 @@ rule "CBI.23.1: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\" (ve
         FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
         $document: Document(containsStringIgnoreCase("AUTHOR(S)"), containsAnyStringIgnoreCase("COMPLETION DATE", "STUDY COMPLETION DATE"))
     then
-        entityCreationService.shortestBetweenAnyStringIgnoreCase(List.of("AUTHOR(S)", "AUTHOR(S):"), List.of("COMPLETION DATE", "COMPLETION DATE:", "STUDY COMPLETION DATE", "STUDY COMPLETION DATE:"), "CBI_author", EntityType.ENTITY, $document)
+        entityCreationService.shortestBetweenAnyStringIgnoreCase(List.of("AUTHOR(S)", "AUTHOR(S):"), List.of("COMPLETION DATE", "COMPLETION DATE:", "STUDY COMPLETION DATE", "STUDY COMPLETION DATE:"), "CBI_author", EntityType.ENTITY, $document, 200)
         .forEach(authorEntity -> authorEntity.redact("CBI.23.1", "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
     end
 
@@ -439,7 +424,7 @@ rule "PII.0.0: Redact all PII"
     when
         $pii: TextEntity(type() == "PII", dictionaryEntry)
     then
-        $pii.redact("PII.0.0", "Personal Information found", "Reg (EC) No 1107/2009 Art. 63 (2g)");
+        $pii.redact("PII.0.0", "Personal Information found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
     end
 
 rule "PII.0.1: Redact all PII (non vertebrate study)"
@@ -524,7 +509,7 @@ rule "PII.2.0: Redact Phone and Fax by RegEx"
                               containsString("Fel") ||
                               containsString("Fer"))
     then
-        entityCreationService.byRegexIgnoreCase("\\b(contact|telephone|phone|ph\\.|fax|tel|ter|mobile|fel|fer)[a-zA-Z\\s]{0,10}[:.\\s]{0,3}([\\+\\d\\(][\\s\\d\\(\\)\\-\\/\\.]{4,100}\\d)\\b", "PII", EntityType.ENTITY, 2, $section)
+        entityCreationService.byRegexIgnoreCase("\\b(contact|telephone|phone|ph\\.|fax|tel|ter[^\\w]|mobile|fel[^\\w]|fer[^\\w])[a-zA-Z\\s]{0,10}[:.\\s]{0,3}([\\+\\d\\(][\\s\\d\\(\\)\\-\\/\\.]{4,100}\\d)\\b", "PII", EntityType.ENTITY, 2, $section)
             .forEach(contactEntity -> contactEntity.redact("PII.2.0", "Found by Phone and Fax Regex", "Reg (EC) No 1107/2009 Art. 63 (2e)"));
     end
 
@@ -534,7 +519,6 @@ rule "PII.2.1: Redact Phone and Fax by RegEx (non vertebrate study)"
         $section: Section(containsString("Contact") ||
                               containsString("Telephone") ||
                               containsString("Phone") ||
-                              containsString("Ph.") ||
                               containsString("Fax") ||
                               containsString("Tel") ||
                               containsString("Ter") ||
@@ -542,7 +526,7 @@ rule "PII.2.1: Redact Phone and Fax by RegEx (non vertebrate study)"
                               containsString("Fel") ||
                               containsString("Fer"))
     then
-        entityCreationService.byRegexIgnoreCase("\\b(contact|telephone|phone|ph\\.|fax|tel|ter[^m]|mobile|fel|fer)[a-zA-Z\\s]{0,10}[:.\\s]{0,3}([\\+\\d\\(][\\s\\d\\(\\)\\-\\/\\.]{4,100}\\d)\\b", "PII", EntityType.ENTITY, 2, $section)
+        entityCreationService.byRegexIgnoreCase("\\b(contact|telephone|phone|ph\\.|fax|tel|ter[^\\w]|mobile|fel[^\\w]|fer[^\\w])[a-zA-Z\\s]{0,10}[:.\\s]{0,3}([\\+\\d\\(][\\s\\d\\(\\)\\-\\/\\.]{4,100}\\d)\\b", "PII", EntityType.ENTITY, 2, $section)
             .forEach(contactEntity -> contactEntity.redact("PII.2.1", "Found by Phone and Fax Regex", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
     end
 
@@ -552,7 +536,6 @@ rule "PII.2.2: Redact Phone and Fax by RegEx (vertebrate study)"
         $section: Section(containsString("Contact") ||
                               containsString("Telephone") ||
                               containsString("Phone") ||
-                              containsString("Ph.") ||
                               containsString("Fax") ||
                               containsString("Tel") ||
                               containsString("Ter") ||
@@ -560,7 +543,7 @@ rule "PII.2.2: Redact Phone and Fax by RegEx (vertebrate study)"
                               containsString("Fel") ||
                               containsString("Fer"))
     then
-        entityCreationService.byRegexIgnoreCase("\\b(contact|telephone|phone|ph\\.|fax|tel|ter[^m]|mobile|fel|fer)[a-zA-Z\\s]{0,10}[:.\\s]{0,3}([\\+\\d\\(][\\s\\d\\(\\)\\-\\/\\.]{4,100}\\d)\\b", "PII", EntityType.ENTITY, 2, $section)
+        entityCreationService.byRegexIgnoreCase("\\b(contact|telephone|phone|ph\\.|fax|tel|ter[^\\w]|mobile|fel[^\\w]|fer[^\\w])[a-zA-Z\\s]{0,10}[:.\\s]{0,3}([\\+\\d\\(][\\s\\d\\(\\)\\-\\/\\.]{4,100}\\d)\\b", "PII", EntityType.ENTITY, 2, $section)
             .forEach(contactEntity -> contactEntity.redact("PII.2.2", "Found by Phone and Fax Regex", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
     end
 
@@ -568,7 +551,7 @@ rule "PII.2.2: Redact Phone and Fax by RegEx (vertebrate study)"
 // Rule unit: PII.3
 rule "PII.3.0: Redact telephone numbers by RegEx"
     when
-        $section: Section(matchesRegex("[+]\\d{1,}"))
+        $section: Section(!hasTables(), matchesRegex("[+]\\d{1,}"))
     then
         entityCreationService.byRegex("((([+]\\d{1,3} (\\d{7,12})\\b)|([+]\\d{1,3}(\\d{3,12})\\b|[+]\\d{1,3}([ -]\\(?\\d{1,6}\\)?){2,4})|[+]\\d{1,3} ?((\\d{2,6}\\)?)([ -]\\d{2,6}){1,4}))(-\\d{1,3})?\\b)", "PII", EntityType.ENTITY, 1, $section)
         .forEach(entity -> entity.redact("PII.3.0", "Telephone number found by regex", "Reg (EC) No 1107/2009 Art. 63 (2e)"));
@@ -577,7 +560,7 @@ rule "PII.3.0: Redact telephone numbers by RegEx"
 rule "PII.3.1: Redact telephone numbers by RegEx (Non vertebrate study)"
     when
         not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
-        $section: Section(matchesRegex("[+]\\d{1,}"))
+        $section: Section(!hasTables(), matchesRegex("[+]\\d{1,}"))
     then
         entityCreationService.byRegex("((([+]\\d{1,3} (\\d{7,12})\\b)|([+]\\d{1,3}(\\d{3,12})\\b|[+]\\d{1,3}([ -]\\(?\\d{1,6}\\)?){2,4})|[+]\\d{1,3} ?((\\d{2,6}\\)?)([ -]\\d{2,6}){1,4}))(-\\d{1,3})?\\b)", "PII", EntityType.ENTITY, 1, $section)
         .forEach(entity -> entity.redact("PII.3.1", "Telephone number found by regex", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
@@ -586,12 +569,30 @@ rule "PII.3.1: Redact telephone numbers by RegEx (Non vertebrate study)"
 rule "PII.3.2: Redact telephone numbers by RegEx (vertebrate study)"
     when
         FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
-        $section: Section(matchesRegex("[+]\\d{1,}"))
+        $section: Section(!hasTables(), matchesRegex("[+]\\d{1,}"))
     then
         entityCreationService.byRegex("((([+]\\d{1,3} (\\d{7,12})\\b)|([+]\\d{1,3}(\\d{3,12})\\b|[+]\\d{1,3}([ -]\\(?\\d{1,6}\\)?){2,4})|[+]\\d{1,3} ?((\\d{2,6}\\)?)([ -]\\d{2,6}){1,4}))(-\\d{1,3})?\\b)", "PII", EntityType.ENTITY, 1, $section)
         .forEach(entity -> entity.redact("PII.3.2", "Telephone number found by regex", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
     end
 
+rule "PII.3.4: Redact telephone numbers by RegEx (Non vertebrate study)"
+    when
+        not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
+        $rowCell: TableCell(matchesRegex("[+]\\d{1,}"))
+    then
+        entityCreationService.byRegex("((([+]\\d{1,3} (\\d{7,12})\\b)|([+]\\d{1,3}(\\d{3,12})\\b|[+]\\d{1,3}([ -]\\(?\\d{1,6}\\)?){2,4})|[+]\\d{1,3} ?((\\d{2,6}\\)?)([ -]\\d{2,6}){1,4}))(-\\d{1,3})?\\b)", "PII", EntityType.ENTITY, 1, $rowCell)
+        .forEach(entity -> entity.redact("PII.3.4", "Telephone number found by regex", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
+    end
+
+rule "PII.3.5: Redact telephone numbers by RegEx (vertebrate study)"
+    when
+        FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
+        $rowCell: TableCell(matchesRegex("[+]\\d{1,}"))
+    then
+        entityCreationService.byRegex("((([+]\\d{1,3} (\\d{7,12})\\b)|([+]\\d{1,3}(\\d{3,12})\\b|[+]\\d{1,3}([ -]\\(?\\d{1,6}\\)?){2,4})|[+]\\d{1,3} ?((\\d{2,6}\\)?)([ -]\\d{2,6}){1,4}))(-\\d{1,3})?\\b)", "PII", EntityType.ENTITY, 1, $rowCell)
+        .forEach(entity -> entity.redact("PII.3.5", "Telephone number found by regex", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
+    end
+
 
 // Rule unit: PII.4
 rule "PII.4.0: Redact line after contact information keywords"
@@ -600,8 +601,6 @@ rule "PII.4.0: Redact line after contact information keywords"
                                          "Contact:",
                                          "Alternative contact:",
                                          "European contact:",
-                                         "No:",
-                                         "Contact:",
                                          "Tel.:",
                                          "Tel:",
                                          "Telephone number:",
@@ -648,62 +647,6 @@ rule "PII.4.1: Redact line after contact information keywords"
             .forEach(contactEntity -> contactEntity.redact("PII.4.1", "Found after \"" + $contactKeyword + "\" contact keyword", "Article 4(1)(b), Regulation (EC) No 1049/2001 (Personal data)"));
     end
 
-rule "PII.4.2: Redact line after contact information keywords (Non vertebrate study)"
-    when
-        not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
-        $contactKeyword: String() from List.of("Contact point:",
-                                         "Contact:",
-                                         "Alternative contact:",
-                                         "European contact:",
-                                         "No:",
-                                         "Contact:",
-                                         "Tel.:",
-                                         "Tel:",
-                                         "Telephone number:",
-                                         "Telephone No:",
-                                         "Telephone:",
-                                         "Phone No.",
-                                         "Phone:",
-                                         "Fax number:",
-                                         "Fax:",
-                                         "E-mail:",
-                                         "Email:",
-                                         "e-mail:",
-                                         "E-mail address:")
-        $section: Section(containsString($contactKeyword))
-    then
-        entityCreationService.lineAfterString($contactKeyword, "PII", EntityType.ENTITY, $section)
-            .forEach(contactEntity -> contactEntity.redact("PII.4.2", "Found after \"" + $contactKeyword + "\" contact keyword", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
-    end
-
-rule "PII.4.3: Redact line after contact information keywords (Vertebrate study)"
-    when
-        FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
-        $contactKeyword: String() from List.of("Contact point:",
-                                         "Contact:",
-                                         "Alternative contact:",
-                                         "European contact:",
-                                         "No:",
-                                         "Contact:",
-                                         "Tel.:",
-                                         "Tel:",
-                                         "Telephone number:",
-                                         "Telephone No:",
-                                         "Telephone:",
-                                         "Phone No.",
-                                         "Phone:",
-                                         "Fax number:",
-                                         "Fax:",
-                                         "E-mail:",
-                                         "Email:",
-                                         "e-mail:",
-                                         "E-mail address:")
-        $section: Section(containsString($contactKeyword))
-    then
-        entityCreationService.lineAfterString($contactKeyword, "PII", EntityType.ENTITY, $section)
-            .forEach(contactEntity -> contactEntity.redact("PII.4.3", "Found after \"" + $contactKeyword + "\" contact keyword", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
-    end
-
 
 // Rule unit: PII.5
 rule "PII.5.0: Redact line after contact information keywords reduced"
@@ -803,12 +746,7 @@ rule "PII.7.0: Redact contact information if applicant is found"
                           containsString("Applicant") ||
                           containsString("Telephone number:"))
     then
-        Stream.concat(entityCreationService.lineAfterStrings(List.of("Contact point:", "Contact:", "Alternative contact:", "European contact:", "No:", "Contact:", "Tel.:", "Tel:", "Telephone number:",
+        entityCreationService.lineAfterStrings(List.of("Contact point:", "Contact:", "Alternative contact:", "European contact:", "Tel.:", "Tel:", "Telephone number:", "Telephone No:", "Telephone:", "Phone No.", "Phone:", "Fax number:", "Fax:", "E-mail:", "Email:", "e-mail:", "E-mail address:"), "PII", EntityType.ENTITY, $section)
-                            "Telephone No:", "Telephone:", "Phone No.", "Phone:", "Fax number:", "Fax:", "E-mail:", "Email:", "e-mail:", "E-mail address:"), "PII", EntityType.ENTITY, $section),
-             Stream.concat(
-                 entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section),
-                 entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section)
-             ))
             .forEach(entity -> entity.redact("PII.7.0", "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)"));
     end
 
@@ -909,7 +847,7 @@ rule "PII.9.0: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\""
     when
         $document: Document(containsStringIgnoreCase("AUTHOR(S)"), containsAnyStringIgnoreCase("COMPLETION DATE", "STUDY COMPLETION DATE"))
     then
-        entityCreationService.shortestBetweenAnyStringIgnoreCase(List.of("AUTHOR(S)", "AUTHOR(S):"), List.of("COMPLETION DATE", "COMPLETION DATE:", "STUDY COMPLETION DATE", "STUDY COMPLETION DATE:"), "PII", EntityType.ENTITY, $document)
+        entityCreationService.shortestBetweenAnyStringIgnoreCase(List.of("AUTHOR(S)", "AUTHOR(S):"), List.of("COMPLETION DATE", "COMPLETION DATE:", "STUDY COMPLETION DATE", "STUDY COMPLETION DATE:"), "PII", EntityType.ENTITY, $document, 200)
          .forEach(authorEntity -> authorEntity.redact("PII.9.0", "AUTHOR(S) was found", "Reg (EC) No 1107/2009 Art. 63 (2e)"));
     end
 
@@ -923,8 +861,9 @@ rule "PII.9.3: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\""
 
 
 // Rule unit: PII.10
-rule "PII.10.0: Redact study director abbreviation"
+rule "PII.10.0: Redact study director abbreviation (non vertebrate study)"
     when
+        not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
         $section: Section(containsString("KATH") || containsString("BECH") || containsString("KML"))
     then
         entityCreationService.byRegexIgnoreCase("((KATH)|(BECH)|(KML)) ?(\\d{4})","PII", EntityType.ENTITY, 1, $section)
@@ -1027,7 +966,7 @@ rule "ETC.3.2: Redact logos (vertebrate study)"
         FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
         $logo: Image(imageType == ImageType.LOGO)
     then
-        $logo.redact("ETC.3.2", "Logo Found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        $logo.redact("ETC.3.2", "Logo Found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
     end
 
 rule "ETC.3.3: Redact logos"
@@ -1321,12 +1260,23 @@ rule "MAN.4.1: Apply legal basis change"
 rule "X.0.0: Remove Entity contained by Entity of same type"
     salience 65
     when
-        $larger: TextEntity($type: type(), $entityType: entityType, !removed())
+        $larger: TextEntity($type: type(), $entityType: entityType, !removed(), !hasManualChanges())
-        $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges(), !removed())
+        not TextEntity(getTextRange().equals($larger.getTextRange()), type() == $type, entityType == EntityType.DICTIONARY_REMOVAL, engines contains Engine.DOSSIER_DICTIONARY, !hasManualChanges())
+        $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges())
+    then
+        $contained.remove("X.0.0", "remove Entity contained by Entity of same type");
+        retract($contained);
+    end
+
+rule "X.0.1: Remove Entity contained by Entity of same type with manual changes"
+    salience 65
+    when
+        $larger: TextEntity($type: type(), $entityType: entityType, !removed(), hasManualChanges())
+        $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges())
     then
         $contained.getIntersectingNodes().forEach(node -> update(node));
-        $contained.remove("X.0.0", "remove Entity contained by Entity of same type");
+        $contained.remove("X.0.1", "remove Entity contained by Entity of same type with manual changes");
-        update($contained);
+        retract($contained);
     end
 
 
@@ -1335,10 +1285,21 @@ rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
     salience 64
     when
         $falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active())
-        $entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges())
+        $entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.ENTITY), !hasManualChanges())
+    then
+        $entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE");
+        retract($entity)
+    end
+
+
+rule "X.2.1: Remove Entity of type HINT when contained by FALSE_POSITIVE"
+    salience 64
+    when
+        $falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active())
+        $entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.HINT), !hasManualChanges())
     then
         $entity.getIntersectingNodes().forEach(node -> update(node));
-        $entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE");
+        $entity.remove("X.2.1", "remove Entity of type ENTITY when contained by FALSE_POSITIVE");
         retract($entity)
     end
 
@@ -1459,29 +1420,41 @@ rule "X.10.0: remove false positives of ai"
 
 
 // Rule unit: X.11
-rule "X.11.0: Remove dictionary entity which intersects with a manual entity"
+rule "X.11.1: Remove non manual entity which intersects with a manual entity"
     salience 64
     when
         $manualEntity: TextEntity(engines contains Engine.MANUAL, active())
-        $dictionaryEntity: TextEntity(intersects($manualEntity), dictionaryEntry, engines not contains Engine.MANUAL)
+        $nonManualEntity: TextEntity(intersects($manualEntity), engines not contains Engine.MANUAL)
     then
-        $dictionaryEntity.remove("X.11.0", "remove dictionary entity which intersects with a manual entity");
+        $nonManualEntity.remove("X.11.1", "remove entity which intersects with a manual entity");
-        retract($dictionaryEntity);
+        retract($nonManualEntity);
+    end
+
+
+rule "X.11.2: Remove non manual entity which are equal to manual entity"
+    salience 70
+    when
+        $manualEntity: TextEntity(engines contains Engine.MANUAL, active(), $type: type())
+        $nonManualEntity: TextEntity(getTextRange().equals($manualEntity.getTextRange()), type() == $type, entityType == EntityType.ENTITY, !hasManualChanges(), engines not contains Engine.MANUAL)
+    then
+        $manualEntity.addEngines($nonManualEntity.getEngines());
+        $nonManualEntity.remove("X.11.2", "remove non manual entity which are equal to manual entity");
+        retract($nonManualEntity);
     end
 
 
 //------------------------------------ Dictionary merging rules ------------------------------------
 
 // Rule unit: DICT.0
-rule "DICT.0.0: Ignore Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL"
+rule "DICT.0.0: Remove Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL"
     salience 64
     when
         $dictionaryRemoval: TextEntity($type: type(), entityType == EntityType.DICTIONARY_REMOVAL, engines contains Engine.DOSSIER_DICTIONARY)
         $entity: TextEntity(getTextRange().equals($dictionaryRemoval.getTextRange()), engines contains Engine.DICTIONARY, type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges())
-        not TextEntity(containedBy($entity), engines contains Engine.DOSSIER_DICTIONARY, type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges())
     then
         $entity.getIntersectingNodes().forEach(node -> update(node));
-        $entity.ignore("DICT.0.0", "Ignore Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL");
+        $entity.remove("DICT.0.0", "Remove Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL");
+        $entity.addEngine(Engine.DOSSIER_DICTIONARY);
     end
 
 

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora.drl

@@ -1388,12 +1388,23 @@ rule "MAN.4.1: Apply legal basis change"
 rule "X.0.0: Remove Entity contained by Entity of same type"
     salience 65
     when
-        $larger: TextEntity($type: type(), $entityType: entityType, !removed())
+        $larger: TextEntity($type: type(), $entityType: entityType, !removed(), !hasManualChanges())
-        $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges(), !removed())
+        not TextEntity(getTextRange().equals($larger.getTextRange()), type() == $type, entityType == EntityType.DICTIONARY_REMOVAL, engines contains Engine.DOSSIER_DICTIONARY, !hasManualChanges())
+        $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges())
+    then
+        $contained.remove("X.0.0", "remove Entity contained by Entity of same type");
+        retract($contained);
+    end
+
+rule "X.0.1: Remove Entity contained by Entity of same type with manual changes"
+    salience 65
+    when
+        $larger: TextEntity($type: type(), $entityType: entityType, !removed(), hasManualChanges())
+        $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges())
     then
         $contained.getIntersectingNodes().forEach(node -> update(node));
-        $contained.remove("X.0.0", "remove Entity contained by Entity of same type");
+        $contained.remove("X.0.1", "remove Entity contained by Entity of same type with manual changes");
-        update($contained);
+        retract($contained);
     end
 
 
@@ -1402,10 +1413,21 @@ rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
     salience 64
     when
         $falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active())
-        $entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges())
+        $entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.ENTITY), !hasManualChanges())
+    then
+        $entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE");
+        retract($entity)
+    end
+
+
+rule "X.2.1: Remove Entity of type HINT when contained by FALSE_POSITIVE"
+    salience 64
+    when
+        $falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active())
+        $entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.HINT), !hasManualChanges())
     then
         $entity.getIntersectingNodes().forEach(node -> update(node));
-        $entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE");
+        $entity.remove("X.2.1", "remove Entity of type ENTITY when contained by FALSE_POSITIVE");
         retract($entity)
     end
 
@@ -1527,15 +1549,15 @@ rule "X.11.0: Remove dictionary entity which intersects with a manual entity"
 //------------------------------------ Dictionary merging rules ------------------------------------
 
 // Rule unit: DICT.0
-rule "DICT.0.0: Ignore Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL"
+rule "DICT.0.0: Remove Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL"
     salience 64
     when
         $dictionaryRemoval: TextEntity($type: type(), entityType == EntityType.DICTIONARY_REMOVAL, engines contains Engine.DOSSIER_DICTIONARY)
         $entity: TextEntity(getTextRange().equals($dictionaryRemoval.getTextRange()), engines contains Engine.DICTIONARY, type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges())
-        not TextEntity(containedBy($entity), engines contains Engine.DOSSIER_DICTIONARY, type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges())
     then
         $entity.getIntersectingNodes().forEach(node -> update(node));
-        $entity.ignore("DICT.0.0", "Ignore Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL");
+        $entity.remove("DICT.0.0", "Remove Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL");
+        $entity.addEngine(Engine.DOSSIER_DICTIONARY);
     end
 
 

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/efsa_sanitisation.drl

@@ -85,23 +85,60 @@ rule "SYN.1.0: Recommend CTL/BL laboratory that start with BL or CTL"
 //------------------------------------ CBI rules ------------------------------------
 
 // Rule unit: CBI.0
-rule "CBI.0.0: Redact CBI Authors (non vertebrate Study)"
+rule "CBI.0.0: Add CBI_author with \"et al.\" RegEx"
+    agenda-group "LOCAL_DICTIONARY_ADDS"
+    when
+        $section: Section(containsString("et al."))
+    then
+        entityCreationService.byRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, 1, $section)
+            .forEach(entity -> {
+                entity.skip("CBI.0.0", "Author found by \"et al\" regex");
+                dictionary.recommendEverywhere(entity);
+            });
+    end
+
+rule "CBI.0.1: Add CBI_author with \"et al.\" RegEx (non vertebrate study)"
+    agenda-group "LOCAL_DICTIONARY_ADDS"
+    when
+        not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
+        $section: Section(containsString("et al."))
+    then
+        entityCreationService.byRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, 1, $section)
+            .forEach(entity -> {
+                entity.redact("CBI.0.1", "Author found by \"et al\" regex", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+                dictionary.recommendEverywhere(entity);
+            });
+    end
+
+rule "CBI.0.2: Add CBI_author with \"et al.\" RegEx (vertebrate study)"
+    agenda-group "LOCAL_DICTIONARY_ADDS"
+    when
+        FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
+        $section: Section(containsString("et al."))
+    then
+        entityCreationService.byRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, 1, $section)
+            .forEach(entity -> {
+                entity.redact("CBI.0.2", "Author found by \"et al\" regex", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+                dictionary.recommendEverywhere(entity);
+            });
+    end
+
+rule "CBI.0.3: Redact CBI Authors (non vertebrate Study)"
     when
         not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
         $entity: TextEntity(type() == "CBI_author", dictionaryEntry)
     then
-        $entity.redact("CBI.0.0", "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        $entity.redact("CBI.0.3", "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
     end
 
-rule "CBI.0.1: Redact CBI Authors (vertebrate Study)"
+rule "CBI.0.4: Redact CBI Authors (vertebrate Study)"
     when
         FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
         $entity: TextEntity(type() == "CBI_author", dictionaryEntry)
     then
-        $entity.redact("CBI.0.1", "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        $entity.redact("CBI.0.4", "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
     end
 
-
 // Rule unit: CBI.1
 rule "CBI.1.0: Do not redact CBI Address (non vertebrate Study)"
     when
@@ -130,34 +167,6 @@ rule "CBI.2.0: Do not redact genitive CBI Author"
     end
 
 
-// Rule unit: CBI.7
-rule "CBI.7.1: Add CBI_author with \"et al.\" RegEx (non vertebrate study)"
-    agenda-group "LOCAL_DICTIONARY_ADDS"
-    when
-        not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
-        $section: Section(containsString("et al."))
-    then
-        entityCreationService.byRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, 1, $section)
-            .forEach(entity -> {
-                entity.redact("CBI.7.1", "Author found by \"et al\" regex", "Article 39(e)(3) of Regulation (EC) No 178/2002");
-                dictionary.recommendEverywhere(entity);
-            });
-    end
-
-rule "CBI.7.2: Add CBI_author with \"et al.\" RegEx (vertebrate study)"
-    agenda-group "LOCAL_DICTIONARY_ADDS"
-    when
-        FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
-        $section: Section(containsString("et al."))
-    then
-        entityCreationService.byRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, 1, $section)
-            .forEach(entity -> {
-                entity.redact("CBI.7.2", "Author found by \"et al\" regex", "Article 39(e)(2) of Regulation (EC) No 178/2002");
-                dictionary.recommendEverywhere(entity);
-            });
-    end
-
-
 // Rule unit: CBI.9
 rule "CBI.9.0: Redact all cells with Header Author(s) as CBI_author (non vertebrate study)"
     agenda-group "LOCAL_DICTIONARY_ADDS"
@@ -305,7 +314,7 @@ rule "CBI.23.0: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\" (no
         not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
         $document: Document(containsStringIgnoreCase("AUTHOR(S)"), containsAnyStringIgnoreCase("COMPLETION DATE", "STUDY COMPLETION DATE"))
     then
-        entityCreationService.shortestBetweenAnyStringIgnoreCase(List.of("AUTHOR(S)", "AUTHOR(S):"), List.of("COMPLETION DATE", "COMPLETION DATE:", "STUDY COMPLETION DATE", "STUDY COMPLETION DATE:"), "CBI_author", EntityType.ENTITY, $document)
+        entityCreationService.shortestBetweenAnyStringIgnoreCase(List.of("AUTHOR(S)", "AUTHOR(S):"), List.of("COMPLETION DATE", "COMPLETION DATE:", "STUDY COMPLETION DATE", "STUDY COMPLETION DATE:"), "CBI_author", EntityType.ENTITY, $document, 200)
          .forEach(authorEntity -> authorEntity.redact("CBI.23.0", "AUTHOR(S) was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
     end
 
@@ -314,7 +323,7 @@ rule "CBI.23.1: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\" (ve
         FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
         $document: Document(containsStringIgnoreCase("AUTHOR(S)"), containsAnyStringIgnoreCase("COMPLETION DATE", "STUDY COMPLETION DATE"))
     then
-        entityCreationService.shortestBetweenAnyStringIgnoreCase(List.of("AUTHOR(S)", "AUTHOR(S):"), List.of("COMPLETION DATE", "COMPLETION DATE:", "STUDY COMPLETION DATE", "STUDY COMPLETION DATE:"), "CBI_author", EntityType.ENTITY, $document)
+        entityCreationService.shortestBetweenAnyStringIgnoreCase(List.of("AUTHOR(S)", "AUTHOR(S):"), List.of("COMPLETION DATE", "COMPLETION DATE:", "STUDY COMPLETION DATE", "STUDY COMPLETION DATE:"), "CBI_author", EntityType.ENTITY, $document, 200)
         .forEach(authorEntity -> authorEntity.redact("CBI.23.1", "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
     end
 
@@ -374,7 +383,6 @@ rule "PII.2.1: Redact Phone and Fax by RegEx (non vertebrate study)"
         $section: Section(containsString("Contact") ||
                               containsString("Telephone") ||
                               containsString("Phone") ||
-                              containsString("Ph.") ||
                               containsString("Fax") ||
                               containsString("Tel") ||
                               containsString("Ter") ||
@@ -382,7 +390,7 @@ rule "PII.2.1: Redact Phone and Fax by RegEx (non vertebrate study)"
                               containsString("Fel") ||
                               containsString("Fer"))
     then
-        entityCreationService.byRegexIgnoreCase("\\b(contact|telephone|phone|ph\\.|fax|tel|ter[^m]|mobile|fel|fer)[a-zA-Z\\s]{0,10}[:.\\s]{0,3}([\\+\\d\\(][\\s\\d\\(\\)\\-\\/\\.]{4,100}\\d)\\b", "PII", EntityType.ENTITY, 2, $section)
+        entityCreationService.byRegexIgnoreCase("\\b(contact|telephone|phone|ph\\.|fax|tel|ter[^\\w]|mobile|fel[^\\w]|fer[^\\w])[a-zA-Z\\s]{0,10}[:.\\s]{0,3}([\\+\\d\\(][\\s\\d\\(\\)\\-\\/\\.]{4,100}\\d)\\b", "PII", EntityType.ENTITY, 2, $section)
             .forEach(contactEntity -> contactEntity.redact("PII.2.1", "Found by Phone and Fax Regex", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
     end
 
@@ -392,7 +400,6 @@ rule "PII.2.2: Redact Phone and Fax by RegEx (vertebrate study)"
         $section: Section(containsString("Contact") ||
                               containsString("Telephone") ||
                               containsString("Phone") ||
-                              containsString("Ph.") ||
                               containsString("Fax") ||
                               containsString("Tel") ||
                               containsString("Ter") ||
@@ -400,7 +407,7 @@ rule "PII.2.2: Redact Phone and Fax by RegEx (vertebrate study)"
                               containsString("Fel") ||
                               containsString("Fer"))
     then
-        entityCreationService.byRegexIgnoreCase("\\b(contact|telephone|phone|ph\\.|fax|tel|ter[^m]|mobile|fel|fer)[a-zA-Z\\s]{0,10}[:.\\s]{0,3}([\\+\\d\\(][\\s\\d\\(\\)\\-\\/\\.]{4,100}\\d)\\b", "PII", EntityType.ENTITY, 2, $section)
+        entityCreationService.byRegexIgnoreCase("\\b(contact|telephone|phone|ph\\.|fax|tel|ter[^\\w]|mobile|fel[^\\w]|fer[^\\w])[a-zA-Z\\s]{0,10}[:.\\s]{0,3}([\\+\\d\\(][\\s\\d\\(\\)\\-\\/\\.]{4,100}\\d)\\b", "PII", EntityType.ENTITY, 2, $section)
             .forEach(contactEntity -> contactEntity.redact("PII.2.2", "Found by Phone and Fax Regex", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
     end
 
@@ -417,7 +424,7 @@ rule "PII.2.3: Redact phone numbers without indicators"
 rule "PII.3.1: Redact telephone numbers by RegEx (Non vertebrate study)"
     when
         not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
-        $section: Section(matchesRegex("[+]\\d{1,}"))
+        $section: Section(!hasTables(), matchesRegex("[+]\\d{1,}"))
     then
         entityCreationService.byRegex("((([+]\\d{1,3} (\\d{7,12})\\b)|([+]\\d{1,3}(\\d{3,12})\\b|[+]\\d{1,3}([ -]\\(?\\d{1,6}\\)?){2,4})|[+]\\d{1,3} ?((\\d{2,6}\\)?)([ -]\\d{2,6}){1,4}))(-\\d{1,3})?\\b)", "PII", EntityType.ENTITY, 1, $section)
         .forEach(entity -> entity.redact("PII.3.1", "Telephone number found by regex", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
@@ -426,13 +433,86 @@ rule "PII.3.1: Redact telephone numbers by RegEx (Non vertebrate study)"
 rule "PII.3.2: Redact telephone numbers by RegEx (vertebrate study)"
     when
         FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
-        $section: Section(matchesRegex("[+]\\d{1,}"))
+        $section: Section(!hasTables(), matchesRegex("[+]\\d{1,}"))
     then
         entityCreationService.byRegex("((([+]\\d{1,3} (\\d{7,12})\\b)|([+]\\d{1,3}(\\d{3,12})\\b|[+]\\d{1,3}([ -]\\(?\\d{1,6}\\)?){2,4})|[+]\\d{1,3} ?((\\d{2,6}\\)?)([ -]\\d{2,6}){1,4}))(-\\d{1,3})?\\b)", "PII", EntityType.ENTITY, 1, $section)
         .forEach(entity -> entity.redact("PII.3.2", "Telephone number found by regex", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
     end
 
 
+rule "PII.3.4: Redact telephone numbers by RegEx (Non vertebrate study)"
+    when
+        not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
+        $rowCell: TableCell(matchesRegex("[+]\\d{1,}"))
+    then
+        entityCreationService.byRegex("((([+]\\d{1,3} (\\d{7,12})\\b)|([+]\\d{1,3}(\\d{3,12})\\b|[+]\\d{1,3}([ -]\\(?\\d{1,6}\\)?){2,4})|[+]\\d{1,3} ?((\\d{2,6}\\)?)([ -]\\d{2,6}){1,4}))(-\\d{1,3})?\\b)", "PII", EntityType.ENTITY, 1, $rowCell)
+        .forEach(entity -> entity.redact("PII.3.4", "Telephone number found by regex", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
+    end
+
+rule "PII.3.5: Redact telephone numbers by RegEx (vertebrate study)"
+    when
+        FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
+        $rowCell: TableCell(matchesRegex("[+]\\d{1,}"))
+    then
+        entityCreationService.byRegex("((([+]\\d{1,3} (\\d{7,12})\\b)|([+]\\d{1,3}(\\d{3,12})\\b|[+]\\d{1,3}([ -]\\(?\\d{1,6}\\)?){2,4})|[+]\\d{1,3} ?((\\d{2,6}\\)?)([ -]\\d{2,6}){1,4}))(-\\d{1,3})?\\b)", "PII", EntityType.ENTITY, 1, $rowCell)
+        .forEach(entity -> entity.redact("PII.3.5", "Telephone number found by regex", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
+    end
+
+
+// Rule unit: PII.5
+rule "PII.5.1: Redact line after contact information keywords reduced (non vertebrate study)"
+    when
+        not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
+        $contactKeyword: String() from List.of("Contact point:",
+                                         "Contact:",
+                                         "Alternative contact:",
+                                         "European contact:")
+        $section: Section(containsString($contactKeyword))
+    then
+        entityCreationService.lineAfterString($contactKeyword, "PII", EntityType.ENTITY, $section)
+            .forEach(contactEntity -> contactEntity.redact("PII.5.1", "Found after \"" + $contactKeyword + "\" contact keyword", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
+    end
+
+rule "PII.5.2: Redact line after contact information keywords reduced (Vertebrate study)"
+    when
+        FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
+        $contactKeyword: String() from List.of("Contact point:",
+                                               "Contact:",
+                                               "Alternative contact:",
+                                               "European contact:")
+        $section: Section(containsString($contactKeyword))
+    then
+        entityCreationService.lineAfterString($contactKeyword, "PII", EntityType.ENTITY, $section)
+            .forEach(contactEntity -> contactEntity.redact("PII.5.2", "Found after \"" + $contactKeyword + "\" contact keyword", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
+    end
+
+
+// Rule unit: PII.6
+rule "PII.6.1: Redact line between contact keywords (non vertebrate study)"
+    when
+        not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
+        $section: Section((containsString("No:") && containsString("Fax")) || (containsString("Contact:") && containsString("Tel")))
+    then
+        Stream.concat(
+            entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section),
+            entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section)
+        )
+            .forEach(contactEntity -> contactEntity.redact("PII.6.1", "Found between contact keywords", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
+    end
+
+rule "PII.6.2: Redact line between contact keywords (vertebrate study)"
+    when
+        FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
+        $section: Section((containsString("No:") && containsString("Fax")) || (containsString("Contact:") && containsString("Tel")))
+    then
+        Stream.concat(
+            entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section),
+            entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section)
+        )
+            .forEach(contactEntity -> contactEntity.redact("PII.6.2", "Found between contact keywords", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
+    end
+
+
 // Rule unit: PII.7
 rule "PII.7.1: Redact contact information if applicant is found (non vertebrate study)"
     when
@@ -510,8 +590,9 @@ rule "PII.8.2: Redact contact information if producer is found (vertebrate study
 
 
 // Rule unit: PII.10
-rule "PII.10.0: Redact study director abbreviation"
+rule "PII.10.0: Redact study director abbreviation (non vertebrate study)"
     when
+        not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
         $section: Section(containsString("KATH") || containsString("BECH") || containsString("KML"))
     then
         entityCreationService.byRegexIgnoreCase("((KATH)|(BECH)|(KML)) ?(\\d{4})","PII", EntityType.ENTITY, 1, $section)
@@ -519,6 +600,16 @@ rule "PII.10.0: Redact study director abbreviation"
      end
 
 
+rule "PII.10.1: Redact study director abbreviation (vertebrate study)"
+    when
+        FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
+        $section: Section(containsString("KATH") || containsString("BECH") || containsString("KML"))
+    then
+        entityCreationService.byRegexIgnoreCase("((KATH)|(BECH)|(KML)) ?(\\d{4})","PII", EntityType.ENTITY, 1, $section)
+        .forEach(entity -> entity.redact("PII.10.1", "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
+     end
+
+
 // Rule unit: PII.11
 rule "PII.11.0: Redact On behalf of Sequani Ltd.:"
     when
@@ -593,7 +684,7 @@ rule "ETC.3.2: Redact logos (vertebrate study)"
         FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
         $logo: Image(imageType == ImageType.LOGO)
     then
-        $logo.redact("ETC.3.2", "Logo Found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        $logo.redact("ETC.3.2", "Logo Found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
     end
 
 
@@ -617,6 +708,22 @@ rule "ETC.5.1: Remove dossier_redaction entries if confidentiality is not 'confi
         retract($dossierRedaction);
     end
 
+// Rule unit: ETC.12
+rule "ETC.12.2: Skip dossier_redaction (Non vertebrate study)"
+    when
+        not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
+        $dossierRedaction: TextEntity(type() == "dossier_redaction")
+    then
+        $dossierRedaction.skip("ETC.12.2", "Dossier dictionary entry found");
+    end
+
+rule "ETC.12.3: Skip dossier_redaction (Vertebrate study)"
+    when
+        FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
+        $dossierRedaction: TextEntity(type() == "dossier_redaction")
+    then
+        $dossierRedaction.skip("ETC.12.3", "Dossier dictionary entry found");
+    end
 
 //------------------------------------ AI rules ------------------------------------
 
@@ -880,12 +987,23 @@ rule "MAN.4.1: Apply legal basis change"
 rule "X.0.0: Remove Entity contained by Entity of same type"
     salience 65
     when
-        $larger: TextEntity($type: type(), $entityType: entityType, !removed())
+        $larger: TextEntity($type: type(), $entityType: entityType, !removed(), !hasManualChanges())
-        $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges(), !removed())
+        not TextEntity(getTextRange().equals($larger.getTextRange()), type() == $type, entityType == EntityType.DICTIONARY_REMOVAL, engines contains Engine.DOSSIER_DICTIONARY, !hasManualChanges())
+        $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges())
+    then
+        $contained.remove("X.0.0", "remove Entity contained by Entity of same type");
+        retract($contained);
+    end
+
+rule "X.0.1: Remove Entity contained by Entity of same type with manual changes"
+    salience 65
+    when
+        $larger: TextEntity($type: type(), $entityType: entityType, !removed(), hasManualChanges())
+        $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges())
     then
         $contained.getIntersectingNodes().forEach(node -> update(node));
-        $contained.remove("X.0.0", "remove Entity contained by Entity of same type");
+        $contained.remove("X.0.1", "remove Entity contained by Entity of same type with manual changes");
-        update($contained);
+        retract($contained);
     end
 
 
@@ -894,10 +1012,21 @@ rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
     salience 64
     when
         $falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active())
-        $entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges())
+        $entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.ENTITY), !hasManualChanges())
+    then
+        $entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE");
+        retract($entity)
+    end
+
+
+rule "X.2.1: Remove Entity of type HINT when contained by FALSE_POSITIVE"
+    salience 64
+    when
+        $falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active())
+        $entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.HINT), !hasManualChanges())
     then
         $entity.getIntersectingNodes().forEach(node -> update(node));
-        $entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE");
+        $entity.remove("X.2.1", "remove Entity of type ENTITY when contained by FALSE_POSITIVE");
         retract($entity)
     end
 
@@ -1018,29 +1147,41 @@ rule "X.10.0: remove false positives of ai"
 
 
 // Rule unit: X.11
-rule "X.11.0: Remove dictionary entity which intersects with a manual entity"
+rule "X.11.1: Remove non manual entity which intersects with a manual entity"
     salience 64
     when
         $manualEntity: TextEntity(engines contains Engine.MANUAL, active())
-        $dictionaryEntity: TextEntity(intersects($manualEntity), dictionaryEntry, engines not contains Engine.MANUAL)
+        $nonManualEntity: TextEntity(intersects($manualEntity), engines not contains Engine.MANUAL)
     then
-        $dictionaryEntity.remove("X.11.0", "remove dictionary entity which intersects with a manual entity");
+        $nonManualEntity.remove("X.11.1", "remove entity which intersects with a manual entity");
-        retract($dictionaryEntity);
+        retract($nonManualEntity);
+    end
+
+
+rule "X.11.2: Remove non manual entity which are equal to manual entity"
+    salience 70
+    when
+        $manualEntity: TextEntity(engines contains Engine.MANUAL, active(), $type: type())
+        $nonManualEntity: TextEntity(getTextRange().equals($manualEntity.getTextRange()), type() == $type, entityType == EntityType.ENTITY, !hasManualChanges(), engines not contains Engine.MANUAL)
+    then
+        $manualEntity.addEngines($nonManualEntity.getEngines());
+        $nonManualEntity.remove("X.11.2", "remove non manual entity which are equal to manual entity");
+        retract($nonManualEntity);
     end
 
 
 //------------------------------------ Dictionary merging rules ------------------------------------
 
 // Rule unit: DICT.0
-rule "DICT.0.0: Ignore Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL"
+rule "DICT.0.0: Remove Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL"
     salience 64
     when
         $dictionaryRemoval: TextEntity($type: type(), entityType == EntityType.DICTIONARY_REMOVAL, engines contains Engine.DOSSIER_DICTIONARY)
         $entity: TextEntity(getTextRange().equals($dictionaryRemoval.getTextRange()), engines contains Engine.DICTIONARY, type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges())
-        not TextEntity(containedBy($entity), engines contains Engine.DOSSIER_DICTIONARY, type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges())
     then
         $entity.getIntersectingNodes().forEach(node -> update(node));
-        $entity.ignore("DICT.0.0", "Ignore Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL");
+        $entity.remove("DICT.0.0", "Remove Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL");
+        $entity.addEngine(Engine.DOSSIER_DICTIONARY);
     end
 
 

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/manual_redaction_rules.drl

@@ -149,7 +149,7 @@ rule "AI.7.0: Add all NER Entities of type Address"
     end
 
 
-//------------------------------------ Manual changes rules ------------------------------------
+//------------------------------------ Manual redaction rules ------------------------------------
 
 // Rule unit: MAN.0
 rule "MAN.0.0: Apply manual resize redaction"
@@ -304,7 +304,90 @@ rule "MAN.4.1: Apply legal basis change"
 
 //------------------------------------ Entity merging rules ------------------------------------
 
+// Rule unit: X.0
+rule "X.0.0: Remove Entity contained by Entity of same type"
+    salience 65
+    when
+        $larger: TextEntity($type: type(), $entityType: entityType, !removed(), !hasManualChanges())
+        not TextEntity(getTextRange().equals($larger.getTextRange()), type() == $type, entityType == EntityType.DICTIONARY_REMOVAL, engines contains Engine.DOSSIER_DICTIONARY, !hasManualChanges())
+        $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges())
+    then
+        $contained.remove("X.0.0", "remove Entity contained by Entity of same type");
+        retract($contained);
+    end
+
+rule "X.0.1: Remove Entity contained by Entity of same type with manual changes"
+    salience 65
+    when
+        $larger: TextEntity($type: type(), $entityType: entityType, !removed(), hasManualChanges())
+        $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges())
+    then
+        $contained.getIntersectingNodes().forEach(node -> update(node));
+        $contained.remove("X.0.1", "remove Entity contained by Entity of same type with manual changes");
+        retract($contained);
+    end
+
+
+// Rule unit: X.2
+rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
+    salience 64
+    when
+        $falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active())
+        $entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.ENTITY), !hasManualChanges())
+    then
+        $entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE");
+        retract($entity)
+    end
+
+
+rule "X.2.1: Remove Entity of type HINT when contained by FALSE_POSITIVE"
+    salience 64
+    when
+        $falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active())
+        $entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.HINT), !hasManualChanges())
+    then
+        $entity.getIntersectingNodes().forEach(node -> update(node));
+        $entity.remove("X.2.1", "remove Entity of type ENTITY when contained by FALSE_POSITIVE");
+        retract($entity)
+    end
+
+
+// Rule unit: X.3
+rule "X.3.0: Remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"
+    salience 64
+    when
+        $falseRecommendation: TextEntity($type: type(), entityType == EntityType.FALSE_RECOMMENDATION, active())
+        $recommendation: TextEntity(containedBy($falseRecommendation), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges())
+    then
+        $recommendation.remove("X.3.0", "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION");
+        retract($recommendation);
+    end
+
+
+// Rule unit: X.4
+rule "X.4.0: Remove Entity of type RECOMMENDATION when text range equals ENTITY with same type"
+    salience 256
+    when
+        $entity: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
+        $recommendation: TextEntity(getTextRange().equals($entity.getTextRange()), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges())
+    then
+        $entity.addEngines($recommendation.getEngines());
+        $recommendation.remove("X.4.0", "remove Entity of type RECOMMENDATION when text range equals ENTITY with same type");
+        retract($recommendation);
+    end
+
+
 // Rule unit: X.5
+rule "X.5.0: Remove Entity of type RECOMMENDATION when intersected by ENTITY"
+    salience 256
+    when
+        $entity: TextEntity((entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
+        $recommendation: TextEntity(intersects($entity), entityType == EntityType.RECOMMENDATION, !hasManualChanges())
+    then
+        $recommendation.remove("X.5.0", "remove Entity of type RECOMMENDATION when intersected by ENTITY");
+        retract($recommendation);
+    end
+
 rule "X.5.1: Remove Entity of type RECOMMENDATION when contained by RECOMMENDATION"
     salience 256
     when
@@ -316,6 +399,30 @@ rule "X.5.1: Remove Entity of type RECOMMENDATION when contained by RECOMMENDATI
     end
 
 
+// Rule unit: X.6
+rule "X.6.0: Remove Entity of lower rank, when contained by entity of type ENTITY or HINT"
+    salience 32
+    when
+        $higherRank: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
+        $lowerRank: TextEntity(containedBy($higherRank), type() != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges())
+    then
+        $lowerRank.getIntersectingNodes().forEach(node -> update(node));
+        $lowerRank.remove("X.6.0", "remove Entity of lower rank, when contained by entity of type ENTITY or HINT");
+        retract($lowerRank);
+    end
+
+rule "X.6.1: remove Entity, when contained in another entity of type ENTITY or HINT with larger text range"
+    salience 32
+    when
+         $outer: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
+         $inner: TextEntity(containedBy($outer), type() != $type, $outer.getTextRange().length > getTextRange().length(), !hasManualChanges())
+    then
+        $inner.getIntersectingNodes().forEach(node -> update(node));
+        $inner.remove("X.6.1", "remove Entity, when contained in another entity of type ENTITY or HINT with larger text range");
+        retract($inner);
+    end
+
+
 // Rule unit: X.8
 rule "X.8.0: Remove Entity when text range and type equals to imported Entity"
     salience 257
@@ -361,29 +468,54 @@ rule "X.10.0: remove false positives of ai"
 
 
 // Rule unit: X.11
-rule "X.11.0: Remove dictionary entity which intersects with a manual entity"
+rule "X.11.1: Remove non manual entity which intersects with a manual entity"
     salience 64
     when
         $manualEntity: TextEntity(engines contains Engine.MANUAL, active())
-        $dictionaryEntity: TextEntity(intersects($manualEntity), dictionaryEntry, engines not contains Engine.MANUAL)
+        $nonManualEntity: TextEntity(intersects($manualEntity), engines not contains Engine.MANUAL)
     then
-        $dictionaryEntity.remove("X.11.0", "remove dictionary entity which intersects with a manual entity");
+        $nonManualEntity.remove("X.11.1", "remove entity which intersects with a manual entity");
-        retract($dictionaryEntity);
+        retract($nonManualEntity);
+    end
+
+
+rule "X.11.2: Remove non manual entity which are equal to manual entity"
+    salience 70
+    when
+        $manualEntity: TextEntity(engines contains Engine.MANUAL, active(), $type: type())
+        $nonManualEntity: TextEntity(getTextRange().equals($manualEntity.getTextRange()), type() == $type, entityType == EntityType.ENTITY, !hasManualChanges(), engines not contains Engine.MANUAL)
+    then
+        $manualEntity.addEngines($nonManualEntity.getEngines());
+        $nonManualEntity.remove("X.11.2", "remove non manual entity which are equal to manual entity");
+        retract($nonManualEntity);
     end
 
 
 //------------------------------------ Dictionary merging rules ------------------------------------
 
 // Rule unit: DICT.0
-rule "DICT.0.0: Ignore Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL"
+rule "DICT.0.0: Remove Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL"
     salience 64
     when
         $dictionaryRemoval: TextEntity($type: type(), entityType == EntityType.DICTIONARY_REMOVAL, engines contains Engine.DOSSIER_DICTIONARY)
         $entity: TextEntity(getTextRange().equals($dictionaryRemoval.getTextRange()), engines contains Engine.DICTIONARY, type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges())
-        not TextEntity(containedBy($entity), engines contains Engine.DOSSIER_DICTIONARY, type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges())
     then
         $entity.getIntersectingNodes().forEach(node -> update(node));
-        $entity.ignore("DICT.0.0", "Ignore Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL");
+        $entity.remove("DICT.0.0", "Remove Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL");
+        $entity.addEngine(Engine.DOSSIER_DICTIONARY);
+    end
+
+
+//------------------------------------ File attributes rules ------------------------------------
+
+// Rule unit: FA.1
+rule "FA.1.0: Remove duplicate FileAttributes"
+    salience 64
+    when
+        $fileAttribute: FileAttribute($label: label, $value: value)
+        $duplicate: FileAttribute(this != $fileAttribute, label == $label, value == $value)
+    then
+        retract($duplicate);
     end
 
 

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl

@@ -85,151 +85,20 @@ rule "SYN.0.0: Redact if CTL/* or BL/* was found (Non Vertebrate Study)"
 
 
 //------------------------------------ CBI rules ------------------------------------
-
+// Rule unit: CBI.0
-// Rule unit: CBI.3
+rule "CBI.0.0: Add CBI_author with \"et al.\" RegEx"
-rule "CBI.3.0: Redacted because Section contains a vertebrate"
-    when
-        $section: Section(!hasTables(), hasEntitiesOfType("vertebrate"), (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
-    then
-        $section.getEntitiesOfType(List.of("CBI_author", "CBI_address"))
-            .forEach(entity -> {
-                entity.applyWithReferences(
-                        "CBI.3.0",
-                        "Vertebrate found",
-                        "Reg (EC) No 1107/2009 Art. 63 (2g)",
-                        $section.getEntitiesOfType("vertebrate")
-                );
-            });
-    end
-
-rule "CBI.3.1: Redacted because table row contains a vertebrate"
-    when
-        $table: Table(hasEntitiesOfType("vertebrate"), (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
-    then
-        $table.streamEntitiesWhereRowContainsEntitiesOfType(List.of("vertebrate"))
-            .filter(entity -> entity.getType().equals("CBI_author") || entity.getType().equals("CBI_address"))
-            .forEach(entity -> {
-                entity.applyWithReferences(
-                        "CBI.3.1",
-                        "Vertebrate found",
-                        "Reg (EC) No 1107/2009 Art. 63 (2g)",
-                        $table.getEntitiesOfTypeInSameRow("vertebrate", entity)
-                );
-            });
-    end
-
-rule "CBI.3.2: Do not redact because Section does not contain a vertebrate"
-    when
-        $section: Section(!hasTables(), !hasEntitiesOfType("vertebrate"), (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
-    then
-        $section.getEntitiesOfType(List.of("CBI_author", "CBI_address"))
-            .forEach(entity -> entity.skip("CBI.3.2", "No vertebrate found"));
-    end
-
-rule "CBI.3.3: Do not redact because table row does not contain a vertebrate"
-    when
-        $table: Table(hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address"))
-    then
-        $table.streamEntitiesWhereRowContainsNoEntitiesOfType(List.of("vertebrate"))
-            .filter(entity -> entity.getType().equals("CBI_author") || entity.getType().equals("CBI_address"))
-            .forEach(entity -> entity.skip("CBI.3.3", "No vertebrate found"));
-    end
-
-
-// Rule unit: CBI.4
-rule "CBI.4.0: Do not redact Names and Addresses if no_redaction_indicator is found in Section"
-    when
-        $section: Section(!hasTables(),
-                          hasEntitiesOfType("vertebrate"),
-                          hasEntitiesOfType("no_redaction_indicator"),
-                          (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
-    then
-        $section.getEntitiesOfType(List.of("CBI_author", "CBI_address"))
-            .forEach(entity -> {
-                entity.skipWithReferences(
-                        "CBI.4.0",
-                        "Vertebrate but a no redaction indicator found",
-                        $section.getEntitiesOfType("no_redaction_indicator")
-                );
-            });
-    end
-
-rule "CBI.4.1: Don't redact authors or addresses which appear in the same row as a vertebrate and a no_redaction_indicator"
-    when
-        $table: Table(hasEntitiesOfType("no_redaction_indicator"),
-                      hasEntitiesOfType("vertebrate"),
-                      (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
-    then
-        $table.streamEntitiesWhereRowContainsEntitiesOfType(List.of("vertebrate", "no-redaction_indicator"))
-            .filter(entity -> entity.getType().equals("CBI_author") || entity.getType().equals("CBI_address"))
-            .forEach(entity -> {
-                entity.skipWithReferences(
-                        "CBI.4.1",
-                        "Vertebrate but a no redaction indicator found",
-                        Stream.concat(
-                            $table.getEntitiesOfTypeInSameRow("vertebrate", entity).stream(),
-                            $table.getEntitiesOfTypeInSameRow("no_redaction_indicator", entity).stream()).toList()
-                );
-            });
-    end
-
-
-// Rule unit: CBI.5
-rule "CBI.5.0: Redact Names and Addresses if no_redaction_indicator but also redaction_indicator is found in Section"
-    when
-        $section: Section(!hasTables(),
-                          hasEntitiesOfType("redaction_indicator"),
-                          hasEntitiesOfType("no_redaction_indicator"),
-                          (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
-    then
-        $section.getEntitiesOfType(List.of("CBI_author", "CBI_address"))
-            .forEach(entity -> {
-                entity.applyWithReferences(
-                        "CBI.5.0",
-                        "no_redaction_indicator but also redaction_indicator found",
-                        "Reg (EC) No 1107/2009 Art. 63 (2g)",
-                        Stream.concat(
-                            $section.getEntitiesOfType("redaction_indicator").stream(),
-                            $section.getEntitiesOfType("no_redaction_indicator").stream()).toList()
-                );
-            });
-    end
-
-rule "CBI.5.1: Redact Names and Addresses if no_redaction_indicator but also redaction_indicator is found in table row"
-    when
-        $table: Table(hasEntitiesOfType("no_redaction_indicator"),
-                      hasEntitiesOfType("redaction_indicator"),
-                      (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
-    then
-        $table.streamEntitiesWhereRowContainsEntitiesOfType(List.of("redaction_indicator", "no_redaction_indicator"))
-            .filter(entity -> entity.getType().equals("CBI_author") || entity.getType().equals("CBI_address"))
-            .forEach(entity -> {
-                entity.applyWithReferences(
-                        "CBI.5.1",
-                        "no_redaction_indicator but also redaction_indicator found",
-                        "Reg (EC) No 1107/2009 Art. 63 (2g)",
-                        Stream.concat(
-                            $table.getEntitiesOfTypeInSameRow("redaction_indicator", entity).stream(),
-                            $table.getEntitiesOfTypeInSameRow("no_redaction_indicator", entity).stream()).toList()
-                );
-            });
-    end
-
-
-// Rule unit: CBI.7
-rule "CBI.7.0: Add CBI_author with \"et al.\" RegEx"
     agenda-group "LOCAL_DICTIONARY_ADDS"
     when
         $section: Section(containsString("et al."))
     then
         entityCreationService.byRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, 1, $section)
             .forEach(entity -> {
-                entity.redact("CBI.7.0", "Author found by \"et al\" regex", "Reg (EC) No 1107/2009 Art. 63 (2g)");
+                entity.skip("CBI.0.0", "Author found by \"et al\" regex");
                 dictionary.recommendEverywhere(entity);
             });
     end
 
-rule "CBI.7.1: Add CBI_author with \"et al.\" RegEx (non vertebrate study)"
+rule "CBI.0.1: Add CBI_author with \"et al.\" RegEx (non vertebrate study)"
     agenda-group "LOCAL_DICTIONARY_ADDS"
     when
         not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
@@ -237,12 +106,12 @@ rule "CBI.7.1: Add CBI_author with \"et al.\" RegEx (non vertebrate study)"
     then
         entityCreationService.byRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, 1, $section)
             .forEach(entity -> {
-                entity.redact("CBI.7.1", "Author found by \"et al\" regex", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+                entity.redact("CBI.0.1", "Author found by \"et al\" regex", "Article 39(e)(3) of Regulation (EC) No 178/2002");
                 dictionary.recommendEverywhere(entity);
             });
     end
 
-rule "CBI.7.2: Add CBI_author with \"et al.\" RegEx (vertebrate study)"
+rule "CBI.0.2: Add CBI_author with \"et al.\" RegEx (vertebrate study)"
     agenda-group "LOCAL_DICTIONARY_ADDS"
     when
         FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
@@ -250,56 +119,11 @@ rule "CBI.7.2: Add CBI_author with \"et al.\" RegEx (vertebrate study)"
     then
         entityCreationService.byRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, 1, $section)
             .forEach(entity -> {
-                entity.redact("CBI.7.2", "Author found by \"et al\" regex", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+                entity.redact("CBI.0.2", "Author found by \"et al\" regex", "Article 39(e)(2) of Regulation (EC) No 178/2002");
                 dictionary.recommendEverywhere(entity);
             });
     end
 
-rule "CBI.7.3: Add CBI_author with \"et al.\" RegEx"
-    agenda-group "LOCAL_DICTIONARY_ADDS"
-    when
-        $section: Section(containsString("et al."))
-    then
-        entityCreationService.byRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, 1, $section)
-            .forEach(entity -> {
-                entity.redact("CBI.7.3", "Author found by \"et al\" regex", "Article 4(1)(b), Regulation (EC) No 1049/2001 (Personal data)");
-                dictionary.recommendEverywhere(entity);
-            });
-    end
-
-
-// Rule unit: CBI.8
-rule "CBI.8.0: Redacted because Section contains must_redact entity"
-    when
-        $section: Section(!hasTables(), hasEntitiesOfType("must_redact"), (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
-    then
-        $section.getEntitiesOfType(List.of("CBI_author", "CBI_address"))
-            .forEach(entity -> {
-                entity.applyWithReferences(
-                        "CBI.8.0",
-                        "must_redact entity found",
-                        "Reg (EC) No 1107/2009 Art. 63 (2g)",
-                        $section.getEntitiesOfType("must_redact")
-                );
-            });
-    end
-
-rule "CBI.8.1: Redacted because table row contains must_redact entity"
-    when
-        $table: Table(hasEntitiesOfType("must_redact"), (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
-    then
-        $table.streamEntitiesWhereRowContainsEntitiesOfType(List.of("must_redact"))
-            .filter(entity -> entity.getType().equals("CBI_author") || entity.getType().equals("CBI_address"))
-            .forEach(entity -> {
-                entity.applyWithReferences(
-                       "CBI.8.1",
-                       "Must_redact found",
-                       "Reg (EC) No 1107/2009 Art. 63 (2g)",
-                       $table.getEntitiesOfTypeInSameRow("must_redact", entity)
-               );
-            });
-    end
-
 
 // Rule unit: CBI.9
 rule "CBI.9.0: Redact all cells with Header Author(s) as CBI_author (non vertebrate study)"
@@ -353,19 +177,19 @@ rule "CBI.11.0: Recommend all CBI_author entities in Table with Vertebrate Study
     end
 
 
-// Rule unit: CBI.12
+// Rule unit: CBI.12 - table rules remains
 rule "CBI.12.0: Redact and recommend TableCell with header 'Author' or 'Author(s)' and header 'Vertebrate study Y/N' with value 'Yes'"
     agenda-group "LOCAL_DICTIONARY_ADDS"
     when
         $table: Table(hasHeader("Author(s)") || hasHeader("Author"), hasHeaderIgnoreCase("Vertebrate Study Y/N"))
         TableCell(header, containsAnyStringIgnoreCase("Author", "Author(s)"), $authorCol: col) from $table.streamHeaders().toList()
         TableCell(header, containsStringIgnoreCase("Vertebrate study Y/N"), $vertebrateCol: col) from $table.streamHeaders().toList()
-        $rowCell: TableCell(!header, containsAnyString("Yes", "Y"), $rowWithYes: row) from $table.streamCol($vertebrateCol).toList()
+        $rowCell: TableCell(!header, containsAnyString("Yes", "Y", "yes"), !containsAnyString("Y/N"), $rowWithYes: row) from $table.streamCol($vertebrateCol).toList()
         TableCell(row == $rowWithYes) from $table.streamCol($authorCol).toList()
     then
-        entityCreationService.bySemanticNode($rowCell, "must_redact", EntityType.HINT)
+        entityCreationService.bySemanticNode($rowCell, "redaction_indicator", EntityType.HINT)
             .ifPresent(yesEntity -> {
-                yesEntity.skip("CBI.12.0", "must_redact");
+                yesEntity.skip("CBI.12.0", "redaction_indicator");
             });
     end
 
@@ -376,7 +200,7 @@ rule "CBI.12.1: Redact and recommend TableCell with header 'Author' or 'Author(s
         $table: Table(hasHeader("Author(s)") || hasHeader("Author"), hasHeaderIgnoreCase("Vertebrate Study Y/N"))
         TableCell(header, containsAnyStringIgnoreCase("Author", "Author(s)"), $authorCol: col) from $table.streamHeaders().toList()
         TableCell(header, containsStringIgnoreCase("Vertebrate study Y/N"), $vertebrateCol: col) from $table.streamHeaders().toList()
-        TableCell(!header, containsAnyString("Yes", "Y"), $rowWithYes: row) from $table.streamCol($vertebrateCol).toList()
+        TableCell(!header, containsAnyString("Yes", "Y", "yes"), !containsAnyString("Y/N"), $rowWithYes: row) from $table.streamCol($vertebrateCol).toList()
         $authorCell: TableCell(row == $rowWithYes) from $table.streamCol($authorCol).toList()
     then
         entityCreationService.bySemanticNode($authorCell, "CBI_author", EntityType.ENTITY)
@@ -384,7 +208,6 @@ rule "CBI.12.1: Redact and recommend TableCell with header 'Author' or 'Author(s
               authorEntity.redact("CBI.12.1", "Redacted because it's row belongs to a vertebrate study", "Article 39(e)(3) of Regulation (EC) No 178/2002");
               dictionary.addMultipleAuthorsAsRecommendation(authorEntity);
           });
-
     end
 
 rule "CBI.12.2: Redact and recommend TableCell with header 'Author' or 'Author(s)' and header 'Vertebrate study Y/N' with value 'Yes' (vertebrate study)"
@@ -419,6 +242,126 @@ rule "CBI.12.3: Skip TableCell with header 'Author' or 'Author(s)' and header 'V
     end
 
 
+//from CBI.3.3
+rule "CBI.12.4: Redacted because table row contains a redaction_indicator"
+    when
+        $table: Table(hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address"))
+    then
+        $table.streamEntitiesWhereRowContainsEntitiesOfType(List.of("redaction_indicator"))
+            .filter(entity -> entity.getType().equals("CBI_author") || entity.getType().equals("CBI_address"))
+            .forEach(entity -> {
+                entity.applyWithReferences(
+                        "CBI.12.4",
+                        "Redaction_indicator found",
+                        "Reg (EC) No 1107/2009 Art. 63 (2g)",
+                        $table.getEntitiesOfTypeInSameRow("redaction_indicator", entity)
+                );
+            });
+    end
+
+
+//from CBI.3.1
+rule "CBI.12.5: Redacted because table row contains a vertebrate"
+    when
+        $table: Table(hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address"))
+    then
+        $table.streamEntitiesWhereRowContainsEntitiesOfType(List.of("vertebrate"))
+            .filter(entity -> entity.getType().equals("CBI_author") || entity.getType().equals("CBI_address"))
+            .forEach(entity -> {
+                entity.applyWithReferences(
+                        "CBI.12.5",
+                        "Vertebrate found",
+                        "Reg (EC) No 1107/2009 Art. 63 (2g)",
+                        $table.getEntitiesOfTypeInSameRow("vertebrate", entity)
+                );
+            });
+    end
+
+
+// Rule unit: CBI.13 - section rules
+// from CBI.3.0
+rule "CBI.13.1: Redacted because Section contains a vertebrate"
+    when
+        $section: Section(!hasTables(), hasEntitiesOfType("vertebrate"), (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
+    then
+        $section.getEntitiesOfType(List.of("CBI_author", "CBI_address"))
+            .forEach(entity -> {
+                entity.applyWithReferences(
+                        "CBI.13.1",
+                        "Vertebrate found",
+                        "Reg (EC) No 1107/2009 Art. 63 (2g)",
+                        $section.getEntitiesOfType("vertebrate")
+                );
+            });
+    end
+
+//from CBI.3.2
+rule "CBI.13.2: Do not redact because Section does not contain a vertebrate"
+    when
+        $section: Section(!hasTables(), !hasEntitiesOfType("vertebrate"), (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
+    then
+        $section.getEntitiesOfType(List.of("CBI_author", "CBI_address"))
+            .forEach(entity -> entity.skip("CBI.13.2", "No vertebrate found"));
+    end
+
+
+// from CBI.4.0
+rule "CBI.13.3: Do not redact Names and Addresses if vertebrate and no_redaction_indicator is found in Section"
+    when
+        $section: Section(!hasTables(),
+                          hasEntitiesOfType("vertebrate"),
+                          hasEntitiesOfType("no_redaction_indicator"),
+                          (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
+    then
+        $section.getEntitiesOfType(List.of("CBI_author", "CBI_address"))
+            .forEach(entity -> {
+                entity.skipWithReferences(
+                        "CBI.13.3",
+                        "Vertebrate but a no redaction indicator found",
+                        $section.getEntitiesOfType("no_redaction_indicator")
+                );
+            });
+    end
+
+
+// from CBI.5.0
+rule "CBI.13.4: Redact Names and Addresses if vertebrate and no_redaction_indicator but also redaction_indicator is found in Section"
+    when
+        $section: Section(!hasTables(),
+                          hasEntitiesOfType("vertebrate"),
+                          hasEntitiesOfType("redaction_indicator"),
+                          hasEntitiesOfType("no_redaction_indicator"),
+                          (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
+    then
+        $section.getEntitiesOfType(List.of("CBI_author", "CBI_address"))
+            .forEach(entity -> {
+                entity.applyWithReferences(
+                        "CBI.13.4",
+                        "no_redaction_indicator but also redaction_indicator found",
+                        "Reg (EC) No 1107/2009 Art. 63 (2g)",
+                        Stream.concat(
+                            $section.getEntitiesOfType("redaction_indicator").stream(),
+                            $section.getEntitiesOfType("no_redaction_indicator").stream()).toList()
+                );
+            });
+    end
+
+// From CBI.8.0
+rule "CBI.13.5: Redacted because Section contains must_redact entity"
+    when
+        $section: Section(!hasTables(), hasEntitiesOfType("must_redact"), (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
+    then
+        $section.getEntitiesOfType(List.of("CBI_author", "CBI_address"))
+            .forEach(entity -> {
+                entity.applyWithReferences(
+                        "CBI.13.5",
+                        "must_redact entity found",
+                        "Reg (EC) No 1107/2009 Art. 63 (2b)",
+                        $section.getEntitiesOfType("must_redact")
+                );
+            });
+    end
+
 // Rule unit: CBI.14
 rule "CBI.14.0: Redact CBI_sponsor entities if preceded by \"batches produced at\""
     when
@@ -450,7 +393,8 @@ rule "CBI.15.0: Redact row if row contains \"determination of residues\" and liv
     then
         entityCreationService.byString($keyword, "must_redact", EntityType.HINT, $section)
             .forEach(entity -> entity.skip("CBI.15.0", "must_redact"));
-
+        entityCreationService.byString($residueKeyword, "must_redact", EntityType.HINT, $section)
+            .forEach(entity -> entity.skip("CBI.15.0", "must_redact"));
         $section.getEntitiesOfType(List.of("CBI_author", "CBI_address"))
             .forEach(redactionEntity -> redactionEntity.redact("CBI.15.0", "Determination of residues and keyword \"" + $keyword + "\" was found.", "Reg (EC) No 1107/2009 Art. 63 (2g)"));
     end
@@ -471,7 +415,8 @@ rule "CBI.15.1: Redact CBI_author and CBI_address if row contains \"determinatio
     then
         entityCreationService.byString($keyword, "must_redact", EntityType.HINT, $table)
             .forEach(entity -> entity.skip("CBI.15.1", "must_redact"));
-
+        entityCreationService.byString($residueKeyword, "must_redact", EntityType.HINT, $table)
+            .forEach(entity -> entity.skip("CBI.15.1", "must_redact"));
         $table.streamEntitiesWhereRowContainsStringsIgnoreCase(List.of($keyword, $residueKeyword))
             .filter(redactionEntity -> redactionEntity.isAnyType(List.of("CBI_author", "CBI_address")))
             .forEach(redactionEntity -> redactionEntity.redact("CBI.15.1", "Determination of residues and keyword \"" + $keyword + "\" was found.", "Reg (EC) No 1107/2009 Art. 63 (2g)"));
@@ -536,9 +481,9 @@ rule "CBI.20.0: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJEC
     when
         $section: Section(!hasTables(), containsString("PERFORMING LABORATORY:"), containsString("LABORATORY PROJECT ID:"))
     then
-        entityCreationService.betweenStrings("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", EntityType.ENTITY, $section)
+        entityCreationService.betweenStrings("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "PII", EntityType.ENTITY, $section)
          .forEach(laboratoryEntity -> {
-             laboratoryEntity.redact("CBI.20.0", "PERFORMING LABORATORY was found", "Reg (EC) No 1107/2009 Art. 63 (2g)");
+             laboratoryEntity.redact("CBI.20.0", "PERFORMING LABORATORY was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
              dictionary.recommendEverywhere(laboratoryEntity);
          });
     end
@@ -588,7 +533,7 @@ rule "CBI.23.0: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\" (no
         not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
         $document: Document(containsStringIgnoreCase("AUTHOR(S)"), containsAnyStringIgnoreCase("COMPLETION DATE", "STUDY COMPLETION DATE"))
     then
-        entityCreationService.shortestBetweenAnyStringIgnoreCase(List.of("AUTHOR(S)", "AUTHOR(S):"), List.of("COMPLETION DATE", "COMPLETION DATE:", "STUDY COMPLETION DATE", "STUDY COMPLETION DATE:"), "CBI_author", EntityType.ENTITY, $document)
+        entityCreationService.shortestBetweenAnyStringIgnoreCase(List.of("AUTHOR(S)", "AUTHOR(S):"), List.of("COMPLETION DATE", "COMPLETION DATE:", "STUDY COMPLETION DATE", "STUDY COMPLETION DATE:"), "CBI_author", EntityType.ENTITY, $document, 200)
          .forEach(authorEntity -> authorEntity.redact("CBI.23.0", "AUTHOR(S) was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
     end
 
@@ -597,7 +542,7 @@ rule "CBI.23.1: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\" (ve
         FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
         $document: Document(containsStringIgnoreCase("AUTHOR(S)"), containsAnyStringIgnoreCase("COMPLETION DATE", "STUDY COMPLETION DATE"))
     then
-        entityCreationService.shortestBetweenAnyStringIgnoreCase(List.of("AUTHOR(S)", "AUTHOR(S):"), List.of("COMPLETION DATE", "COMPLETION DATE:", "STUDY COMPLETION DATE", "STUDY COMPLETION DATE:"), "CBI_author", EntityType.ENTITY, $document)
+        entityCreationService.shortestBetweenAnyStringIgnoreCase(List.of("AUTHOR(S)", "AUTHOR(S):"), List.of("COMPLETION DATE", "COMPLETION DATE:", "STUDY COMPLETION DATE", "STUDY COMPLETION DATE:"), "CBI_author", EntityType.ENTITY, $document, 200)
         .forEach(authorEntity -> authorEntity.redact("CBI.23.1", "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
     end
 
@@ -609,7 +554,7 @@ rule "PII.0.0: Redact all PII"
     when
         $pii: TextEntity(type() == "PII", dictionaryEntry)
     then
-        $pii.redact("PII.0.0", "Personal Information found", "Reg (EC) No 1107/2009 Art. 63 (2g)");
+        $pii.redact("PII.0.0", "Personal Information found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
     end
 
 rule "PII.0.1: Redact all PII (non vertebrate study)"
@@ -687,8 +632,6 @@ rule "PII.4.0: Redact line after contact information keywords"
                                          "Contact:",
                                          "Alternative contact:",
                                          "European contact:",
-                                         "No:",
-                                         "Contact:",
                                          "Tel.:",
                                          "Tel:",
                                          "Telephone number:",
@@ -735,62 +678,6 @@ rule "PII.4.1: Redact line after contact information keywords"
             .forEach(contactEntity -> contactEntity.redact("PII.4.1", "Found after \"" + $contactKeyword + "\" contact keyword", "Article 4(1)(b), Regulation (EC) No 1049/2001 (Personal data)"));
     end
 
-rule "PII.4.2: Redact line after contact information keywords (Non vertebrate study)"
-    when
-        not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
-        $contactKeyword: String() from List.of("Contact point:",
-                                         "Contact:",
-                                         "Alternative contact:",
-                                         "European contact:",
-                                         "No:",
-                                         "Contact:",
-                                         "Tel.:",
-                                         "Tel:",
-                                         "Telephone number:",
-                                         "Telephone No:",
-                                         "Telephone:",
-                                         "Phone No.",
-                                         "Phone:",
-                                         "Fax number:",
-                                         "Fax:",
-                                         "E-mail:",
-                                         "Email:",
-                                         "e-mail:",
-                                         "E-mail address:")
-        $section: Section(containsString($contactKeyword))
-    then
-        entityCreationService.lineAfterString($contactKeyword, "PII", EntityType.ENTITY, $section)
-            .forEach(contactEntity -> contactEntity.redact("PII.4.2", "Found after \"" + $contactKeyword + "\" contact keyword", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
-    end
-
-rule "PII.4.3: Redact line after contact information keywords (Vertebrate study)"
-    when
-        FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
-        $contactKeyword: String() from List.of("Contact point:",
-                                         "Contact:",
-                                         "Alternative contact:",
-                                         "European contact:",
-                                         "No:",
-                                         "Contact:",
-                                         "Tel.:",
-                                         "Tel:",
-                                         "Telephone number:",
-                                         "Telephone No:",
-                                         "Telephone:",
-                                         "Phone No.",
-                                         "Phone:",
-                                         "Fax number:",
-                                         "Fax:",
-                                         "E-mail:",
-                                         "Email:",
-                                         "e-mail:",
-                                         "E-mail address:")
-        $section: Section(containsString($contactKeyword))
-    then
-        entityCreationService.lineAfterString($contactKeyword, "PII", EntityType.ENTITY, $section)
-            .forEach(contactEntity -> contactEntity.redact("PII.4.3", "Found after \"" + $contactKeyword + "\" contact keyword", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
-    end
-
 
 // Rule unit: PII.6
 rule "PII.6.0: Redact line between contact keywords"
@@ -850,12 +737,7 @@ rule "PII.7.0: Redact contact information if applicant is found"
                           containsString("Applicant") ||
                           containsString("Telephone number:"))
     then
-        Stream.concat(entityCreationService.lineAfterStrings(List.of("Contact point:", "Contact:", "Alternative contact:", "European contact:", "No:", "Contact:", "Tel.:", "Tel:", "Telephone number:",
+        entityCreationService.lineAfterStrings(List.of("Contact point:", "Contact:", "Alternative contact:", "European contact:", "Tel.:", "Tel:", "Telephone number:", "Telephone No:", "Telephone:", "Phone No.", "Phone:", "Fax number:", "Fax:", "E-mail:", "Email:", "e-mail:", "E-mail address:"), "PII", EntityType.ENTITY, $section)
-                            "Telephone No:", "Telephone:", "Phone No.", "Phone:", "Fax number:", "Fax:", "E-mail:", "Email:", "e-mail:", "E-mail address:"), "PII", EntityType.ENTITY, $section),
-             Stream.concat(
-                 entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section),
-                 entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section)
-             ))
             .forEach(entity -> entity.redact("PII.7.0", "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)"));
     end
 
@@ -956,7 +838,7 @@ rule "PII.9.0: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\""
     when
         $document: Document(containsStringIgnoreCase("AUTHOR(S)"), containsAnyStringIgnoreCase("COMPLETION DATE", "STUDY COMPLETION DATE"))
     then
-        entityCreationService.shortestBetweenAnyStringIgnoreCase(List.of("AUTHOR(S)", "AUTHOR(S):"), List.of("COMPLETION DATE", "COMPLETION DATE:", "STUDY COMPLETION DATE", "STUDY COMPLETION DATE:"), "PII", EntityType.ENTITY, $document)
+        entityCreationService.shortestBetweenAnyStringIgnoreCase(List.of("AUTHOR(S)", "AUTHOR(S):"), List.of("COMPLETION DATE", "COMPLETION DATE:", "STUDY COMPLETION DATE", "STUDY COMPLETION DATE:"), "PII", EntityType.ENTITY, $document, 200)
          .forEach(authorEntity -> authorEntity.redact("PII.9.0", "AUTHOR(S) was found", "Reg (EC) No 1107/2009 Art. 63 (2e)"));
     end
 
@@ -1064,7 +946,7 @@ rule "ETC.3.2: Redact logos (vertebrate study)"
         FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
         $logo: Image(imageType == ImageType.LOGO)
     then
-        $logo.redact("ETC.3.2", "Logo Found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        $logo.redact("ETC.3.2", "Logo Found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
     end
 
 rule "ETC.3.3: Redact logos"
@@ -1251,7 +1133,7 @@ rule "AI.7.0: Add all NER Entities of type Address"
     end
 
 
-//------------------------------------ Manual changes rules ------------------------------------
+//------------------------------------ Manual redaction rules ------------------------------------
 
 // Rule unit: MAN.0
 rule "MAN.0.0: Apply manual resize redaction"
@@ -1410,12 +1292,23 @@ rule "MAN.4.1: Apply legal basis change"
 rule "X.0.0: Remove Entity contained by Entity of same type"
     salience 65
     when
-        $larger: TextEntity($type: type(), $entityType: entityType, !removed())
+        $larger: TextEntity($type: type(), $entityType: entityType, !removed(), !hasManualChanges())
-        $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges(), !removed())
+        not TextEntity(getTextRange().equals($larger.getTextRange()), type() == $type, entityType == EntityType.DICTIONARY_REMOVAL, engines contains Engine.DOSSIER_DICTIONARY, !hasManualChanges())
+        $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges())
+    then
+        $contained.remove("X.0.0", "remove Entity contained by Entity of same type");
+        retract($contained);
+    end
+
+rule "X.0.1: Remove Entity contained by Entity of same type with manual changes"
+    salience 65
+    when
+        $larger: TextEntity($type: type(), $entityType: entityType, !removed(), hasManualChanges())
+        $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges())
     then
         $contained.getIntersectingNodes().forEach(node -> update(node));
-        $contained.remove("X.0.0", "remove Entity contained by Entity of same type");
+        $contained.remove("X.0.1", "remove Entity contained by Entity of same type with manual changes");
-        update($contained);
+        retract($contained);
     end
 
 
@@ -1424,10 +1317,21 @@ rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
     salience 64
     when
         $falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active())
-        $entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges())
+        $entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.ENTITY), !hasManualChanges())
+    then
+        $entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE");
+        retract($entity)
+    end
+
+
+rule "X.2.1: Remove Entity of type HINT when contained by FALSE_POSITIVE"
+    salience 64
+    when
+        $falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active())
+        $entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.HINT), !hasManualChanges())
     then
         $entity.getIntersectingNodes().forEach(node -> update(node));
-        $entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE");
+        $entity.remove("X.2.1", "remove Entity of type ENTITY when contained by FALSE_POSITIVE");
         retract($entity)
     end
 
@@ -1548,29 +1452,41 @@ rule "X.10.0: remove false positives of ai"
 
 
 // Rule unit: X.11
-rule "X.11.0: Remove dictionary entity which intersects with a manual entity"
+rule "X.11.1: Remove non manual entity which intersects with a manual entity"
     salience 64
     when
         $manualEntity: TextEntity(engines contains Engine.MANUAL, active())
-        $dictionaryEntity: TextEntity(intersects($manualEntity), dictionaryEntry, engines not contains Engine.MANUAL)
+        $nonManualEntity: TextEntity(intersects($manualEntity), engines not contains Engine.MANUAL)
     then
-        $dictionaryEntity.remove("X.11.0", "remove dictionary entity which intersects with a manual entity");
+        $nonManualEntity.remove("X.11.1", "remove entity which intersects with a manual entity");
-        retract($dictionaryEntity);
+        retract($nonManualEntity);
+    end
+
+
+rule "X.11.2: Remove non manual entity which are equal to manual entity"
+    salience 70
+    when
+        $manualEntity: TextEntity(engines contains Engine.MANUAL, active(), $type: type())
+        $nonManualEntity: TextEntity(getTextRange().equals($manualEntity.getTextRange()), type() == $type, entityType == EntityType.ENTITY, !hasManualChanges(), engines not contains Engine.MANUAL)
+    then
+        $manualEntity.addEngines($nonManualEntity.getEngines());
+        $nonManualEntity.remove("X.11.2", "remove non manual entity which are equal to manual entity");
+        retract($nonManualEntity);
     end
 
 
 //------------------------------------ Dictionary merging rules ------------------------------------
 
 // Rule unit: DICT.0
-rule "DICT.0.0: Ignore Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL"
+rule "DICT.0.0: Remove Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL"
     salience 64
     when
         $dictionaryRemoval: TextEntity($type: type(), entityType == EntityType.DICTIONARY_REMOVAL, engines contains Engine.DOSSIER_DICTIONARY)
         $entity: TextEntity(getTextRange().equals($dictionaryRemoval.getTextRange()), engines contains Engine.DICTIONARY, type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges())
-        not TextEntity(containedBy($entity), engines contains Engine.DOSSIER_DICTIONARY, type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges())
     then
         $entity.getIntersectingNodes().forEach(node -> update(node));
-        $entity.ignore("DICT.0.0", "Ignore Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL");
+        $entity.remove("DICT.0.0", "Remove Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL");
+        $entity.addEngine(Engine.DOSSIER_DICTIONARY);
     end
 
 

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_v2.drl

@@ -72,12 +72,46 @@ query "getFileAttributes"
 //------------------------------------ CBI rules ------------------------------------
 
 // Rule unit: CBI.0
-rule "CBI.0.0: Redact CBI Authors (non vertebrate Study)"
+rule "CBI.0.1: Add CBI_author with \"et al.\" RegEx (non vertebrate study)"
+    agenda-group "LOCAL_DICTIONARY_ADDS"
+    when
+        not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
+        $section: Section(containsString("et al."))
+    then
+        entityCreationService.byRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, 1, $section)
+            .forEach(entity -> {
+                entity.redact("CBI.0.1", "Author found by \"et al\" regex", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+                dictionary.recommendEverywhere(entity);
+            });
+    end
+
+rule "CBI.0.2: Add CBI_author with \"et al.\" RegEx (vertebrate study)"
+    agenda-group "LOCAL_DICTIONARY_ADDS"
+    when
+        FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
+        $section: Section(containsString("et al."))
+    then
+        entityCreationService.byRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, 1, $section)
+            .forEach(entity -> {
+                entity.redact("CBI.0.2", "Author found by \"et al\" regex", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+                dictionary.recommendEverywhere(entity);
+            });
+    end
+
+rule "CBI.0.3: Redact CBI Authors (non vertebrate Study)"
     when
         not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
         $entity: TextEntity(type() == "CBI_author", dictionaryEntry)
     then
-        $entity.redact("CBI.0.0", "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        $entity.redact("CBI.0.3", "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+    end
+
+rule "CBI.0.4: Redact CBI Authors (vertebrate Study)"
+    when
+        FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
+        $entity: TextEntity(type() == "CBI_author", dictionaryEntry)
+    then
+        $entity.redact("CBI.0.4", "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
     end
 
 
@@ -332,12 +366,23 @@ rule "MAN.4.1: Apply legal basis change"
 rule "X.0.0: Remove Entity contained by Entity of same type"
     salience 65
     when
-        $larger: TextEntity($type: type(), $entityType: entityType, !removed())
+        $larger: TextEntity($type: type(), $entityType: entityType, !removed(), !hasManualChanges())
-        $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges(), !removed())
+        not TextEntity(getTextRange().equals($larger.getTextRange()), type() == $type, entityType == EntityType.DICTIONARY_REMOVAL, engines contains Engine.DOSSIER_DICTIONARY, !hasManualChanges())
+        $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges())
+    then
+        $contained.remove("X.0.0", "remove Entity contained by Entity of same type");
+        retract($contained);
+    end
+
+rule "X.0.1: Remove Entity contained by Entity of same type with manual changes"
+    salience 65
+    when
+        $larger: TextEntity($type: type(), $entityType: entityType, !removed(), hasManualChanges())
+        $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges())
     then
         $contained.getIntersectingNodes().forEach(node -> update(node));
-        $contained.remove("X.0.0", "remove Entity contained by Entity of same type");
+        $contained.remove("X.0.1", "remove Entity contained by Entity of same type with manual changes");
-        update($contained);
+        retract($contained);
     end
 
 
@@ -346,10 +391,21 @@ rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
     salience 64
     when
         $falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active())
-        $entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges())
+        $entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.ENTITY), !hasManualChanges())
+    then
+        $entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE");
+        retract($entity)
+    end
+
+
+rule "X.2.1: Remove Entity of type HINT when contained by FALSE_POSITIVE"
+    salience 64
+    when
+        $falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active())
+        $entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.HINT), !hasManualChanges())
     then
         $entity.getIntersectingNodes().forEach(node -> update(node));
-        $entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE");
+        $entity.remove("X.2.1", "remove Entity of type ENTITY when contained by FALSE_POSITIVE");
         retract($entity)
     end
 
@@ -470,29 +526,41 @@ rule "X.10.0: remove false positives of ai"
 
 
 // Rule unit: X.11
-rule "X.11.0: Remove dictionary entity which intersects with a manual entity"
+rule "X.11.1: Remove non manual entity which intersects with a manual entity"
     salience 64
     when
         $manualEntity: TextEntity(engines contains Engine.MANUAL, active())
-        $dictionaryEntity: TextEntity(intersects($manualEntity), dictionaryEntry, engines not contains Engine.MANUAL)
+        $nonManualEntity: TextEntity(intersects($manualEntity), engines not contains Engine.MANUAL)
     then
-        $dictionaryEntity.remove("X.11.0", "remove dictionary entity which intersects with a manual entity");
+        $nonManualEntity.remove("X.11.1", "remove entity which intersects with a manual entity");
-        retract($dictionaryEntity);
+        retract($nonManualEntity);
+    end
+
+
+rule "X.11.2: Remove non manual entity which are equal to manual entity"
+    salience 70
+    when
+        $manualEntity: TextEntity(engines contains Engine.MANUAL, active(), $type: type())
+        $nonManualEntity: TextEntity(getTextRange().equals($manualEntity.getTextRange()), type() == $type, entityType == EntityType.ENTITY, !hasManualChanges(), engines not contains Engine.MANUAL)
+    then
+        $manualEntity.addEngines($nonManualEntity.getEngines());
+        $nonManualEntity.remove("X.11.2", "remove non manual entity which are equal to manual entity");
+        retract($nonManualEntity);
     end
 
 
 //------------------------------------ Dictionary merging rules ------------------------------------
 
 // Rule unit: DICT.0
-rule "DICT.0.0: Ignore Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL"
+rule "DICT.0.0: Remove Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL"
     salience 64
     when
         $dictionaryRemoval: TextEntity($type: type(), entityType == EntityType.DICTIONARY_REMOVAL, engines contains Engine.DOSSIER_DICTIONARY)
         $entity: TextEntity(getTextRange().equals($dictionaryRemoval.getTextRange()), engines contains Engine.DICTIONARY, type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges())
-        not TextEntity(containedBy($entity), engines contains Engine.DOSSIER_DICTIONARY, type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges())
     then
         $entity.getIntersectingNodes().forEach(node -> update(node));
-        $entity.ignore("DICT.0.0", "Ignore Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL");
+        $entity.remove("DICT.0.0", "Remove Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL");
+        $entity.addEngine(Engine.DOSSIER_DICTIONARY);
     end
 
 

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/table_demo.drl

@@ -458,12 +458,23 @@ rule "MAN.4.1: Apply legal basis change"
 rule "X.0.0: Remove Entity contained by Entity of same type"
     salience 65
     when
-        $larger: TextEntity($type: type(), $entityType: entityType, !removed())
+        $larger: TextEntity($type: type(), $entityType: entityType, !removed(), !hasManualChanges())
-        $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges(), !removed())
+        not TextEntity(getTextRange().equals($larger.getTextRange()), type() == $type, entityType == EntityType.DICTIONARY_REMOVAL, engines contains Engine.DOSSIER_DICTIONARY, !hasManualChanges())
+        $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges())
+    then
+        $contained.remove("X.0.0", "remove Entity contained by Entity of same type");
+        retract($contained);
+    end
+
+rule "X.0.1: Remove Entity contained by Entity of same type with manual changes"
+    salience 65
+    when
+        $larger: TextEntity($type: type(), $entityType: entityType, !removed(), hasManualChanges())
+        $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges())
     then
         $contained.getIntersectingNodes().forEach(node -> update(node));
-        $contained.remove("X.0.0", "remove Entity contained by Entity of same type");
+        $contained.remove("X.0.1", "remove Entity contained by Entity of same type with manual changes");
-        update($contained);
+        retract($contained);
     end
 
 
@@ -472,10 +483,21 @@ rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
     salience 64
     when
         $falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active())
-        $entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges())
+        $entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.ENTITY), !hasManualChanges())
+    then
+        $entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE");
+        retract($entity)
+    end
+
+
+rule "X.2.1: Remove Entity of type HINT when contained by FALSE_POSITIVE"
+    salience 64
+    when
+        $falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active())
+        $entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.HINT), !hasManualChanges())
     then
         $entity.getIntersectingNodes().forEach(node -> update(node));
-        $entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE");
+        $entity.remove("X.2.1", "remove Entity of type ENTITY when contained by FALSE_POSITIVE");
         retract($entity)
     end
 
@@ -607,29 +629,41 @@ rule "X.10.0: remove false positives of ai"
 
 
 // Rule unit: X.11
-rule "X.11.0: Remove dictionary entity which intersects with a manual entity"
+rule "X.11.1: Remove non manual entity which intersects with a manual entity"
     salience 64
     when
         $manualEntity: TextEntity(engines contains Engine.MANUAL, active())
-        $dictionaryEntity: TextEntity(intersects($manualEntity), dictionaryEntry, engines not contains Engine.MANUAL)
+        $nonManualEntity: TextEntity(intersects($manualEntity), engines not contains Engine.MANUAL)
     then
-        $dictionaryEntity.remove("X.11.0", "remove dictionary entity which intersects with a manual entity");
+        $nonManualEntity.remove("X.11.1", "remove entity which intersects with a manual entity");
-        retract($dictionaryEntity);
+        retract($nonManualEntity);
+    end
+
+
+rule "X.11.2: Remove non manual entity which are equal to manual entity"
+    salience 70
+    when
+        $manualEntity: TextEntity(engines contains Engine.MANUAL, active(), $type: type())
+        $nonManualEntity: TextEntity(getTextRange().equals($manualEntity.getTextRange()), type() == $type, entityType == EntityType.ENTITY, !hasManualChanges(), engines not contains Engine.MANUAL)
+    then
+        $manualEntity.addEngines($nonManualEntity.getEngines());
+        $nonManualEntity.remove("X.11.2", "remove non manual entity which are equal to manual entity");
+        retract($nonManualEntity);
     end
 
 
 //------------------------------------ Dictionary merging rules ------------------------------------
 
 // Rule unit: DICT.0
-rule "DICT.0.0: Ignore Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL"
+rule "DICT.0.0: Remove Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL"
     salience 64
     when
         $dictionaryRemoval: TextEntity($type: type(), entityType == EntityType.DICTIONARY_REMOVAL, engines contains Engine.DOSSIER_DICTIONARY)
         $entity: TextEntity(getTextRange().equals($dictionaryRemoval.getTextRange()), engines contains Engine.DICTIONARY, type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges())
-        not TextEntity(containedBy($entity), engines contains Engine.DOSSIER_DICTIONARY, type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges())
     then
         $entity.getIntersectingNodes().forEach(node -> update(node));
-        $entity.ignore("DICT.0.0", "Ignore Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL");
+        $entity.remove("DICT.0.0", "Remove Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL");
+        $entity.addEngine(Engine.DOSSIER_DICTIONARY);
     end
 
 

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/test_rules.drl

@@ -358,12 +358,23 @@ rule "MAN.4.1: Apply legal basis change"
 rule "X.0.0: Remove Entity contained by Entity of same type"
     salience 65
     when
-        $larger: TextEntity($type: type(), $entityType: entityType, !removed())
+        $larger: TextEntity($type: type(), $entityType: entityType, !removed(), !hasManualChanges())
-        $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges(), !removed())
+        not TextEntity(getTextRange().equals($larger.getTextRange()), type() == $type, entityType == EntityType.DICTIONARY_REMOVAL, engines contains Engine.DOSSIER_DICTIONARY, !hasManualChanges())
+        $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges())
+    then
+        $contained.remove("X.0.0", "remove Entity contained by Entity of same type");
+        retract($contained);
+    end
+
+rule "X.0.1: Remove Entity contained by Entity of same type with manual changes"
+    salience 65
+    when
+        $larger: TextEntity($type: type(), $entityType: entityType, !removed(), hasManualChanges())
+        $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges())
     then
         $contained.getIntersectingNodes().forEach(node -> update(node));
-        $contained.remove("X.0.0", "remove Entity contained by Entity of same type");
+        $contained.remove("X.0.1", "remove Entity contained by Entity of same type with manual changes");
-        update($contained);
+        retract($contained);
     end
 
 
@@ -372,14 +383,49 @@ rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
     salience 64
     when
         $falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active())
-        $entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges())
+        $entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.ENTITY), !hasManualChanges())
+    then
+        $entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE");
+        retract($entity)
+    end
+
+
+rule "X.2.1: Remove Entity of type HINT when contained by FALSE_POSITIVE"
+    salience 64
+    when
+        $falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active())
+        $entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.HINT), !hasManualChanges())
     then
         $entity.getIntersectingNodes().forEach(node -> update(node));
+        $entity.remove("X.2.1", "remove Entity of type ENTITY when contained by FALSE_POSITIVE");
+        retract($entity)
+    end
+
+
+// Rule unit: X.2
+rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
+    salience 64
+    when
+        $falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active())
+        $entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.ENTITY), !hasManualChanges())
+    then
         $entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE");
         retract($entity)
     end
 
 
+rule "X.2.1: Remove Entity of type HINT when contained by FALSE_POSITIVE"
+    salience 64
+    when
+        $falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active())
+        $entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.HINT), !hasManualChanges())
+    then
+        $entity.getIntersectingNodes().forEach(node -> update(node));
+        $entity.remove("X.2.1", "remove Entity of type ENTITY when contained by FALSE_POSITIVE");
+        retract($entity)
+    end
+
+
 // Rule unit: X.3
 rule "X.3.0: Remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"
     salience 64
@@ -507,29 +553,41 @@ rule "X.10.0: remove false positives of ai"
 
 
 // Rule unit: X.11
-rule "X.11.0: Remove dictionary entity which intersects with a manual entity"
+rule "X.11.1: Remove non manual entity which intersects with a manual entity"
     salience 64
     when
         $manualEntity: TextEntity(engines contains Engine.MANUAL, active())
-        $dictionaryEntity: TextEntity(intersects($manualEntity), dictionaryEntry, engines not contains Engine.MANUAL)
+        $nonManualEntity: TextEntity(intersects($manualEntity), engines not contains Engine.MANUAL)
     then
-        $dictionaryEntity.remove("X.11.0", "remove dictionary entity which intersects with a manual entity");
+        $nonManualEntity.remove("X.11.1", "remove entity which intersects with a manual entity");
-        retract($dictionaryEntity);
+        retract($nonManualEntity);
+    end
+
+
+rule "X.11.2: Remove non manual entity which are equal to manual entity"
+    salience 70
+    when
+        $manualEntity: TextEntity(engines contains Engine.MANUAL, active(), $type: type())
+        $nonManualEntity: TextEntity(getTextRange().equals($manualEntity.getTextRange()), type() == $type, entityType == EntityType.ENTITY, !hasManualChanges(), engines not contains Engine.MANUAL)
+    then
+        $manualEntity.addEngines($nonManualEntity.getEngines());
+        $nonManualEntity.remove("X.11.2", "remove non manual entity which are equal to manual entity");
+        retract($nonManualEntity);
     end
 
 
 //------------------------------------ Dictionary merging rules ------------------------------------
 
 // Rule unit: DICT.0
-rule "DICT.0.0: Ignore Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL"
+rule "DICT.0.0: Remove Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL"
     salience 64
     when
         $dictionaryRemoval: TextEntity($type: type(), entityType == EntityType.DICTIONARY_REMOVAL, engines contains Engine.DOSSIER_DICTIONARY)
         $entity: TextEntity(getTextRange().equals($dictionaryRemoval.getTextRange()), engines contains Engine.DICTIONARY, type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges())
-        not TextEntity(containedBy($entity), engines contains Engine.DOSSIER_DICTIONARY, type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges())
     then
         $entity.getIntersectingNodes().forEach(node -> update(node));
-        $entity.ignore("DICT.0.0", "Ignore Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL");
+        $entity.remove("DICT.0.0", "Remove Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL");
+        $entity.addEngine(Engine.DOSSIER_DICTIONARY);
     end
 
 

redaction-service-v1/redaction-service-server-v1/src/test/resources/mappings/GuidelineMapping.csv.metadata.json

@@ -11,5 +11,6 @@
   "storageId": "2c4cfafc-b3c3-48de-8706-a979dea764aa/f48e8ab0-250b-4a73-aa4d-436a7176841f_GuidelineMapping_GuidelineMapping.csv",
   "numberOfLines": 30,
   "encoding": "UTF-8",
-  "delimiter": ","
+  "delimiter": ",",
+  "quoteChar": "\""
 }

redaction-service-v1/redaction-service-server-v1/src/test/resources/mappings/GuidelineMapping2.csv.metadata.json

@@ -11,5 +11,6 @@
   "storageId": "2c4cfafc-b3c3-48de-8706-a979dea764aa/f48e8ab0-250b-4a73-aa4d-436a7176841f_GuidelineMapping_GuidelineMapping.csv",
   "numberOfLines": 28,
   "encoding": "UTF-8",
-  "delimiter": ","
+  "delimiter": ",",
+  "quoteChar": "\""
 }

redaction-service-v1/rules-management/src/main/resources/all_rules_documine.drl

@@ -1454,12 +1454,23 @@ rule "MAN.4.1: Apply legal basis change"
 rule "X.0.0: Remove Entity contained by Entity of same type"
     salience 65
     when
-        $larger: TextEntity($type: type(), $entityType: entityType, !removed())
+        $larger: TextEntity($type: type(), $entityType: entityType, !removed(), !hasManualChanges())
-        $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges(), !removed())
+        not TextEntity(getTextRange().equals($larger.getTextRange()), type() == $type, entityType == EntityType.DICTIONARY_REMOVAL, engines contains Engine.DOSSIER_DICTIONARY, !hasManualChanges())
+        $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges())
+    then
+        $contained.remove("X.0.0", "remove Entity contained by Entity of same type");
+        retract($contained);
+    end
+
+rule "X.0.1: Remove Entity contained by Entity of same type with manual changes"
+    salience 65
+    when
+        $larger: TextEntity($type: type(), $entityType: entityType, !removed(), hasManualChanges())
+        $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges())
     then
         $contained.getIntersectingNodes().forEach(node -> update(node));
-        $contained.remove("X.0.0", "remove Entity contained by Entity of same type");
+        $contained.remove("X.0.1", "remove Entity contained by Entity of same type with manual changes");
-        update($contained);
+        retract($contained);
     end
 
 
@@ -1468,10 +1479,21 @@ rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
     salience 64
     when
         $falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active())
-        $entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges())
+        $entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.ENTITY), !hasManualChanges())
+    then
+        $entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE");
+        retract($entity)
+    end
+
+
+rule "X.2.1: Remove Entity of type HINT when contained by FALSE_POSITIVE"
+    salience 64
+    when
+        $falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active())
+        $entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.HINT), !hasManualChanges())
     then
         $entity.getIntersectingNodes().forEach(node -> update(node));
-        $entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE");
+        $entity.remove("X.2.1", "remove Entity of type ENTITY when contained by FALSE_POSITIVE");
         retract($entity)
     end
 
@@ -1593,15 +1615,15 @@ rule "X.11.0: Remove dictionary entity which intersects with a manual entity"
 //------------------------------------ Dictionary merging rules ------------------------------------
 
 // Rule unit: DICT.0
-rule "DICT.0.0: Ignore Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL"
+rule "DICT.0.0: Remove Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL"
     salience 64
     when
         $dictionaryRemoval: TextEntity($type: type(), entityType == EntityType.DICTIONARY_REMOVAL, engines contains Engine.DOSSIER_DICTIONARY)
         $entity: TextEntity(getTextRange().equals($dictionaryRemoval.getTextRange()), engines contains Engine.DICTIONARY, type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges())
-        not TextEntity(containedBy($entity), engines contains Engine.DOSSIER_DICTIONARY, type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges())
     then
         $entity.getIntersectingNodes().forEach(node -> update(node));
-        $entity.ignore("DICT.0.0", "Ignore Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL");
+        $entity.remove("DICT.0.0", "Remove Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL");
+        $entity.addEngine(Engine.DOSSIER_DICTIONARY);
     end
 
 

redaction-service-v1/rules-management/src/test/java/com/knecon/fforesight/utility/rules/management/factory/RuleFileFactoryTest.java

@@ -97,7 +97,7 @@ class RuleFileFactoryTest {
     void createFileFromIdentifiersForRedactionService() {
 
         // This is exactly the string used for the current rules.drl in the redaction-service
-        String identifiers = "CBI.5.*, CBI.9.*, CBI.11.*, AI.1.*, PII.4.*, ETC.8.*, PII.0.*, ETC.6.*, SYN.0.*, CBI.3.*, ETC.4.*, ETC.3.*, PII.12.*, ETC.1.*, PII.9.*, PII.7.*, CBI.12.*, X.*.*, CBI.14.*, CBI.16.*, CBI.18.*, CBI.4.*, AI.0.*, CBI.8.*, PII.1.*, ETC.7.*, LDS.*.*, MAN.*.*, ETC.5.*, PII.11.*, ETC.2.*, CBI.20.*, FA.*.*, PII.8.*, PII.6.*, CBI.15.*, CBI.17.*, CBI.19.*";
+        String identifiers = "CBI.9.*, CBI.11.*, AI.1.*, PII.4.*, ETC.8.*, PII.0.*, ETC.6.*, SYN.0.*, ETC.4.*, ETC.3.*, PII.12.*, ETC.1.*, PII.9.*, PII.7.*, CBI.12.*, X.*.*, CBI.14.*, CBI.16.*, CBI.18.*, AI.0.*, PII.1.*, ETC.7.*, LDS.*.*, MAN.*.*, ETC.5.*, PII.11.*, ETC.2.*, CBI.20.*, FA.*.*, PII.8.*, PII.6.*, CBI.15.*, CBI.17.*, CBI.19.*";
         String result = RuleFileFactory.createFileFromIdentifiers(identifiers, ApplicationType.RM);
         System.out.println(result);
         try (var out = new FileOutputStream(