Merge branch 'RED-8610' into 'master'

RED-8610: Dictionary remove on dossier level should be displayed as skipped

Closes RED-8610

See merge request redactmanager/redaction-service!335

redaction-service-v1/redaction-service-server-v1/build.gradle.kts

@@ -16,7 +16,7 @@ val layoutParserVersion = "0.96.0"
 val jacksonVersion = "2.15.2"
 val droolsVersion = "9.44.0.Final"
 val pdfBoxVersion = "3.0.0"
-val persistenceServiceVersion = "2.377.0"
+val persistenceServiceVersion = "2.379.0"
 val springBootStarterVersion = "3.1.5"
 
 configurations {

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/MigrationEntity.java

@@ -166,6 +166,7 @@ public final class MigrationEntity {
             case FALSE_POSITIVE -> EntryType.FALSE_POSITIVE;
             case RECOMMENDATION -> EntryType.RECOMMENDATION;
             case FALSE_RECOMMENDATION -> EntryType.FALSE_RECOMMENDATION;
+            default -> EntryType.FALSE_POSITIVE;
         };
     }
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/dictionary/DictionaryModel.java

@@ -2,7 +2,6 @@ package com.iqser.red.service.redaction.v1.server.model.dictionary;
 
 import java.io.Serializable;
 import java.util.HashMap;
-import java.util.List;
 import java.util.Locale;
 import java.util.Set;
 import java.util.stream.Collectors;
@@ -11,11 +10,11 @@ import com.iqser.red.service.dictionarymerge.commons.DictionaryEntry;
 import com.iqser.red.service.dictionarymerge.commons.DictionaryEntryModel;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule;
 
-import lombok.AllArgsConstructor;
 import lombok.Data;
+import lombok.extern.slf4j.Slf4j;
 
 @Data
-@AllArgsConstructor
+@Slf4j
 public class DictionaryModel implements Serializable {
 
     private final String type;
@@ -29,6 +28,7 @@ public class DictionaryModel implements Serializable {
     private final Set<DictionaryEntryModel> falseRecommendations;
 
     private transient SearchImplementation entriesSearch;
+    private transient SearchImplementation deletionEntriesSearch;
     private transient SearchImplementation falsePositiveSearch;
     private transient SearchImplementation falseRecommendationsSearch;
 
@@ -56,20 +56,6 @@ public class DictionaryModel implements Serializable {
         this.entries = entries;
         this.falsePositives = falsePositives;
         this.falseRecommendations = falseRecommendations;
-
-        this.entriesSearch = new SearchImplementation(this.entries.stream()
-                                                              .filter(e -> !e.isDeleted())
-                                                              .map(DictionaryEntryModel::getValue)
-                                                              .collect(Collectors.toList()), caseInsensitive);
-        this.falsePositiveSearch = new SearchImplementation(this.falsePositives.stream()
-                                                                    .filter(e -> !e.isDeleted())
-                                                                    .map(DictionaryEntryModel::getValue)
-                                                                    .collect(Collectors.toList()), caseInsensitive);
-        this.falseRecommendationsSearch = new SearchImplementation(this.falseRecommendations.stream()
-                                                                           .filter(e -> !e.isDeleted())
-                                                                           .map(DictionaryEntry::getValue)
-                                                                           .collect(Collectors.toList()), caseInsensitive);
-
     }
 
 
@@ -94,6 +80,18 @@ public class DictionaryModel implements Serializable {
     }
 
 
+    public SearchImplementation getDeletionEntriesSearch() {
+
+        if (deletionEntriesSearch == null) {
+            this.deletionEntriesSearch = new SearchImplementation(this.entries.stream()
+                                                                          .filter(DictionaryEntry::isDeleted)
+                                                                          .map(DictionaryEntry::getValue)
+                                                                          .collect(Collectors.toList()), caseInsensitive);
+        }
+        return deletionEntriesSearch;
+    }
+
+
     public SearchImplementation getFalsePositiveSearch() {
 
         if (falsePositiveSearch == null) {

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/EntityType.java

@@ -5,5 +5,6 @@ public enum EntityType {
     HINT,
     RECOMMENDATION,
     FALSE_POSITIVE,
-    FALSE_RECOMMENDATION
+    FALSE_RECOMMENDATION,
+    DICTIONARY_REMOVAL
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/TextEntity.java

@@ -40,7 +40,7 @@ public class TextEntity implements IEntity {
     TextRange textRange;
     @Builder.Default
     List<TextRange> duplicateTextRanges = new ArrayList<>();
-    String type; // TODO: make final once ManualChangesApplicatioService recategorize is deleted
+    String type; // TODO: make final once ManualChangesApplicationService::recategorize is deleted
     final EntityType entityType;
 
     @Builder.Default
@@ -182,25 +182,37 @@ public class TextEntity implements IEntity {
 
     public boolean containedBy(TextEntity textEntity) {
 
-        return this.textRange.containedBy(textEntity.getTextRange())
+        return this.textRange.containedBy(textEntity.getTextRange()) //
-                || duplicateTextRanges.stream().anyMatch(duplicateTextRange -> duplicateTextRange.containedBy(textEntity.textRange))
+               || duplicateTextRanges.stream()
-                || duplicateTextRanges.stream().anyMatch(duplicateTextRange -> textEntity.getDuplicateTextRanges().stream().anyMatch(duplicateTextRange::containedBy));
+                       .anyMatch(duplicateTextRange -> duplicateTextRange.containedBy(textEntity.textRange)) //
+               || duplicateTextRanges.stream()
+                       .anyMatch(duplicateTextRange -> textEntity.getDuplicateTextRanges()
+                               .stream()
+                               .anyMatch(duplicateTextRange::containedBy));
     }
 
 
     public boolean contains(TextEntity textEntity) {
 
-        return this.textRange.contains(textEntity.getTextRange())
+        return this.textRange.contains(textEntity.getTextRange()) //
-                || duplicateTextRanges.stream().anyMatch(duplicateTextRange -> duplicateTextRange.contains(textEntity.textRange))
+               || duplicateTextRanges.stream()
-                || duplicateTextRanges.stream().anyMatch(duplicateTextRange -> textEntity.getDuplicateTextRanges().stream().anyMatch(duplicateTextRange::contains));
+                       .anyMatch(duplicateTextRange -> duplicateTextRange.contains(textEntity.textRange)) //
+               || duplicateTextRanges.stream()
+                       .anyMatch(duplicateTextRange -> textEntity.getDuplicateTextRanges()
+                               .stream()
+                               .anyMatch(duplicateTextRange::contains));
     }
 
 
     public boolean intersects(TextEntity textEntity) {
 
-        return this.textRange.intersects(textEntity.getTextRange())
+        return this.textRange.intersects(textEntity.getTextRange()) //
-                || duplicateTextRanges.stream().anyMatch(duplicateTextRange -> duplicateTextRange.intersects(textEntity.textRange))
+               || duplicateTextRanges.stream()
-                || duplicateTextRanges.stream().anyMatch(duplicateTextRange -> textEntity.getDuplicateTextRanges().stream().anyMatch(duplicateTextRange::intersects));
+                       .anyMatch(duplicateTextRange -> duplicateTextRange.intersects(textEntity.textRange)) //
+               || duplicateTextRanges.stream()
+                       .anyMatch(duplicateTextRange -> textEntity.getDuplicateTextRanges()
+                               .stream()
+                               .anyMatch(duplicateTextRange::intersects));
     }
 
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/DictionarySearchService.java

@@ -6,10 +6,11 @@ import java.util.Set;
 import org.springframework.stereotype.Service;
 
 import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine;
+import com.iqser.red.service.redaction.v1.server.model.dictionary.Dictionary;
+import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryModel;
+import com.iqser.red.service.redaction.v1.server.model.dictionary.SearchImplementation;
 import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
 import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
-import com.iqser.red.service.redaction.v1.server.model.dictionary.SearchImplementation;
-import com.iqser.red.service.redaction.v1.server.model.dictionary.Dictionary;
 import com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService;
 import com.iqser.red.service.redaction.v1.server.service.document.EntityEnrichmentService;
 
@@ -38,10 +39,13 @@ public class DictionarySearchService {
     @Observed(name = "DictionarySearchService", contextualName = "add-dictionary-entries")
     public void addDictionaryEntities(Dictionary dictionary, SemanticNode node) {
 
-        for (var model : dictionary.getDictionaryModels()) {
+        for (DictionaryModel model : dictionary.getDictionaryModels()) {
             bySearchImplementationAsDictionary(model.getEntriesSearch(), model.getType(), model.isHint() ? EntityType.HINT : EntityType.ENTITY, node, model.isDossierDictionary());
             bySearchImplementationAsDictionary(model.getFalsePositiveSearch(), model.getType(), EntityType.FALSE_POSITIVE, node, model.isDossierDictionary());
             bySearchImplementationAsDictionary(model.getFalseRecommendationsSearch(), model.getType(), EntityType.FALSE_RECOMMENDATION, node, model.isDossierDictionary());
+            if (model.isDossierDictionary()) {
+                bySearchImplementationAsDictionary(model.getDeletionEntriesSearch(), model.getType(), EntityType.DICTIONARY_REMOVAL, node, model.isDossierDictionary());
+            }
         }
     }
 
@@ -52,11 +56,12 @@ public class DictionarySearchService {
                                                    SemanticNode node,
                                                    boolean isDossierDictionaryEntry) {
 
+        Set<Engine> engines = Set.of(Engine.DICTIONARY);
         EntityCreationService entityCreationService = new EntityCreationService(entityEnrichmentService);
         searchImplementation.getBoundaries(node.getTextBlock(), node.getTextRange())
                 .stream()
                 .filter(boundary -> entityCreationService.isValidEntityTextRange(node.getTextBlock(), boundary))
-                .forEach(bounds -> entityCreationService.byTextRangeWithEngine(bounds, type, entityType, node, Set.of(Engine.DICTIONARY))
+                .forEach(bounds -> entityCreationService.byTextRangeWithEngine(bounds, type, entityType, node, engines)
                         .ifPresent(entity -> {
                             entity.setDictionaryEntry(true);
                             entity.setDossierDictionaryEntry(isDossierDictionaryEntry);

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/DictionaryService.java

@@ -1,9 +1,9 @@
 package com.iqser.red.service.redaction.v1.server.service;
 
 import java.awt.Color;
-import java.util.ArrayList;
 import java.util.Comparator;
 import java.util.HashSet;
+import java.util.LinkedList;
 import java.util.List;
 import java.util.Locale;
 import java.util.Optional;
@@ -368,30 +368,23 @@ public class DictionaryService {
     @Observed(name = "DictionaryService", contextualName = "deep-copy-dictionary")
     public Dictionary getDeepCopyDictionary(String dossierTemplateId, String dossierId) {
 
-        List<DictionaryModel> mergedDictionaries;
+        List<DictionaryModel> mergedDictionaries = new LinkedList<>();
 
-        var dossierTemplateRepresentation = getDossierTemplateDictionary(dossierTemplateId);
+        DictionaryRepresentation dossierTemplateRepresentation = getDossierTemplateDictionary(dossierTemplateId);
-        var dossierTemplateDictionaries = dossierTemplateRepresentation.getDictionary();
+        List<DictionaryModel> dossierTemplateDictionaries = dossierTemplateRepresentation.getDictionary();
+        dossierTemplateDictionaries.forEach(dm -> mergedDictionaries.add(SerializationUtils.clone(dm)));
 
-        // merge dictionaries if they have same names
+        // add dossier
-        long dossierDictionaryVersion = -1;
+        DictionaryRepresentation dossierRepresentation = getDossierDictionary(dossierId);
-        if (dossierDictionaryExists(dossierId)) {
+        List<DictionaryModel> dossierDictionaries = dossierRepresentation.getDictionary();
-            var dossierRepresentation = getDossierDictionary(dossierId);
+        dossierDictionaries.forEach(dm -> mergedDictionaries.add(SerializationUtils.clone(dm)));
-            var dossierDictionaries = dossierRepresentation.getDictionary();
-            mergedDictionaries = convertCommonsDictionaryModel(dictionaryMergeService.getMergedDictionary(convertDictionaryModel(dossierTemplateDictionaries),
-                                                                                                          convertDictionaryModel(dossierDictionaries)));
-            dossierDictionaryVersion = dossierRepresentation.getDictionaryVersion();
-        } else {
-            mergedDictionaries = new ArrayList<>();
-            dossierTemplateDictionaries.forEach(dm -> mergedDictionaries.add(SerializationUtils.clone(dm)));
-        }
 
         return new Dictionary(mergedDictionaries.stream()
                                       .sorted(Comparator.comparingInt(DictionaryModel::getRank).reversed())
                                       .collect(Collectors.toList()),
                               DictionaryVersion.builder()
                                       .dossierTemplateVersion(dossierTemplateRepresentation.getDictionaryVersion())
-                                      .dossierVersion(dossierDictionaryVersion)
+                                      .dossierVersion(dossierRepresentation.getDictionaryVersion())
                                       .build());
     }
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java

@@ -52,9 +52,11 @@ public class EntityLogCreatorService {
     RedactionStorageService redactionStorageService;
 
 
-    private static boolean notFalsePositiveOrFalseRecommendation(TextEntity textEntity) {
+    private static boolean notFalsePositiveOrFalseRecommendationOrRemoval(TextEntity textEntity) {
 
-        return !(textEntity.getEntityType().equals(EntityType.FALSE_POSITIVE) || textEntity.getEntityType().equals(EntityType.FALSE_RECOMMENDATION));
+        return !(textEntity.getEntityType().equals(EntityType.FALSE_POSITIVE) //
+                 || textEntity.getEntityType().equals(EntityType.FALSE_RECOMMENDATION) //
+                 || textEntity.getEntityType().equals(EntityType.DICTIONARY_REMOVAL));
     }
 
 
@@ -145,7 +147,7 @@ public class EntityLogCreatorService {
         document.getEntities()
                 .stream()
                 .filter(entity -> !entity.getValue().isEmpty())
-                .filter(EntityLogCreatorService::notFalsePositiveOrFalseRecommendation)
+                .filter(EntityLogCreatorService::notFalsePositiveOrFalseRecommendationOrRemoval)
                 .filter(entity -> !entity.removed())
                 .forEach(entityNode -> entries.addAll(toEntityLogEntries(entityNode)));
         document.streamAllImages()
@@ -341,6 +343,7 @@ public class EntityLogCreatorService {
             case FALSE_POSITIVE -> EntryType.FALSE_POSITIVE;
             case RECOMMENDATION -> EntryType.RECOMMENDATION;
             case FALSE_RECOMMENDATION -> EntryType.FALSE_RECOMMENDATION;
+            case DICTIONARY_REMOVAL -> EntryType.FALSE_POSITIVE;
         };
     }
 

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DictionaryServiceTest.java

@@ -233,18 +233,13 @@ public class DictionaryServiceTest {
 
         dictionaryService.updateDictionary("dtId", "dossierId");
         var dict = dictionaryService.getDeepCopyDictionary("dtId", "dossierId");
-        assertThat(dict.getDictionaryModels().size()).isEqualTo(1);
+        assertThat(dict.getDictionaryModels().size()).isEqualTo(2);
         var dictModel = dict.getDictionaryModels()
                 .get(0);
         assertThat(dictModel.getType()).isEqualTo(type);
-        assertThat(dictModel.getEntries().size()).isEqualTo(4);
+        assertThat(dictModel.getEntries().size()).isEqualTo(3);
         dictModel.getEntries()
-                .forEach(entry -> {
+                .forEach(entry -> assertThat(entry.getTypeId()).isEqualTo(dtType.getTypeId()));
-                    switch (entry.getValue()) {
-                        case "aa", "dd", "bb" -> assertThat(entry.getTypeId()).isEqualTo(dossierType.getTypeId());
-                        case "cc" -> assertThat(entry.getTypeId()).isEqualTo(dtType.getTypeId());
-                    }
-                });
     }
 
 }

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/acceptance_rules.drl

@@ -1335,6 +1335,20 @@ rule "X.10.0: remove false positives of ai"
     end
 
 
+//------------------------------------ Dictionary merging rules ------------------------------------
+
+// Rule unit: DICT.0
+rule "DICT.0.0: Ignore Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL"
+    salience 64
+    when
+        $dictionaryRemoval: TextEntity($type: type(), entityType == EntityType.DICTIONARY_REMOVAL, engines contains Engine.DOSSIER_DICTIONARY, active())
+        $entity: TextEntity(getTextRange().equals($dictionaryRemoval.getTextRange()), engines contains Engine.DICTIONARY, type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges())
+    then
+        $entity.getIntersectingNodes().forEach(node -> update(node));
+        $entity.ignore("DICT.0.0", "Ignore Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL");
+    end
+
+
 //------------------------------------ File attributes rules ------------------------------------
 
 // Rule unit: FA.1

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/all_redact_manager_rules.drl

@@ -1037,6 +1037,7 @@ rule "PII.3.2: Redact telephone numbers by RegEx (vertebrate study)"
         .forEach(entity -> entity.redact("PII.3.2", "Telephone number found by regex", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
     end
 
+
 // Rule unit: PII.4
 rule "PII.4.0: Redact line after contact information keywords"
     when
@@ -1340,8 +1341,6 @@ rule "PII.12.0: Expand PII entities with salutation prefix"
         .ifPresent(expandedEntity -> expandedEntity.apply("PII.12.0", "Expanded PII with salutation prefix", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
     end
 
-
-// Rule unit: PII.12
 rule "PII.12.1: Expand PII entities with salutation prefix"
     when
         FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
@@ -1351,6 +1350,7 @@ rule "PII.12.1: Expand PII entities with salutation prefix"
         .ifPresent(expandedEntity -> expandedEntity.apply("PII.12.1", "Expanded PII with salutation prefix", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
     end
 
+
 // Rule unit: PII.13
 rule "PII.13.0: Add recommendation for PII after Contact Person"
     when
@@ -1542,6 +1542,7 @@ rule "ETC.8.1: Redact formulas (non vertebrate study)"
         $logo.redact("ETC.8.1", "Logo Found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
     end
 
+
 // Rule unit: ETC.9
 rule "ETC.9.0: Redact skipped impurities"
     when
@@ -1559,6 +1560,7 @@ rule "ETC.9.1: Redact impurities"
         $skippedImpurities.redact("ETC.9.1", "Impurity found", "Article 63(2)(b) of Regulation (EC) No 1107/2009");
     end
 
+
 // Rule unit: ETC.10
 rule "ETC.10.0: Redact Product Composition Information"
     when
@@ -1567,6 +1569,7 @@ rule "ETC.10.0: Redact Product Composition Information"
         $compositionInformation.redact("ETC.10.0", "Product Composition Information found", "Article 63(2)(d) of Regulation (EC) No 1107/2009");
     end
 
+
 // Rule unit: ETC.11
 rule "ETC.11.0: Recommend first line in table cell with name and address of owner"
     when
@@ -1596,6 +1599,7 @@ rule "ETC.12.1: Redact dossier_redaction (Vertebrate study)"
         $dossierRedaction.redact("ETC.12.1", "Dossier dictionary entry found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
     end
 
+
 //------------------------------------ AI rules ------------------------------------
 
 // Rule unit: AI.0
@@ -1641,7 +1645,8 @@ rule "AI.3.0: Recommend authors from AI as PII"
             .forEach(nerEntity -> entityCreationService.optionalByNerEntity(nerEntity, "PII", EntityType.RECOMMENDATION, document));
     end
 
-//------------------------------------ Manual redaction rules ------------------------------------
+
+//------------------------------------ Manual changes rules ------------------------------------
 
 // Rule unit: MAN.0
 rule "MAN.0.0: Apply manual resize redaction"
@@ -1761,7 +1766,6 @@ rule "MAN.3.2: Apply image recategorization"
         retract($recategorization);
     end
 
-
 rule "MAN.3.3: Apply recategorization entities by default"
     salience 128
     when
@@ -1770,6 +1774,7 @@ rule "MAN.3.3: Apply recategorization entities by default"
         $entity.apply("MAN.3.3", "Recategorized entities are applied by default.", $entity.legalBasis());
     end
 
+
 // Rule unit: MAN.4
 rule "MAN.4.0: Apply legal basis change"
     salience 128
@@ -1930,6 +1935,7 @@ rule "X.8.1: Remove Entity when intersected by imported Entity"
         retract($other);
     end
 
+
 // Rule unit: X.9
 rule "X.9.0: Merge mostly contained signatures"
     when
@@ -1940,6 +1946,7 @@ rule "X.9.0: Merge mostly contained signatures"
         $signature.addEngine(LayoutEngine.AI);
     end
 
+
 // Rule unit: X.10
 rule "X.10.0: remove false positives of ai"
     when
@@ -1949,6 +1956,21 @@ rule "X.10.0: remove false positives of ai"
         $aiSignature.remove("X.10.0", "Removed because false positive");
     end
 
+
+//------------------------------------ Dictionary merging rules ------------------------------------
+
+// Rule unit: DICT.0
+rule "DICT.0.0: Ignore Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL"
+    salience 64
+    when
+        $dictionaryRemoval: TextEntity($type: type(), entityType == EntityType.DICTIONARY_REMOVAL, engines contains Engine.DOSSIER_DICTIONARY, active())
+        $entity: TextEntity(getTextRange().equals($dictionaryRemoval.getTextRange()), engines contains Engine.DICTIONARY, type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges())
+    then
+        $entity.getIntersectingNodes().forEach(node -> update(node));
+        $entity.ignore("DICT.0.0", "Ignore Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL");
+    end
+
+
 //------------------------------------ File attributes rules ------------------------------------
 
 // Rule unit: FA.1

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora.drl

@@ -1437,6 +1437,20 @@ rule "X.10.0: remove false positives of ai"
     end
 
 
+//------------------------------------ Dictionary merging rules ------------------------------------
+
+// Rule unit: DICT.0
+rule "DICT.0.0: Ignore Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL"
+    salience 64
+    when
+        $dictionaryRemoval: TextEntity($type: type(), entityType == EntityType.DICTIONARY_REMOVAL, engines contains Engine.DOSSIER_DICTIONARY, active())
+        $entity: TextEntity(getTextRange().equals($dictionaryRemoval.getTextRange()), engines contains Engine.DICTIONARY, type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges())
+    then
+        $entity.getIntersectingNodes().forEach(node -> update(node));
+        $entity.ignore("DICT.0.0", "Ignore Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL");
+    end
+
+
 //------------------------------------ File attributes rules ------------------------------------
 
 // Rule unit: FA.1

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/efsa_sanitisation.drl

@@ -296,6 +296,7 @@ rule "CBI.20.2: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJEC
          });
     end
 
+
 // Rule unit: CBI.23
 rule "CBI.23.0: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\" (non vertebrate study)"
     when
@@ -315,6 +316,7 @@ rule "CBI.23.1: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\" (ve
         .forEach(authorEntity -> authorEntity.redact("CBI.23.1", "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
     end
 
+
 //------------------------------------ PII rules ------------------------------------
 
 // Rule unit: PII.0
@@ -504,6 +506,7 @@ rule "PII.8.2: Redact contact information if producer is found (vertebrate study
             .forEach(entity -> entity.redact("PII.8.2", "Producer was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
     end
 
+
 // Rule unit: PII.10
 rule "PII.10.0: Redact study director abbreviation"
     when
@@ -947,6 +950,20 @@ rule "X.10.0: remove false positives of ai"
     end
 
 
+//------------------------------------ Dictionary merging rules ------------------------------------
+
+// Rule unit: DICT.0
+rule "DICT.0.0: Ignore Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL"
+    salience 64
+    when
+        $dictionaryRemoval: TextEntity($type: type(), entityType == EntityType.DICTIONARY_REMOVAL, engines contains Engine.DOSSIER_DICTIONARY, active())
+        $entity: TextEntity(getTextRange().equals($dictionaryRemoval.getTextRange()), engines contains Engine.DICTIONARY, type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges())
+    then
+        $entity.getIntersectingNodes().forEach(node -> update(node));
+        $entity.ignore("DICT.0.0", "Ignore Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL");
+    end
+
+
 //------------------------------------ File attributes rules ------------------------------------
 
 // Rule unit: FA.1

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/manual_redaction_rules.drl

@@ -278,6 +278,20 @@ rule "X.10.0: remove false positives of ai"
     end
 
 
+//------------------------------------ Dictionary merging rules ------------------------------------
+
+// Rule unit: DICT.0
+rule "DICT.0.0: Ignore Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL"
+    salience 64
+    when
+        $dictionaryRemoval: TextEntity($type: type(), entityType == EntityType.DICTIONARY_REMOVAL, engines contains Engine.DOSSIER_DICTIONARY, active())
+        $entity: TextEntity(getTextRange().equals($dictionaryRemoval.getTextRange()), engines contains Engine.DICTIONARY, type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges())
+    then
+        $entity.getIntersectingNodes().forEach(node -> update(node));
+        $entity.ignore("DICT.0.0", "Ignore Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL");
+    end
+
+
 //------------------------------------ Local dictionary search rules ------------------------------------
 
 // Rule unit: LDS.0

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl

@@ -274,7 +274,6 @@ rule "CBI.9.1: Redact all cells with Header Author as CBI_author (non vertebrate
             .forEach(redactionEntity -> redactionEntity.redact("CBI.9.1", "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
     end
 
-
 rule "CBI.9.2: Redact all cells with Header Author(s) as CBI_author (non vertebrate study)"
     agenda-group "LOCAL_DICTIONARY_ADDS"
     when
@@ -602,7 +601,6 @@ rule "CBI.23.1: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\" (ve
 
 //------------------------------------ PII rules ------------------------------------
 
-
 // Rule unit: PII.0
 rule "PII.0.0: Redact all PII"
     when
@@ -1425,6 +1423,20 @@ rule "X.10.0: remove false positives of ai"
     end
 
 
+//------------------------------------ Dictionary merging rules ------------------------------------
+
+// Rule unit: DICT.0
+rule "DICT.0.0: Ignore Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL"
+    salience 64
+    when
+        $dictionaryRemoval: TextEntity($type: type(), entityType == EntityType.DICTIONARY_REMOVAL, engines contains Engine.DOSSIER_DICTIONARY, active())
+        $entity: TextEntity(getTextRange().equals($dictionaryRemoval.getTextRange()), engines contains Engine.DICTIONARY, type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges())
+    then
+        $entity.getIntersectingNodes().forEach(node -> update(node));
+        $entity.ignore("DICT.0.0", "Ignore Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL");
+    end
+
+
 //------------------------------------ File attributes rules ------------------------------------
 
 // Rule unit: FA.1

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_v2.drl

@@ -402,6 +402,20 @@ rule "X.10.0: remove false positives of ai"
     end
 
 
+//------------------------------------ Dictionary merging rules ------------------------------------
+
+// Rule unit: DICT.0
+rule "DICT.0.0: Ignore Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL"
+    salience 64
+    when
+        $dictionaryRemoval: TextEntity($type: type(), entityType == EntityType.DICTIONARY_REMOVAL, engines contains Engine.DOSSIER_DICTIONARY, active())
+        $entity: TextEntity(getTextRange().equals($dictionaryRemoval.getTextRange()), engines contains Engine.DICTIONARY, type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges())
+    then
+        $entity.getIntersectingNodes().forEach(node -> update(node));
+        $entity.ignore("DICT.0.0", "Ignore Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL");
+    end
+
+
 //------------------------------------ File attributes rules ------------------------------------
 
 // Rule unit: FA.1

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/table_demo.drl

@@ -523,6 +523,20 @@ rule "X.10.0: remove false positives of ai"
     end
 
 
+//------------------------------------ Dictionary merging rules ------------------------------------
+
+// Rule unit: DICT.0
+rule "DICT.0.0: Ignore Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL"
+    salience 64
+    when
+        $dictionaryRemoval: TextEntity($type: type(), entityType == EntityType.DICTIONARY_REMOVAL, engines contains Engine.DOSSIER_DICTIONARY, active())
+        $entity: TextEntity(getTextRange().equals($dictionaryRemoval.getTextRange()), engines contains Engine.DICTIONARY, type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges())
+    then
+        $entity.getIntersectingNodes().forEach(node -> update(node));
+        $entity.ignore("DICT.0.0", "Ignore Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL");
+    end
+
+
 //------------------------------------ File attributes rules ------------------------------------
 
 // Rule unit: FA.1

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/test_rules.drl

@@ -439,6 +439,20 @@ rule "X.10.0: remove false positives of ai"
     end
 
 
+//------------------------------------ Dictionary merging rules ------------------------------------
+
+// Rule unit: DICT.0
+rule "DICT.0.0: Ignore Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL"
+    salience 64
+    when
+        $dictionaryRemoval: TextEntity($type: type(), entityType == EntityType.DICTIONARY_REMOVAL, engines contains Engine.DOSSIER_DICTIONARY, active())
+        $entity: TextEntity(getTextRange().equals($dictionaryRemoval.getTextRange()), engines contains Engine.DICTIONARY, type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges())
+    then
+        $entity.getIntersectingNodes().forEach(node -> update(node));
+        $entity.ignore("DICT.0.0", "Ignore Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL");
+    end
+
+
 //------------------------------------ File attributes rules ------------------------------------
 
 // Rule unit: FA.1

redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/migration/RuleFileMigrator.java

@@ -33,11 +33,7 @@ public class RuleFileMigrator {
             rulesToAdd.forEach(ruleFileBluePrint::addRule);
         }
 
-        RuleFileBluePrint newBluePrint = new RuleFileBluePrint(combinedBluePrint.imports(),
+        String migratedRulesString = RuleFileFactory.buildRuleString(ruleFileBluePrint);
-                                                               combinedBluePrint.globals(),
-                                                               combinedBluePrint.queries(),
-                                                               ruleFileBluePrint.ruleClasses());
-        String migratedRulesString = RuleFileFactory.buildRuleString(newBluePrint);
         String migratedFilePath = ruleFile.getAbsolutePath();
         try (var out = new FileOutputStream(migratedFilePath)) {
             out.write(migratedRulesString.getBytes(StandardCharsets.UTF_8));

redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/RuleType.java

@@ -17,6 +17,7 @@ public class RuleType {
                                                     "ETC", "Other",//
                                                     "MAN", "Manual changes",//
                                                     "X", "Entity merging",//
+                                                    "DICT", "Dictionary merging",//
                                                     "FA", "File attributes",//
                                                     "LDS", "Local dictionary search",//
                                                     "TAB", "Table extraction",//

redaction-service-v1/rules-management/src/main/resources/all_redact_manager_rules.drl

@@ -1949,6 +1949,21 @@ rule "X.10.0: remove false positives of ai"
         $aiSignature.remove("X.10.0", "Removed because false positive");
     end
 
+
+//------------------------------------ Dictionary merging rules ------------------------------------
+
+// Rule unit: DICT.0
+rule "DICT.0.0: Ignore Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL"
+    salience 64
+    when
+        $dictionaryRemoval: TextEntity($type: type(), entityType == EntityType.DICTIONARY_REMOVAL, engines contains Engine.DOSSIER_DICTIONARY, active())
+        $entity: TextEntity(getTextRange().equals($dictionaryRemoval.getTextRange()), engines contains Engine.DICTIONARY, type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges())
+    then
+        $entity.getIntersectingNodes().forEach(node -> update(node));
+        $entity.ignore("DICT.0.0", "Ignore Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL");
+    end
+
+
 //------------------------------------ File attributes rules ------------------------------------
 
 // Rule unit: FA.1

redaction-service-v1/rules-management/src/main/resources/all_rules_documine.drl

@@ -1582,6 +1582,19 @@ rule "X.10.0: remove false positives of ai"
     end
 
 
+//------------------------------------ Dictionary merging rules ------------------------------------
+
+// Rule unit: DICT.0
+rule "DICT.0.0: Ignore Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL"
+    salience 64
+    when
+        $dictionaryRemoval: TextEntity($type: type(), entityType == EntityType.DICTIONARY_REMOVAL, engines contains Engine.DOSSIER_DICTIONARY, active())
+        $entity: TextEntity(getTextRange().equals($dictionaryRemoval.getTextRange()), engines contains Engine.DICTIONARY, type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges())
+    then
+        $entity.getIntersectingNodes().forEach(node -> update(node));
+        $entity.ignore("DICT.0.0", "Ignore Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL");
+    end
+
 
 //------------------------------------ File attributes rules ------------------------------------
 

redaction-service-v1/rules-management/src/main/resources/default_rule_identifiers.txt

@@ -8,5 +8,6 @@ X.5.0
 X.5.1
 X.6.*
 X.8.*
+DICT.*.*
 FA.*.*
 LDS.*.*

redaction-service-v1/rules-management/src/main/resources/default_rule_identifiers_dm.txt

@@ -7,5 +7,6 @@ X.5.0
 X.5.1
 X.7.0
 X.8.*
+DICT.*.*
 FA.*.*
 LDS.*.*

redaction-service-v1/rules-management/src/main/resources/order_template.txt

@@ -9,5 +9,6 @@ ETC
 AI
 MAN
 X
+DICT
 FA
 LDS