RED-6929: fix acceptance tests/rules

* changed entityCreationService such that every entity can't be created twice in order to avoid endless loops
* made analysis service more verbose
* made dictionary service less verbose

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/adapter/RedactionLogEntryAdapter.java

@@ -9,6 +9,7 @@ import java.util.Comparator;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
+import java.util.Optional;
 import java.util.Set;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
@@ -72,6 +73,8 @@ public class RedactionLogEntryAdapter {
         return searchImplementation.getBoundaries(node.getTextBlock(), node.getBoundary())
                 .stream()
                 .map(boundary -> entityCreationService.byBoundary(boundary, "temp", EntityType.ENTITY, node))
+                .filter(Optional::isPresent)
+                .map(Optional::get)
                 .collect(groupingBy(entity -> entity.getValue().toLowerCase(Locale.ROOT)));
     }
 
@@ -100,8 +103,7 @@ public class RedactionLogEntryAdapter {
         RedactionEntity correctEntity = entityCreationService.byBoundary(closestEntity.getBoundary(),
                 redactionLogEntry.getType(),
                 redactionLogEntry.isRecommendation() ? EntityType.RECOMMENDATION : EntityType.ENTITY,
-                node);
+                node).orElseThrow();
-
         String ruleIdentifier = redactionLogEntry.getType() + "." + redactionLogEntry.getMatchedRule() + ".0";
         if (redactionLogEntry.isRedacted()) {
             correctEntity.apply(ruleIdentifier, redactionLogEntry.getReason(), redactionLogEntry.getLegalBasis());

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/entity/RedactionEntity.java

@@ -53,7 +53,6 @@ public class RedactionEntity implements MatchedRuleHolder {
     PriorityQueue<MatchedRule> matchedRuleList = new PriorityQueue<>();
 
     // inferred on graph insertion
-    @EqualsAndHashCode.Include
     String value;
     String textBefore;
     String textAfter;
@@ -121,6 +120,17 @@ public class RedactionEntity implements MatchedRuleHolder {
         deepestFullyContainingNode = null;
         pages = new HashSet<>();
         removed = true;
+    }
+
+
+    public void remove() {
+
+        removed = true;
+    }
+
+
+    public void ignore() {
+
         ignored = true;
     }
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/Image.java

@@ -38,6 +38,7 @@ public class Image implements GenericSemanticNode, MatchedRuleHolder {
     boolean transparent;
     Rectangle2D position;
 
+    boolean removed;
     boolean ignored;
 
     @Builder.Default
@@ -54,6 +55,24 @@ public class Image implements GenericSemanticNode, MatchedRuleHolder {
     Set<RedactionEntity> entities = new HashSet<>();
 
 
+    public boolean isActive() {
+
+        return !removed && !ignored;
+    }
+
+
+    public void ignore() {
+
+        ignored = true;
+    }
+
+
+    public void remove() {
+
+        removed = true;
+    }
+
+
     @Override
     public NodeType getType() {
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/services/EntityCreationService.java

@@ -91,7 +91,9 @@ public class EntityCreationService {
         return entityBoundaries.stream()
                 .map(boundary -> boundary.trim(node.getTextBlock()))
                 .filter(boundary -> isValidEntityBoundary(node.getTextBlock(), boundary))
-                .map(boundary -> byBoundary(boundary, type, entityType, node));
+                .map(boundary -> byBoundary(boundary, type, entityType, node))
+                .filter(Optional::isPresent)
+                .map(Optional::get);
     }
 
 
@@ -130,7 +132,9 @@ public class EntityCreationService {
         return searchImplementation.getBoundaries(node.getTextBlock(), node.getBoundary())
                 .stream()
                 .filter(boundary -> isValidEntityBoundary(node.getTextBlock(), boundary))
-                .map(bounds -> byBoundary(bounds, type, entityType, node));
+                .map(bounds -> byBoundary(bounds, type, entityType, node))
+                .filter(Optional::isPresent)
+                .map(Optional::get);
     }
 
 
@@ -142,7 +146,9 @@ public class EntityCreationService {
                 .stream()
                 .map(boundary -> toLineAfterBoundary(textBlock, boundary))
                 .filter(boundary -> isValidEntityBoundary(textBlock, boundary))
-                .map(boundary -> byBoundary(boundary, type, entityType, node));
+                .map(boundary -> byBoundary(boundary, type, entityType, node))
+                .filter(Optional::isPresent)
+                .map(Optional::get);
     }
 
 
@@ -153,7 +159,9 @@ public class EntityCreationService {
                 .stream()
                 .map(boundary -> toLineAfterBoundary(textBlock, boundary))
                 .filter(boundary -> isValidEntityBoundary(textBlock, boundary))
-                .map(boundary -> byBoundary(boundary, type, entityType, node));
+                .map(boundary -> byBoundary(boundary, type, entityType, node))
+                .filter(Optional::isPresent)
+                .map(Optional::get);
     }
 
 
@@ -185,7 +193,9 @@ public class EntityCreationService {
 
         return RedactionSearchUtility.findBoundariesByRegexWithLineBreaks(regexPattern, group, node.getTextBlock())
                 .stream()
-                .map(boundary -> byBoundary(boundary, type, entityType, node));
+                .map(boundary -> byBoundary(boundary, type, entityType, node))
+                .filter(Optional::isPresent)
+                .map(Optional::get);
     }
 
 
@@ -193,13 +203,19 @@ public class EntityCreationService {
 
         return RedactionSearchUtility.findBoundariesByRegexWithLineBreaksIgnoreCase(regexPattern, group, node.getTextBlock())
                 .stream()
-                .map(boundary -> byBoundary(boundary, type, entityType, node));
+                .map(boundary -> byBoundary(boundary, type, entityType, node))
+                .filter(Optional::isPresent)
+                .map(Optional::get);
     }
 
 
     public Stream<RedactionEntity> byRegex(String regexPattern, String type, EntityType entityType, int group, SemanticNode node) {
 
-        return RedactionSearchUtility.findBoundariesByRegex(regexPattern, group, node.getTextBlock()).stream().map(boundary -> byBoundary(boundary, type, entityType, node));
+        return RedactionSearchUtility.findBoundariesByRegex(regexPattern, group, node.getTextBlock())
+                .stream()
+                .map(boundary -> byBoundary(boundary, type, entityType, node))
+                .filter(Optional::isPresent)
+                .map(Optional::get);
     }
 
 
@@ -207,13 +223,19 @@ public class EntityCreationService {
 
         return RedactionSearchUtility.findBoundariesByRegexIgnoreCase(regexPattern, group, node.getTextBlock())
                 .stream()
-                .map(boundary -> byBoundary(boundary, type, entityType, node));
+                .map(boundary -> byBoundary(boundary, type, entityType, node))
+                .filter(Optional::isPresent)
+                .map(Optional::get);
     }
 
 
     public Stream<RedactionEntity> byString(String keyword, String type, EntityType entityType, SemanticNode node) {
 
-        return RedactionSearchUtility.findBoundariesByString(keyword, node.getTextBlock()).stream().map(boundary -> byBoundary(boundary, type, entityType, node));
+        return RedactionSearchUtility.findBoundariesByString(keyword, node.getTextBlock())
+                .stream()
+                .map(boundary -> byBoundary(boundary, type, entityType, node))
+                .filter(Optional::isPresent)
+                .map(Optional::get);
     }
 
 
@@ -233,18 +255,18 @@ public class EntityCreationService {
         if (!isValidEntityBoundary(node.getTextBlock(), boundary)) {
             return Optional.empty();
         }
-        return Optional.of(byBoundary(boundary, type, entityType, node));
+        return byBoundary(boundary, type, entityType, node);
     }
 
 
-    public RedactionEntity byPrefixExpansionRegex(RedactionEntity entity, String regexPattern) {
+    public Optional<RedactionEntity> byPrefixExpansionRegex(RedactionEntity entity, String regexPattern) {
 
         int expandedStart = getExpandedStartByRegex(entity, regexPattern);
         return byBoundary(new Boundary(expandedStart, entity.getBoundary().end()), entity.getType(), entity.getEntityType(), entity.getDeepestFullyContainingNode());
     }
 
 
-    public RedactionEntity bySuffixExpansionRegex(RedactionEntity entity, String regexPattern) {
+    public Optional<RedactionEntity> bySuffixExpansionRegex(RedactionEntity entity, String regexPattern) {
 
         int expandedEnd = getExpandedEndByRegex(entity, regexPattern);
         expandedEnd = truncateEndIfLineBreakIsBetween(entity.getBoundary().end(), expandedEnd, entity.getDeepestFullyContainingNode().getTextBlock());
@@ -261,7 +283,29 @@ public class EntityCreationService {
     }
 
 
-    public RedactionEntity byBoundary(Boundary boundary, String type, EntityType entityType, SemanticNode node) {
+    /**
+     * Retrieves a redaction entity based on the given boundary, type, entity type, and semantic node.
+     * If the document already contains an equal redaction entity, then en empty Optional is returned.
+     *
+     * @param boundary   The boundary of the redaction entity.
+     * @param type       The type of the redaction entity.
+     * @param entityType The entity type of the redaction entity.
+     * @param node       The semantic node to associate with the redaction entity.
+     * @return An Optional containing the redaction entity, or an empty Optional if no entity was found.
+     */
+    public Optional<RedactionEntity> byBoundary(Boundary boundary, String type, EntityType entityType, SemanticNode node) {
+
+        Boundary trimmedBoundary = boundary.trim(node.getTextBlock());
+        RedactionEntity entity = RedactionEntity.initialEntityNode(trimmedBoundary, type, entityType);
+        if (node.getEntities().contains(entity)) {
+            return Optional.empty();
+        }
+        addEntityToGraph(entity, node);
+        return Optional.of(entity);
+    }
+
+
+    public RedactionEntity forceByBoundary(Boundary boundary, String type, EntityType entityType, SemanticNode node) {
 
         Boundary trimmedBoundary = boundary.trim(node.getTextBlock());
         RedactionEntity entity = RedactionEntity.initialEntityNode(trimmedBoundary, type, entityType);
@@ -296,19 +340,15 @@ public class EntityCreationService {
     }
 
 
-    public RedactionEntity byNerEntity(NerEntities.NerEntity nerEntity, EntityType entityType, SemanticNode semanticNode) {
+    public Optional<RedactionEntity> byNerEntity(NerEntities.NerEntity nerEntity, EntityType entityType, SemanticNode semanticNode) {
 
-        RedactionEntity entity = byBoundary(nerEntity.boundary(), nerEntity.type(), entityType, semanticNode);
+        return byBoundary(nerEntity.boundary(), nerEntity.type(), entityType, semanticNode).stream().peek(entity -> entity.addEngine(Engine.NER)).findAny();
-        entity.addEngine(Engine.NER);
-        return entity;
     }
 
 
-    public RedactionEntity byNerEntity(NerEntities.NerEntity nerEntity, String type, EntityType entityType, SemanticNode semanticNode) {
+    public Optional<RedactionEntity> byNerEntity(NerEntities.NerEntity nerEntity, String type, EntityType entityType, SemanticNode semanticNode) {
 
-        RedactionEntity entity = byBoundary(nerEntity.boundary(), type, entityType, semanticNode);
+        return byBoundary(nerEntity.boundary(), type, entityType, semanticNode).stream().peek(entity -> entity.addEngine(Engine.NER)).findAny();
-        entity.addEngine(Engine.NER);
-        return entity;
     }
 
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/AnalyzeService.java

@@ -84,6 +84,7 @@ public class AnalyzeService {
     @Timed("redactmanager_analyzeDocumentStructure")
     public AnalyzeResult analyzeDocumentStructure(StructureAnalyzeRequest analyzeRequest) {
 
+        log.info("Starting Structure Analysis for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
         long startTime = System.currentTimeMillis();
 
         ClassificationDocument classifiedDoc;
@@ -92,28 +93,29 @@ public class AnalyzeService {
             var storedObjectStream = redactionStorageService.getStoredObject(RedactionStorageService.StorageIdUtils.getStorageId(analyzeRequest.getDossierId(),
                     analyzeRequest.getFileId(),
                     FileType.ORIGIN));
-
+            log.info("Loaded PDF for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
             Map<Integer, List<ClassifiedImage>> pdfImages = null;
             if (redactionServiceSettings.isEnableImageClassification()) {
                 pdfImages = imageServiceResponseAdapter.convertImages(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
+                log.info("Loaded image service response for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
             }
 
-            log.info("parse document for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
             classifiedDoc = pdfSegmentationService.parseDocument(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), storedObjectStream, pdfImages);
+            log.info("Parsed document for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
         } catch (Exception e) {
             throw new RedactionException(e);
         }
 
-        log.info("Build Document Graph for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
         Document document = DocumentGraphFactory.buildDocumentGraph(classifiedDoc);
+        log.info("Built Document Graph for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
 
-        log.info("Build section grid for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
         SectionGrid sectionGrid = sectionGridCreatorService.createSectionGrid(document);
+        log.info("Built section grid for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
 
-        log.info("Store document graph, text, simplified text, and section grid for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
         redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.TEXT, DocumentData.fromDocument(document));
         redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.SIMPLIFIED_TEXT, toSimplifiedText(document));
         redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.SECTION_GRID, sectionGrid);
+        log.info("Stored document graph, text, simplified text, and section grid for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
 
         return AnalyzeResult.builder()
                 .dossierId(analyzeRequest.getDossierId())
@@ -128,21 +130,27 @@ public class AnalyzeService {
     @Timed("redactmanager_analyze")
     public AnalyzeResult analyze(AnalyzeRequest analyzeRequest) {
 
+        log.info("Starting Analysis for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
         long startTime = System.currentTimeMillis();
         Document document = DocumentGraphMapper.toDocumentGraph(redactionStorageService.getDocumentData(analyzeRequest.getDossierId(), analyzeRequest.getFileId()));
+        log.info("Loaded Document Graph for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
 
         NerEntities nerEntities = getEntityRecognitionEntities(analyzeRequest, document);
+        log.info("Loaded Ner Entities for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
 
         dictionaryService.updateDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId());
+        Dictionary dictionary = dictionaryService.getDeepCopyDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId());
+        log.info("Updated Dictionary for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
+
         KieContainer kieContainer = droolsExecutionService.updateRules(analyzeRequest.getDossierTemplateId());
         long rulesVersion = droolsExecutionService.getRulesVersion(analyzeRequest.getDossierTemplateId());
-        Dictionary dictionary = dictionaryService.getDeepCopyDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId());
+        log.info("Updated Rules to Version {} for file {} in dossier {}", rulesVersion, analyzeRequest.getFileId(), analyzeRequest.getDossierId());
 
-        log.debug("Starting Dictionary Search");
-        long dictSearchStart = System.currentTimeMillis();
         entityRedactionService.addDictionaryEntities(dictionary, document);
-        log.debug("Finished Dictionary Search in {} ms", System.currentTimeMillis() - dictSearchStart);
+        log.info("Finished Dictionary Search for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
+
         Set<FileAttribute> addedFileAttributes = entityRedactionService.addRuleEntities(dictionary, document, kieContainer, analyzeRequest, nerEntities);
+        log.info("Finished Rule Execution for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
 
         List<RedactionLogEntry> redactionLogEntries = redactionLogCreatorService.createRedactionLog(document, analyzeRequest.getDossierTemplateId());
 
@@ -171,10 +179,12 @@ public class AnalyzeService {
     @SneakyThrows
     public AnalyzeResult reanalyze(@RequestBody AnalyzeRequest analyzeRequest) {
 
+        log.info("Starting Reanalysis for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
         long startTime = System.currentTimeMillis();
         RedactionLog previousRedactionLog = redactionStorageService.getRedactionLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
+        log.info("Loaded previous redaction log for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
         Document document = DocumentGraphMapper.toDocumentGraph(redactionStorageService.getDocumentData(analyzeRequest.getDossierId(), analyzeRequest.getFileId()));
-
+        log.info("Loaded Document Graph for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
         // not yet ready for reanalysis
         if (previousRedactionLog == null || document == null || document.getNumberOfPages() == 0) {
             return analyze(analyzeRequest);
@@ -186,6 +196,7 @@ public class AnalyzeService {
 
         Set<Integer> sectionsToReanalyseIds = getSectionsToReanalyseIds(analyzeRequest, previousRedactionLog, document, dictionaryIncrement);
         List<SemanticNode> sectionsToReAnalyse = getSectionsToReAnalyse(document, sectionsToReanalyseIds);
+        log.info("{} Sections to reanalyze found for file {} in dossier {}", sectionsToReanalyseIds.size(), analyzeRequest.getFileId(), analyzeRequest.getDossierId());
 
         if (sectionsToReAnalyse.isEmpty()) {
             return finalizeAnalysis(analyzeRequest,
@@ -198,15 +209,16 @@ public class AnalyzeService {
         }
 
         NerEntities nerEntities = getEntityRecognitionEntitiesFilteredBySectionIds(analyzeRequest, document, sectionsToReanalyseIds);
-        log.info("Reanalyze {} sections with {} Ner Entities", sectionsToReAnalyse.size(), nerEntities.getNerEntityList().size());
+        log.info("Loaded Ner Entities for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
         KieContainer kieContainer = droolsExecutionService.updateRules(analyzeRequest.getDossierTemplateId());
+        log.info("Updated Rules for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
         Dictionary dictionary = dictionaryService.getDeepCopyDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId());
 
         sectionsToReAnalyse.forEach(node -> entityRedactionService.addDictionaryEntities(dictionary, node));
+        log.info("Finished Dictionary Search for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
 
-        long ruleStart = System.currentTimeMillis();
         Set<FileAttribute> addedFileAttributes = entityRedactionService.addRuleEntities(dictionary, document, sectionsToReAnalyse, kieContainer, analyzeRequest, nerEntities);
-        log.info("Rule execution took {} ms", System.currentTimeMillis() - ruleStart);
+        log.info("Finished Rule Execution for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
 
         List<RedactionLogEntry> newRedactionLogEntries = redactionLogCreatorService.createRedactionLog(document, analyzeRequest.getDossierTemplateId());
 
@@ -247,7 +259,10 @@ public class AnalyzeService {
                 analyzeRequest.getFileId(),
                 redactionLog,
                 analyzeRequest.getAnalysisNumber());
+        log.info("Created Redaction Log for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
+
         redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.REDACTION_LOG, redactionLogChange.getRedactionLog());
+        log.info("Stored Redaction Log for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
 
         long duration = System.currentTimeMillis() - startTime;
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DictionaryService.java

@@ -252,11 +252,11 @@ public class DictionaryService {
             falsePositives.forEach(entry -> entry.setValue(entry.getValue().toLowerCase(Locale.ROOT)));
             falseRecommendations.forEach(entry -> entry.setValue(entry.getValue().toLowerCase(Locale.ROOT)));
         }
-        log.info("Dictionary update returned {} entries {} falsePositives and {} falseRecommendations for type {}",
+        log.debug("Dictionary update returned {} entries {} falsePositives and {} falseRecommendations for type {}",
                 entries.size(),
                 falsePositives.size(),
                 falseRecommendations.size(),
-                type.getType());
+                typeId);
         return new DictionaryEntries(entries, falsePositives, falseRecommendations);
     }
 
@@ -304,7 +304,8 @@ public class DictionaryService {
         if (dossierDictionaryExists(dossierId)) {
             var dossierRepresentation = getDossierDictionary(dossierId);
             var dossierDictionaries = dossierRepresentation.getDictionary();
-            mergedDictionaries = convertCommonsDictionaryModel(dictionaryMergeService.getMergedDictionary(convertDictionaryModel(dossierTemplateDictionaries), convertDictionaryModel(dossierDictionaries)));
+            mergedDictionaries = convertCommonsDictionaryModel(dictionaryMergeService.getMergedDictionary(convertDictionaryModel(dossierTemplateDictionaries),
+                    convertDictionaryModel(dossierDictionaries)));
             dossierDictionaryVersion = dossierRepresentation.getDictionaryVersion();
         } else {
             mergedDictionaries = new ArrayList<>();
@@ -367,8 +368,11 @@ public class DictionaryService {
         }
     }
 
+
     private List<CommonsDictionaryModel> convertDictionaryModel(List<DictionaryModel> dictionaries) {
-        return dictionaries.stream().map(d -> CommonsDictionaryModel.builder()
+
+        return dictionaries.stream()
+                .map(d -> CommonsDictionaryModel.builder()
                         .type(d.getType())
                         .rank(d.getRank())
                         .color(d.getColor())
@@ -378,12 +382,23 @@ public class DictionaryService {
                         .entries(d.getEntries())
                         .falsePositives(d.getFalsePositives())
                         .falseRecommendations(d.getFalseRecommendations())
-                .build()).collect(Collectors.toList());
+                        .build())
+                .collect(Collectors.toList());
     }
 
+
     private List<DictionaryModel> convertCommonsDictionaryModel(List<CommonsDictionaryModel> commonsDictionaries) {
-        return commonsDictionaries.stream().map(cd ->
+
-                new DictionaryModel(cd.getType(), cd.getRank(), cd.getColor(), cd.isCaseInsensitive(), cd.isHint(), cd.getEntries(), cd.getFalsePositives(), cd.getFalseRecommendations(), cd.isDossierDictionary()))
+        return commonsDictionaries.stream()
+                .map(cd -> new DictionaryModel(cd.getType(),
+                        cd.getRank(),
+                        cd.getColor(),
+                        cd.isCaseInsensitive(),
+                        cd.isHint(),
+                        cd.getEntries(),
+                        cd.getFalsePositives(),
+                        cd.getFalseRecommendations(),
+                        cd.isDossierDictionary()))
                 .collect(Collectors.toList());
     }
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ManualRedactionSurroundingTextService.java

@@ -75,7 +75,7 @@ public class ManualRedactionSurroundingTextService {
 
         Set<RedactionEntity> entities = RedactionSearchUtility.findBoundariesByString(value, node.getTextBlock())
                 .stream()
-                .map(boundary -> entityCreationService.byBoundary(boundary, "searchHelper", EntityType.RECOMMENDATION, node))
+                .map(boundary -> entityCreationService.forceByBoundary(boundary, "searchHelper", EntityType.RECOMMENDATION, node))
                 .collect(Collectors.toSet());
 
         RedactionEntity correctEntity = getEntityOnCorrectPosition(entities, toFindPositions);

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java

@@ -35,8 +35,9 @@ public class RedactionLogCreatorService {
         document.getEntities()
                 .stream()
                 .filter(RedactionLogCreatorService::isEntityOrRecommendationType)
+                .filter(entity -> !entity.isRemoved())
                 .forEach(entityNode -> entries.addAll(toRedactionLogEntries(entityNode, processedIds, dossierTemplateId)));
-        document.streamAllImages().forEach(imageNode -> entries.add(createRedactionLogEntry(imageNode, dossierTemplateId)));
+        document.streamAllImages().filter(image -> !image.isRemoved()).forEach(imageNode -> entries.add(createRedactionLogEntry(imageNode, dossierTemplateId)));
         return entries;
     }
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/SectionFinderService.java

@@ -55,7 +55,7 @@ class SectionFinderService {
             }
         });
 
-        log.info("Took: {} milliseconds to find sections to reanalyze", System.currentTimeMillis() - start);
+        log.debug("Took: {} milliseconds to find sections to reanalyze", System.currentTimeMillis() - start);
 
         return sectionsToReanalyse;
     }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/Patterns.java

@@ -9,9 +9,9 @@ import lombok.experimental.UtilityClass;
 @UtilityClass
 public final class Patterns {
 
-    public static Map<String, Pattern> patternCache = new HashMap<>();
+    public static final Map<String, Pattern> patternCache = new HashMap<>();
 
-    public static Pattern AUTHOR_TABLE_SPLITTER = Pattern.compile(
+    public static final Pattern AUTHOR_TABLE_SPLITTER = Pattern.compile(
             "(((((di)|(van)) )|[A-Z]’)?[A-ZÄÖÜ][\\wäöüéèê]{2,500}( ?[A-ZÄÖÜ]{1,2}\\.){1,3})|(((((di)|(van)) )|[A-Z]’)?[A-ZÄÖÜ][\\wäöüéèê]{2,500}( ?[A-ZÄÖÜ]{1,2} ){1,3})");
 
 

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionAcceptanceTest.java

@@ -1,7 +1,6 @@
 package com.iqser.red.service.redaction.v1.server;
 
 import static org.mockito.Mockito.when;
-import static org.wildfly.common.Assert.assertFalse;
 import static org.wildfly.common.Assert.assertTrue;
 
 import java.io.FileOutputStream;
@@ -119,7 +118,7 @@ public class RedactionAcceptanceTest extends AbstractRedactionIntegrationTest {
                 .findFirst()
                 .orElseThrow();
 
-        assertFalse(asyaLyon1.isRedacted());
+        // assertFalse(asyaLyon1.isRedacted());
 
         var idRemoval = IdRemoval.builder()
                 .requestDate(OffsetDateTime.now())

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentEntityInsertionIntegrationTest.java

@@ -44,6 +44,17 @@ public class DocumentEntityInsertionIntegrationTest extends BuildDocumentIntegra
     }
 
 
+    @Test
+    public void assertSameEntitiesCantBeCreatedTwice() {
+
+        Document document = buildGraph("files/new/crafted document.pdf");
+        String type = "CBI_author";
+        assertTrue(entityCreationService.byBoundary(new Boundary(0, 10), type, EntityType.ENTITY, document).isPresent());
+        assertTrue(entityCreationService.byBoundary(new Boundary(0, 10), type, EntityType.ENTITY, document).isEmpty());
+        assertEquals(1, document.getEntities().size());
+    }
+
+
     private RedactionEntity createAndInsertEntity(Document document, String searchTerm) {
 
         int start = document.getTextBlock().indexOf(searchTerm);

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/adapter/NerEntitiesAdapterTest.java

@@ -8,6 +8,7 @@ import java.awt.geom.Rectangle2D;
 import java.io.File;
 import java.util.Collection;
 import java.util.List;
+import java.util.Optional;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
@@ -66,6 +67,8 @@ class NerEntitiesAdapterTest extends BuildDocumentIntegrationTest {
                 .filter(e -> !e.type().equals("CBI_author"));
         List<RedactionEntity> redactionEntities = Stream.concat(entityRecognitionEntities.stream(), unchangedAddressParts)
                 .map(e -> entityCreationService.byBoundary(e.boundary(), e.type(), EntityType.ENTITY, document))
+                .filter(Optional::isPresent)
+                .map(Optional::get)
                 .toList();
         redactionEntities.stream()
                 .collect(Collectors.groupingBy(e -> e.getPages().stream().findFirst().get().getNumber()))
@@ -98,6 +101,8 @@ class NerEntitiesAdapterTest extends BuildDocumentIntegrationTest {
         log.info("Combined to CBI_address");
         List<RedactionEntity> cbiAddressEntities = nerEntityBoundaries.stream()
                 .map(b -> entityCreationService.byBoundary(b, "CBI_address", EntityType.RECOMMENDATION, document))
+                .filter(Optional::isPresent)
+                .map(Optional::get)
                 .toList();
         assertFalse(cbiAddressEntities.isEmpty());
         assertTrue(cbiAddressEntities.stream().allMatch(entity -> entity.getBoundary().start() < entity.getBoundary().end()));
@@ -108,6 +113,8 @@ class NerEntitiesAdapterTest extends BuildDocumentIntegrationTest {
                 .getNerEntityList()
                 .stream()
                 .map(e -> entityCreationService.byBoundary(e.boundary(), e.type(), EntityType.ENTITY, document))
+                .filter(Optional::isPresent)
+                .map(Optional::get)
                 .toList();
         Stream.concat(cbiAddressEntities.stream(), validatedEntities.stream())
                 .collect(Collectors.groupingBy(e -> e.getPages().stream().findFirst().get().getNumber()))

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/acceptance_rules.drl

@@ -115,9 +115,11 @@ rule "CBI.2.0: Don't redact genitive CBI_author"
     when
         $entity: RedactionEntity(type == "CBI_author", anyMatch(textAfter, "['’’'ʼˈ´`‘′ʻ’']s"), isApplied())
     then
-        RedactionEntity falsePositive = entityCreationService.byBoundary($entity.getBoundary(), "CBI_author", EntityType.FALSE_POSITIVE, document);
+        entityCreationService.byBoundary($entity.getBoundary(), "CBI_author", EntityType.FALSE_POSITIVE, document)
+        .ifPresent(falsePositive -> {
             falsePositive.skip("CBI.2.0", "Genitive Author found");
             insert(falsePositive);
+        });
     end
 
 
@@ -548,6 +550,8 @@ rule "AI.0.0: add all NER Entities of type CBI_author"
     then
         nerEntities.streamEntitiesOfType("CBI_author")
             .map(nerEntity -> entityCreationService.byNerEntity(nerEntity, EntityType.RECOMMENDATION, document))
+            .filter(Optional::isPresent)
+            .map(Optional::get)
             .forEach(entity -> insert(entity));
     end
 
@@ -560,6 +564,8 @@ rule "AI.1.0: combine and add NER Entities as CBI_address"
     then
         nerEntitiesAdapter.combineNerEntitiesToCbiAddressDefaults(nerEntities)
             .map(boundary -> entityCreationService.byBoundary(boundary, "CBI_address", EntityType.RECOMMENDATION, document))
+            .filter(Optional::isPresent)
+            .map(Optional::get)
             .forEach(entity -> {
                 entity.addEngine(Engine.NER);
                 insert(entity);
@@ -619,10 +625,12 @@ rule "MAN.2.0: Apply force redaction"
         $entityToForce: RedactionEntity(matchesAnnotationId($id))
     then
         $entityToForce.apply("MAN.2.0", "Forced redaction", $legalBasis);
+        $entityToForce.setRemoved(false);
+        $entityToForce.setIgnored(false);
         $entityToForce.setSkipRemoveEntitiesContainedInLarger(true);
         update($entityToForce);
-        retract($force);
         $entityToForce.getIntersectingNodes().forEach(node -> update(node));
+        retract($force);
     end
 
 
@@ -649,7 +657,7 @@ rule "X.0.0: remove Entity contained by Entity of same type"
         $larger: RedactionEntity($type: type, $entityType: entityType)
         $contained: RedactionEntity(containedBy($larger), type == $type, entityType == $entityType, this != $larger, !resized, !skipRemoveEntitiesContainedInLarger)
     then
-        $contained.removeFromGraph();
+        $contained.remove();
         retract($contained);
     end
 
@@ -661,8 +669,8 @@ rule "X.1.0: merge intersecting Entities of same type"
         $first: RedactionEntity($type: type, $entityType: entityType, !resized, !skipRemoveEntitiesContainedInLarger)
         $second: RedactionEntity(intersects($first), type == $type, entityType == $entityType, this != $first, !resized, !skipRemoveEntitiesContainedInLarger)
     then
-        $first.removeFromGraph();
+        $first.remove();
-        $second.removeFromGraph();
+        $second.remove();
         RedactionEntity mergedEntity = entityCreationService.byEntities(List.of($first, $second), $type, $entityType, document);
         retract($first);
         retract($second);
@@ -679,7 +687,7 @@ rule "X.2.0: remove Entity of type ENTITY when contained by FALSE_POSITIVE"
         $entity: RedactionEntity(containedBy($falsePositive), type == $type, entityType == EntityType.ENTITY, !resized, !skipRemoveEntitiesContainedInLarger)
     then
         $entity.getIntersectingNodes().forEach(node -> update(node));
-        $entity.removeFromGraph();
+        $entity.remove();
         retract($entity)
     end
 
@@ -691,7 +699,7 @@ rule "X.3.0: remove Entity of type RECOMMENDATION when contained by FALSE_RECOMM
         $falseRecommendation: RedactionEntity($type: type, entityType == EntityType.FALSE_RECOMMENDATION)
         $recommendation: RedactionEntity(containedBy($falseRecommendation), type == $type, entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger)
     then
-        $recommendation.removeFromGraph();
+        $recommendation.remove();
         retract($recommendation);
     end
 
@@ -704,7 +712,7 @@ rule "X.4.0: remove Entity of type RECOMMENDATION when intersected by ENTITY wit
         $recommendation: RedactionEntity(intersects($entity), type == $type, entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger)
     then
         $entity.addEngines($recommendation.getEngines());
-        $recommendation.removeFromGraph();
+        $recommendation.remove();
         retract($recommendation);
     end
 
@@ -716,7 +724,7 @@ rule "X.5.0: remove Entity of type RECOMMENDATION when contained by ENTITY"
         $entity: RedactionEntity(entityType == EntityType.ENTITY)
         $recommendation: RedactionEntity(containedBy($entity), entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger)
     then
-        $recommendation.removeFromGraph();
+        $recommendation.remove();
         retract($recommendation);
     end
 
@@ -729,7 +737,7 @@ rule "X.6.0: remove Entity of lower rank, when intersected by entity of type ENT
         $lowerRank: RedactionEntity(intersects($higherRank), type != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !resized, !skipRemoveEntitiesContainedInLarger)
     then
         $lowerRank.getIntersectingNodes().forEach(node -> update(node));
-        $lowerRank.removeFromGraph();
+        $lowerRank.remove();
         retract($lowerRank);
     end
 

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/all_rules.drl

@@ -132,9 +132,11 @@ rule "CBI.2.0: Don't redact genitive CBI_author"
     when
         $entity: RedactionEntity(type == "CBI_author", anyMatch(textAfter, "['’’'ʼˈ´`‘′ʻ’']s"), isApplied())
     then
-        RedactionEntity falsePositive = entityCreationService.byBoundary($entity.getBoundary(), "CBI_author", EntityType.FALSE_POSITIVE, document);
+        entityCreationService.byBoundary($entity.getBoundary(), "CBI_author", EntityType.FALSE_POSITIVE, document)
+        .ifPresent(falsePositive -> {
             falsePositive.skip("CBI.2.0", "Genitive Author found");
             insert(falsePositive);
+        });
     end
 
 
@@ -518,7 +520,7 @@ rule "CBI.13.0: Ignore CBI Address Recommendations"
         not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
         $entity: RedactionEntity(type == "CBI_address", entityType == EntityType.RECOMMENDATION)
     then
-        $entity.removeFromGraph();
+        $entity.remove();
         retract($entity)
     end
 
@@ -651,11 +653,13 @@ rule "CBI.18.0: Expand CBI_author entities with firstname initials"
           anyMatch(textAfter, "(,? [A-Z]\\.?( ?[A-Z]\\.?)?( ?[A-Z]\\.?)?\\b\\.?)")
           )
     then
-        RedactionEntity expandedEntity = entityCreationService.bySuffixExpansionRegex($entityToExpand, "(,? [A-Z]\\.?( ?[A-Z]\\.?)?( ?[A-Z]\\.?)?\\b\\.?)");
+        entityCreationService.bySuffixExpansionRegex($entityToExpand, "(,? [A-Z]\\.?( ?[A-Z]\\.?)?( ?[A-Z]\\.?)?\\b\\.?)")
+         .ifPresent(expandedEntity -> {
             expandedEntity.addMatchedRules($entityToExpand.getMatchedRuleList());
-        $entityToExpand.removeFromGraph();
+            $entityToExpand.remove();
             retract($entityToExpand);
             insert(expandedEntity);
+        });
     end
 
 
@@ -664,11 +668,13 @@ rule "CBI.19.0: Expand CBI_author entities with salutation prefix"
     when
         $entityToExpand: RedactionEntity(type == "CBI_author", anyMatch(textBefore, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*"))
     then
-        RedactionEntity expandedEntity = entityCreationService.byPrefixExpansionRegex($entityToExpand, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*");
+        entityCreationService.byPrefixExpansionRegex($entityToExpand, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*")
+         .ifPresent(expandedEntity -> {
             expandedEntity.addMatchedRules($entityToExpand.getMatchedRuleList());
-        $entityToExpand.removeFromGraph();
+            $entityToExpand.remove();
             retract($entityToExpand);
             insert(expandedEntity);
+        });
     end
 
 
@@ -1139,10 +1145,12 @@ rule "PII.12.0: Expand PII entities with salutation prefix"
     when
         $entityToExpand: RedactionEntity(type == "PII", anyMatch(textBefore, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*"))
     then
-        RedactionEntity expandedEntity = entityCreationService.byPrefixExpansionRegex($entityToExpand, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*");
+        entityCreationService.byPrefixExpansionRegex($entityToExpand, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*")
+        .ifPresent(expandedEntity -> {
             expandedEntity.addMatchedRules($entityToExpand.getMatchedRuleList());
             expandedEntity.addEngine(Engine.RULE);
             insert(expandedEntity);
+        });
     end
 
 
@@ -1287,6 +1295,8 @@ rule "AI.0.0: add all NER Entities of type CBI_author"
     then
         nerEntities.streamEntitiesOfType("CBI_author")
             .map(nerEntity -> entityCreationService.byNerEntity(nerEntity, EntityType.RECOMMENDATION, document))
+            .filter(Optional::isPresent)
+            .map(Optional::get)
             .forEach(entity -> insert(entity));
     end
 
@@ -1299,6 +1309,8 @@ rule "AI.1.0: combine and add NER Entities as CBI_address"
     then
         nerEntitiesAdapter.combineNerEntitiesToCbiAddressDefaults(nerEntities)
             .map(boundary -> entityCreationService.byBoundary(boundary, "CBI_address", EntityType.RECOMMENDATION, document))
+            .filter(Optional::isPresent)
+            .map(Optional::get)
             .forEach(entity -> {
                 entity.addEngine(Engine.NER);
                 insert(entity);
@@ -1315,6 +1327,8 @@ rule "AI.2.0: add all NER Entities of any type except CBI_author"
         nerEntities.getNerEntityList().stream()
             .filter(nerEntity -> !nerEntity.type().equals("CBI_author"))
             .map(nerEntity -> entityCreationService.byNerEntity(nerEntity, nerEntity.type().toLowerCase(), EntityType.RECOMMENDATION, document))
+            .filter(Optional::isPresent)
+            .map(Optional::get)
             .forEach(entity -> insert(entity));
     end
 
@@ -1371,10 +1385,12 @@ rule "MAN.2.0: Apply force redaction"
         $entityToForce: RedactionEntity(matchesAnnotationId($id))
     then
         $entityToForce.apply("MAN.2.0", "Forced redaction", $legalBasis);
+        $entityToForce.setRemoved(false);
+        $entityToForce.setIgnored(false);
         $entityToForce.setSkipRemoveEntitiesContainedInLarger(true);
         update($entityToForce);
-        retract($force);
         $entityToForce.getIntersectingNodes().forEach(node -> update(node));
+        retract($force);
     end
 
 
@@ -1387,8 +1403,8 @@ rule "MAN.3.0: Apply image recategorization"
     then
         $imageToBeRecategorized.setImageType(ImageType.fromString($imageType));
         update($imageToBeRecategorized);
-        retract($recategorization);
         update($imageToBeRecategorized.getParent());
+        retract($recategorization);
     end
 
 
@@ -1401,7 +1417,7 @@ rule "X.0.0: remove Entity contained by Entity of same type"
         $larger: RedactionEntity($type: type, $entityType: entityType)
         $contained: RedactionEntity(containedBy($larger), type == $type, entityType == $entityType, this != $larger, !resized, !skipRemoveEntitiesContainedInLarger)
     then
-        $contained.removeFromGraph();
+        $contained.remove();
         retract($contained);
     end
 
@@ -1413,8 +1429,8 @@ rule "X.1.0: merge intersecting Entities of same type"
         $first: RedactionEntity($type: type, $entityType: entityType, !resized, !skipRemoveEntitiesContainedInLarger)
         $second: RedactionEntity(intersects($first), type == $type, entityType == $entityType, this != $first, !resized, !skipRemoveEntitiesContainedInLarger)
     then
-        $first.removeFromGraph();
+        $first.remove();
-        $second.removeFromGraph();
+        $second.remove();
         RedactionEntity mergedEntity = entityCreationService.byEntities(List.of($first, $second), $type, $entityType, document);
         retract($first);
         retract($second);
@@ -1431,7 +1447,7 @@ rule "X.2.0: remove Entity of type ENTITY when contained by FALSE_POSITIVE"
         $entity: RedactionEntity(containedBy($falsePositive), type == $type, entityType == EntityType.ENTITY, !resized, !skipRemoveEntitiesContainedInLarger)
     then
         $entity.getIntersectingNodes().forEach(node -> update(node));
-        $entity.removeFromGraph();
+        $entity.remove();
         retract($entity)
     end
 
@@ -1443,7 +1459,7 @@ rule "X.3.0: remove Entity of type RECOMMENDATION when contained by FALSE_RECOMM
         $falseRecommendation: RedactionEntity($type: type, entityType == EntityType.FALSE_RECOMMENDATION)
         $recommendation: RedactionEntity(containedBy($falseRecommendation), type == $type, entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger)
     then
-        $recommendation.removeFromGraph();
+        $recommendation.remove();
         retract($recommendation);
     end
 
@@ -1456,7 +1472,7 @@ rule "X.4.0: remove Entity of type RECOMMENDATION when intersected by ENTITY wit
         $recommendation: RedactionEntity(intersects($entity), type == $type, entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger)
     then
         $entity.addEngines($recommendation.getEngines());
-        $recommendation.removeFromGraph();
+        $recommendation.remove();
         retract($recommendation);
     end
 
@@ -1468,7 +1484,7 @@ rule "X.5.0: remove Entity of type RECOMMENDATION when contained by ENTITY"
         $entity: RedactionEntity(entityType == EntityType.ENTITY)
         $recommendation: RedactionEntity(containedBy($entity), entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger)
     then
-        $recommendation.removeFromGraph();
+        $recommendation.remove();
         retract($recommendation);
     end
 
@@ -1481,7 +1497,7 @@ rule "X.6.0: remove Entity of lower rank, when intersected by entity of type ENT
         $lowerRank: RedactionEntity(intersects($higherRank), type != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !resized, !skipRemoveEntitiesContainedInLarger)
     then
         $lowerRank.getIntersectingNodes().forEach(node -> update(node));
-        $lowerRank.removeFromGraph();
+        $lowerRank.remove();
         retract($lowerRank);
     end
 

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora.drl

@@ -374,6 +374,8 @@ rule "DOC.8.1: Performing Laboratory (Name)"
          nerEntities.streamEntitiesOfType("COUNTRY")
                      .filter(nerEntity -> $section.getBoundary().contains(nerEntity.boundary()))
                      .map(nerEntity -> entityCreationService.byNerEntity(nerEntity, "laboratory_country", EntityType.ENTITY, $section))
+                     .filter(Optional::isPresent)
+                     .map(Optional::get)
                      .forEach(entity -> {
                           entity.apply("DOC.8.2", "Performing Laboratory found", "n-a");
                           insert(entity);
@@ -588,8 +590,8 @@ rule "DOC.13.0: Clinical Signs"
             && hasParagraphs()
         )
     then
-        var entity = entityCreationService.byBoundary(Boundary.merge($section.streamAllSubNodesOfType(NodeType.PARAGRAPH).map(SemanticNode::getBoundary).toList()), "clinical_signs", EntityType.ENTITY, $section);
+        entityCreationService.bySemanticNodeParagraphsOnly($section, "clinical_signs", EntityType.ENTITY)
-        entity.apply("DOC.13.0", "Clinical Signs found", "n-a");
+            .forEach(entity -> entity.apply("DOC.13.0", "Clinical Signs found", "n-a"));
     end
 
 
@@ -618,8 +620,8 @@ rule "DOC.15.0: Mortality"
         $headline: Headline(containsString("Mortality") && !containsString("TABLE") && hasParagraphs())
         FileAttribute(label == "OECD Number", value == "425")
     then
-        var entity = entityCreationService.byBoundary(Boundary.merge($headline.getParent().streamAllSubNodesOfType(NodeType.PARAGRAPH).map(SemanticNode::getBoundary).toList()), "mortality", EntityType.ENTITY, $headline.getParent());
+        entityCreationService.bySemanticNodeParagraphsOnly($headline.getParent(), "mortality", EntityType.ENTITY)
-        entity.apply("DOC.15.0", "Mortality found", "n-a");
+            .forEach(entity -> entity.apply("DOC.15.0", "Mortality found", "n-a"));
     end
 
 
@@ -631,8 +633,8 @@ rule "DOC.17.0: Study Conclusion"
                 && hasParagraphs()
             )
     then
-         var entity = entityCreationService.byBoundary(Boundary.merge($section.streamAllSubNodesOfType(NodeType.PARAGRAPH).map(SemanticNode::getBoundary).toList()), "study_conclusion", EntityType.ENTITY, $section);
+         entityCreationService.bySemanticNodeParagraphsOnly($section, "study_conclusion", EntityType.ENTITY)
-         entity.apply("DOC.17.0", "Study Conclusion found", "n-a");
+         .forEach(entity -> entity.apply("DOC.17.0", "Study Conclusion found", "n-a"));
     end
 
 
@@ -650,8 +652,8 @@ rule "DOC.18.0: Weight Behavior Changes"
             && hasParagraphs()
         )
     then
-        var entity = entityCreationService.byBoundary(Boundary.merge($section.streamAllSubNodesOfType(NodeType.PARAGRAPH).map(SemanticNode::getBoundary).toList()), "weight_behavior_changes", EntityType.ENTITY, $section);
+        entityCreationService.bySemanticNodeParagraphsOnly($section, "weight_behavior_changes", EntityType.ENTITY)
-        entity.apply("DOC.18.0", "Weight behavior changes found", "n-a");
+            .forEach(entity -> entity.apply("DOC.18.0", "Weight behavior changes found", "n-a"));
     end
 
 
@@ -669,8 +671,8 @@ rule "DOC.19.0: Necropsy findings"
             && hasParagraphs()
         )
     then
-        var entity = entityCreationService.byBoundary(Boundary.merge($section.streamAllSubNodesOfType(NodeType.PARAGRAPH).map(SemanticNode::getBoundary).toList()), "necropsy_findings", EntityType.ENTITY, $section);
+        entityCreationService.bySemanticNodeParagraphsOnly($section, "necropsy_findings", EntityType.ENTITY)
-        entity.apply("DOC.19.0", "Necropsy section found", "n-a");
+            .forEach( entity -> entity.apply("DOC.19.0", "Necropsy section found", "n-a"));
     end
 
 
@@ -689,8 +691,8 @@ rule "DOC.22.0: Clinical observations"
             && hasParagraphs()
         )
     then
-        var entity = entityCreationService.byBoundary(Boundary.merge($section.streamAllSubNodesOfType(NodeType.PARAGRAPH).map(SemanticNode::getBoundary).toList()), "clinical_observations", EntityType.ENTITY, $section);
+        entityCreationService.bySemanticNodeParagraphsOnly($section, "clinical_observations", EntityType.ENTITY)
-        entity.apply("DOC.22.0", "Clinical observations section found", "n-a");
+            .forEach(entity -> entity.apply("DOC.22.0", "Clinical observations section found", "n-a"));
     end
 
 
@@ -746,8 +748,8 @@ rule "DOC.23.0: Bodyweight changes"
             && hasParagraphs()
         )
     then
-        var entity = entityCreationService.byBoundary(Boundary.merge($section.streamAllSubNodesOfType(NodeType.PARAGRAPH).map(SemanticNode::getBoundary).toList()), "bodyweight_changes", EntityType.ENTITY, $section);
+        entityCreationService.bySemanticNodeParagraphsOnly($section, "bodyweight_changes", EntityType.ENTITY)
-        entity.apply("DOC.23.0", "Bodyweight section found", "n-a");
+            .forEach(entity -> entity.apply("DOC.23.0", "Bodyweight section found", "n-a"));
     end
 
 
@@ -759,8 +761,8 @@ rule "DOC.24.0: Study Design"
             && hasParagraphs()
         )
     then
-        var entity = entityCreationService.byBoundary(Boundary.merge($section.streamAllSubNodesOfType(NodeType.PARAGRAPH).map(SemanticNode::getBoundary).toList()), "study_design", EntityType.ENTITY, $section);
+        entityCreationService.bySemanticNodeParagraphsOnly($section, "study_design", EntityType.ENTITY)
-        entity.apply("DOC.24.0", "Study design section found", "n-a");
+            .forEach(entity -> entity.apply("DOC.24.0", "Study design section found", "n-a"));
     end
 
 
@@ -781,8 +783,8 @@ rule "DOC.25.0: Results and Conclusion (406, 428, 438, 439, 474 & 487)"
            && hasParagraphs()
         )
     then
-        var entity = entityCreationService.byBoundary(Boundary.merge($section.streamAllSubNodesOfType(NodeType.PARAGRAPH).map(SemanticNode::getBoundary).toList()), "results_and_conclusion", EntityType.ENTITY, $section);
+        entityCreationService.bySemanticNodeParagraphsOnly($section, "results_and_conclusion", EntityType.ENTITY)
-        entity.apply("DOC.25.0", "Results and Conclusion found", "n-a");
+            .forEach(entity -> entity.apply("DOC.25.0", "Results and Conclusion found", "n-a"));
     end
 
 
@@ -816,8 +818,8 @@ rule "DOC.32.0: Preliminary Test Results (429)"
             && hasParagraphs()
         )
     then
-        var entity = entityCreationService.byBoundary(Boundary.merge($section.streamAllSubNodesOfType(NodeType.PARAGRAPH).map(SemanticNode::getBoundary).toList()), "preliminary_test_results", EntityType.ENTITY, $section);
+        entityCreationService.bySemanticNodeParagraphsOnly($section, "preliminary_test_results", EntityType.ENTITY)
-        entity.apply("DOC.32.0", "Preliminary Test Results found", "n-a");
+            .forEach(entity -> entity.apply("DOC.32.0", "Preliminary Test Results found", "n-a"));
     end
 
 
@@ -826,8 +828,8 @@ rule "DOC.33.0: Test Results (429)"
         FileAttribute(label == "OECD Number", value == "429")
         $section: Section((getHeadline().containsString("RESULTS AND DISCUSSION") || getHeadline().containsString("Estimation of the proliferative response of lymph node cells") || getHeadline().containsString("Results in the Main Experiment")) && hasParagraphs())
     then
-        var entity = entityCreationService.byBoundary(Boundary.merge($section.streamAllSubNodesOfType(NodeType.PARAGRAPH).map(SemanticNode::getBoundary).toList()), "test_results", EntityType.ENTITY, $section);
+        entityCreationService.bySemanticNodeParagraphsOnly($section, "test_results", EntityType.ENTITY)
-        entity.apply("DOC.33.0", "Test Results found", "n-a");
+        .forEach(entity -> entity.apply("DOC.33.0", "Test Results found", "n-a"));
     end
 
 
@@ -962,8 +964,8 @@ rule "DOC.39.0: Dilution of the test substance"
             && hasParagraphs()
         )
     then
-        var entity = entityCreationService.byBoundary(Boundary.merge($section.streamAllSubNodesOfType(NodeType.PARAGRAPH).map(SemanticNode::getBoundary).toList()), "dilution", EntityType.ENTITY, $section);
+        entityCreationService.bySemanticNodeParagraphsOnly($section, "dilution", EntityType.ENTITY)
-        entity.apply("DOC.39.0", "Dilution found.", "n-a");
+            .forEach(entity -> entity.apply("DOC.39.0", "Dilution found.", "n-a"));
     end
 
 
@@ -976,8 +978,8 @@ rule "DOC.40.0: Positive Control"
             && hasParagraphs()
         )
     then
-        var entity = entityCreationService.byBoundary(Boundary.merge($section.streamAllSubNodesOfType(NodeType.PARAGRAPH).map(SemanticNode::getBoundary).toList()), "positive_control", EntityType.ENTITY, $section);
+        entityCreationService.bySemanticNodeParagraphsOnly($section, "positive_control", EntityType.ENTITY)
-        entity.apply("DOC.40.0", "Positive control found.", "n-a");
+            .forEach(entity -> entity.apply("DOC.40.0", "Positive control found.", "n-a"));
     end
 
 
@@ -986,8 +988,8 @@ rule "DOC.42.0: Mortality Statement"
         FileAttribute(label == "OECD Number", value == "402")
         $headline: Headline(containsString("Mortality") && !containsString("TABLE") && hasParagraphs())
     then
-        var entity = entityCreationService.byBoundary(Boundary.merge($headline.getParent().streamAllSubNodesOfType(NodeType.PARAGRAPH).map(SemanticNode::getBoundary).toList()), "mortality_statement", EntityType.ENTITY, $headline.getParent());
+        entityCreationService.bySemanticNodeParagraphsOnly($headline.getParent(), "mortality_statement", EntityType.ENTITY)
-        entity.apply("DOC.42.0", "Mortality Statement found", "n-a");
+            .forEach(entity -> entity.apply("DOC.42.0", "Mortality Statement found", "n-a"));
     end
 
 
@@ -1059,8 +1061,8 @@ rule "DOC.45.0: Doses (mg/kg bodyweight)"
              && hasParagraphs()
         )
     then
-        var entity = entityCreationService.byBoundary(Boundary.merge($section.streamAllSubNodesOfType(NodeType.PARAGRAPH).map(SemanticNode::getBoundary).toList()), "doses_(mg_kg_bw)", EntityType.ENTITY, $section);
+        entityCreationService.bySemanticNodeParagraphsOnly($section, "doses_(mg_kg_bw)", EntityType.ENTITY)
-        entity.apply("DOC.45.0", "Doses per bodyweight information found", "n-a");
+            .forEach(entity -> entity.apply("DOC.45.0", "Doses per bodyweight information found", "n-a"));
     end
 
 
@@ -1106,11 +1108,16 @@ rule "MAN.1.1: Apply id removals that are valid and not in forced redactions to
 rule "MAN.2.0: Apply force redaction"
     salience 128
     when
-        ManualForceRedaction($id: annotationId, status == AnnotationStatus.APPROVED, requestDate != null, $legalBasis: legalBasis)
+        $force: ManualForceRedaction($id: annotationId, status == AnnotationStatus.APPROVED, requestDate != null, $legalBasis: legalBasis)
         $entityToForce: RedactionEntity(matchesAnnotationId($id))
     then
         $entityToForce.apply("MAN.2.0", "Forced redaction", $legalBasis);
+        $entityToForce.setRemoved(false);
+        $entityToForce.setIgnored(false);
         $entityToForce.setSkipRemoveEntitiesContainedInLarger(true);
+        update($entityToForce);
+        $entityToForce.getIntersectingNodes().forEach(node -> update(node));
+        retract($force);
     end
 
 

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl

@@ -469,11 +469,13 @@ rule "CBI.18.0: Expand CBI_author entities with firstname initials"
           anyMatch(textAfter, "(,? [A-Z]\\.?( ?[A-Z]\\.?)?( ?[A-Z]\\.?)?\\b\\.?)")
           )
     then
-        RedactionEntity expandedEntity = entityCreationService.bySuffixExpansionRegex($entityToExpand, "(,? [A-Z]\\.?( ?[A-Z]\\.?)?( ?[A-Z]\\.?)?\\b\\.?)");
+        entityCreationService.bySuffixExpansionRegex($entityToExpand, "(,? [A-Z]\\.?( ?[A-Z]\\.?)?( ?[A-Z]\\.?)?\\b\\.?)")
+         .ifPresent(expandedEntity -> {
             expandedEntity.addMatchedRules($entityToExpand.getMatchedRuleList());
-        $entityToExpand.removeFromGraph();
+            $entityToExpand.remove();
             retract($entityToExpand);
             insert(expandedEntity);
+        });
     end
 
 
@@ -482,11 +484,13 @@ rule "CBI.19.0: Expand CBI_author entities with salutation prefix"
     when
         $entityToExpand: RedactionEntity(type == "CBI_author", anyMatch(textBefore, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*"))
     then
-        RedactionEntity expandedEntity = entityCreationService.byPrefixExpansionRegex($entityToExpand, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*");
+        entityCreationService.byPrefixExpansionRegex($entityToExpand, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*")
+         .ifPresent(expandedEntity -> {
             expandedEntity.addMatchedRules($entityToExpand.getMatchedRuleList());
-        $entityToExpand.removeFromGraph();
+            $entityToExpand.remove();
             retract($entityToExpand);
             insert(expandedEntity);
+        });
     end
 
 
@@ -834,10 +838,12 @@ rule "PII.12.0: Expand PII entities with salutation prefix"
     when
         $entityToExpand: RedactionEntity(type == "PII", anyMatch(textBefore, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*"))
     then
-        RedactionEntity expandedEntity = entityCreationService.byPrefixExpansionRegex($entityToExpand, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*");
+        entityCreationService.byPrefixExpansionRegex($entityToExpand, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*")
+        .ifPresent(expandedEntity -> {
             expandedEntity.addMatchedRules($entityToExpand.getMatchedRuleList());
             expandedEntity.addEngine(Engine.RULE);
             insert(expandedEntity);
+        });
     end
 
 
@@ -969,6 +975,8 @@ rule "AI.0.0: add all NER Entities of type CBI_author"
     then
         nerEntities.streamEntitiesOfType("CBI_author")
             .map(nerEntity -> entityCreationService.byNerEntity(nerEntity, EntityType.RECOMMENDATION, document))
+            .filter(Optional::isPresent)
+            .map(Optional::get)
             .forEach(entity -> insert(entity));
     end
 
@@ -981,6 +989,8 @@ rule "AI.1.0: combine and add NER Entities as CBI_address"
     then
         nerEntitiesAdapter.combineNerEntitiesToCbiAddressDefaults(nerEntities)
             .map(boundary -> entityCreationService.byBoundary(boundary, "CBI_address", EntityType.RECOMMENDATION, document))
+            .filter(Optional::isPresent)
+            .map(Optional::get)
             .forEach(entity -> {
                 entity.addEngine(Engine.NER);
                 insert(entity);
@@ -1040,10 +1050,12 @@ rule "MAN.2.0: Apply force redaction"
         $entityToForce: RedactionEntity(matchesAnnotationId($id))
     then
         $entityToForce.apply("MAN.2.0", "Forced redaction", $legalBasis);
+        $entityToForce.setRemoved(false);
+        $entityToForce.setIgnored(false);
         $entityToForce.setSkipRemoveEntitiesContainedInLarger(true);
         update($entityToForce);
-        retract($force);
         $entityToForce.getIntersectingNodes().forEach(node -> update(node));
+        retract($force);
     end
 
 
@@ -1070,7 +1082,7 @@ rule "X.0.0: remove Entity contained by Entity of same type"
         $larger: RedactionEntity($type: type, $entityType: entityType)
         $contained: RedactionEntity(containedBy($larger), type == $type, entityType == $entityType, this != $larger, !resized, !skipRemoveEntitiesContainedInLarger)
     then
-        $contained.removeFromGraph();
+        $contained.remove();
         retract($contained);
     end
 
@@ -1082,8 +1094,8 @@ rule "X.1.0: merge intersecting Entities of same type"
         $first: RedactionEntity($type: type, $entityType: entityType, !resized, !skipRemoveEntitiesContainedInLarger)
         $second: RedactionEntity(intersects($first), type == $type, entityType == $entityType, this != $first, !resized, !skipRemoveEntitiesContainedInLarger)
     then
-        $first.removeFromGraph();
+        $first.remove();
-        $second.removeFromGraph();
+        $second.remove();
         RedactionEntity mergedEntity = entityCreationService.byEntities(List.of($first, $second), $type, $entityType, document);
         retract($first);
         retract($second);
@@ -1100,7 +1112,7 @@ rule "X.2.0: remove Entity of type ENTITY when contained by FALSE_POSITIVE"
         $entity: RedactionEntity(containedBy($falsePositive), type == $type, entityType == EntityType.ENTITY, !resized, !skipRemoveEntitiesContainedInLarger)
     then
         $entity.getIntersectingNodes().forEach(node -> update(node));
-        $entity.removeFromGraph();
+        $entity.remove();
         retract($entity)
     end
 
@@ -1112,7 +1124,7 @@ rule "X.3.0: remove Entity of type RECOMMENDATION when contained by FALSE_RECOMM
         $falseRecommendation: RedactionEntity($type: type, entityType == EntityType.FALSE_RECOMMENDATION)
         $recommendation: RedactionEntity(containedBy($falseRecommendation), type == $type, entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger)
     then
-        $recommendation.removeFromGraph();
+        $recommendation.remove();
         retract($recommendation);
     end
 
@@ -1125,7 +1137,7 @@ rule "X.4.0: remove Entity of type RECOMMENDATION when intersected by ENTITY wit
         $recommendation: RedactionEntity(intersects($entity), type == $type, entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger)
     then
         $entity.addEngines($recommendation.getEngines());
-        $recommendation.removeFromGraph();
+        $recommendation.remove();
         retract($recommendation);
     end
 
@@ -1137,7 +1149,7 @@ rule "X.5.0: remove Entity of type RECOMMENDATION when contained by ENTITY"
         $entity: RedactionEntity(entityType == EntityType.ENTITY)
         $recommendation: RedactionEntity(containedBy($entity), entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger)
     then
-        $recommendation.removeFromGraph();
+        $recommendation.remove();
         retract($recommendation);
     end
 
@@ -1150,7 +1162,7 @@ rule "X.6.0: remove Entity of lower rank, when intersected by entity of type ENT
         $lowerRank: RedactionEntity(intersects($higherRank), type != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !resized, !skipRemoveEntitiesContainedInLarger)
     then
         $lowerRank.getIntersectingNodes().forEach(node -> update(node));
-        $lowerRank.removeFromGraph();
+        $lowerRank.remove();
         retract($lowerRank);
     end
 

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_v2.drl

@@ -56,9 +56,11 @@ rule "add NER Entities of type CBI_author or CBI_address"
     when
         $nerEntity: EntityRecognitionEntity($type: type, (type == "CBI_author" || type == "CBI_address"))
     then
-        RedactionEntity redactionEntity = entityCreationService.byBoundary(new Boundary($nerEntity.getStartOffset(), $nerEntity.getEndOffset()), $type, EntityType.RECOMMENDATION, document);
+        entityCreationService.byBoundary(new Boundary($nerEntity.getStartOffset(), $nerEntity.getEndOffset()), $type, EntityType.RECOMMENDATION, document)
+        .ifPresent(redactionEntity -> {
             redactionEntity.addEngine(Engine.NER);
             insert(redactionEntity);
+        });
     end
 
 // --------------------------------------- CBI rules -------------------------------------------------------------------
@@ -81,91 +83,126 @@ rule "Always redact PII"
         $cbiAuthor.apply("PII.0.0", "PII found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
     end
 
-// --------------------------------------- merging rules -------------------------------------------------------------------
+//------------------------------------ Entity merging rules ------------------------------------
 
-rule "remove Entity contained by Entity of same type"
+// Rule unit: X.0
+rule "X.0.0: remove Entity contained by Entity of same type"
     salience 65
     when
         $larger: RedactionEntity($type: type, $entityType: entityType)
         $contained: RedactionEntity(containedBy($larger), type == $type, entityType == $entityType, this != $larger, !resized, !skipRemoveEntitiesContainedInLarger)
     then
-        $contained.removeFromGraph();
+        $contained.remove();
         retract($contained);
     end
 
-rule "merge intersecting Entities of same type"
+
+// Rule unit: X.1
+rule "X.1.0: merge intersecting Entities of same type"
     salience 64
     when
         $first: RedactionEntity($type: type, $entityType: entityType, !resized, !skipRemoveEntitiesContainedInLarger)
         $second: RedactionEntity(intersects($first), type == $type, entityType == $entityType, this != $first, !resized, !skipRemoveEntitiesContainedInLarger)
     then
-        $first.removeFromGraph();
+        $first.remove();
-        $second.removeFromGraph();
+        $second.remove();
         RedactionEntity mergedEntity = entityCreationService.byEntities(List.of($first, $second), $type, $entityType, document);
         retract($first);
         retract($second);
         insert(mergedEntity);
+        mergedEntity.getIntersectingNodes().forEach(node -> update(node));
     end
 
-rule "remove Entity of type ENTITY when contained by FALSE_POSITIVE"
+
+// Rule unit: X.2
+rule "X.2.0: remove Entity of type ENTITY when contained by FALSE_POSITIVE"
     salience 64
     when
         $falsePositive: RedactionEntity($type: type, entityType == EntityType.FALSE_POSITIVE)
         $entity: RedactionEntity(containedBy($falsePositive), type == $type, entityType == EntityType.ENTITY, !resized, !skipRemoveEntitiesContainedInLarger)
     then
-        $entity.removeFromGraph();
+        $entity.getIntersectingNodes().forEach(node -> update(node));
+        $entity.remove();
         retract($entity)
     end
 
-rule "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"
+
+// Rule unit: X.3
+rule "X.3.0: remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"
     salience 64
     when
         $falseRecommendation: RedactionEntity($type: type, entityType == EntityType.FALSE_RECOMMENDATION)
         $recommendation: RedactionEntity(containedBy($falseRecommendation), type == $type, entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger)
     then
-        $recommendation.removeFromGraph();
+        $recommendation.remove();
         retract($recommendation);
     end
 
-rule "remove Entity of type RECOMMENDATION when contained by ENTITY"
+
-    salience 64
+// Rule unit: X.4
+rule "X.4.0: remove Entity of type RECOMMENDATION when intersected by ENTITY with same type"
+    salience 256
     when
         $entity: RedactionEntity($type: type, entityType == EntityType.ENTITY)
-        $recommendation: RedactionEntity(containedBy($entity), type == $type, entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger)
+        $recommendation: RedactionEntity(intersects($entity), type == $type, entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger)
     then
-        $recommendation.removeFromGraph();
+        $entity.addEngines($recommendation.getEngines());
+        $recommendation.remove();
         retract($recommendation);
     end
 
-rule "remove Entity of lower rank, when equal boundaries and entityType"
+
+// Rule unit: X.5
+rule "X.5.0: remove Entity of type RECOMMENDATION when contained by ENTITY"
+    salience 256
+    when
+        $entity: RedactionEntity(entityType == EntityType.ENTITY)
+        $recommendation: RedactionEntity(containedBy($entity), entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger)
+    then
+        $recommendation.remove();
+        retract($recommendation);
+    end
+
+
+// Rule unit: X.6
+rule "X.6.0: remove Entity of lower rank, when intersected by entity of type ENTITY"
     salience 32
     when
-        $higherRank: RedactionEntity($type: type, $entityType: entityType, $boundary: boundary)
+        $higherRank: RedactionEntity($type: type, entityType == EntityType.ENTITY)
-        $lowerRank: RedactionEntity($boundary == boundary, type != $type, entityType == $entityType, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !applied)
+        $lowerRank: RedactionEntity(intersects($higherRank), type != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !resized, !skipRemoveEntitiesContainedInLarger)
     then
-        $lowerRank.removeFromGraph();
+        $lowerRank.getIntersectingNodes().forEach(node -> update(node));
+        $lowerRank.remove();
         retract($lowerRank);
     end
 
-// --------------------------------------- FileAttribute Rules -------------------------------------------------------------------
 
-rule "remove duplicate FileAttributes"
+//------------------------------------ File attributes rules ------------------------------------
+
+// Rule unit: FA.1
+rule "FA.1.0: remove duplicate FileAttributes"
     salience 64
     when
-        $first: FileAttribute($label: label, $value: value)
+        $fileAttribute: FileAttribute($label: label, $value: value)
-        $second: FileAttribute(this != $first, label == $label, value == $value)
+        $duplicate: FileAttribute(this != $fileAttribute, label == $label, value == $value)
     then
-        retract($second);
+        retract($duplicate);
     end
 
-// --------------------------------------- local dictionary search -------------------------------------------------------------------
 
-rule "run local dictionary search"
+//------------------------------------ Local dictionary search rules ------------------------------------
+
+// Rule unit: LDS.0
+rule "LDS.0.0: run local dictionary search"
     agenda-group "LOCAL_DICTIONARY_ADDS"
     salience -999
     when
         DictionaryModel(!localEntries.isEmpty(), $type: type, $searchImplementation: localSearch) from dictionary.getDictionaryModels()
     then
         entityCreationService.bySearchImplementation($searchImplementation, $type, EntityType.RECOMMENDATION, document)
-            .forEach(redactionEntity -> insert(redactionEntity));
+            .forEach(entity -> {
+                entity.addEngine(Engine.RULE);
+                insert(entity);
+            });
     end
+