RED-6929: fix acceptance tests/rules

* updated rules
* added SectionIdentifier
* inserted document into kieSession
* renames
* added tests

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/factory/SectionNodeFactory.java

@@ -80,7 +80,7 @@ public class SectionNodeFactory {
             remainingBlocks.removeAll(alreadyMerged);
 
             if (abstractPageBlock instanceof TextPageBlock) {
-                List<TextPageBlock> textBlocks = findTextBlocksWithSameClassificationAndAlignsY(abstractPageBlock, remainingBlocks);
+                List<TextPageBlock> textBlocks = findTextBlocksWithSameClassificationAndAlignsYAndSameOrientation(abstractPageBlock, remainingBlocks);
                 alreadyMerged.addAll(textBlocks);
                 DocumentGraphFactory.addParagraphOrHeadline(section, (TextPageBlock) abstractPageBlock, context, textBlocks);
             } else if (abstractPageBlock instanceof TablePageBlock tablePageBlock) {
@@ -162,14 +162,15 @@ public class SectionNodeFactory {
     }
 
 
-    private List<TextPageBlock> findTextBlocksWithSameClassificationAndAlignsY(AbstractPageBlock atc, List<AbstractPageBlock> pageBlocks) {
+    private List<TextPageBlock> findTextBlocksWithSameClassificationAndAlignsYAndSameOrientation(AbstractPageBlock atc, List<AbstractPageBlock> pageBlocks) {
 
         return pageBlocks.stream()
-                .filter(abstractTextContainer -> !abstractTextContainer.equals(atc))
-                .filter(abstractTextContainer -> abstractTextContainer.getPage() == atc.getPage())
-                .filter(abstractTextContainer -> abstractTextContainer instanceof TextPageBlock)
-                .filter(abstractTextContainer -> abstractTextContainer.intersectsY(atc))
-                .map(abstractTextContainer -> (TextPageBlock) abstractTextContainer)
+                .filter(abstractPageBlock -> !abstractPageBlock.equals(atc))
+                .filter(abstractPageBlock -> abstractPageBlock.getPage() == atc.getPage())
+                .filter(abstractPageBlock -> abstractPageBlock.getOrientation().equals(atc.getOrientation()))
+                .filter(abstractPageBlock -> abstractPageBlock.intersectsY(atc))
+                .filter(abstractPageBlock -> abstractPageBlock instanceof TextPageBlock)
+                .map(abstractPageBlock -> (TextPageBlock) abstractPageBlock)
                 .toList();
     }
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/Document.java

@@ -12,6 +12,7 @@ import java.util.stream.Stream;
 
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.DocumentTree;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionEntity;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.sectionidentifiers.SectionIdentifier;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.TextBlock;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.TextBlockCollector;
 
@@ -35,6 +36,8 @@ public class Document implements GenericSemanticNode {
     TextBlock textBlock;
     @Builder.Default
     Set<RedactionEntity> entities = new HashSet<>();
+    @Builder.Default
+    static final SectionIdentifier sectionIdentifier = SectionIdentifier.document();
 
 
     @Override
@@ -79,6 +82,13 @@ public class Document implements GenericSemanticNode {
     }
 
 
+    @Override
+    public SectionIdentifier getSectionIdentifier() {
+
+        return sectionIdentifier;
+    }
+
+
     @Override
     public Headline getHeadline() {
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/Footer.java

@@ -6,6 +6,7 @@ import java.util.Set;
 
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.DocumentTree;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionEntity;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.sectionidentifiers.SectionIdentifier;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.TextBlock;
 
 import lombok.AccessLevel;
@@ -23,6 +24,9 @@ import lombok.experimental.FieldDefaults;
 @FieldDefaults(level = AccessLevel.PRIVATE)
 public class Footer implements GenericSemanticNode {
 
+    @Builder.Default
+    final static SectionIdentifier sectionIdentifier = SectionIdentifier.empty();
+
     List<Integer> treeId;
     TextBlock leafTextBlock;
 
@@ -55,6 +59,13 @@ public class Footer implements GenericSemanticNode {
     }
 
 
+    @Override
+    public SectionIdentifier getSectionIdentifier() {
+
+        return sectionIdentifier;
+    }
+
+
     @Override
     public String toString() {
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/Header.java

@@ -6,6 +6,7 @@ import java.util.Set;
 
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.DocumentTree;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionEntity;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.sectionidentifiers.SectionIdentifier;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.TextBlock;
 
 import lombok.AccessLevel;
@@ -23,6 +24,9 @@ import lombok.experimental.FieldDefaults;
 @FieldDefaults(level = AccessLevel.PRIVATE)
 public class Header implements GenericSemanticNode {
 
+    @Builder.Default
+    final static SectionIdentifier sectionIdentifier = SectionIdentifier.empty();
+
     List<Integer> treeId;
     TextBlock leafTextBlock;
 
@@ -55,6 +59,13 @@ public class Header implements GenericSemanticNode {
     }
 
 
+    @Override
+    public SectionIdentifier getSectionIdentifier() {
+
+        return sectionIdentifier;
+    }
+
+
     @Override
     public String toString() {
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/Headline.java

@@ -6,6 +6,7 @@ import java.util.Set;
 
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.DocumentTree;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionEntity;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.sectionidentifiers.SectionIdentifier;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.AtomicTextBlock;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.TextBlock;
 
@@ -26,6 +27,7 @@ public class Headline implements GenericSemanticNode {
 
     List<Integer> treeId;
     TextBlock leafTextBlock;
+    SectionIdentifier sectionIdentifier;
 
     @EqualsAndHashCode.Exclude
     DocumentTree documentTree;
@@ -70,12 +72,24 @@ public class Headline implements GenericSemanticNode {
     }
 
 
+    @Override
+    public SectionIdentifier getSectionIdentifier() {
+
+        if (sectionIdentifier == null) {
+            sectionIdentifier = SectionIdentifier.fromSearchText(getTextBlock().getSearchText());
+        }
+        return sectionIdentifier;
+    }
+
+
     public static Headline empty() {
 
         return Headline.builder().leafTextBlock(AtomicTextBlock.empty(-1L, 0, new Page(), -1, null)).build();
     }
 
-    public boolean hasParagraphs(){
+
+    public boolean hasParagraphs() {
+
         return getParent().streamAllSubNodesOfType(NodeType.PARAGRAPH).findFirst().isPresent();
     }
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/Section.java

@@ -6,9 +6,9 @@ import java.util.Set;
 
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.DocumentTree;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionEntity;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.sectionidentifiers.SectionIdentifier;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.TextBlock;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.TextBlockCollector;
-import java.util.stream.Stream;
 
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
@@ -49,6 +49,13 @@ public class Section implements GenericSemanticNode {
     }
 
 
+    @Override
+    public SectionIdentifier getSectionIdentifier() {
+
+        return getHeadline().getSectionIdentifier();
+    }
+
+
     @Override
     public TextBlock getTextBlock() {
 
@@ -75,19 +82,22 @@ public class Section implements GenericSemanticNode {
     }
 
 
-    public boolean anyHeadlineContainsString(String value){
+    public boolean anyHeadlineContainsString(String value) {
+
         return streamChildrenOfType(NodeType.HEADLINE)//
                 .map(node -> (Headline) node).anyMatch(h -> h.containsString(value));
     }
 
 
-    public boolean anyHeadlineContainsStringIgnoreCase(String value){
+    public boolean anyHeadlineContainsStringIgnoreCase(String value) {
+
         return streamChildrenOfType(NodeType.HEADLINE)//
                 .map(node -> (Headline) node).anyMatch(h -> h.containsStringIgnoreCase(value));
     }
 
 
-    public boolean hasParagraphs(){
+    public boolean hasParagraphs() {
+
         return streamAllSubNodesOfType(NodeType.PARAGRAPH).findFirst().isPresent();
     }
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/SemanticNode.java

@@ -13,6 +13,7 @@ import java.util.stream.Stream;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.Boundary;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.DocumentTree;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionEntity;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.sectionidentifiers.SectionIdentifier;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.AtomicTextBlock;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.TextBlock;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RectangleTransformations;
@@ -115,6 +116,17 @@ public interface SemanticNode {
     }
 
 
+    /**
+     * Returns a SectionIdentifier, such that it acts as a child of the first Headline associated with this SemanticNode.
+     *
+     * @return The SectionIdentifier from the first Headline.
+     */
+    default SectionIdentifier getSectionIdentifier() {
+
+        return SectionIdentifier.asChildOf(getHeadline().getSectionIdentifier());
+    }
+
+
     /**
      * Checks if its TreeId has a length greater than zero.
      *

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/sectionidentifiers/SectionIdentifier.java

@@ -0,0 +1,123 @@
+package com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.sectionidentifiers;
+
+import java.util.Collections;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import lombok.AccessLevel;
+import lombok.AllArgsConstructor;
+import lombok.experimental.FieldDefaults;
+
+@AllArgsConstructor
+@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
+public class SectionIdentifier {
+
+    static Pattern numericalIdentifierPattern = Pattern.compile("^[\\s]?(\\d+)[\\s.,;]?(\\d+)?[\\s.,;]?(\\d+)?[\\s.,;]?(\\d+)?");
+
+    private enum Format {
+        EMPTY,
+        NUMERICAL,
+        DOCUMENT
+    }
+
+    Format format;
+    String identifierString;
+    List<Integer> identifiers;
+    boolean asChild;
+
+
+    public static SectionIdentifier fromSearchText(String headline) {
+
+        if (headline == null || headline.isEmpty() || headline.isBlank()) {
+            return SectionIdentifier.empty();
+        }
+
+        Matcher numericalIdentifierMatcher = numericalIdentifierPattern.matcher(headline);
+        if (numericalIdentifierMatcher.find()) {
+            return buildNumericalSectionIdentifier(headline, numericalIdentifierMatcher);
+        }
+        // more formats here
+        return SectionIdentifier.empty();
+    }
+
+
+    public static SectionIdentifier asChildOf(SectionIdentifier sectionIdentifier) {
+
+        return new SectionIdentifier(sectionIdentifier.format, sectionIdentifier.toString(), sectionIdentifier.identifiers, true);
+    }
+
+
+    public static SectionIdentifier document() {
+
+        return new SectionIdentifier(Format.DOCUMENT, "document", Collections.emptyList(), false);
+    }
+
+
+    public static SectionIdentifier empty() {
+
+        return new SectionIdentifier(Format.EMPTY, "empty", Collections.emptyList(), false);
+    }
+
+
+    private static SectionIdentifier buildNumericalSectionIdentifier(String headline, Matcher numericalIdentifierMatcher) {
+
+        String identifierString = headline.substring(numericalIdentifierMatcher.start(), numericalIdentifierMatcher.end());
+        List<Integer> identifiers = new LinkedList<>();
+        for (int i = 1; i <= 4; i++) {
+            String numericalIdentifier = numericalIdentifierMatcher.group(i);
+            if (numericalIdentifier == null || numericalIdentifier.equals("0") || numericalIdentifier.isEmpty() || numericalIdentifier.isBlank()) {
+                break;
+            }
+            identifiers.add(Integer.parseInt(numericalIdentifier.trim()));
+        }
+        return new SectionIdentifier(Format.NUMERICAL, identifierString, identifiers.stream().toList(), false);
+    }
+
+
+    /**
+     * Determines if the current section is the parent of the given section.
+     *
+     * @param sectionIdentifier The section identifier to compare against.
+     * @return true if the current section is the parent of the given section, false otherwise.
+     */
+    public boolean isParentOf(SectionIdentifier sectionIdentifier) {
+
+        if (this.format.equals(Format.EMPTY)) {
+            return false;
+        }
+        if (this.format.equals(Format.DOCUMENT)) {
+            return true;
+        }
+        if (!this.format.equals(sectionIdentifier.format)) {
+            return false;
+        }
+        if (this.identifiers.size() >= sectionIdentifier.identifiers.size() && !(this.identifiers.size() == sectionIdentifier.identifiers.size() && sectionIdentifier.asChild)) {
+            return false;
+        }
+        for (int i = 0; i < this.identifiers.size(); i++) {
+            if (!this.identifiers.get(i).equals(sectionIdentifier.identifiers.get(i))) {
+                return false;
+            }
+        }
+        return true;
+    }
+
+
+    public boolean isChildOf(SectionIdentifier sectionIdentifier) {
+
+        if (this.format.equals(Format.DOCUMENT) || this.format.equals(Format.EMPTY)) {
+            return false;
+        }
+        return sectionIdentifier.isParentOf(this);
+    }
+
+
+    @Override
+    public String toString() {
+
+        return identifierString;
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/utils/RectangleTransformations.java

@@ -116,7 +116,7 @@ public class RectangleTransformations {
         @Override
         public BiConsumer<BBox, Rectangle2D> accumulator() {
 
-            return (bb, rect) -> bb.addRectangle(rect.getMinX(), rect.getMinY(), rect.getMaxX(), rect.getMaxY());
+            return BBox::addRectangle;
         }
 
 
@@ -154,7 +154,12 @@ public class RectangleTransformations {
             Double upperRightY;
 
 
-            public void addRectangle(double lowerLeftX, double lowerLeftY, double upperRightX, double upperRightY) {
+            public void addRectangle(Rectangle2D rectangle2D) {
+
+                double lowerLeftX = Math.min(rectangle2D.getMinX(), rectangle2D.getMaxX());
+                double lowerLeftY = Math.min(rectangle2D.getMinY(), rectangle2D.getMaxY());
+                double upperRightX = Math.max(rectangle2D.getMinX(), rectangle2D.getMaxX());
+                double upperRightY = Math.max(rectangle2D.getMinY(), rectangle2D.getMaxY());
 
                 if (this.lowerLeftX == null) {
                     this.lowerLeftX = lowerLeftX;

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/AnalyzeService.java

@@ -98,13 +98,16 @@ public class AnalyzeService {
                 pdfImages = imageServiceResponseAdapter.convertImages(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
             }
 
+            log.info("parse document for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
             classifiedDoc = pdfSegmentationService.parseDocument(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), storedObjectStream, pdfImages);
         } catch (Exception e) {
             throw new RedactionException(e);
         }
 
+        log.info("Build Document Graph for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
         Document document = DocumentGraphFactory.buildDocumentGraph(classifiedDoc);
 
+        log.info("Build section grid for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
         SectionGrid sectionGrid = sectionGridCreatorService.createSectionGrid(document);
 
         log.info("Store document graph, text, simplified text, and section grid for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DroolsExecutionService.java

@@ -23,7 +23,6 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribu
 import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
 import com.iqser.red.service.redaction.v1.server.client.RulesClient;
 import com.iqser.red.service.redaction.v1.server.exception.RulesValidationException;
-import com.knecon.fforesight.tenantcommons.TenantContext;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.Document;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.SemanticNode;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.services.EntityCreationService;
@@ -91,6 +90,7 @@ public class DroolsExecutionService {
         kieSession.setGlobal("dictionary", dictionary);
         kieSession.setGlobal("nerEntitiesAdapter", nerEntitiesAdapter);
 
+        kieSession.insert(document);
         document.getEntities().forEach(kieSession::insert);
         sectionsToAnalyze.forEach(kieSession::insert);
         sectionsToAnalyze.stream().flatMap(SemanticNode::streamAllSubNodes).forEach(kieSession::insert);

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java

@@ -80,10 +80,7 @@ public class RedactionLogCreatorService {
     private RedactionLogEntry createRedactionLogEntry(RedactionEntity entity, String dossierTemplateId) {
 
         Set<String> referenceIds = new HashSet<>();
-        entity.getReferences()
-                .stream()
-                .filter(redactionEntity -> !redactionEntity.isRemoved() && !redactionEntity.isIgnored())
-                .forEach(ref -> ref.getRedactionPositionsPerPage().forEach(pos -> referenceIds.add(pos.getId())));
+        entity.getReferences().stream().filter(RedactionEntity::isActive).forEach(ref -> ref.getRedactionPositionsPerPage().forEach(pos -> referenceIds.add(pos.getId())));
         int sectionNumber = entity.getDeepestFullyContainingNode().getTreeId().isEmpty() ? 0 : entity.getDeepestFullyContainingNode().getTreeId().get(0);
 
         return RedactionLogEntry.builder()

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionAcceptanceTest.java

@@ -0,0 +1,169 @@
+package com.iqser.red.service.redaction.v1.server;
+
+import static org.mockito.Mockito.when;
+import static org.wildfly.common.Assert.assertFalse;
+import static org.wildfly.common.Assert.assertTrue;
+
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.time.OffsetDateTime;
+import java.util.List;
+import java.util.Set;
+
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
+import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration;
+import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.ComponentScan;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.context.annotation.FilterType;
+import org.springframework.context.annotation.Import;
+import org.springframework.context.annotation.Primary;
+import org.springframework.test.context.junit.jupiter.SpringExtension;
+
+import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeResult;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type;
+import com.iqser.red.service.redaction.v1.model.StructureAnalyzeRequest;
+import com.iqser.red.service.redaction.v1.server.annotate.AnnotateRequest;
+import com.iqser.red.service.redaction.v1.server.annotate.AnnotateResponse;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.document.services.EntityCreationService;
+import com.iqser.red.service.redaction.v1.server.redaction.utils.OsUtils;
+import com.iqser.red.storage.commons.StorageAutoConfiguration;
+import com.iqser.red.storage.commons.service.StorageService;
+import com.knecon.fforesight.tenantcommons.TenantContext;
+
+@ExtendWith(SpringExtension.class)
+@SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
+@Import(RedactionIntegrationTest.RedactionIntegrationTestConfiguration.class)
+public class RedactionAcceptanceTest extends AbstractRedactionIntegrationTest {
+
+    private static final String RULES = loadFromClassPath("drools/acceptance_rules.drl");
+
+    @Autowired
+    private EntityCreationService entityCreationService;
+
+    @Configuration
+    @EnableAutoConfiguration(exclude = {RabbitAutoConfiguration.class})
+    @ComponentScan(excludeFilters = {@ComponentScan.Filter(type = FilterType.ASSIGNABLE_TYPE, value = StorageAutoConfiguration.class)})
+    public static class RedactionIntegrationTestConfiguration {
+
+        @Bean
+        @Primary
+        public StorageService inmemoryStorage() {
+
+            return new FileSystemBackedStorageService();
+        }
+
+    }
+
+
+    @BeforeEach
+    public void stubClients() {
+
+        TenantContext.setTenantId("redaction");
+
+        when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L);
+        when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(JSONPrimitive.of(RULES));
+
+        loadDictionaryForTest();
+        loadTypeForTest();
+        loadNerForTest();
+        when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L);
+        when(dictionaryClient.getAllTypesForDossierTemplate(TEST_DOSSIER_TEMPLATE_ID, false)).thenReturn(getTypeResponse());
+
+        when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L);
+        when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, false)).thenReturn(List.of(Type.builder()
+                .id(DOSSIER_REDACTIONS_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID)
+                .type(DOSSIER_REDACTIONS_INDICATOR)
+                .dossierTemplateId(TEST_DOSSIER_ID)
+                .hexColor("#ffe187")
+                .isHint(hintTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
+                .isCaseInsensitive(caseInSensitiveMap.get(DOSSIER_REDACTIONS_INDICATOR))
+                .isRecommendation(recommendationTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
+                .rank(rankTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
+                .build()));
+
+        mockDictionaryCalls(null);
+
+        when(dictionaryClient.getColors(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(colors);
+    }
+
+
+    @Test
+    public void acceptanceTests() throws IOException {
+
+        AnalyzeRequest request = uploadFileToStorage("files/new/SYNGENTA_EFSA_sanitisation_GFL_v1_moreSections.pdf");
+        System.out.println("Start Full integration test");
+        analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
+        System.out.println("Finished structure analysis");
+        AnalyzeResult result = analyzeService.analyze(request);
+        System.out.println("Finished analysis");
+        var redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID);
+
+        var publishedInformationEntry1 = redactionLog.getRedactionLogEntry()
+                .stream()
+                .filter(entry -> entry.getType().equals("published_information"))
+                .filter(entry -> entry.getValue().equals("Oxford University Press"))
+                .findFirst()
+                .orElseThrow();
+
+        var asyaLyon1 = redactionLog.getRedactionLogEntry()
+                .stream()
+                .filter(entry -> entry.getType().equals("CBI_author"))
+                .filter(entry -> entry.getValue().equals("Asya Lyon"))
+                .filter(entry -> entry.getSectionNumber() == publishedInformationEntry1.getSectionNumber())
+                .findFirst()
+                .orElseThrow();
+
+        assertFalse(asyaLyon1.isRedacted());
+
+        var idRemoval = IdRemoval.builder()
+                .requestDate(OffsetDateTime.now())
+                .annotationId(publishedInformationEntry1.getId())
+                .status(AnnotationStatus.APPROVED)
+                .fileId(TEST_FILE_ID)
+                .build();
+
+        var manualRedactions = ManualRedactions.builder().idsToRemove(Set.of(idRemoval)).build();
+        request.setManualRedactions(manualRedactions);
+        analyzeService.reanalyze(request);
+
+        redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID);
+
+        var publishedInformationEntry2 = redactionLog.getRedactionLogEntry()
+                .stream()
+                .filter(entry -> entry.getType().equals("published_information"))
+                .filter(entry -> entry.getValue().equals("Oxford University Press"))
+                .findFirst()
+                .orElseThrow();
+
+        var asyaLyon2 = redactionLog.getRedactionLogEntry()
+                .stream()
+                .filter(entry -> entry.getType().equals("CBI_author"))
+                .filter(entry -> entry.getValue().equals("Asya Lyon"))
+                .filter(entry -> entry.getSectionNumber() == publishedInformationEntry2.getSectionNumber())
+                .findFirst()
+                .orElseThrow();
+
+        assertTrue(asyaLyon2.isRedacted());
+
+        AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder().dossierId(TEST_DOSSIER_ID).fileId(TEST_FILE_ID).build());
+
+        String outputFileName = OsUtils.getTemporaryDirectory() + "/AcceptanceTest.pdf";
+
+        try (FileOutputStream fileOutputStream = new FileOutputStream(outputFileName)) {
+            fileOutputStream.write(annotateResponse.getDocument());
+        }
+
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/sectionidentifiers/SectionIdentifierTest.java

@@ -0,0 +1,56 @@
+package com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.sectionidentifiers;
+
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import org.junit.jupiter.api.Test;
+
+class SectionIdentifierTest {
+
+    @Test
+    public void testParentOf() {
+
+        var headline = SectionIdentifier.fromSearchText("1 Did you ever hear the tragedy of Darth Plagueis The Wise?");
+        var headline1 = SectionIdentifier.fromSearchText("1.0 I thought not. It’s not a story the Jedi would tell you.");
+        var headline2 = SectionIdentifier.fromSearchText("1.1  It’s a Sith legend. Darth Plagueis was a Dark Lord of the Sith, ");
+        var headline3 = SectionIdentifier.fromSearchText("1.2.3 so powerful and so wise he could use the Force to influence the midichlorians to create life…");
+        var headline4 = SectionIdentifier.fromSearchText("1.2.3.4 He had such a knowledge of the dark side that he could even keep the ones he cared about from dying.");
+        var headline5 = SectionIdentifier.fromSearchText("1.2.3.4.5 The dark side of the Force is a pathway to many abilities some consider to be unnatural.");
+        var headline6 = SectionIdentifier.fromSearchText("2.0 He became so powerful…");
+        var headline7 = SectionIdentifier.fromSearchText("10000.0 the only thing he was afraid of was losing his power,");
+        var headline8 = SectionIdentifier.fromSearchText("A.0 which eventually, of course, he did.");
+        var headline9 = SectionIdentifier.fromSearchText("Unfortunately, he taught his apprentice everything he knew, then his apprentice killed him in his sleep.");
+        var headline10 = SectionIdentifier.fromSearchText("2.1.2 Ironic.");
+        var headline11 = SectionIdentifier.fromSearchText("2.He could save others from death,");
+        var headline12 = SectionIdentifier.fromSearchText(" 2. but not himself.");
+
+        var paragraph1 = SectionIdentifier.asChildOf(headline);
+        assertTrue(paragraph1.isChildOf(headline));
+        assertTrue(headline.isParentOf(paragraph1));
+        assertFalse(paragraph1.isParentOf(headline));
+
+        assertFalse(headline.isParentOf(headline1));
+        assertTrue(headline.isParentOf(headline2));
+        assertTrue(headline.isParentOf(headline3));
+        assertTrue(headline.isParentOf(headline4));
+        assertTrue(headline.isParentOf(headline5));
+        assertTrue(headline1.isParentOf(headline2));
+        assertFalse(headline1.isParentOf(headline1));
+        assertTrue(headline3.isParentOf(headline4));
+        assertFalse(headline4.isParentOf(headline5));
+        assertFalse(headline2.isParentOf(headline3));
+        assertFalse(headline2.isParentOf(headline4));
+        assertTrue(headline1.isParentOf(headline3));
+        assertTrue(headline1.isParentOf(headline4));
+        assertFalse(headline1.isParentOf(headline6));
+        assertFalse(headline1.isParentOf(headline7));
+        assertFalse(headline8.isParentOf(headline1));
+        assertFalse(headline8.isParentOf(headline2));
+        assertFalse(headline8.isParentOf(headline3));
+        assertFalse(headline8.isParentOf(headline4));
+        assertFalse(headline9.isParentOf(headline9));
+        assertTrue(headline10.isChildOf(headline11));
+        assertTrue(headline10.isChildOf(headline12));
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/utils/RectangleTransformationsTest.java

@@ -0,0 +1,90 @@
+package com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+import java.awt.geom.Rectangle2D;
+import java.util.List;
+
+import org.junit.jupiter.api.Test;
+
+class RectangleTransformationsTest {
+
+    @Test
+    public void testRectangle2DBBox() {
+
+        var r1 = new Rectangle2D.Double(0, 0, 1, 1);
+        var r2 = new Rectangle2D.Double(1, 1, 1, 1);
+        var result = RectangleTransformations.rectangle2DBBox(List.of(r1, r2));
+        assertEquals(0, result.getX());
+        assertEquals(0, result.getY());
+        assertEquals(2, result.getWidth());
+        assertEquals(2, result.getHeight());
+    }
+
+
+    @Test
+    public void testRectangle2DBBox2() {
+
+        var r1 = new Rectangle2D.Double(0, 0, -1, -1);
+        var r2 = new Rectangle2D.Double(1, 1, 1, 1);
+        var result = RectangleTransformations.rectangle2DBBox(List.of(r1, r2));
+        assertEquals(-1, result.getX());
+        assertEquals(-1, result.getY());
+        assertEquals(3, result.getWidth());
+        assertEquals(3, result.getHeight());
+    }
+
+
+    @Test
+    public void testRectangle2DBBox3() {
+
+        var r1 = new Rectangle2D.Double(0, 0, -1, -1);
+        var r2 = new Rectangle2D.Double(1, 1, 1, 1);
+        var result = RectangleTransformations.rectangle2DBBox(List.of(r2, r1));
+        assertEquals(-1, result.getX());
+        assertEquals(-1, result.getY());
+        assertEquals(3, result.getWidth());
+        assertEquals(3, result.getHeight());
+    }
+
+
+    @Test
+    public void testRectangle2DBBox4() {
+
+        var r1 = new Rectangle2D.Double(2, 0, -1, -1);
+        var r2 = new Rectangle2D.Double(0, 2, 1, -1);
+        var result = RectangleTransformations.rectangle2DBBox(List.of(r2, r1));
+        assertEquals(0, result.getX());
+        assertEquals(-1, result.getY());
+        assertEquals(2, result.getWidth());
+        assertEquals(3, result.getHeight());
+    }
+
+
+    @Test
+    public void testRectangle2DBBox5() {
+
+        var r1 = new Rectangle2D.Double(2, 0, -1, -1);
+        var r2 = new Rectangle2D.Double(0, 2, 1, -1);
+        var r3 = new Rectangle2D.Double(3, 2, 1, 1);
+        var result = RectangleTransformations.rectangle2DBBox(List.of(r2, r1, r3));
+        assertEquals(0, result.getX());
+        assertEquals(-1, result.getY());
+        assertEquals(4, result.getWidth());
+        assertEquals(4, result.getHeight());
+    }
+
+
+    @Test
+    public void testRectangle2DBBox6() {
+
+        var r1 = new Rectangle2D.Double(0, 0, -1, -1);
+        var r2 = new Rectangle2D.Double(-1, -1, -1, -1);
+        var result = RectangleTransformations.rectangle2DBBox(List.of(r1, r2));
+        assertEquals(-2, result.getX());
+        assertEquals(-2, result.getY());
+        assertEquals(2, result.getWidth());
+        assertEquals(2, result.getHeight());
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/CBI_author.txt

@@ -143,6 +143,7 @@ Allen T.
 Allen T.R.
 Almeida A
 Almeida A.
+Asya Lyon
 Almeida A.A.
 Almeida A.A.|Vassilieff I.
 Almeida|A.A.|Vassilieff|I.

redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/published_information.txt

@@ -87,3 +87,4 @@ Toxicol Sci.
 Toxicol Sci. 1
 Test Ignored Hint Published Information
 Workshop
+Oxford University Press

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/acceptance_rules.drl

@@ -0,0 +1,761 @@
+package drools
+
+import static java.lang.String.format;
+import static com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RedactionSearchUtility.anyMatch;
+import static com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RedactionSearchUtility.exactMatch;
+
+import java.util.List;
+import java.util.LinkedList;
+import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.Collection;
+import java.util.stream.Stream;
+import java.util.Optional;
+
+import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.*;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.*;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.Section;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.Table;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.SemanticNode;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.Document;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.Paragraph;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.Image;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.*;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.*;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.EntityType;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.ImageType;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.document.services.EntityCreationService;
+import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.Dictionary;
+import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.DictionaryModel;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualImageRecategorization;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.document.services.ManualRedactionApplicationService;
+import com.iqser.red.service.redaction.v1.server.client.model.EntityRecognitionEntity;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.Boundary;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionEntity;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.Boundary;
+import com.iqser.red.service.redaction.v1.server.redaction.adapter.NerEntitiesAdapter;
+import com.iqser.red.service.redaction.v1.server.redaction.adapter.NerEntities;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RedactionSearchUtility;
+
+global Document document
+global EntityCreationService entityCreationService
+global ManualRedactionApplicationService manualRedactionApplicationService
+global NerEntitiesAdapter nerEntitiesAdapter
+global Dictionary dictionary
+
+//------------------------------------ queries ------------------------------------
+
+query "getFileAttributes"
+    $fileAttribute: FileAttribute()
+    end
+
+//------------------------------------ Syngenta specific rules ------------------------------------
+
+// Rule unit: SYN.1
+rule "SYN.1.0: Recommend CTL/BL laboratory that start with BL or CTL"
+    when
+        $section: Section(containsString("CT") || containsString("BL"))
+    then
+        /* Regular expression: ((\b((([Cc]T(([1ILli\/])| L|~P))|(BL))[\. ]?([\dA-Ziltphz~\/.:!]| ?[\(',][Ppi](\(e)?|([\(-?']\/))+( ?[\(\/\dA-Znasieg]+)?)\b( ?\/? ?\d+)?)|(\bCT[L1i]\b)) */
+        entityCreationService.byRegexIgnoreCase("((\\b((([Cc]T(([1ILli\\/])| L|~P))|(BL))[\\. ]?([\\dA-Ziltphz~\\/.:!]| ?[\\(',][Ppi](\\(e)?|([\\(-?']\\/))+( ?[\\(\\/\\dA-Znasieg]+)?)\\b( ?\\/? ?\\d+)?)|(\\bCT[L1i]\\b))", "CBI_address", EntityType.RECOMMENDATION, $section)
+            .forEach(entity -> {
+                entity.skip("SYN.1.0", "");
+                entity.addEngine(Engine.RULE);
+                insert(entity);
+            });
+    end
+
+
+//------------------------------------ CBI rules ------------------------------------
+
+// Rule unit: CBI.0
+rule "CBI.0.0: Redact CBI Authors (Non Vertebrate Study)"
+    when
+        not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
+        $entity: RedactionEntity(type == "CBI_author", dictionaryEntry)
+    then
+        $entity.apply("CBI.0.0", "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+    end
+
+rule "CBI.0.1: Redact CBI Authors (Vertebrate Study)"
+    when
+        FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
+        $entity: RedactionEntity(type == "CBI_author", dictionaryEntry)
+    then
+        $entity.apply("CBI.0.1", "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+    end
+
+
+// Rule unit: CBI.1
+rule "CBI.1.0: Don't redact CBI Address (Non Vertebrate Study)"
+    when
+        not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
+        $entity: RedactionEntity(type == "CBI_address", dictionaryEntry)
+    then
+        $entity.skip("CBI.1.0", "Address found for Non Vertebrate Study");
+    end
+
+rule "CBI.1.1: Redact CBI Address (Vertebrate Study)"
+    when
+        FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
+        $entity: RedactionEntity(type == "CBI_address", dictionaryEntry)
+    then
+        $entity.apply("CBI.1.1", "Address found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+    end
+
+
+// Rule unit: CBI.2
+rule "CBI.2.0: Don't redact genitive CBI_author"
+    when
+        $entity: RedactionEntity(type == "CBI_author", anyMatch(textAfter, "['’’'ʼˈ´`‘′ʻ’']s"), isApplied())
+    then
+        RedactionEntity falsePositive = entityCreationService.byBoundary($entity.getBoundary(), "CBI_author", EntityType.FALSE_POSITIVE, document);
+        falsePositive.skip("CBI.2.0", "Genitive Author found");
+        insert(falsePositive);
+    end
+
+
+// Rule unit: CBI.7
+rule "CBI.7.0: Do not redact Names and Addresses if published information found in section without tables"
+    when
+        $section: Section(!hasTables(),
+                            hasEntitiesOfType("published_information"),
+                            (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
+    then
+        $section.getEntitiesOfType(List.of("CBI_author", "CBI_address"))
+            .forEach(redactionEntity -> {
+                redactionEntity.skipWithReferences(
+                        "CBI.7.0",
+                        "Published Information found in section",
+                        $section.getEntitiesOfType("published_information")
+                );
+            });
+    end
+
+rule "CBI.7.1: Do not redact Names and Addresses if published information found in same table row"
+    when
+        $table: Table(hasEntitiesOfType("published_information"),
+                         (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
+    then
+        $table.streamEntitiesWhereRowContainsEntitiesOfType(List.of("CBI_author", "CBI_address"))
+            .forEach(redactionEntity -> {
+                redactionEntity.skipWithReferences(
+                        "CBI.7.1",
+                        "Published Information found in row",
+                        $table.getEntitiesOfTypeInSameRow("published_information", redactionEntity)
+                );
+            });
+    end
+
+
+// Rule unit: CBI.9
+rule "CBI.9.0: Redact all Cell's with Header Author(s) as CBI_author (non vertebrate study)"
+    agenda-group "LOCAL_DICTIONARY_ADDS"
+    when
+        not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
+        $table: Table(hasHeader("Author(s)"))
+    then
+        $table.streamTableCellsWithHeader("Author(s)")
+            .map(tableCell -> entityCreationService.bySemanticNode(tableCell, "CBI_author", EntityType.ENTITY))
+            .filter(Optional::isPresent)
+            .map(Optional::get)
+            .forEach(redactionEntity -> {
+                redactionEntity.apply("CBI.9.0", "Author(s) found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+                redactionEntity.addEngine(Engine.RULE);
+                insert(redactionEntity);
+            });
+    end
+
+rule "CBI.9.1: Redact all Cell's with Header Author as CBI_author (non vertebrate study)"
+    agenda-group "LOCAL_DICTIONARY_ADDS"
+    when
+        not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
+        $table: Table(hasHeader("Author"))
+    then
+        $table.streamTableCellsWithHeader("Author")
+            .map(tableCell -> entityCreationService.bySemanticNode(tableCell, "CBI_author", EntityType.ENTITY))
+            .filter(Optional::isPresent)
+            .map(Optional::get)
+            .forEach(redactionEntity -> {
+                redactionEntity.apply("CBI.9.1", "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+                redactionEntity.addEngine(Engine.RULE);
+                insert(redactionEntity);
+            });
+    end
+
+
+// Rule unit: CBI.10
+rule "CBI.10.0: Redact all Cell's with Header Author(s) as CBI_author (vertebrate study)"
+    agenda-group "LOCAL_DICTIONARY_ADDS"
+    when
+        FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
+        $table: Table(hasHeader("Author(s)"))
+    then
+        $table.streamTableCellsWithHeader("Author(s)")
+            .map(tableCell -> entityCreationService.bySemanticNode(tableCell, "CBI_author", EntityType.ENTITY))
+            .filter(Optional::isPresent)
+            .map(Optional::get)
+            .forEach(redactionEntity -> {
+                redactionEntity.apply("CBI.10.0", "Author(s) found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+                redactionEntity.addEngine(Engine.RULE);
+                insert(redactionEntity);
+            });
+    end
+
+rule "CBI.10.1: Redact all Cell's with Header Author as CBI_author (vertebrate study)"
+    agenda-group "LOCAL_DICTIONARY_ADDS"
+    when
+        FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
+        $table: Table(hasHeader("Author"))
+    then
+        $table.streamTableCellsWithHeader("Author")
+            .map(tableCell -> entityCreationService.bySemanticNode(tableCell, "CBI_author", EntityType.ENTITY))
+            .filter(Optional::isPresent)
+            .map(Optional::get)
+            .forEach(redactionEntity -> {
+                redactionEntity.apply("CBI.10.1", "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+                redactionEntity.addEngine(Engine.RULE);
+                insert(redactionEntity);
+            });
+    end
+
+
+// Rule unit: CBI.11
+rule "CBI.11.0: Recommend all CBI_author entities in Table with Vertebrate Study Y/N Header"
+    agenda-group "LOCAL_DICTIONARY_ADDS"
+    salience -1
+    when
+        $table: Table(hasHeader("Author(s)") && hasHeader("Vertebrate Study Y/N"))
+    then
+        $table.getEntitiesOfType("CBI_author").forEach(entity -> dictionary.addMultipleAuthorsAsRecommendation(entity));
+    end
+
+
+// Rule unit: CBI.16
+rule "CBI.16.0: Add CBI_author with \"et al.\" Regex (non vertebrate study)"
+    agenda-group "LOCAL_DICTIONARY_ADDS"
+    when
+        not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
+        $section: Section(containsString("et al."))
+    then
+        entityCreationService.byRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, 1, $section)
+            .forEach(entity -> {
+                entity.apply("CBI.16.0", "Author found by \"et al\" regex", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+                entity.addEngine(Engine.RULE);
+                dictionary.addLocalDictionaryEntry("CBI_author", entity.getValue(), false);
+                insert(entity);
+            });
+    end
+
+rule "CBI.16.1: Add CBI_author with \"et al.\" Regex (vertebrate study)"
+    agenda-group "LOCAL_DICTIONARY_ADDS"
+    when
+        FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
+        $section: Section(containsString("et al."))
+    then
+        entityCreationService.byRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, 1, $section)
+            .forEach(entity -> {
+                entity.apply("CBI.16.1", "Author found by \"et al\" regex", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+                entity.addEngine(Engine.RULE);
+                insert(entity);
+                dictionary.addLocalDictionaryEntry("CBI_author", entity.getValue(), false);
+            });
+    end
+
+
+// Rule unit: CBI.17
+rule "CBI.17.0: Add recommendation for Addresses in Test Organism sections, without colon"
+    when
+        $section: Section(!hasTables(), containsString("Species") && containsString("Source") && !containsString("Species:") && !containsString("Source:"))
+    then
+        entityCreationService.lineAfterString("Source",  "CBI_address", EntityType.RECOMMENDATION, $section)
+            .forEach(entity -> {
+                entity.addEngine(Engine.RULE);
+                entity.skip("CBI.17.0", "Line after \"Source\" in Test Organism Section");
+                insert(entity);
+            });
+    end
+
+rule "CBI.17.1: Add recommendation for Addresses in Test Organism sections, with colon"
+    when
+        $section: Section(!hasTables(), containsString("Species:"), containsString("Source:"))
+    then
+        entityCreationService.lineAfterString("Source:",  "CBI_address", EntityType.RECOMMENDATION, $section)
+            .forEach(entity -> {
+                entity.addEngine(Engine.RULE);
+                entity.skip("CBI.17.1", "Line after \"Source:\" in Test Animals Section");
+                insert(entity);
+            });
+    end
+
+
+// Rule unit: CBI.20
+rule "CBI.20.0: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJECT ID:\" (non vertebrate study)"
+    agenda-group "LOCAL_DICTIONARY_ADDS"
+    when
+        not FileAttribute(label == "Vertebrate Study", value == "Yes")
+        $section: Section(!hasTables(), containsString("PERFORMING LABORATORY:"), containsString("LABORATORY PROJECT ID:"))
+    then
+        entityCreationService.betweenStrings("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", EntityType.ENTITY, $section)
+         .forEach(laboratoryEntity -> {
+             laboratoryEntity.skip("CBI.20.0", "PERFORMING LABORATORY was found for non vertebrate study");
+             laboratoryEntity.addEngine(Engine.RULE);
+             dictionary.addLocalDictionaryEntry(laboratoryEntity);
+             insert(laboratoryEntity);
+         });
+    end
+
+rule "CBI.20.1: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJECT ID:\" (vertebrate study)"
+    agenda-group "LOCAL_DICTIONARY_ADDS"
+    when
+        FileAttribute(label == "Vertebrate Study", value == "Yes")
+        $section: Section(!hasTables(), containsString("PERFORMING LABORATORY:"), containsString("LABORATORY PROJECT ID:"))
+    then
+         entityCreationService.betweenStrings("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", EntityType.ENTITY, $section)
+         .forEach(laboratoryEntity -> {
+             laboratoryEntity.apply("CBI.20.1", "PERFORMING LABORATORY was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+             laboratoryEntity.addEngine(Engine.RULE);
+             dictionary.addLocalDictionaryEntry(laboratoryEntity);
+             insert(laboratoryEntity);
+         });
+    end
+
+
+//------------------------------------ PII rules ------------------------------------
+
+// Rule unit: PII.0
+rule "PII.0.0: Redact all PII (non vertebrate study)"
+    when
+        not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
+        $pii: RedactionEntity(type == "PII", dictionaryEntry)
+    then
+        $pii.apply("PII.0.0", "Personal Information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+    end
+
+rule "PII.0.1: Redact all PII (vertebrate study)"
+    when
+        FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
+        $pii: RedactionEntity(type == "PII", dictionaryEntry)
+    then
+        $pii.apply("PII.0.1", "Personal Information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+    end
+
+
+// Rule unit: PII.1
+rule "PII.1.0: Redact Emails by RegEx (Non vertebrate study)"
+    when
+        not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
+        $section: Section(containsString("@"))
+    then
+        entityCreationService.byRegex("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", "PII", EntityType.ENTITY, 1, $section)
+            .forEach(emailEntity -> {
+                emailEntity.addEngine(Engine.RULE);
+                emailEntity.apply("PII.1.0", "Found by Email Regex", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+                insert(emailEntity);
+            });
+    end
+
+rule "PII.1.1: Redact Emails by RegEx (vertebrate study)"
+    when
+        FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
+        $section: Section(containsString("@"))
+    then
+        entityCreationService.byRegex("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", "PII", EntityType.ENTITY, 1, $section)
+            .forEach(emailEntity -> {
+                emailEntity.addEngine(Engine.RULE);
+                emailEntity.apply("PII.1.1", "Found by Email Regex", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+                insert(emailEntity);
+            });
+    end
+
+
+// Rule unit: PII.2
+rule "PII.2.0: Redact Phone and Fax by RegEx (non vertebrate study)"
+    when
+        not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
+        $section: Section(containsString("Contact") ||
+                              containsString("Telephone") ||
+                              containsString("Phone") ||
+                              containsString("Ph.") ||
+                              containsString("Fax") ||
+                              containsString("Tel") ||
+                              containsString("Ter") ||
+                              containsString("Mobile") ||
+                              containsString("Fel") ||
+                              containsString("Fer"))
+    then
+        entityCreationService.byRegexIgnoreCase("\\b(contact|telephone|phone|ph\\.|fax|tel|ter|mobile|fel|fer)[a-zA-Z\\s]{0,10}[:.\\s]{0,3}([\\+\\d\\(][\\s\\d\\(\\)\\-\\/\\.]{4,100}\\d)\\b", "PII", EntityType.ENTITY, 2, $section)
+            .forEach(contactEntity -> {
+                contactEntity.addEngine(Engine.RULE);
+                contactEntity.apply("PII.2.0", "Found by Phone and Fax Regex", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+                insert(contactEntity);
+            });
+    end
+
+rule "PII.2.1: Redact Phone and Fax by RegEx (vertebrate study)"
+    when
+        FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
+        $section: Section(containsString("Contact") ||
+                              containsString("Telephone") ||
+                              containsString("Phone") ||
+                              containsString("Ph.") ||
+                              containsString("Fax") ||
+                              containsString("Tel") ||
+                              containsString("Ter") ||
+                              containsString("Mobile") ||
+                              containsString("Fel") ||
+                              containsString("Fer"))
+    then
+        entityCreationService.byRegexIgnoreCase("\\b(contact|telephone|phone|ph\\.|fax|tel|ter|mobile|fel|fer)[a-zA-Z\\s]{0,10}[:.\\s]{0,3}([\\+\\d\\(][\\s\\d\\(\\)\\-\\/\\.]{4,100}\\d)\\b", "PII", EntityType.ENTITY, 2, $section)
+            .forEach(contactEntity -> {
+                contactEntity.addEngine(Engine.RULE);
+                contactEntity.apply("PII.2.1", "Found by Phone and Fax Regex", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+                insert(contactEntity);
+            });
+    end
+
+
+// Rule unit: PII.9
+rule "PII.9.0: Redact between \"AUTHOR(S)\" and \"COMPLETION DATE\" (non vertebrate study)"
+    when
+        not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
+        $section: Section(!hasTables(), containsString("AUTHOR(S):"), containsString("COMPLETION DATE:"), !containsString("STUDY COMPLETION DATE:"))
+    then
+        entityCreationService.betweenStrings("AUTHOR(S):", "COMPLETION DATE:", "PII", EntityType.ENTITY, $section)
+         .forEach(authorEntity -> {
+             authorEntity.apply("PII.9.0", "AUTHOR(S) was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+             authorEntity.addEngine(Engine.RULE);
+             insert(authorEntity);
+         });
+    end
+
+rule "PII.9.1: Redact between \"AUTHOR(S)\" and \"STUDY COMPLETION DATE\" (non vertebrate study)"
+    when
+        FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
+        $section: Section(!hasTables(), containsString("AUTHOR(S):"), containsString("COMPLETION DATE:"), !containsString("STUDY COMPLETION DATE:"))
+    then
+        entityCreationService.betweenStrings("AUTHOR(S):", "COMPLETION DATE:", "PII", EntityType.ENTITY, $section)
+         .forEach(authorEntity -> {
+             authorEntity.apply("PII.9.1", "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+             authorEntity.addEngine(Engine.RULE);
+             insert(authorEntity);
+         });
+    end
+
+rule "PII.9.2: Redact between \"AUTHOR(S)\" and \"COMPLETION DATE\" (non vertebrate study)"
+    when
+        not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
+        $section: Section(!hasTables(), containsString("AUTHOR(S):"), containsString("STUDY COMPLETION DATE:"))
+    then
+        entityCreationService.betweenStrings("AUTHOR(S):", "STUDY COMPLETION DATE:", "PII", EntityType.ENTITY, $section)
+         .forEach(authorEntity -> {
+             authorEntity.apply("PII.9.2", "AUTHOR(S) was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+             authorEntity.addEngine(Engine.RULE);
+             insert(authorEntity);
+         });
+    end
+
+rule "PII.9.3: Redact between \"AUTHOR(S)\" and \"STUDY COMPLETION DATE\" (vertebrate study)"
+    when
+        FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
+        $section: Section(!hasTables(), containsString("AUTHOR(S):"), containsString("STUDY COMPLETION DATE:"))
+    then
+        entityCreationService.betweenStrings("AUTHOR(S):", "STUDY COMPLETION DATE:", "PII", EntityType.ENTITY, $section)
+         .forEach(authorEntity -> {
+             authorEntity.apply("PII.9.3", "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+             authorEntity.addEngine(Engine.RULE);
+             insert(authorEntity);
+         });
+    end
+
+
+//------------------------------------ Other rules ------------------------------------
+
+// Rule unit: ETC.0
+rule "ETC.0.0: Purity Hint"
+    when
+        $section: Section(containsStringIgnoreCase("purity"))
+    then
+        entityCreationService.byRegexIgnoreCase("(purity ?( of|\\(.{1,20}\\))?( ?:)?) .{0,5}[\\d\\.]+( .{0,4}\\.)? ?%", "hint_only", EntityType.ENTITY, 1, $section)
+            .forEach(hint -> {
+                hint.addEngine(Engine.RULE);
+                hint.skip("ETC.0.0", "");
+            });
+    end
+
+
+// Rule unit: ETC.2
+rule "ETC.2.0: Redact signatures (non vertebrate study)"
+    when
+        not FileAttribute(label == "Vertebrate Study", value == "Yes")
+        $signature: Image(imageType == ImageType.SIGNATURE)
+    then
+        $signature.apply("ETC.2.0", "Signature Found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+    end
+
+rule "ETC.2.0: Redact signatures (vertebrate study)"
+    when
+        FileAttribute(label == "Vertebrate Study", value == "Yes")
+        $signature: Image(imageType == ImageType.SIGNATURE)
+    then
+        $signature.apply("ETC.2.0", "Signature Found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+    end
+
+
+// Rule unit: ETC.3
+rule "ETC.3.0: Redact logos (vertebrate study)"
+    when
+        not FileAttribute(label == "Vertebrate Study", value == "Yes")
+        $logo: Image(imageType == ImageType.LOGO)
+    then
+        $logo.apply("ETC.3.0", "Logo Found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+    end
+
+rule "ETC.3.1: Redact logos (non vertebrate study)"
+    when
+        FileAttribute(label == "Vertebrate Study", value == "Yes")
+        $logo: Image(imageType == ImageType.LOGO)
+    then
+        $logo.apply("ETC.3.1", "Logo Found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+    end
+
+
+// Rule unit: ETC.5
+rule "ETC.5.0: Ignore dossier_redaction entries if confidentiality is not 'confidential'"
+    when
+        not FileAttribute(label == "Confidentiality", value == "confidential")
+        $dossierRedaction: RedactionEntity(type == "dossier_redaction")
+    then
+        $dossierRedaction.setIgnored(true);
+        update($dossierRedaction);
+        $dossierRedaction.getIntersectingNodes().forEach(node -> update(node));
+    end
+
+
+//------------------------------------ AI rules ------------------------------------
+
+// Rule unit: AI.0
+rule "AI.0.0: add all NER Entities of type CBI_author"
+    salience 999
+    when
+        nerEntities: NerEntities(hasEntitiesOfType("CBI_author"))
+    then
+        nerEntities.streamEntitiesOfType("CBI_author")
+            .map(nerEntity -> entityCreationService.byNerEntity(nerEntity, EntityType.RECOMMENDATION, document))
+            .forEach(entity -> insert(entity));
+    end
+
+
+// Rule unit: AI.1
+rule "AI.1.0: combine and add NER Entities as CBI_address"
+    salience 999
+    when
+        nerEntities: NerEntities(hasEntitiesOfType("ORG") || hasEntitiesOfType("STREET") || hasEntitiesOfType("CITY"))
+    then
+        nerEntitiesAdapter.combineNerEntitiesToCbiAddressDefaults(nerEntities)
+            .map(boundary -> entityCreationService.byBoundary(boundary, "CBI_address", EntityType.RECOMMENDATION, document))
+            .forEach(entity -> {
+                entity.addEngine(Engine.NER);
+                insert(entity);
+            });
+    end
+
+
+//------------------------------------ Manual redaction rules ------------------------------------
+
+// Rule unit: MAN.0
+rule "MAN.0.0: Apply manual resize redaction"
+    salience 128
+    when
+        $resizeRedaction: ManualResizeRedaction($id: annotationId)
+        $entityToBeResized: RedactionEntity(matchesAnnotationId($id))
+    then
+        manualRedactionApplicationService.resizeEntityAndReinsert($entityToBeResized, $resizeRedaction);
+        retract($resizeRedaction);
+        update($entityToBeResized);
+        $entityToBeResized.getIntersectingNodes().forEach(node -> update(node));
+    end
+
+
+// Rule unit: MAN.1
+rule "MAN.1.0: Apply id removals that are valid and not in forced redactions to Entity"
+    salience 128
+    when
+        $idRemoval: IdRemoval(status == AnnotationStatus.APPROVED, !removeFromDictionary, requestDate != null, $id: annotationId)
+        not ManualForceRedaction($id == annotationId, status == AnnotationStatus.APPROVED, requestDate != null)
+        $entityToBeRemoved: RedactionEntity(matchesAnnotationId($id))
+    then
+        $entityToBeRemoved.setIgnored(true);
+        update($entityToBeRemoved);
+        retract($idRemoval);
+        $entityToBeRemoved.getIntersectingNodes().forEach(node -> update(node));
+    end
+
+rule "MAN.1.1: Apply id removals that are valid and not in forced redactions to Image"
+    salience 128
+    when
+        $idRemoval: IdRemoval(status == AnnotationStatus.APPROVED, !removeFromDictionary, requestDate != null, $id: annotationId)
+        not ManualForceRedaction($id == annotationId, status == AnnotationStatus.APPROVED, requestDate != null)
+        $imageEntityToBeRemoved: Image($id == id)
+    then
+        $imageEntityToBeRemoved.setIgnored(true);
+        update($imageEntityToBeRemoved);
+        retract($idRemoval);
+        update($imageEntityToBeRemoved.getParent());
+    end
+
+
+// Rule unit: MAN.2
+rule "MAN.2.0: Apply force redaction"
+    salience 128
+    when
+        $force: ManualForceRedaction($id: annotationId, status == AnnotationStatus.APPROVED, requestDate != null, $legalBasis: legalBasis)
+        $entityToForce: RedactionEntity(matchesAnnotationId($id))
+    then
+        $entityToForce.apply("MAN.2.0", "Forced redaction", $legalBasis);
+        $entityToForce.setSkipRemoveEntitiesContainedInLarger(true);
+        update($entityToForce);
+        retract($force);
+        $entityToForce.getIntersectingNodes().forEach(node -> update(node));
+    end
+
+
+// Rule unit: MAN.3
+rule "MAN.3.0: Apply image recategorization"
+    salience 128
+    when
+        $recategorization: ManualImageRecategorization($id: annotationId, status == AnnotationStatus.APPROVED, $imageType: type)
+        $imageToBeRecategorized: Image($id == id)
+    then
+        $imageToBeRecategorized.setImageType(ImageType.fromString($imageType));
+        update($imageToBeRecategorized);
+        retract($recategorization);
+        update($imageToBeRecategorized.getParent());
+    end
+
+
+//------------------------------------ Entity merging rules ------------------------------------
+
+// Rule unit: X.0
+rule "X.0.0: remove Entity contained by Entity of same type"
+    salience 65
+    when
+        $larger: RedactionEntity($type: type, $entityType: entityType)
+        $contained: RedactionEntity(containedBy($larger), type == $type, entityType == $entityType, this != $larger, !resized, !skipRemoveEntitiesContainedInLarger)
+    then
+        $contained.removeFromGraph();
+        retract($contained);
+    end
+
+
+// Rule unit: X.1
+rule "X.1.0: merge intersecting Entities of same type"
+    salience 64
+    when
+        $first: RedactionEntity($type: type, $entityType: entityType, !resized, !skipRemoveEntitiesContainedInLarger)
+        $second: RedactionEntity(intersects($first), type == $type, entityType == $entityType, this != $first, !resized, !skipRemoveEntitiesContainedInLarger)
+    then
+        $first.removeFromGraph();
+        $second.removeFromGraph();
+        RedactionEntity mergedEntity = entityCreationService.byEntities(List.of($first, $second), $type, $entityType, document);
+        retract($first);
+        retract($second);
+        insert(mergedEntity);
+    end
+
+
+// Rule unit: X.2
+rule "X.2.0: remove Entity of type ENTITY when contained by FALSE_POSITIVE"
+    salience 64
+    when
+        $falsePositive: RedactionEntity($type: type, entityType == EntityType.FALSE_POSITIVE)
+        $entity: RedactionEntity(containedBy($falsePositive), type == $type, entityType == EntityType.ENTITY, !resized, !skipRemoveEntitiesContainedInLarger)
+    then
+        $entity.removeFromGraph();
+        retract($entity)
+    end
+
+
+// Rule unit: X.3
+rule "X.3.0: remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"
+    salience 64
+    when
+        $falseRecommendation: RedactionEntity($type: type, entityType == EntityType.FALSE_RECOMMENDATION)
+        $recommendation: RedactionEntity(containedBy($falseRecommendation), type == $type, entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger)
+    then
+        $recommendation.removeFromGraph();
+        retract($recommendation);
+    end
+
+
+// Rule unit: X.4
+rule "X.4.0: remove Entity of type RECOMMENDATION when intersected by ENTITY with same type"
+    salience 256
+    when
+        $entity: RedactionEntity($type: type, entityType == EntityType.ENTITY)
+        $recommendation: RedactionEntity(intersects($entity), type == $type, entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger)
+    then
+        $entity.addEngines($recommendation.getEngines());
+        $recommendation.removeFromGraph();
+        retract($recommendation);
+    end
+
+
+// Rule unit: X.5
+rule "X.5.0: remove Entity of type RECOMMENDATION when contained by ENTITY"
+    salience 256
+    when
+        $entity: RedactionEntity(entityType == EntityType.ENTITY)
+        $recommendation: RedactionEntity(containedBy($entity), entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger)
+    then
+        $recommendation.removeFromGraph();
+        retract($recommendation);
+    end
+
+
+// Rule unit: X.6
+rule "X.6.0: remove Entity of lower rank, when intersected by entity of type ENTITY"
+    salience 32
+    when
+        $higherRank: RedactionEntity($type: type, entityType == EntityType.ENTITY)
+        $lowerRank: RedactionEntity(intersects($higherRank), type != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !resized, !skipRemoveEntitiesContainedInLarger)
+    then
+        $lowerRank.removeFromGraph();
+        retract($lowerRank);
+    end
+
+
+//------------------------------------ File attributes rules ------------------------------------
+
+// Rule unit: FA.1
+rule "FA.1.0: remove duplicate FileAttributes"
+    salience 64
+    when
+        $fileAttribute: FileAttribute($label: label, $value: value)
+        $duplicate: FileAttribute(this != $fileAttribute, label == $label, value == $value)
+    then
+        retract($duplicate);
+    end
+
+
+//------------------------------------ Local dictionary search rules ------------------------------------
+
+// Rule unit: LDS.0
+rule "LDS.0.0: run local dictionary search"
+    agenda-group "LOCAL_DICTIONARY_ADDS"
+    salience -999
+    when
+        DictionaryModel(!localEntries.isEmpty(), $type: type, $searchImplementation: localSearch) from dictionary.getDictionaryModels()
+    then
+        entityCreationService.bySearchImplementation($searchImplementation, $type, EntityType.RECOMMENDATION, document)
+            .forEach(entity -> {
+                entity.addEngine(Engine.RULE);
+                insert(entity);
+            });
+    end

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/all_rules.drl

@@ -1341,21 +1341,27 @@ rule "MAN.0.0: Apply manual resize redaction"
 rule "MAN.1.0: Apply id removals that are valid and not in forced redactions to Entity"
     salience 128
     when
-        IdRemoval(status == AnnotationStatus.APPROVED, !removeFromDictionary, requestDate != null, $id: annotationId)
+        $idRemoval: IdRemoval(status == AnnotationStatus.APPROVED, !removeFromDictionary, requestDate != null, $id: annotationId)
         not ManualForceRedaction($id == annotationId, status == AnnotationStatus.APPROVED, requestDate != null)
         $entityToBeRemoved: RedactionEntity(matchesAnnotationId($id))
     then
         $entityToBeRemoved.setIgnored(true);
+        update($entityToBeRemoved);
+        retract($idRemoval);
+        $entityToBeRemoved.getIntersectingNodes().forEach(node -> update(node));
     end
 
 rule "MAN.1.1: Apply id removals that are valid and not in forced redactions to Image"
     salience 128
     when
-        IdRemoval(status == AnnotationStatus.APPROVED, !removeFromDictionary, requestDate != null, $id: annotationId)
+        $idRemoval: IdRemoval(status == AnnotationStatus.APPROVED, !removeFromDictionary, requestDate != null, $id: annotationId)
         not ManualForceRedaction($id == annotationId, status == AnnotationStatus.APPROVED, requestDate != null)
         $imageEntityToBeRemoved: Image($id == id)
     then
         $imageEntityToBeRemoved.setIgnored(true);
+        update($imageEntityToBeRemoved);
+        retract($idRemoval);
+        update($imageEntityToBeRemoved.getParent());
     end