RED-6369: Rules Refactor

* refactored DocumentData to 4 different jsons to make reading only text possible
* wip integration of new rules into workflow

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/data/AtomicPositionBlockData.java

@@ -0,0 +1,19 @@
+package com.iqser.red.service.redaction.v1.server.document.data;
+
+import lombok.AccessLevel;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.experimental.FieldDefaults;
+
+@Data
+@Builder
+@AllArgsConstructor
+@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
+public class AtomicPositionBlockData {
+
+    Long id;
+    int[] stringIdxToPositionIdx;
+    float[][] positions;
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/data/AtomicTextBlockData.java

@@ -15,10 +15,9 @@ public class AtomicTextBlockData {
     Long id;
     Long page;
     String searchText;
+    int numberOnPage;
     int start;
     int end;
     int[] lineBreaks;
-    int[] stringIdxToPositionIdx;
-    float[][] positions;
 
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/data/DocumentData.java

@@ -1,7 +1,5 @@
 package com.iqser.red.service.redaction.v1.server.document.data;
 
-import java.util.List;
-
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
 import lombok.Builder;
@@ -14,8 +12,9 @@ import lombok.experimental.FieldDefaults;
 @FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
 public class DocumentData {
 
-    List<PageData> pages;
+    PageData[] pages;
-    List<AtomicTextBlockData> atomicTextBlocks;
+    AtomicTextBlockData[] atomicTextBlocks;
+    AtomicPositionBlockData[] atomicPositionBlocks;
     TableOfContentsData tableOfContents;
 
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/data/PageData.java

@@ -15,5 +15,6 @@ public class PageData {
     int number;
     int height;
     int width;
+    int rotation;
 
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/data/TableOfContentsData.java

@@ -15,12 +15,14 @@ import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;
+import lombok.NoArgsConstructor;
 import lombok.experimental.FieldDefaults;
 
 @Data
 @Builder
 @AllArgsConstructor
-@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
+@NoArgsConstructor
+@FieldDefaults(level = AccessLevel.PRIVATE)
 public class TableOfContentsData {
 
     List<EntryData> entries;
@@ -64,7 +66,7 @@ public class TableOfContentsData {
 
 
     @Builder
-    public record EntryData(int[] tocId, List<EntryData> subEntries, NodeType type, Long[] atomicTextBlocks, Long[] pages, int numberOnPage, Map<String, String> properties) {
+    public record EntryData(NodeType type, int[] tocId, Long[] atomicBlocks, Long[] pages, Map<String, String> properties, List<EntryData> subEntries) {
 
         @Override
         public String toString() {
@@ -80,7 +82,7 @@ public class TableOfContentsData {
 
             sb.append(type);
             sb.append(" atbs = ");
-            sb.append(atomicTextBlocks.length);
+            sb.append(atomicBlocks.length);
 
             return sb.toString();
         }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/TableOfContents.java

@@ -3,8 +3,6 @@ package com.iqser.red.service.redaction.v1.server.document.graph;
 import static java.lang.String.format;
 
 import java.nio.charset.StandardCharsets;
-import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.Collections;
 import java.util.LinkedList;
 import java.util.List;
@@ -153,17 +151,7 @@ public class TableOfContents {
         @Override
         public String toString() {
 
-            return tocId + ": " + type + ".: " + buildSummary(node().buildTextBlock().getSearchText());
+            return node().toString();
-        }
-
-
-        private static String buildSummary(CharSequence text) {
-
-            String[] words = text.toString().split(" ");
-            int bound = Math.min(words.length, 4);
-            List<String> list = new ArrayList<>(Arrays.asList(words).subList(0, bound));
-
-            return String.join(" ", list);
         }
 
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/entity/EntityNode.java

@@ -25,28 +25,6 @@ import lombok.experimental.FieldDefaults;
 @FieldDefaults(level = AccessLevel.PRIVATE)
 public class EntityNode {
 
-    public static EntityNode initialEntityNode(Boundary boundary, String type, EntityType entityType) {
-
-        return EntityNode.builder()
-                .type(type)
-                .entityType(entityType)
-                .boundary(boundary)
-                .redaction(false)
-                .removed(false)
-                .ignored(false)
-                .resized(false)
-                .skipRemoveEntitiesContainedInLarger(false)
-                .dictionaryEntry(false)
-                .dossierDictionaryEntry(false)
-                .engines(new HashSet<>())
-                .references(new HashSet<>())
-                .matchedRule(-1)
-                .redactionReason("")
-                .legalBasis("")
-                .build();
-    }
-
-
     // initial values
     final Boundary boundary;
     final String type;
@@ -72,12 +50,34 @@ public class EntityNode {
     CharSequence textAfter;
     @Builder.Default
     Set<PageNode> pages = new HashSet<>();
-    List<EntityPosition> entityPositions;
+    List<EntityPosition> entityPositionsPerPage;
     @Builder.Default
     List<SemanticNode> intersectingNodes = new LinkedList<>();
     SemanticNode deepestFullyContainingNode;
 
 
+    public static EntityNode initialEntityNode(Boundary boundary, String type, EntityType entityType) {
+
+        return EntityNode.builder()
+                .type(type)
+                .entityType(entityType)
+                .boundary(boundary)
+                .redaction(false)
+                .removed(false)
+                .ignored(false)
+                .resized(false)
+                .skipRemoveEntitiesContainedInLarger(false)
+                .dictionaryEntry(false)
+                .dossierDictionaryEntry(false)
+                .engines(new HashSet<>())
+                .references(new HashSet<>())
+                .matchedRule(-1)
+                .redactionReason("")
+                .legalBasis("")
+                .build();
+    }
+
+
     public void addIntersectingNode(SemanticNode containingNode) {
 
         intersectingNodes.add(containingNode);
@@ -93,13 +93,13 @@ public class EntityNode {
     }
 
 
-    public List<EntityPosition> getEntityPositions() {
+    public List<EntityPosition> getEntityPositionsPerPage() {
 
-        if (entityPositions == null || entityPositions.isEmpty()) {
+        if (entityPositionsPerPage == null || entityPositionsPerPage.isEmpty()) {
-            entityPositions = deepestFullyContainingNode.buildTextBlock().getEntityPositions(boundary);
+            entityPositionsPerPage = deepestFullyContainingNode.buildTextBlock().getEntityPositionsPerPage(boundary);
         }
 
-        return entityPositions;
+        return entityPositionsPerPage;
     }
 
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/factory/DocumentGraphFactory.java

@@ -28,10 +28,10 @@ import com.iqser.red.service.redaction.v1.server.classification.model.Page;
 import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
 import com.iqser.red.service.redaction.v1.server.document.graph.DocumentGraph;
 import com.iqser.red.service.redaction.v1.server.document.graph.TableOfContents;
-import com.iqser.red.service.redaction.v1.server.document.graph.entity.ImageNode;
 import com.iqser.red.service.redaction.v1.server.document.graph.nodes.FooterNode;
 import com.iqser.red.service.redaction.v1.server.document.graph.nodes.HeaderNode;
 import com.iqser.red.service.redaction.v1.server.document.graph.nodes.HeadlineNode;
+import com.iqser.red.service.redaction.v1.server.document.graph.nodes.ImageNode;
 import com.iqser.red.service.redaction.v1.server.document.graph.nodes.NodeType;
 import com.iqser.red.service.redaction.v1.server.document.graph.nodes.PageNode;
 import com.iqser.red.service.redaction.v1.server.document.graph.nodes.ParagraphNode;
@@ -40,7 +40,6 @@ import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SemanticNo
 import com.iqser.red.service.redaction.v1.server.document.graph.nodes.TableCellNode;
 import com.iqser.red.service.redaction.v1.server.document.graph.nodes.TableNode;
 import com.iqser.red.service.redaction.v1.server.document.graph.textblock.AtomicTextBlock;
-import com.iqser.red.service.redaction.v1.server.document.services.ImageSortService;
 import com.iqser.red.service.redaction.v1.server.exception.NotFoundException;
 import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
 import com.iqser.red.service.redaction.v1.server.redaction.model.PdfImage;
@@ -48,20 +47,14 @@ import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractT
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.Cell;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;
 
-import lombok.RequiredArgsConstructor;
-
 @Service
-@RequiredArgsConstructor
 public class DocumentGraphFactory {
 
     public static final double TABLE_CELL_MERGE_CONTENTS_SIZE_THRESHOLD = 0.05;
 
-    private final ImageSortService imageSortService;
-
 
     public DocumentGraph buildDocumentGraph(Document document) {
 
-        ImageSortService.SortedImages sortedImages = imageSortService.sortImagesIntoStructure(document);
         TextBlockFactory textBlockFactory = new TextBlockFactory();
         Context context = new Context(new TableOfContents(), new HashMap<>(), new LinkedList<>(), new LinkedList<>(), textBlockFactory);
 
@@ -121,13 +114,13 @@ public class DocumentGraphFactory {
     }
 
 
-    private List<TextBlock> findTextBlocksWithSameClassificationAndAlignsY(AbstractTextContainer atc, List<AbstractTextContainer> pageBlocks) {
+    private static List<TextBlock> findTextBlocksWithSameClassificationAndAlignsY(AbstractTextContainer atc, List<AbstractTextContainer> pageBlocks) {
 
         return pageBlocks.stream()
                 .filter(abstractTextContainer -> !abstractTextContainer.equals(atc))
+                .filter(abstractTextContainer -> abstractTextContainer.getPage() == atc.getPage())
                 .filter(abstractTextContainer -> abstractTextContainer instanceof TextBlock)
                 .filter(abstractTextContainer -> abstractTextContainer.intersectsY(atc))
-                .filter(abstractTextContainer -> abstractTextContainer.getPage() == atc.getPage())
                 .map(abstractTextContainer -> (TextBlock) abstractTextContainer)
                 .toList();
     }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/factory/SearchTextWithTextPositionFactory.java

@@ -7,13 +7,10 @@ import java.util.LinkedList;
 import java.util.List;
 import java.util.Objects;
 
-import org.springframework.stereotype.Service;
-
 import com.iqser.red.service.redaction.v1.server.parsing.model.RedTextPosition;
 import com.iqser.red.service.redaction.v1.server.parsing.model.TextDirection;
 import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
 
-@Service
 public class SearchTextWithTextPositionFactory {
 
     public static final int HEIGHT_PADDING = 2;

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/FooterNode.java

@@ -47,7 +47,7 @@ public class FooterNode implements SemanticNode {
     @Override
     public String toString() {
 
-        return tocId + ": " + NodeType.FOOTER + ": " + terminalTextBlock.getSearchText();
+        return tocId + ": " + NodeType.FOOTER + ": " + terminalTextBlock.buildSummary();
     }
 
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/HeaderNode.java

@@ -47,7 +47,7 @@ public class HeaderNode implements SemanticNode {
     @Override
     public String toString() {
 
-        return tocId + ": " + NodeType.HEADER + ": " + terminalTextBlock.getSearchText();
+        return tocId + ": " + NodeType.HEADER + ": " + terminalTextBlock.buildSummary();
     }
 
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/HeadlineNode.java

@@ -47,7 +47,7 @@ public class HeadlineNode implements SemanticNode {
     @Override
     public String toString() {
 
-        return tocId + ": " + NodeType.HEADLINE + ": " + terminalTextBlock.toString();
+        return tocId + ": " + NodeType.HEADLINE + ": " + terminalTextBlock.buildSummary();
     }
 
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/ImageNode.java

@@ -1,4 +1,4 @@
-package com.iqser.red.service.redaction.v1.server.document.graph.entity;
+package com.iqser.red.service.redaction.v1.server.document.graph.nodes;
 
 import java.awt.geom.Rectangle2D;
 import java.util.Collections;
@@ -9,9 +9,7 @@ import java.util.Map;
 import java.util.Set;
 
 import com.iqser.red.service.redaction.v1.server.document.graph.TableOfContents;
-import com.iqser.red.service.redaction.v1.server.document.graph.nodes.NodeType;
+import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityNode;
-import com.iqser.red.service.redaction.v1.server.document.graph.nodes.PageNode;
-import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SemanticNode;
 import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock;
 import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlockCollector;
 import com.iqser.red.service.redaction.v1.server.redaction.model.ImageType;

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/NodeType.java

@@ -1,52 +1,12 @@
 package com.iqser.red.service.redaction.v1.server.document.graph.nodes;
 
 public enum NodeType {
-    SECTION {
+    SECTION,
-        public String toString() {
+    HEADLINE,
-
+    PARAGRAPH,
-            return "Section";
+    TABLE,
-        }
+    TABLE_CELL,
-    },
+    IMAGE,
-    HEADLINE {
+    HEADER,
-        public String toString() {
+    FOOTER
-
-            return "Headline";
-        }
-    },
-    PARAGRAPH {
-        public String toString() {
-
-            return "Paragraph";
-        }
-    },
-    TABLE {
-        public String toString() {
-
-            return "Table";
-        }
-    },
-    TABLE_CELL {
-        public String toString() {
-
-            return "Cell";
-        }
-    },
-    IMAGE {
-        public String toString() {
-
-            return "Image";
-        }
-    },
-    HEADER {
-        public String toString() {
-
-            return "Header";
-        }
-    },
-    FOOTER {
-        public String toString() {
-
-            return "Footer";
-        }
-    }
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/PageNode.java

@@ -5,7 +5,6 @@ import java.util.List;
 import java.util.Set;
 
 import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityNode;
-import com.iqser.red.service.redaction.v1.server.document.graph.entity.ImageNode;
 import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock;
 import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlockCollector;
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/ParagraphNode.java

@@ -45,7 +45,7 @@ public class ParagraphNode implements SemanticNode {
     @Override
     public String toString() {
 
-        return tocId + ": " + NodeType.PARAGRAPH + ": " + terminalTextBlock.toString();
+        return tocId + ": " + NodeType.PARAGRAPH + ": " + terminalTextBlock.buildSummary();
     }
 
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/SectionNode.java

@@ -49,7 +49,7 @@ public class SectionNode implements SemanticNode {
     @Override
     public String toString() {
 
-        return tocId.toString() + ": " + NodeType.SECTION + ": " + buildTextBlock().getSearchText();
+        return tocId.toString() + ": " + NodeType.SECTION + ": " + buildTextBlock().buildSummary();
     }
 
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/SemanticNode.java

@@ -168,6 +168,16 @@ public interface SemanticNode {
     }
 
 
+    /**
+     * @param strings A List of Strings which the TextBlock might contain
+     * @return true, if this node's TextBlock contains any of the strings
+     */
+    default boolean containsAnyString(List<String> strings) {
+
+        return strings.stream().anyMatch(this::containsString);
+    }
+
+
     /**
      * This function is used during insertion of EntityNodes into the graph, it checks if the boundary of the Entity intersects or even contains the Entity.
      * It sets the fields accordingly and recursively calls this function on all its children.
@@ -221,7 +231,8 @@ public interface SemanticNode {
 
 
     /**
-     * If this Node is Terminal it will calculate the boundingBox of its TerminalTextBlock, otherwise it will calcuate the Union of the BoundingBoxes of all Children
+     * If this Node is Terminal it will calculate the boundingBox of its TerminalTextBlock, otherwise it will calculate the Union of the BoundingBoxes of all its Children.
+     * If called on the Document, it will return the cropbox of each page
      *
      * @return Rectangle2D fully encapsulating this Node for each page.
      */
@@ -237,6 +248,8 @@ public interface SemanticNode {
 
 
     /**
+     * TODO this does not yet work for sections spanning multiple columns
+     *
      * @param bBoxPerPage initial empty BoundingBox
      * @return The union of the BoundingBoxes of all children
      */

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/TableCellNode.java

@@ -73,7 +73,7 @@ public class TableCellNode implements SemanticNode {
     @Override
     public String toString() {
 
-        return tocId + ": " + NodeType.TABLE_CELL + ": " + buildTextBlock().getSearchText();
+        return tocId + ": " + NodeType.TABLE_CELL + ": " + buildTextBlock().buildSummary();
     }
 
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/TableNode.java

@@ -67,7 +67,7 @@ public class TableNode implements SemanticNode {
     @Override
     public String toString() {
 
-        return tocId.toString() + ": " + NodeType.TABLE + ": " + buildTextBlock().getSearchText();
+        return tocId.toString() + ": " + NodeType.TABLE + ": " + buildTextBlock().buildSummary();
     }
 
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/textblock/AtomicTextBlock.java

@@ -112,7 +112,7 @@ public class AtomicTextBlock implements TextBlock {
     }
 
 
-    public List<EntityPosition> getEntityPositions(Boundary stringBoundary) {
+    public List<EntityPosition> getEntityPositionsPerPage(Boundary stringBoundary) {
 
         List<Rectangle2D> positionsPerLine = stringBoundary.split(getLineBreaks().stream().map(lb -> lb + boundary.start()).filter(stringBoundary::contains).toList())
                 .stream()

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/textblock/ConcatenatedTextBlock.java

@@ -139,23 +139,23 @@ public class ConcatenatedTextBlock implements TextBlock {
 
 
     @Override
-    public List<EntityPosition> getEntityPositions(Boundary stringBoundary) {
+    public List<EntityPosition> getEntityPositionsPerPage(Boundary stringBoundary) {
 
         List<AtomicTextBlock> textBlocks = getAllAtomicTextBlocksPartiallyInStringBoundary(stringBoundary);
 
         if (textBlocks.size() == 1) {
-            return textBlocks.get(0).getEntityPositions(stringBoundary);
+            return textBlocks.get(0).getEntityPositionsPerPage(stringBoundary);
         }
 
         AtomicTextBlock firstTextBlock = textBlocks.get(0);
-        List<EntityPosition> positions = new LinkedList<>(firstTextBlock.getEntityPositions(new Boundary(stringBoundary.start(), firstTextBlock.getBoundary().end())));
+        List<EntityPosition> positions = new LinkedList<>(firstTextBlock.getEntityPositionsPerPage(new Boundary(stringBoundary.start(), firstTextBlock.getBoundary().end())));
 
         for (AtomicTextBlock textBlock : textBlocks.subList(1, textBlocks.size() - 1)) {
-            positions.addAll(textBlock.getEntityPositions(textBlock.getBoundary()));
+            positions.addAll(textBlock.getEntityPositionsPerPage(textBlock.getBoundary()));
         }
 
         AtomicTextBlock lastTextBlock = textBlocks.get(textBlocks.size() - 1);
-        positions.addAll(lastTextBlock.getEntityPositions(new Boundary(lastTextBlock.getBoundary().start(), stringBoundary.end())));
+        positions.addAll(lastTextBlock.getEntityPositionsPerPage(new Boundary(lastTextBlock.getBoundary().start(), stringBoundary.end())));
 
         return mergeEntityPositionsWithSamePageNode(positions);
     }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/textblock/TextBlock.java

@@ -3,6 +3,8 @@ package com.iqser.red.service.redaction.v1.server.document.graph.textblock;
 import static java.lang.String.format;
 
 import java.awt.geom.Rectangle2D;
+import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.List;
 import java.util.Set;
 import java.util.stream.Collectors;
@@ -27,6 +29,7 @@ public interface TextBlock extends CharSequence {
 
     int getPreviousLinebreak(int fromIndex);
 
+
     List<Integer> getLineBreaks();
 
 
@@ -36,16 +39,18 @@ public interface TextBlock extends CharSequence {
     List<Rectangle2D> getPositions(Boundary stringBoundary);
 
 
-    List<EntityPosition> getEntityPositions(Boundary stringBoundary);
+    List<EntityPosition> getEntityPositionsPerPage(Boundary stringBoundary);
 
 
     int numberOfLines();
 
+
     default int indexOf(String searchTerm) {
 
         return indexOf(searchTerm, getBoundary().start());
     }
 
+
     default Set<PageNode> getPages() {
 
         return getAtomicTextBlocks().stream().map(AtomicTextBlock::getPage).collect(Collectors.toUnmodifiableSet());
@@ -89,6 +94,16 @@ public interface TextBlock extends CharSequence {
     }
 
 
+    default String buildSummary() {
+
+        String[] words = getSearchText().split(" ");
+        int bound = Math.min(words.length, 4);
+        List<String> list = new ArrayList<>(Arrays.asList(words).subList(0, bound));
+
+        return String.join(" ", list);
+    }
+
+
     @Override
     default CharSequence subSequence(int start, int end) {
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/mapper/DocumentDataMapper.java

@@ -7,13 +7,14 @@ import java.util.Map;
 
 import org.springframework.stereotype.Service;
 
+import com.iqser.red.service.redaction.v1.server.document.data.AtomicPositionBlockData;
 import com.iqser.red.service.redaction.v1.server.document.data.AtomicTextBlockData;
 import com.iqser.red.service.redaction.v1.server.document.data.DocumentData;
 import com.iqser.red.service.redaction.v1.server.document.data.PageData;
 import com.iqser.red.service.redaction.v1.server.document.data.TableOfContentsData;
 import com.iqser.red.service.redaction.v1.server.document.graph.DocumentGraph;
 import com.iqser.red.service.redaction.v1.server.document.graph.TableOfContents;
-import com.iqser.red.service.redaction.v1.server.document.graph.entity.ImageNode;
+import com.iqser.red.service.redaction.v1.server.document.graph.nodes.ImageNode;
 import com.iqser.red.service.redaction.v1.server.document.graph.nodes.PageNode;
 import com.iqser.red.service.redaction.v1.server.document.graph.nodes.TableCellNode;
 import com.iqser.red.service.redaction.v1.server.document.graph.nodes.TableNode;
@@ -30,9 +31,21 @@ public class DocumentDataMapper {
                 .distinct()
                 .map(this::toAtomicTextBlockData)
                 .toList();
+
+        List<AtomicPositionBlockData> atomicPositionBlockData = documentGraph.streamTerminalTextBlocksInOrder()
+                .flatMap(textBlock -> textBlock.getAtomicTextBlocks().stream())
+                .distinct()
+                .map(this::toAtomicPositionBlockData)
+                .toList();
+
         List<PageData> pageData = documentGraph.getPages().stream().map(this::toPageData).toList();
         TableOfContentsData tableOfContentsData = toTableOfContentsData(documentGraph.getTableOfContents());
-        return DocumentData.builder().atomicTextBlocks(atomicTextBlockData).pages(pageData).tableOfContents(tableOfContentsData).build();
+        return DocumentData.builder()
+                .atomicTextBlocks(atomicTextBlockData.toArray(new AtomicTextBlockData[0]))
+                .atomicPositionBlocks(atomicPositionBlockData.toArray(new AtomicPositionBlockData[0]))
+                .pages(pageData.toArray(new PageData[0]))
+                .tableOfContents(tableOfContentsData)
+                .build();
     }
 
 
@@ -65,7 +78,6 @@ public class DocumentDataMapper {
                 .type(entry.type())
                 .atomicTextBlocks(atomicTextBlocks)
                 .pages(entry.node().getPages().stream().map(PageNode::getNumber).map(Integer::longValue).toArray(Long[]::new))
-                .numberOnPage(entry.node().getNumberOnPage())
                 .properties(properties)
                 .build();
     }
@@ -77,9 +89,9 @@ public class DocumentDataMapper {
     }
 
 
-    private PageData toPageData(PageNode pageNode) {
+    private PageData toPageData(PageNode p) {
 
-        return PageData.builder().height(pageNode.getHeight()).width(pageNode.getWidth()).number(pageNode.getNumber()).build();
+        return PageData.builder().rotation(p.getRotation()).height(p.getHeight()).width(p.getWidth()).number(p.getNumber()).build();
     }
 
 
@@ -89,11 +101,20 @@ public class DocumentDataMapper {
                 .id(atomicTextBlock.getId())
                 .page(atomicTextBlock.getPage().getNumber().longValue())
                 .searchText(atomicTextBlock.getSearchText())
+                .numberOnPage(atomicTextBlock.getNumberOnPage())
                 .start(atomicTextBlock.getBoundary().start())
                 .end(atomicTextBlock.getBoundary().end())
                 .lineBreaks(toPrimitiveIntArray(atomicTextBlock.getLineBreaks()))
-                .stringIdxToPositionIdx(toPrimitiveIntArray(atomicTextBlock.getStringIdxToPositionIdx()))
+                .build();
+    }
+
+
+    private AtomicPositionBlockData toAtomicPositionBlockData(AtomicTextBlock atomicTextBlock) {
+
+        return AtomicPositionBlockData.builder()
+                .id(atomicTextBlock.getId())
                 .positions(toPrimitiveFloatMatrix(atomicTextBlock.getPositions()))
+                .stringIdxToPositionIdx(toPrimitiveIntArray(atomicTextBlock.getStringIdxToPositionIdx()))
                 .build();
     }
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/mapper/DocumentGraphMapper.java

@@ -17,6 +17,7 @@ import org.apache.commons.lang3.NotImplementedException;
 import org.springframework.stereotype.Service;
 
 import com.google.common.primitives.Ints;
+import com.iqser.red.service.redaction.v1.server.document.data.AtomicPositionBlockData;
 import com.iqser.red.service.redaction.v1.server.document.data.AtomicTextBlockData;
 import com.iqser.red.service.redaction.v1.server.document.data.DocumentData;
 import com.iqser.red.service.redaction.v1.server.document.data.PageData;
@@ -24,10 +25,10 @@ import com.iqser.red.service.redaction.v1.server.document.data.TableOfContentsDa
 import com.iqser.red.service.redaction.v1.server.document.graph.Boundary;
 import com.iqser.red.service.redaction.v1.server.document.graph.DocumentGraph;
 import com.iqser.red.service.redaction.v1.server.document.graph.TableOfContents;
-import com.iqser.red.service.redaction.v1.server.document.graph.entity.ImageNode;
 import com.iqser.red.service.redaction.v1.server.document.graph.nodes.FooterNode;
 import com.iqser.red.service.redaction.v1.server.document.graph.nodes.HeaderNode;
 import com.iqser.red.service.redaction.v1.server.document.graph.nodes.HeadlineNode;
+import com.iqser.red.service.redaction.v1.server.document.graph.nodes.ImageNode;
 import com.iqser.red.service.redaction.v1.server.document.graph.nodes.NodeType;
 import com.iqser.red.service.redaction.v1.server.document.graph.nodes.PageNode;
 import com.iqser.red.service.redaction.v1.server.document.graph.nodes.ParagraphNode;
@@ -45,14 +46,19 @@ public class DocumentGraphMapper {
 
     public DocumentGraph toDocumentGraph(DocumentData documentData) {
 
-        Context context = new Context(documentData, new TableOfContents(), new LinkedList<>(), new LinkedList<>(), documentData.getAtomicTextBlocks());
+        Context context = new Context(documentData,
+                new TableOfContents(),
+                new LinkedList<>(),
+                new LinkedList<>(),
+                Arrays.stream(documentData.getAtomicTextBlocks()).toList(),
+                Arrays.stream(documentData.getAtomicPositionBlocks()).toList());
 
-        context.pages.addAll(documentData.getPages().stream().map(this::buildPage).toList());
+        context.pages.addAll(Arrays.stream(documentData.getPages()).map(this::buildPage).toList());
 
         context.tableOfContents.setEntries(buildEntries(documentData.getTableOfContents().getEntries(), context));
 
         DocumentGraph documentGraph = DocumentGraph.builder()
-                .numberOfPages(documentData.getPages().size())
+                .numberOfPages(documentData.getPages().length)
                 .pages(new HashSet<>(context.pages))
                 .tableOfContents(context.tableOfContents)
                 .build();
@@ -82,7 +88,7 @@ public class DocumentGraphMapper {
             };
 
             if (node.isTerminal()) {
-                TextBlock textBlock = toTextBlock(entryData.atomicTextBlocks(), context, node);
+                TextBlock textBlock = toTextBlock(entryData.atomicBlocks(), context, node);
                 node.setTerminalTextBlock(textBlock);
             }
             List<Integer> tocId = Arrays.stream(entryData.tocId()).boxed().toList();
@@ -109,7 +115,7 @@ public class DocumentGraphMapper {
 
     private static boolean isTerminal(TableOfContentsData.EntryData entryData) {
 
-        return entryData.atomicTextBlocks().length > 0;
+        return entryData.atomicBlocks().length > 0;
     }
 
 
@@ -165,34 +171,42 @@ public class DocumentGraphMapper {
     private TextBlock toTextBlock(Long[] atomicTextBlockIds, Context context, SemanticNode parent) {
 
         return Arrays.stream(atomicTextBlockIds)
-                .map(atomicTextBlockId -> toAtomicTextBlock(context.atomicTextBlockData.get(toIntExact(atomicTextBlockId)), parent, context))
+                .map(atomicTextBlockId -> toAtomicTextBlock(context.atomicTextBlockData.get(toIntExact(atomicTextBlockId)),
+                        context.atomicPositionBlockData.get(toIntExact(atomicTextBlockId)),
+                        parent,
+                        context))
                 .collect(new TextBlockCollector());
     }
 
 
     private PageNode buildPage(PageData p) {
 
-        return PageNode.builder().height(p.getHeight()).width(p.getWidth()).number(p.getNumber()).mainBody(new LinkedList<>()).build();
+        return PageNode.builder().rotation(p.getRotation()).height(p.getHeight()).width(p.getWidth()).number(p.getNumber()).mainBody(new LinkedList<>()).build();
     }
 
 
-    private AtomicTextBlock toAtomicTextBlock(AtomicTextBlockData atomicTextBlockData, SemanticNode parent, Context context) {
+    private AtomicTextBlock toAtomicTextBlock(AtomicTextBlockData atomicTextBlockData, AtomicPositionBlockData atomicPositionBlockData, SemanticNode parent, Context context) {
 
         return AtomicTextBlock.builder()
                 .id(atomicTextBlockData.getId())
+                .numberOnPage(atomicTextBlockData.getNumberOnPage())
                 .page(getPage(atomicTextBlockData.getPage(), context))
-                .searchText(atomicTextBlockData.getSearchText())
                 .boundary(new Boundary(atomicTextBlockData.getStart(), atomicTextBlockData.getEnd()))
+                .searchText(atomicTextBlockData.getSearchText())
                 .lineBreaks(Ints.asList(atomicTextBlockData.getLineBreaks()))
-                .positions(Arrays.stream(atomicTextBlockData.getPositions())
+                .stringIdxToPositionIdx(Ints.asList(atomicPositionBlockData.getStringIdxToPositionIdx()))
-                        .map(floatArr -> (Rectangle2D) new Rectangle2D.Float(floatArr[0], floatArr[1], floatArr[2], floatArr[3]))
+                .positions(toRectangle2DList(atomicPositionBlockData.getPositions()))
-                        .toList())
-                .stringIdxToPositionIdx(Ints.asList(atomicTextBlockData.getStringIdxToPositionIdx()))
                 .parent(parent)
                 .build();
     }
 
 
+    private static List<Rectangle2D> toRectangle2DList(float[][] positions) {
+
+        return Arrays.stream(positions).map(floatArr -> (Rectangle2D) new Rectangle2D.Float(floatArr[0], floatArr[1], floatArr[2], floatArr[3])).toList();
+    }
+
+
     private PageNode getPage(Long pageIndex, Context context) {
 
         return context.pages.stream()
@@ -202,7 +216,13 @@ public class DocumentGraphMapper {
     }
 
 
-    record Context(DocumentData documentData, TableOfContents tableOfContents, List<PageNode> pages, List<SectionNode> sections, List<AtomicTextBlockData> atomicTextBlockData) {
+    record Context(
+            DocumentData documentData,
+            TableOfContents tableOfContents,
+            List<PageNode> pages,
+            List<SectionNode> sections,
+            List<AtomicTextBlockData> atomicTextBlockData,
+            List<AtomicPositionBlockData> atomicPositionBlockData) {
 
     }
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/mapper/PropertiesMapper.java

@@ -5,8 +5,8 @@ import static com.iqser.red.service.redaction.v1.server.document.graph.factory.R
 import java.util.HashMap;
 import java.util.Map;
 
-import com.iqser.red.service.redaction.v1.server.document.graph.entity.ImageNode;
 import com.iqser.red.service.redaction.v1.server.document.graph.factory.RectangleTransformations;
+import com.iqser.red.service.redaction.v1.server.document.graph.nodes.ImageNode;
 import com.iqser.red.service.redaction.v1.server.document.graph.nodes.TableCellNode;
 import com.iqser.red.service.redaction.v1.server.document.graph.nodes.TableNode;
 import com.iqser.red.service.redaction.v1.server.redaction.model.ImageType;

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/CharSequenceSearchUtils.java

@@ -1,6 +1,6 @@
 package com.iqser.red.service.redaction.v1.server.document.services;
 
-import static com.iqser.red.service.redaction.v1.server.redaction.utils.SeparatorUtils.validateBoundaryIsSurroundedBySeparators;
+import static com.iqser.red.service.redaction.v1.server.redaction.utils.SeparatorUtils.boundaryIsSurroundedBySeparators;
 
 import java.util.LinkedList;
 import java.util.List;
@@ -43,7 +43,7 @@ public class CharSequenceSearchUtils {
         while (matcher.find()) {
             boundaries.add(new Boundary(matcher.start() + textBlock.getBoundary().start(), matcher.end() + textBlock.getBoundary().start()));
         }
-        return boundaries.stream().filter(boundary -> validateBoundaryIsSurroundedBySeparators(textBlock, boundary)).toList();
+        return boundaries.stream().filter(boundary -> boundaryIsSurroundedBySeparators(textBlock, boundary)).toList();
     }
 
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/EntityCreationService.java

@@ -1,7 +1,8 @@
 package com.iqser.red.service.redaction.v1.server.document.services;
 
 import static com.iqser.red.service.redaction.v1.server.document.services.CharSequenceSearchUtils.findBoundariesByString;
-import static com.iqser.red.service.redaction.v1.server.redaction.utils.SeparatorUtils.validateBoundaryIsSurroundedBySeparators;
+import static com.iqser.red.service.redaction.v1.server.redaction.utils.SeparatorUtils.boundaryIsSurroundedBySeparators;
+import static com.iqser.red.service.redaction.v1.server.redaction.utils.SeparatorUtils.isWhiteSpacesOrSeparatorsOnly;
 
 import java.util.Collections;
 import java.util.LinkedList;
@@ -30,7 +31,7 @@ import lombok.extern.slf4j.Slf4j;
 @RequiredArgsConstructor
 public class EntityCreationService {
 
-    private final EntityTextEnrichmentService entityEnrichmentService;
+    private final EntityEnrichmentService entityEnrichmentService;
 
 
     public Set<EntityNode> betweenStrings(String start, String stop, String type, EntityType entityType, SemanticNode node) {
@@ -52,7 +53,7 @@ public class EntityCreationService {
             entityBoundaries.add(new Boundary(startBoundary.end(), optionalFirstStopBoundary.get().start()));
         }
         return entityBoundaries.stream()
-                .filter(boundary -> validateBoundaryIsSurroundedBySeparators(textBlock, boundary))
+                .filter(boundary -> isValidEntityBoundary(textBlock, boundary))
                 .map(boundary -> byBoundary(boundary, type, entityType, node))
                 .collect(Collectors.toUnmodifiableSet());
     }
@@ -62,19 +63,32 @@ public class EntityCreationService {
 
         return searchImplementation.getBoundaries(node.buildTextBlock(), node.getBoundary())
                 .stream()
-                .filter(boundary -> validateBoundaryIsSurroundedBySeparators(node.buildTextBlock(), boundary))
+                .filter(boundary -> isValidEntityBoundary(node.buildTextBlock(), boundary))
                 .map(bounds -> byBoundary(bounds, type, entityType, node))
                 .collect(Collectors.toUnmodifiableSet());
     }
 
 
+    public Set<EntityNode> lineAfterStrings(List<String> strings, String type, EntityType entityType, SemanticNode node) {
+
+        TextBlock textBlock = node.buildTextBlock();
+        SearchImplementation searchImplementation = new SearchImplementation(strings, false);
+        return searchImplementation.getBoundaries(textBlock, node.getBoundary())
+                .stream()
+                .map(boundary -> toLineAfterBoundary(textBlock, boundary))
+                .filter(boundary -> isValidEntityBoundary(textBlock, boundary))
+                .map(boundary -> byBoundary(boundary, type, entityType, node))
+                .collect(Collectors.toUnmodifiableSet());
+    }
+
+
     public Set<EntityNode> lineAfterString(String string, String type, EntityType entityType, SemanticNode node) {
 
         TextBlock textBlock = node.buildTextBlock();
         List<Boundary> boundaries = findBoundariesByString(string, textBlock);
         return boundaries.stream()
                 .map(boundary -> toLineAfterBoundary(textBlock, boundary))
-                .filter(boundary -> validateBoundaryIsSurroundedBySeparators(textBlock, boundary))
+                .filter(boundary -> isValidEntityBoundary(textBlock, boundary))
                 .map(boundary -> byBoundary(boundary, type, entityType, node))
                 .collect(Collectors.toUnmodifiableSet());
     }
@@ -103,7 +117,7 @@ public class EntityCreationService {
     }
 
 
-    public EntityNode createMergedEntity(List<EntityNode> entitiesToMerge, String type, EntityType entityType, SemanticNode node) {
+    public EntityNode byEntities(List<EntityNode> entitiesToMerge, String type, EntityType entityType, SemanticNode node) {
 
         if (!allEntitiesIntersectAndHaveSameTypes(entitiesToMerge)) {
             throw new IllegalArgumentException("Provided entities can not be merged!");
@@ -121,6 +135,12 @@ public class EntityCreationService {
     }
 
 
+    public boolean isValidEntityBoundary(TextBlock textBlock, Boundary boundary) {
+
+        return boundaryIsSurroundedBySeparators(textBlock, boundary) && !isWhiteSpacesOrSeparatorsOnly(textBlock, boundary);
+    }
+
+
     public void addEntityToGraph(EntityNode entity, TableOfContents tableOfContents) {
 
         try {

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/EntityEnrichmentService.java

@@ -1,11 +1,13 @@
 package com.iqser.red.service.redaction.v1.server.document.services;
 
 import java.util.Arrays;
+import java.util.Collection;
 import java.util.List;
 import java.util.Objects;
 
 import org.springframework.stereotype.Service;
 
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine;
 import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityNode;
 import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock;
 import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;
@@ -14,17 +16,16 @@ import lombok.RequiredArgsConstructor;
 
 @Service
 @RequiredArgsConstructor
-public class EntityTextEnrichmentService {
+public class EntityEnrichmentService {
 
     private final RedactionServiceSettings redactionServiceSettings;
 
 
-    public EntityNode enrichEntity(EntityNode entity, TextBlock textBlock) {
+    public void enrichEntity(EntityNode entity, TextBlock textBlock) {
 
         entity.setValue(textBlock.subSequence(entity.getBoundary()).toString());
         entity.setTextAfter(findTextAfter(entity.getBoundary().end(), textBlock));
         entity.setTextBefore(findTextBefore(entity.getBoundary().start(), textBlock));
-        return entity;
     }
 
 
@@ -64,7 +65,7 @@ public class EntityTextEnrichmentService {
     }
 
 
-    private String concatWordsBefore(List<String> words, boolean endWithSpace) {
+    private static String concatWordsBefore(List<String> words, boolean endWithSpace) {
 
         StringBuilder sb = new StringBuilder();
 
@@ -77,7 +78,7 @@ public class EntityTextEnrichmentService {
     }
 
 
-    private String concatWordsAfter(List<String> words, boolean startWithSpace) {
+    private static String concatWordsAfter(List<String> words, boolean startWithSpace) {
 
         StringBuilder sb = new StringBuilder();
 
@@ -89,4 +90,27 @@ public class EntityTextEnrichmentService {
         return startWithSpace ? " " + result : result;
     }
 
+
+    public static void setFields(EntityNode entityNode, int matchedRule, String redactionReason, String legalBasis, Engine engine) {
+
+        if (entityNode.getMatchedRule() == -1 && matchedRule != -1) {
+            entityNode.setMatchedRule(matchedRule);
+        }
+        if (entityNode.getRedactionReason().equals("") && redactionReason != null) {
+            entityNode.setRedactionReason(redactionReason);
+        }
+        if (entityNode.getLegalBasis().equals("") && legalBasis != null) {
+            entityNode.setLegalBasis(legalBasis);
+        }
+        if (engine != null) {
+            entityNode.addEngine(engine);
+        }
+    }
+
+
+    public static void setFields(Collection<EntityNode> entityNodes, int matchedRule, String redactionReason, String legalBasis, Engine engine) {
+
+        entityNodes.forEach(entityNode -> setFields(entityNode, matchedRule, redactionReason, legalBasis, engine));
+    }
+
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/EntityFieldSetter.java

@@ -1,32 +0,0 @@
-package com.iqser.red.service.redaction.v1.server.document.services;
-
-import java.util.Collection;
-
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine;
-import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityNode;
-
-public class EntityFieldSetter {
-
-    public static void setFields(EntityNode entityNode, int matchedRule, String redactionReason, String legalBasis, Engine engine) {
-
-        if (entityNode.getMatchedRule() == -1 && matchedRule != -1) {
-            entityNode.setMatchedRule(matchedRule);
-        }
-        if (entityNode.getRedactionReason().equals("") && redactionReason != null) {
-            entityNode.setRedactionReason(redactionReason);
-        }
-        if (entityNode.getLegalBasis().equals("") && legalBasis != null) {
-            entityNode.setLegalBasis(legalBasis);
-        }
-        if (engine != null) {
-            entityNode.addEngine(engine);
-        }
-    }
-
-
-    public static void setFields(Collection<EntityNode> entityNodes, int matchedRule, String redactionReason, String legalBasis, Engine engine) {
-
-        entityNodes.forEach(entityNode -> setFields(entityNode, matchedRule, redactionReason, legalBasis, engine));
-    }
-
-}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Dictionary.java

@@ -92,7 +92,7 @@ public class Dictionary {
         if (localAccessMap.get(type).getLocalEntries() == null) {
             throw new IllegalArgumentException(format("DictionaryModel of type %s has no local Entries", type));
         }
-        if (StringUtils.isEmpty(value) && value.length() < 3) {
+        if (StringUtils.isEmpty(value)) {
             throw new IllegalArgumentException(format("%s is not a valid dictionary entry", value));
         }
         localAccessMap.get(type).getLocalEntries().add(value);

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DroolsExecutionService.java

@@ -26,11 +26,15 @@ import org.kie.api.runtime.rule.QueryResults;
 import org.kie.api.runtime.rule.QueryResultsRow;
 import org.springframework.stereotype.Service;
 
+import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;
 import com.iqser.red.service.redaction.v1.server.client.RulesClient;
+import com.iqser.red.service.redaction.v1.server.document.graph.DocumentGraph;
+import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SectionNode;
+import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SemanticNode;
+import com.iqser.red.service.redaction.v1.server.document.services.EntityCreationService;
 import com.iqser.red.service.redaction.v1.server.exception.RulesValidationException;
 import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
 import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
-import com.iqser.red.service.redaction.v1.server.redaction.model.Paragraph;
 import com.iqser.red.service.redaction.v1.server.redaction.model.Section;
 
 import io.micrometer.core.annotation.Timed;
@@ -77,6 +81,8 @@ public class DroolsExecutionService {
                 });
     }
 
+    private final EntityCreationService entityCreationService;
+
 
     public KieContainer getKieContainer(String dossierTemplateId) {
 
@@ -90,18 +96,46 @@ public class DroolsExecutionService {
 
 
     @Timed("redactmanager_executeRules")
-    public List<Entity> executeRules(KieContainer kieContainer, List<Section> sections, List<Paragraph> paragraphs, Dictionary dictionary) {
+    public List<FileAttribute> executeRules(KieContainer kieContainer, DocumentGraph document, Dictionary dictionary, List<FileAttribute> fileAttributes) {
 
         KieSession kieSession = kieContainer.newKieSession();
+        kieSession.setGlobal("document", document);
+        kieSession.setGlobal("entityCreationService", entityCreationService);
         kieSession.setGlobal("dictionary", dictionary);
-        sections.forEach(kieSession::insert);
+
-        paragraphs.forEach(kieSession::insert);
+        document.getEntities().forEach(kieSession::insert);
+        document.getTableOfContents().streamEntriesInOrder().forEach(entry -> kieSession.insert(entry.node()));
+        document.getPages().forEach(kieSession::insert);
+        fileAttributes.forEach(kieSession::insert);
+
+        kieSession.getAgenda().getAgendaGroup("LOCAL_DICTIONARY_ADDS").setFocus();
         kieSession.fireAllRules();
-        List<Entity> entities = getEntities(kieSession);
-        kieSession.dispose();
 
-        return entities;
+        return getFileAttributes(kieSession);
+    }
 
+
+    @Timed("redactmanager_executeRules")
+    public List<FileAttribute> executeRules(KieContainer kieContainer,
+                                            DocumentGraph document,
+                                            List<SectionNode> sectionsToReanalyze,
+                                            Dictionary dictionary,
+                                            List<FileAttribute> fileAttributes) {
+
+        KieSession kieSession = kieContainer.newKieSession();
+        kieSession.setGlobal("document", document);
+        kieSession.setGlobal("entityCreationService", entityCreationService);
+        kieSession.setGlobal("dictionary", dictionary);
+
+        document.getEntities().forEach(kieSession::insert);
+        sectionsToReanalyze.stream().flatMap(SemanticNode::streamAllSubNodes).forEach(kieSession::insert);
+        document.getPages().forEach(kieSession::insert);
+        fileAttributes.forEach(kieSession::insert);
+
+        kieSession.getAgenda().getAgendaGroup("LOCAL_DICTIONARY_ADDS").setFocus();
+        kieSession.fireAllRules();
+
+        return getFileAttributes(kieSession);
     }
 
 
@@ -119,6 +153,17 @@ public class DroolsExecutionService {
     }
 
 
+    public List<FileAttribute> getFileAttributes(KieSession kieSession) {
+
+        List<FileAttribute> fileAttributes = new LinkedList<>();
+        QueryResults entitiesResult = kieSession.getQueryResults("getFileAttributes");
+        for (QueryResultsRow resultsRow : entitiesResult) {
+            fileAttributes.add((FileAttribute) resultsRow.get("$result"));
+        }
+        return fileAttributes;
+    }
+
+
     public List<Entity> getEntities(KieSession ks) {
 
         List<Entity> entities = new LinkedList<>();

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java

@@ -52,7 +52,7 @@ public class RedactionLogCreatorService {
 
         // Duplicates can exist due table extraction columns over multiple rows.
 
-        for (EntityPosition entityPosition : entityNode.getEntityPositions()) {
+        for (EntityPosition entityPosition : entityNode.getEntityPositionsPerPage()) {
 
             RedactionLogEntry redactionLogEntry = createRedactionLogEntry(entityNode, dossierTemplateId);
 
@@ -61,7 +61,6 @@ public class RedactionLogCreatorService {
             }
 
             processedIds.add(entityPosition.getId());
-
             redactionLogEntry.setId(entityPosition.getId());
 
             List<Rectangle> rectanglesPerLine = entityPosition.getRectanglePerLine()
@@ -70,6 +69,7 @@ public class RedactionLogCreatorService {
                     .toList();
 
             redactionLogEntry.setPositions(rectanglesPerLine);
+            redactionLogEntities.add(redactionLogEntry);
         }
 
         return redactionLogEntities;
@@ -79,7 +79,7 @@ public class RedactionLogCreatorService {
     private RedactionLogEntry createRedactionLogEntry(EntityNode entity, String dossierTemplateId) {
 
         Set<String> referenceIds = new HashSet<>();
-        entity.getReferences().forEach(ref -> ref.getEntityPositions().forEach(pos -> referenceIds.add(pos.getId())));
+        entity.getReferences().forEach(ref -> ref.getEntityPositionsPerPage().forEach(pos -> referenceIds.add(pos.getId())));
 
         return RedactionLogEntry.builder()
                 .color(getColor(entity.getType(), dossierTemplateId, entity.isRedaction()))
@@ -91,7 +91,7 @@ public class RedactionLogCreatorService {
                 .isHint(isHint(entity.getType(), dossierTemplateId))
                 .isRecommendation(entity.getEntityType().equals(EntityType.RECOMMENDATION))
                 .isFalsePositive(entity.getEntityType().equals(EntityType.FALSE_POSITIVE) || entity.getEntityType().equals(EntityType.FALSE_RECOMMENDATION))
-                .section(entity.getDeepestFullyContainingNode().getTocId().toString())
+                .section(entity.getDeepestFullyContainingNode().toString())
                 .sectionNumber(entity.getDeepestFullyContainingNode().getTocId().get(0))
                 .matchedRule(entity.getMatchedRule())
                 .isDictionaryEntry(entity.isDictionaryEntry())

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/analyze/AnalyzeService.java

@@ -33,11 +33,15 @@ import com.iqser.red.service.redaction.v1.server.classification.model.Simplified
 import com.iqser.red.service.redaction.v1.server.classification.model.Text;
 import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient;
 import com.iqser.red.service.redaction.v1.server.client.model.NerEntities;
+import com.iqser.red.service.redaction.v1.server.document.graph.DocumentGraph;
+import com.iqser.red.service.redaction.v1.server.document.graph.factory.DocumentGraphFactory;
+import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SectionNode;
+import com.iqser.red.service.redaction.v1.server.document.mapper.DocumentDataMapper;
+import com.iqser.red.service.redaction.v1.server.document.mapper.DocumentGraphMapper;
 import com.iqser.red.service.redaction.v1.server.exception.RedactionException;
 import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
 import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryIncrement;
 import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryVersion;
-import com.iqser.red.service.redaction.v1.server.redaction.model.PageEntities;
 import com.iqser.red.service.redaction.v1.server.redaction.model.PdfImage;
 import com.iqser.red.service.redaction.v1.server.redaction.service.DictionaryService;
 import com.iqser.red.service.redaction.v1.server.redaction.service.DroolsExecutionService;
@@ -81,6 +85,9 @@ public class AnalyzeService {
     ImageService imageService;
     ImportedRedactionService importedRedactionService;
     SectionFinder sectionFinder;
+    DocumentGraphFactory documentGraphFactory;
+    DocumentDataMapper documentDataMapper;
+    DocumentGraphMapper documentGraphMapper;
 
     FunctionTimerValues redactmanagerAnalyzePagewiseValues;
 
@@ -109,6 +116,8 @@ public class AnalyzeService {
             throw new RedactionException(e);
         }
 
+        DocumentGraph documentGraph = documentGraphFactory.buildDocumentGraph(classifiedDoc);
+
         List<SectionText> sectionTexts = sectionTextBuilderService.buildSectionText(classifiedDoc);
 
         sectionGridCreatorService.createSectionGrid(classifiedDoc, pageCount);
@@ -123,8 +132,8 @@ public class AnalyzeService {
                         sectionText.getSectionAreas().stream().map(SectionArea::getPage).collect(Collectors.toSet()),
                         sectionText.getSectionAreas())));
 
-        log.info("Store text, simplified text and section grid for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
+        log.info("Store document graph, text, simplified text, and section grid for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
-        redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.TEXT, text);
+        redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.TEXT, documentDataMapper.toDocumentData(documentGraph));
         redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.SIMPLIFIED_TEXT, convert(text));
         redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.SECTION_GRID, classifiedDoc.getSectionGrid());
 
@@ -144,11 +153,11 @@ public class AnalyzeService {
 
         long startTime = System.currentTimeMillis();
 
-        var redactionLog = redactionStorageService.getRedactionLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
+        RedactionLog redactionLog = redactionStorageService.getRedactionLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
-        var text = redactionStorageService.getText(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
+        DocumentGraph documentGraph = documentGraphMapper.toDocumentGraph(redactionStorageService.getDocumentData(analyzeRequest.getDossierId(), analyzeRequest.getFileId()));
 
         // not yet ready for reanalysis
-        if (redactionLog == null || text == null || text.getNumberOfPages() == 0) {
+        if (redactionLog == null || documentGraph == null || documentGraph.getNumberOfPages() == 0) {
             return analyze(analyzeRequest);
         }
 
@@ -156,28 +165,31 @@ public class AnalyzeService {
                 new DictionaryVersion(redactionLog.getDictionaryVersion(), redactionLog.getDossierDictionaryVersion()),
                 analyzeRequest.getDossierId());
 
-        Set<Integer> sectionsToReanalyse = analyzeRequest.getSectionsToReanalyse().isEmpty() //
+        Set<Integer> sectionsToReanalyseIds = analyzeRequest.getSectionsToReanalyse().isEmpty() //
-                ? sectionFinder.findSectionsToReanalyse(dictionaryIncrement, redactionLog, text, analyzeRequest) //
+                ? sectionFinder.findSectionsToReanalyse(dictionaryIncrement, redactionLog, documentGraph, analyzeRequest) //
                 : analyzeRequest.getSectionsToReanalyse();
-        log.info("Should reanalyze {} sections for request: {}", sectionsToReanalyse.size(), analyzeRequest);
+        log.info("Should reanalyze {} sections for request: {}", sectionsToReanalyseIds.size(), analyzeRequest);
 
-        if (sectionsToReanalyse.isEmpty()) {
+        if (sectionsToReanalyseIds.isEmpty()) {
-            return finalizeAnalysis(analyzeRequest, startTime, redactionLog, text, dictionaryIncrement.getDictionaryVersion(), true, new HashSet<>());
+            return finalizeAnalysis(analyzeRequest, startTime, redactionLog, documentGraph.getNumberOfPages(), dictionaryIncrement.getDictionaryVersion(), true, new HashSet<>());
         }
 
-        Dictionary dictionary = dictionaryService.getDeepCopyDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId());
+        List<SectionNode> sectionsToReAnalyse = documentGraph.getMainSections()
-        List<SectionText> reanalysisSections = text.getSectionTexts()
                 .stream()
-                .filter(sectionText -> sectionsToReanalyse.contains(sectionText.getSectionNumber()))
+                .filter(section -> sectionsToReanalyseIds.contains(section.getTocId().get(0)))
                 .collect(Collectors.toList());
+
+        Dictionary dictionary = dictionaryService.getDeepCopyDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId());
+
         NerEntities nerEntities = redactionServiceSettings.isNerServiceEnabled() //
                 ? redactionStorageService.getNerEntities(analyzeRequest.getDossierId(), analyzeRequest.getFileId()) //
                 : new NerEntities();
+
         KieContainer kieContainer = droolsExecutionService.updateRules(analyzeRequest.getDossierTemplateId());
 
-        PageEntities pageEntities = entityRedactionService.findEntities(dictionary, reanalysisSections, kieContainer, analyzeRequest, nerEntities);
+        Set<FileAttribute> addedFileAttributes = entityRedactionService.addRuleEntities(dictionary, documentGraph, sectionsToReAnalyse, kieContainer, analyzeRequest, nerEntities);
 
-        var newRedactionLogEntries = redactionLogCreatorService.createRedactionLog(pageEntities, text.getNumberOfPages(), analyzeRequest.getDossierTemplateId());
+        List<RedactionLogEntry> newRedactionLogEntries = redactionLogCreatorService.createRedactionLog(documentGraph.getEntities(), analyzeRequest.getDossierTemplateId());
 
         var importedRedactionFilteredEntries = importedRedactionService.processImportedRedactions(analyzeRequest.getDossierTemplateId(),
                 analyzeRequest.getDossierId(),
@@ -185,10 +197,10 @@ public class AnalyzeService {
                 newRedactionLogEntries,
                 false);
 
-        redactionLog.getRedactionLogEntry().removeIf(entry -> sectionsToReanalyse.contains(entry.getSectionNumber()) && !entry.getType().equals(IMPORTED_REDACTION_TYPE));
+        redactionLog.getRedactionLogEntry().removeIf(entry -> sectionsToReanalyseIds.contains(entry.getSectionNumber()) && !entry.getType().equals(IMPORTED_REDACTION_TYPE));
         redactionLog.getRedactionLogEntry().addAll(importedRedactionFilteredEntries);
 
-        return finalizeAnalysis(analyzeRequest, startTime, redactionLog, text, dictionaryIncrement.getDictionaryVersion(), true, pageEntities.getAddedFileAttributes());
+        return finalizeAnalysis(analyzeRequest, startTime, redactionLog, documentGraph.getNumberOfPages(), dictionaryIncrement.getDictionaryVersion(), true, addedFileAttributes);
     }
 
 
@@ -196,7 +208,7 @@ public class AnalyzeService {
     public AnalyzeResult analyze(AnalyzeRequest analyzeRequest) {
 
         long startTime = System.currentTimeMillis();
-        var text = redactionStorageService.getText(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
+        DocumentGraph documentGraph = documentGraphMapper.toDocumentGraph(redactionStorageService.getDocumentData(analyzeRequest.getDossierId(), analyzeRequest.getFileId()));
 
         NerEntities nerEntities;
         if (redactionServiceSettings.isNerServiceEnabled()) {
@@ -209,12 +221,13 @@ public class AnalyzeService {
         long rulesVersion = droolsExecutionService.getRulesVersion(analyzeRequest.getDossierTemplateId());
         Dictionary dictionary = dictionaryService.getDeepCopyDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId());
 
-        PageEntities pageEntities = entityRedactionService.findEntities(dictionary, text.getSectionTexts(), kieContainer, analyzeRequest, nerEntities);
+        entityRedactionService.addDictionaryEntities(dictionary, documentGraph);
+        Set<FileAttribute> addedFileAttributes = entityRedactionService.addRuleEntities(dictionary, documentGraph, kieContainer, analyzeRequest, nerEntities);
 
-        List<RedactionLogEntry> redactionLogEntries = redactionLogCreatorService.createRedactionLog(pageEntities, text.getNumberOfPages(), analyzeRequest.getDossierTemplateId());
+        List<RedactionLogEntry> redactionLogEntries = redactionLogCreatorService.createRedactionLog(documentGraph.getEntities(), analyzeRequest.getDossierTemplateId());
 
-        var legalBasis = legalBasisClient.getLegalBasisMapping(analyzeRequest.getDossierTemplateId());
+        List<LegalBasis> legalBasis = legalBasisClient.getLegalBasisMapping(analyzeRequest.getDossierTemplateId());
-        var redactionLog = new RedactionLog(redactionServiceSettings.getAnalysisVersion(),
+        RedactionLog redactionLog = new RedactionLog(redactionServiceSettings.getAnalysisVersion(),
                 analyzeRequest.getAnalysisNumber(),
                 redactionLogEntries,
                 convert(legalBasis),
@@ -223,21 +236,21 @@ public class AnalyzeService {
                 rulesVersion,
                 legalBasisClient.getVersion(analyzeRequest.getDossierTemplateId()));
 
-        var importedRedactionFilteredEntries = importedRedactionService.processImportedRedactions(analyzeRequest.getDossierTemplateId(),
+        List<RedactionLogEntry> importedRedactionFilteredEntries = importedRedactionService.processImportedRedactions(analyzeRequest.getDossierTemplateId(),
                 analyzeRequest.getDossierId(),
                 analyzeRequest.getFileId(),
                 redactionLog.getRedactionLogEntry(),
                 true);
         redactionLog.setRedactionLogEntry(importedRedactionFilteredEntries);
 
-        return finalizeAnalysis(analyzeRequest, startTime, redactionLog, text, dictionary.getVersion(), false, pageEntities.getAddedFileAttributes());
+        return finalizeAnalysis(analyzeRequest, startTime, redactionLog, documentGraph.getNumberOfPages(), dictionary.getVersion(), false, addedFileAttributes);
     }
 
 
     private AnalyzeResult finalizeAnalysis(AnalyzeRequest analyzeRequest,
                                            long startTime,
                                            RedactionLog redactionLog,
-                                           Text text,
+                                           int numberOfPages,
                                            DictionaryVersion dictionaryVersion,
                                            boolean isReanalysis,
                                            Set<FileAttribute> addedFileAttributes) {
@@ -255,13 +268,13 @@ public class AnalyzeService {
 
         long duration = System.currentTimeMillis() - startTime;
 
-        redactmanagerAnalyzePagewiseValues.increase(text.getNumberOfPages(), duration);
+        redactmanagerAnalyzePagewiseValues.increase(numberOfPages, duration);
 
         return AnalyzeResult.builder()
                 .dossierId(analyzeRequest.getDossierId())
                 .fileId(analyzeRequest.getFileId())
                 .duration(duration)
-                .numberOfPages(text.getNumberOfPages())
+                .numberOfPages(numberOfPages)
                 .hasUpdates(redactionLogChange.isHasChanges())
                 .analysisVersion(redactionServiceSettings.getAnalysisVersion())
                 .analysisNumber(analyzeRequest.getAnalysisNumber())

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/analyze/SectionFinder.java

@@ -19,13 +19,12 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
-import com.iqser.red.service.redaction.v1.server.classification.model.SectionText;
+import com.iqser.red.service.redaction.v1.server.document.graph.DocumentGraph;
-import com.iqser.red.service.redaction.v1.server.classification.model.Text;
+import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SectionNode;
 import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryIncrement;
 import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryIncrementValue;
 import com.iqser.red.service.redaction.v1.server.redaction.model.Image;
 import com.iqser.red.service.redaction.v1.server.redaction.model.RedRectangle2D;
-import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUtils;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.SearchImplementation;
 
 import io.micrometer.core.annotation.Timed;
@@ -39,8 +38,9 @@ import lombok.extern.slf4j.Slf4j;
 @RequiredArgsConstructor
 @FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
 class SectionFinder {
+
     @Timed("redactmanager_findSectionsToReanalyse")
-    public Set<Integer> findSectionsToReanalyse(DictionaryIncrement dictionaryIncrement, RedactionLog redactionLog, Text text, AnalyzeRequest analyzeRequest) {
+    public Set<Integer> findSectionsToReanalyse(DictionaryIncrement dictionaryIncrement, RedactionLog redactionLog, DocumentGraph documentGraph, AnalyzeRequest analyzeRequest) {
 
         long start = System.currentTimeMillis();
         Set<String> relevantManuallyModifiedAnnotationIds = getRelevantManuallyModifiedAnnotationIds(analyzeRequest.getManualRedactions());
@@ -59,10 +59,10 @@ class SectionFinder {
         var dictionaryIncrementsSearch = new SearchImplementation(dictionaryIncrement.getValues().stream().map(DictionaryIncrementValue::getValue).collect(Collectors.toList()),
                 true);
 
-        for (SectionText sectionText : text.getSectionTexts()) {
+        for (SectionNode section : documentGraph.getMainSections()) {
 
-            if (EntitySearchUtils.sectionContainsAny(sectionText.getText(), dictionaryIncrementsSearch)) {
+            if (dictionaryIncrementsSearch.atLeastOneMatches(section.buildTextBlock().getSearchText())) {
-                sectionsToReanalyse.add(sectionText.getSectionNumber());
+                sectionsToReanalyse.add(section.getTocId().get(0));
             }
 
         }
@@ -72,6 +72,7 @@ class SectionFinder {
         return sectionsToReanalyse;
     }
 
+
     private static Set<String> getRelevantManuallyModifiedAnnotationIds(ManualRedactions manualRedactions) {
 
         if (manualRedactions == null) {
@@ -85,6 +86,7 @@ class SectionFinder {
                                         manualRedactions.getForceRedactions().stream().map(ManualForceRedaction::getAnnotationId))))).collect(Collectors.toSet());
     }
 
+
     private static Image convert(RedactionLogEntry entry) {
 
         Rectangle position = entry.getPositions().get(0);
@@ -98,4 +100,5 @@ class SectionFinder {
                 .hasTransparency(entry.isImageHasTransparency())
                 .build();
     }
+
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/entityredaction/EntityRedactionService.java

@@ -1,39 +1,22 @@
 package com.iqser.red.service.redaction.v1.server.redaction.service.entityredaction;
 
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.HashSet;
 import java.util.List;
-import java.util.Map;
 import java.util.Set;
 import java.util.stream.Collectors;
-import java.util.stream.Stream;
 
 import org.kie.api.runtime.KieContainer;
 import org.springframework.stereotype.Service;
 
 import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualImageRecategorization;
-import com.iqser.red.service.redaction.v1.server.classification.model.SectionText;
 import com.iqser.red.service.redaction.v1.server.client.model.NerEntities;
+import com.iqser.red.service.redaction.v1.server.document.graph.DocumentGraph;
+import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SectionNode;
+import com.iqser.red.service.redaction.v1.server.document.services.EntityCreationService;
 import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
-import com.iqser.red.service.redaction.v1.server.redaction.model.Entities;
+import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryModel;
-import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
+import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType;
-import com.iqser.red.service.redaction.v1.server.redaction.model.EntityPositionSequence;
-import com.iqser.red.service.redaction.v1.server.redaction.model.FindEntitiesResult;
-import com.iqser.red.service.redaction.v1.server.redaction.model.Image;
-import com.iqser.red.service.redaction.v1.server.redaction.model.PageEntities;
-import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText;
-import com.iqser.red.service.redaction.v1.server.redaction.model.Section;
-import com.iqser.red.service.redaction.v1.server.redaction.model.SectionSearchableTextPair;
 import com.iqser.red.service.redaction.v1.server.redaction.service.DroolsExecutionService;
-import com.iqser.red.service.redaction.v1.server.redaction.service.SurroundingWordsService;
-import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUtils;
-import com.iqser.red.service.redaction.v1.server.redaction.utils.IdBuilder;
-import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;
 
 import lombok.AccessLevel;
 import lombok.RequiredArgsConstructor;
@@ -47,283 +30,47 @@ import lombok.extern.slf4j.Slf4j;
 public class EntityRedactionService {
 
     DroolsExecutionService droolsExecutionService;
-    EntityFinder entityFinder;
+    EntityCreationService entityCreationService;
-    RedactionServiceSettings redactionServiceSettings;
 
 
-    public PageEntities findEntities(Dictionary dictionary, List<SectionText> sectionTexts, KieContainer kieContainer, AnalyzeRequest analyzeRequest, NerEntities nerEntities) {
+    public Set<FileAttribute> addRuleEntities(Dictionary dictionary,
+                                              DocumentGraph documentGraph,
+                                              KieContainer kieContainer,
+                                              AnalyzeRequest analyzeRequest,
+                                              NerEntities nerEntities) {
 
-        Map<Integer, Set<Image>> imagesPerPage = new HashMap<>();
+        List<FileAttribute> allFileAttributes = droolsExecutionService.executeRules(kieContainer, documentGraph, dictionary, analyzeRequest.getFileAttributes());
-        long start = System.currentTimeMillis();
+        return allFileAttributes.stream().filter(fileAttribute -> !analyzeRequest.getFileAttributes().contains(fileAttribute)).collect(Collectors.toUnmodifiableSet());
-        FindEntitiesResult findEntitiesResult = findEntities(sectionTexts, dictionary, kieContainer, analyzeRequest, false, null, imagesPerPage, nerEntities);
-        System.out.printf("First iteration took %d ms and found %d entities\n", System.currentTimeMillis() - start, findEntitiesResult.getEntities().size());
-        if (dictionary.hasLocalEntries() || !findEntitiesResult.getAddedFileAttributes().isEmpty()) {
-
-            if (!findEntitiesResult.getAddedFileAttributes().isEmpty()) {
-                //AnalyzeRequest provides immutable list.
-                List<FileAttribute> mergedFileAttributes = new ArrayList<>();
-                mergedFileAttributes.addAll(analyzeRequest.getFileAttributes());
-                mergedFileAttributes.addAll(findEntitiesResult.getAddedFileAttributes());
-                analyzeRequest.setFileAttributes(mergedFileAttributes);
-            }
-            long start1 = System.currentTimeMillis();
-            Map<Integer, Set<Entity>> hintsPerSectionNumber = getHintsPerSection(findEntitiesResult.getEntities(), dictionary);
-            FindEntitiesResult foundByLocalEntitiesResult = findEntities(sectionTexts,
-                    dictionary,
-                    kieContainer,
-                    analyzeRequest,
-                    true,
-                    hintsPerSectionNumber,
-                    imagesPerPage,
-                    nerEntities);
-            EntitySearchUtils.addEntitiesWithHigherRank(findEntitiesResult.getEntities(), foundByLocalEntitiesResult.getEntities(), dictionary);
-            EntitySearchUtils.removeEntitiesContainedInLarger(findEntitiesResult.getEntities());
-            System.out.printf("Second iteration took %d ms\n", System.currentTimeMillis() - start1);
-        }
-
-        Map<Integer, List<Entity>> entitiesPerPage = convertToEntitiesPerPage(findEntitiesResult.getEntities());
-        //EntitySearchUtils.removeEntitiesContainedInRedactedLogos(imagesPerPage, entitiesPerPage);
-        System.out.printf("Total time for extracting entities %d ms\n", System.currentTimeMillis() - start);
-        return new PageEntities(entitiesPerPage, imagesPerPage, findEntitiesResult.getAddedFileAttributes());
     }
 
 
-    private FindEntitiesResult findEntities(List<SectionText> reanalysisSections,
+    public Set<FileAttribute> addRuleEntities(Dictionary dictionary,
-                                            Dictionary dictionary,
+                                              DocumentGraph documentGraph,
-                                            KieContainer kieContainer,
+                                              List<SectionNode> sectionsToReanalyze,
-                                            AnalyzeRequest analyzeRequest,
+                                              KieContainer kieContainer,
-                                            boolean local,
+                                              AnalyzeRequest analyzeRequest,
-                                            Map<Integer, Set<Entity>> hintsPerSectionNumber,
+                                              NerEntities nerEntities) {
-                                            Map<Integer, Set<Image>> imagesPerPage,
-                                            NerEntities nerEntities) {
 
-        long start = System.currentTimeMillis();
+        List<FileAttribute> allFileAttributes = droolsExecutionService.executeRules(kieContainer,
-        List<SectionSearchableTextPair> sectionSearchableTextPairs = extractSearchableTextPairs(reanalysisSections,
+                documentGraph,
+                sectionsToReanalyze,
                 dictionary,
-                analyzeRequest,
+                analyzeRequest.getFileAttributes());
-                local,
+        return allFileAttributes.stream().filter(fileAttribute -> !analyzeRequest.getFileAttributes().contains(fileAttribute)).collect(Collectors.toUnmodifiableSet());
-                hintsPerSectionNumber,
-                nerEntities);
-        System.out.printf("Total Dict Search and insertion time %d ms\n", System.currentTimeMillis() - start);
-        Set<FileAttribute> addedFileAttributes = new HashSet<>();
-        Set<Entity> entities = new HashSet<>();
-
-        long start1 = System.currentTimeMillis();
-        sectionSearchableTextPairs.forEach(sectionSearchableTextPair -> {
-
-            if (!addedFileAttributes.isEmpty()) {
-                //Section.Builder provides immutable list.
-                List<FileAttribute> mergedFileAttributes = new ArrayList<>();
-                mergedFileAttributes.addAll(sectionSearchableTextPair.getSection().getAddedFileAttributes());
-                mergedFileAttributes.addAll(addedFileAttributes);
-                sectionSearchableTextPair.getSection().setFileAttributes(mergedFileAttributes);
-            }
-            if (sectionSearchableTextPair.getSection() == null) {
-                return;
-            }
-            Section analysedSection = droolsExecutionService.executeRules(kieContainer, sectionSearchableTextPair.getSection());
-
-            addedFileAttributes.addAll(analysedSection.getAddedFileAttributes());
-
-            EntitySearchUtils.removeEntitiesContainedInLarger(analysedSection.getEntities());
-
-            var entriesWithoutSurroundingText = analysedSection.getEntities()
-                    .stream()
-                    .filter(e -> e.getTextAfter() == null && e.getTextBefore() == null)
-                    .collect(Collectors.toSet());
-
-            addSurroundingText(dictionary, sectionSearchableTextPair.getSearchableText(), sectionSearchableTextPair.getCellStarts(), entriesWithoutSurroundingText);
-
-            entities.addAll(analysedSection.getEntities());
-
-            if (!local) {
-                for (Image image : analysedSection.getImages()) {
-                    imagesPerPage.computeIfAbsent(image.getPage(), (a) -> new HashSet<>()).add(image);
-                }
-                addLocalValuesToDictionary(analysedSection, dictionary);
-            }
-
-        });
-        System.out.printf("rules took %d ms in total\n", System.currentTimeMillis() - start1);
-
-        return FindEntitiesResult.builder().entities(entities).addedFileAttributes(addedFileAttributes).build();
     }
 
 
-    private void addSurroundingText(Dictionary dictionary, SearchableText searchableText, List<Integer> cellStarts, Set<Entity> entriesWithoutSurroundingText) {
+    public void addDictionaryEntities(Dictionary dictionary, DocumentGraph documentGraph) {
 
-        if (cellStarts != null && !cellStarts.isEmpty()) {
+        dictionary.getDictionaryModels().forEach(dictionaryModel -> addDictionaryModelEntities(dictionaryModel, documentGraph));
-            SurroundingWordsService.addSurroundingText(entriesWithoutSurroundingText,
-                    searchableText,
-                    dictionary,
-                    cellStarts,
-                    redactionServiceSettings.getSurroundingWordsOffsetWindow(),
-                    redactionServiceSettings.getNumberOfSurroundingWords());
-
-        } else {
-            SurroundingWordsService.addSurroundingText(entriesWithoutSurroundingText,
-                    searchableText,
-                    dictionary,
-                    redactionServiceSettings.getSurroundingWordsOffsetWindow(),
-                    redactionServiceSettings.getNumberOfSurroundingWords());
-        }
     }
 
 
-    private List<SectionSearchableTextPair> extractSearchableTextPairs(List<SectionText> reanalysisSections,
+    private void addDictionaryModelEntities(DictionaryModel dictionaryModel, DocumentGraph documentGraph) {
-                                                                       Dictionary dictionary,
-                                                                       AnalyzeRequest analyzeRequest,
-                                                                       boolean local,
-                                                                       Map<Integer, Set<Entity>> hintsPerSectionNumber,
-                                                                       NerEntities nerEntities) {
 
-        return reanalysisSections.stream().map(reanalysisSection -> {
+        entityCreationService.bySearchImplementation(dictionaryModel.getEntriesSearch(), dictionaryModel.getType(), EntityType.ENTITY, documentGraph);
-            long start = System.currentTimeMillis();
+        entityCreationService.bySearchImplementation(dictionaryModel.getFalsePositiveSearch(), dictionaryModel.getType(), EntityType.FALSE_POSITIVE, documentGraph);
-            Entities entities = entityFinder.findEntities(reanalysisSection.getSearchableText(),
+        entityCreationService.bySearchImplementation(dictionaryModel.getFalseRecommendationsSearch(), dictionaryModel.getType(), EntityType.FALSE_RECOMMENDATION, documentGraph);
-                    reanalysisSection.getHeadline(),
-                    reanalysisSection.getSectionNumber(),
-                    dictionary,
-                    local,
-                    nerEntities,
-                    reanalysisSection.getCellStarts(),
-                    analyzeRequest.getManualRedactions());
-            addSurroundingText(dictionary, reanalysisSection.getSearchableText(), reanalysisSection.getCellStarts(), entities.getEntities());
-            if (!local && analyzeRequest.getManualRedactions() != null) {
-
-                var approvedForceRedactions = analyzeRequest.getManualRedactions()
-                        .getForceRedactions()
-                        .stream()
-                        .filter(fr -> fr.getStatus() == AnnotationStatus.APPROVED)
-                        .filter(fr -> fr.getRequestDate() != null)
-                        .collect(Collectors.toList());
-                // only approved id removals, that haven't been forced back afterwards
-                var idsToRemove = analyzeRequest.getManualRedactions()
-                        .getIdsToRemove()
-                        .stream()
-                        .filter(idr -> idr.getStatus() == AnnotationStatus.APPROVED && !idr.isRemoveFromDictionary())
-                        .filter(idr -> idr.getRequestDate() != null)
-                        .filter(idr -> approvedForceRedactions.stream()
-                                .noneMatch(forceRedact -> forceRedact.getAnnotationId().equals(idr.getAnnotationId()) && forceRedact.getRequestDate()
-                                        .isAfter(idr.getRequestDate())))
-                        .map(IdRemoval::getAnnotationId)
-                        .collect(Collectors.toSet());
-
-                if (reanalysisSection.getImages() != null && !reanalysisSection.getImages().isEmpty() && analyzeRequest.getManualRedactions().getImageRecategorization() != null) {
-                    for (Image image : reanalysisSection.getImages()) {
-                        String imageId = IdBuilder.buildId(image.getPosition(), image.getPage());
-                        for (ManualImageRecategorization imageRecategorization : analyzeRequest.getManualRedactions().getImageRecategorization()) {
-                            if (imageRecategorization.getStatus().equals(AnnotationStatus.APPROVED) && imageRecategorization.getAnnotationId().equals(imageId)) {
-                                image.setType(imageRecategorization.getType());
-                            }
-                        }
-                        if (idsToRemove.contains(imageId)) {
-                            image.setIgnored(true);
-                        }
-                    }
-                }
-
-                entities.getEntities().forEach(entity -> entity.getPositionSequences().forEach(ps -> {
-                    if (idsToRemove.contains(ps.getId())) {
-                        entity.setIgnored(true);
-                    }
-                }));
-            }
-            log.debug("Section {}, Images: {}", reanalysisSection.getSectionNumber(), reanalysisSection.getImages());
-
-            return toSectionSearchableTextPair(dictionary, analyzeRequest, hintsPerSectionNumber, reanalysisSection, entities);
-        }).collect(Collectors.toList());
-    }
-
-
-    private SectionSearchableTextPair toSectionSearchableTextPair(Dictionary dictionary,
-                                                                  AnalyzeRequest analyzeRequest,
-                                                                  Map<Integer, Set<Entity>> hintsPerSectionNumber,
-                                                                  SectionText reanalysisSection,
-                                                                  Entities entities) {
-
-        return new SectionSearchableTextPair(Section.builder()
-                .isLocal(false)
-                .dictionaryTypes(dictionary.getTypes())
-                .entities(hintsPerSectionNumber != null && hintsPerSectionNumber.containsKey(reanalysisSection.getSectionNumber()) ? Stream.concat(entities.getEntities().stream(),
-                        hintsPerSectionNumber.get(reanalysisSection.getSectionNumber()).stream()).collect(Collectors.toSet()) : entities.getEntities())
-                .nerEntities(entities.getNerEntities())
-                .text(reanalysisSection.getSearchableText().getAsStringWithLinebreaks())
-                .searchText(reanalysisSection.getSearchableText().toString())
-                .headline(reanalysisSection.getHeadline())
-                .sectionNumber(reanalysisSection.getSectionNumber())
-                .tabularData(reanalysisSection.getTabularData())
-                .searchableText(reanalysisSection.getSearchableText())
-                .dictionary(dictionary)
-                .images(reanalysisSection.getImages())
-                .sectionAreas(reanalysisSection.getSectionAreas())
-                .fileAttributes(analyzeRequest.getFileAttributes())
-                .manualRedactions(analyzeRequest.getManualRedactions())
-                .isInTable(reanalysisSection.isTable())
-                .build(), reanalysisSection.getSearchableText(), reanalysisSection.getCellStarts());
-    }
-
-
-    private Map<Integer, List<Entity>> convertToEntitiesPerPage(Set<Entity> entities) {
-
-        Map<Integer, List<Entity>> entitiesPerPage = new HashMap<>();
-        for (Entity entity : entities) {
-            Map<Integer, List<EntityPositionSequence>> sequenceOnPage = new HashMap<>();
-            for (EntityPositionSequence entityPositionSequence : entity.getPositionSequences()) {
-                sequenceOnPage.computeIfAbsent(entityPositionSequence.getPageNumber(), (x) -> new ArrayList<>()).add(entityPositionSequence);
-            }
-
-            for (Map.Entry<Integer, List<EntityPositionSequence>> entry : sequenceOnPage.entrySet()) {
-                entitiesPerPage.computeIfAbsent(entry.getKey(), (x) -> new ArrayList<>())
-                        .add(new Entity(entity.getWord(),
-                                entity.getType(),
-                                entity.isRedaction(),
-                                entity.getRedactionReason(),
-                                entry.getValue(),
-                                entity.getHeadline(),
-                                entity.getMatchedRule(),
-                                entity.getSectionNumber(),
-                                entity.getLegalBasis(),
-                                entity.isDictionaryEntry(),
-                                entity.getTextBefore(),
-                                entity.getTextAfter(),
-                                entity.getStart(),
-                                entity.getEnd(),
-                                entity.isDossierDictionaryEntry(),
-                                entity.getEngines(),
-                                entity.getReferences(),
-                                entity.getEntityType()));
-            }
-        }
-        return entitiesPerPage;
-    }
-
-
-    private Map<Integer, Set<Entity>> getHintsPerSection(Set<Entity> entities, Dictionary dictionary) {
-
-        Map<Integer, Set<Entity>> hintsPerSectionNumber = new HashMap<>();
-        entities.forEach(entity -> {
-            if (dictionary.isHint(entity.getType()) && entity.isDictionaryEntry()) {
-                hintsPerSectionNumber.computeIfAbsent(entity.getSectionNumber(), (x) -> new HashSet<>()).add(entity);
-            }
-        });
-        return hintsPerSectionNumber;
-    }
-
-
-    private void addLocalValuesToDictionary(Section analysedSection, Dictionary dictionary) {
-
-        analysedSection.getLocalDictionaryAdds().keySet().forEach(key -> analysedSection.getLocalDictionaryAdds().get(key).forEach(value -> {
-
-            if (dictionary.getLocalAccessMap().get(key) == null) {
-                log.warn("Dictionary {} is null", key);
-            }
-
-            if (dictionary.getLocalAccessMap().get(key).getLocalEntries() == null) {
-                log.warn("Dictionary {} localEntries is null", key);
-            }
-
-            dictionary.getLocalAccessMap().get(key).getLocalEntries().add(value);
-        }));
     }
 
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/SeparatorUtils.java

@@ -4,6 +4,7 @@ import java.util.Set;
 import java.util.regex.Pattern;
 
 import com.iqser.red.service.redaction.v1.server.document.graph.Boundary;
+import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock;
 
 import lombok.experimental.UtilityClass;
 import lombok.extern.slf4j.Slf4j;
@@ -31,22 +32,38 @@ public final class SeparatorUtils {
     }
 
 
+    public static boolean isWhiteSpacesOrSeparatorsOnly(TextBlock textBlock, Boundary boundary) {
 
-    public static boolean validateBoundaryIsSurroundedBySeparators(CharSequence charSequence, Boundary boundary) {
+        String stringWithoutWhiteSpace = textBlock.subSequence(boundary).toString().replace(" ", "");
-
+        int numberOfSeparators = 0;
-        return validateStart(charSequence, boundary) && validateEnd(charSequence, boundary);
+        for (int i = 0; i < stringWithoutWhiteSpace.length(); i++) {
+            if (isSeparator(stringWithoutWhiteSpace.charAt(i))) {
+                numberOfSeparators++;
+            }
+        }
+        return stringWithoutWhiteSpace.length() == numberOfSeparators;
     }
 
 
-    private static boolean validateEnd(CharSequence charSequence, Boundary boundary) {
+    public static boolean boundaryIsSurroundedBySeparators(TextBlock textBlock, Boundary boundary) {
 
-        return boundary.end() == charSequence.length() || SeparatorUtils.isSeparator(charSequence.charAt(boundary.end())) || SeparatorUtils.isSeparator(charSequence.charAt(boundary.end() - 1));
+        return validateStart(textBlock, boundary) && validateEnd(textBlock, boundary);
     }
 
 
-    private static boolean validateStart(CharSequence charSequence, Boundary boundary) {
+    private static boolean validateEnd(TextBlock textBlock, Boundary boundary) {
 
-        return boundary.start() == 0 || SeparatorUtils.isSeparator(charSequence.charAt(boundary.start() - 1)) || SeparatorUtils.isSeparator(charSequence.charAt(boundary.start()));
+        return boundary.end() == textBlock.getBoundary().end() ||//
+                SeparatorUtils.isSeparator(textBlock.charAt(boundary.end())) ||//
+                SeparatorUtils.isSeparator(textBlock.charAt(boundary.end() - 1));
+    }
+
+
+    private static boolean validateStart(TextBlock textBlock, Boundary boundary) {
+
+        return boundary.start() == textBlock.getBoundary().start() ||//
+                SeparatorUtils.isSeparator(textBlock.charAt(boundary.start() - 1)) ||//
+                SeparatorUtils.isSeparator(textBlock.charAt(boundary.start()));
     }
 
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/storage/RedactionStorageService.java

@@ -10,6 +10,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlo
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.SectionGrid;
 import com.iqser.red.service.redaction.v1.server.classification.model.Text;
 import com.iqser.red.service.redaction.v1.server.client.model.NerEntities;
+import com.iqser.red.service.redaction.v1.server.document.data.DocumentData;
 import com.iqser.red.service.redaction.v1.server.exception.NotFoundException;
 import com.iqser.red.service.redaction.v1.server.multitenancy.TenantContext;
 import com.iqser.red.storage.commons.exception.StorageObjectDoesNotExist;
@@ -55,7 +56,9 @@ public class RedactionStorageService {
     public ImportedRedactions getImportedRedactions(String dossierId, String fileId) {
 
         try {
-            return storageService.readJSONObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.IMPORTED_REDACTIONS), ImportedRedactions.class);
+            return storageService.readJSONObject(TenantContext.getTenantId(),
+                    StorageIdUtils.getStorageId(dossierId, fileId, FileType.IMPORTED_REDACTIONS),
+                    ImportedRedactions.class);
         } catch (StorageObjectDoesNotExist e) {
             log.debug("Imported redactions not available.");
             return null;
@@ -88,6 +91,18 @@ public class RedactionStorageService {
     }
 
 
+    @Timed("redactmanager_getDocumentGraph")
+    public DocumentData getDocumentData(String dossierId, String fileId) {
+
+        try {
+            return storageService.readJSONObject(TenantContext.getTenantId(), StorageIdUtils.getStorageId(dossierId, fileId, FileType.TEXT), DocumentData.class);
+        } catch (StorageObjectDoesNotExist e) {
+            log.debug("Text not available.");
+            return null;
+        }
+    }
+
+
     @Timed("redactmanager_getNerEntities")
     public NerEntities getNerEntities(String dossierId, String fileId) {
 

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DocumentGraphIntegrationTest.java

@@ -1,6 +1,6 @@
 package com.iqser.red.service.redaction.v1.server;
 
-import static com.iqser.red.service.redaction.v1.server.redaction.utils.SeparatorUtils.validateBoundaryIsSurroundedBySeparators;
+import static com.iqser.red.service.redaction.v1.server.redaction.utils.SeparatorUtils.boundaryIsSurroundedBySeparators;
 import static com.iqser.red.service.redaction.v1.server.utils.PdfDraw.drawRectangle2DList;
 
 import java.awt.Color;
@@ -65,7 +65,7 @@ public class DocumentGraphIntegrationTest extends AbstractTestWithDictionaries {
     @SneakyThrows
     public void testDroolsOnDocumentGraph() {
 
-        String filename = "files/new/crafted document";
+        String filename = "files/Metolachlor/S-Metolachlor_RAR_01_Volume_1_2018-09-06";
 
         prepareStorage(filename + ".pdf");
         ClassPathResource fileResource = new ClassPathResource(filename + ".pdf");
@@ -217,7 +217,7 @@ public class DocumentGraphIntegrationTest extends AbstractTestWithDictionaries {
                         .stream()
                         .filter(entityNode -> !entityNode.isRemoved())
                         .filter(EntityNode::isRedaction)
-                        .flatMap(entityNode -> entityNode.getEntityPositions().stream())
+                        .flatMap(entityNode -> entityNode.getEntityPositionsPerPage().stream())
                         .filter(entityPosition -> entityPosition.getPageNode().equals(page))
                         .flatMap(entityPosition -> entityPosition.getRectanglePerLine().stream())
                         .toList();
@@ -231,7 +231,7 @@ public class DocumentGraphIntegrationTest extends AbstractTestWithDictionaries {
                         .stream()
                         .filter(entityNode -> !entityNode.isRemoved())
                         .filter(entityNode -> !entityNode.isRedaction())
-                        .flatMap(entityNode -> entityNode.getEntityPositions().stream())
+                        .flatMap(entityNode -> entityNode.getEntityPositionsPerPage().stream())
                         .filter(entityPosition -> entityPosition.getPageNode().equals(page))
                         .flatMap(entityPosition -> entityPosition.getRectanglePerLine().stream())
                         .toList();
@@ -255,7 +255,7 @@ public class DocumentGraphIntegrationTest extends AbstractTestWithDictionaries {
         TextBlock textBlock = documentGraph.getTextBlock();
         searchImplementation.getBoundaries(textBlock, textBlock.getBoundary())
                 .stream()
-                .filter(boundary -> validateBoundaryIsSurroundedBySeparators(textBlock, boundary))
+                .filter(boundary -> boundaryIsSurroundedBySeparators(textBlock, boundary))
                 .map(bounds -> EntityNode.initialEntityNode(bounds, type, entityType))
                 .forEach(foundEntities::add);
     }

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentGraphMappingTest.java

@@ -5,7 +5,11 @@ import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.core.io.ClassPathResource;
 
 import com.iqser.red.service.redaction.v1.server.AbstractTestWithDictionaries;
+import com.iqser.red.service.redaction.v1.server.document.data.AtomicPositionBlockData;
+import com.iqser.red.service.redaction.v1.server.document.data.AtomicTextBlockData;
 import com.iqser.red.service.redaction.v1.server.document.data.DocumentData;
+import com.iqser.red.service.redaction.v1.server.document.data.PageData;
+import com.iqser.red.service.redaction.v1.server.document.data.TableOfContentsData;
 import com.iqser.red.service.redaction.v1.server.document.graph.factory.DocumentGraphFactory;
 import com.iqser.red.service.redaction.v1.server.document.mapper.DocumentDataMapper;
 import com.iqser.red.service.redaction.v1.server.document.mapper.DocumentGraphMapper;
@@ -33,7 +37,7 @@ public class DocumentGraphMappingTest extends AbstractTestWithDictionaries {
     @SneakyThrows
     public void testGraphMapping() {
 
-        String filename = "files/Metolachlor/S-Metolachlor_RAR_01_Volume_1_2018-09-06";
+        String filename = "files/new/crafted document";
 
         prepareStorage(filename + ".pdf");
         ClassPathResource fileResource = new ClassPathResource(filename + ".pdf");
@@ -41,7 +45,18 @@ public class DocumentGraphMappingTest extends AbstractTestWithDictionaries {
         var classifiedDoc = segmentationService.parseDocument(TEST_DOSSIER_ID, TEST_FILE_ID, fileResource.getInputStream(), null);
         DocumentGraph document = documentGraphFactory.buildDocumentGraph(classifiedDoc);
         DocumentData documentData = documentDataMapper.toDocumentData(document);
-        storageService.storeJSONObject(TenantContext.getTenantId(), filename + ".json", documentData);
+        storageService.storeJSONObject(TenantContext.getTenantId(), filename + "_PAGES" + ".json", documentData.getPages());
+        storageService.storeJSONObject(TenantContext.getTenantId(), filename + "_TEXT" + ".json", documentData.getAtomicTextBlocks());
+        storageService.storeJSONObject(TenantContext.getTenantId(), filename + "_POSITIONS" + ".json", documentData.getAtomicPositionBlocks());
+        storageService.storeJSONObject(TenantContext.getTenantId(), filename + "_STRUCTURE" + ".json", documentData.getTableOfContents());
+
+        PageData[] pageData = storageService.readJSONObject(TenantContext.getTenantId(), filename + "_PAGES" + ".json", PageData[].class);
+        AtomicTextBlockData[] atomicTextBlockData = storageService.readJSONObject(TenantContext.getTenantId(), filename + "_TEXT" + ".json", AtomicTextBlockData[].class);
+        AtomicPositionBlockData[] atomicPositionBlockData = storageService.readJSONObject(TenantContext.getTenantId(),
+                filename + "_POSITIONS" + ".json",
+                AtomicPositionBlockData[].class);
+        TableOfContentsData tableOfContentsData = storageService.readJSONObject(TenantContext.getTenantId(), filename + "_STRUCTURE" + ".json", TableOfContentsData.class);
+
         DocumentGraph newDocumentGraph = documentGraphMapper.toDocumentGraph(documentData);
 
         assert document.toString().equals(newDocumentGraph.toString());

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/entity_rules.drl

@@ -2,7 +2,7 @@ package drools
 
 import static java.lang.String.format;
 import static com.iqser.red.service.redaction.v1.server.document.services.CharSequenceSearchUtils.anyMatch;
-import static com.iqser.red.service.redaction.v1.server.document.services.EntityFieldSetter.setFields;
+import static com.iqser.red.service.redaction.v1.server.document.services.EntityEnrichmentService.setFields;
 
 
 import java.util.List;
@@ -131,6 +131,18 @@ rule "9: Add recommendation for Addresses in Test Animals sections"
         entities.forEach(entity -> insert(entity));
     end
 
+rule "12: Recommend CTL/BL laboratory that start with BL or CTL"
+
+    when
+        $section : SectionNode(containsString("BL") || containsString("CT"))
+    then
+        Set<EntityNode> entities = entityCreationService.byRegex("((\\b((([Cc]T(([1ILli\\/])| L|~P))|(BL))[\\. ]?([\\dA-Ziltphz~\\/.:!]| ?[\\(',][Ppi](\\(e)?|([\\(-?']\\/))+( ?[\\(\\/\\dA-Znasieg]+)?)\\b( ?\\/? ?\\d+)?)|(\\bCT[L1i]\\b))", "CBI_address", EntityType.RECOMMENDATION, $section);
+        setFields(entities, 12, "Syngenta Specific Laboratory keyword", null, Engine.RULE);
+        entities.forEach(entity -> insert(entity));
+        entities.forEach(entity -> dictionary.addLocalDictionaryEntry("PII", entity.getValue(), false));
+    end
+
+
 // --------------------------------------- PII rules -------------------------------------------------------------------
 
 
@@ -182,43 +194,106 @@ rule "13: Redact Emails by RegEx (Vertebrate study)"
         entities.forEach(entity -> insert(entity));
     end
 
-rule "14: Redact contact information (Non vertebrate study)"
+
+rule "14: Redact line after contact information (Non vertebrate study)"
+    agenda-group "LOCAL_DICTIONARY_ADDS"
 
     when
         FileAttribute(label == "Vertebrate Study" , value.toLowerCase() != "yes")
-        $section: SectionNode()
+        $string: String() from List.of("Contact point:",
+                                         "Contact:",
+                                         "Alternative contact:",
+                                         "European contact:",
+                                         "No:",
+                                         "Contact:",
+                                         "Tel.:",
+                                         "Tel:",
+                                         "Telephone number:",
+                                         "Telephone No:",
+                                         "Telephone:",
+                                         "Phone No:",
+                                         "Phone:",
+                                         "Fax number:",
+                                         "Fax:",
+                                         "E-mail:",
+                                         "Email:",
+                                         "e-mail:",
+                                         "E-mail address:")
+        $section: SectionNode(containsString($string))
     then
+
         Set<EntityNode> entities = new HashSet<>();
-        entities.addAll(entityCreationService.lineAfterString("Contact point:", "PII", EntityType.ENTITY, $section));
+        entities.addAll(entityCreationService.lineAfterString($string, "PII", EntityType.ENTITY, $section));
-        entities.addAll(entityCreationService.lineAfterString("Contact:", "PII", EntityType.ENTITY, $section));
+        setFields(entities, 14, "Found by contacts keyword", null, Engine.RULE);
-        entities.addAll(entityCreationService.lineAfterString("Alternative contact:", "PII", EntityType.ENTITY, $section));
-        entities.addAll(entityCreationService.lineAfterString("European contact:", "PII", EntityType.ENTITY, $section));
-        entities.addAll(entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section));
-        entities.addAll(entityCreationService.betweenStrings("Contact:", "Tel.:", "PII", EntityType.ENTITY, $section));
-        setFields(entities, 14, "Found by rule based keywords", null, Engine.RULE);
         entities.forEach(entity -> insert(entity));
         entities.forEach(entity -> dictionary.addLocalDictionaryEntry("PII", entity.getValue(), false));
     end
 
-rule "15: Redact contact information (Vertebrate study)"
+
+rule "15: Redact line after contact information (Vertebrate study)"
+    agenda-group "LOCAL_DICTIONARY_ADDS"
 
     when
         FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "yes")
-        $section: SectionNode()
+        $string: String() from List.of("Contact point:",
+                                         "Contact:",
+                                         "Alternative contact:",
+                                         "European contact:",
+                                         "No:",
+                                         "Contact:",
+                                         "Tel.:",
+                                         "Tel:",
+                                         "Telephone number:",
+                                         "Telephone No:",
+                                         "Telephone:",
+                                         "Phone No:",
+                                         "Phone:",
+                                         "Fax number:",
+                                         "Fax:",
+                                         "E-mail:",
+                                         "Email:",
+                                         "e-mail:",
+                                         "E-mail address:")
+        $section: SectionNode(containsString($string))
     then
-        Set<EntityNode> entities = new HashSet<>();
+        Set<EntityNode> entities = entityCreationService.lineAfterString($string, "PII", EntityType.ENTITY, $section);
-        entities.addAll(entityCreationService.lineAfterString("Contact point:", "PII", EntityType.ENTITY, $section));
+        setFields(entities, 15, "Found by contacts keyword", null, Engine.RULE);
-        entities.addAll(entityCreationService.lineAfterString("Contact:", "PII", EntityType.ENTITY, $section));
-        entities.addAll(entityCreationService.lineAfterString("Alternative contact:", "PII", EntityType.ENTITY, $section));
-        entities.addAll(entityCreationService.lineAfterString("European contact:", "PII", EntityType.ENTITY, $section));
-        entities.addAll(entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section));
-        entities.addAll(entityCreationService.betweenStrings("Contact:", "Tel.:", "PII", EntityType.ENTITY, $section));
-        setFields(entities, 15, "Found by contacts keywords", null, Engine.RULE);
         entities.forEach(entity -> insert(entity));
         entities.forEach(entity -> dictionary.addLocalDictionaryEntry("PII", entity.getValue(), false));
     end
 
-rule "16: Redact Phone and Fax by RegEx"
+
+rule "16: redact line between contact keywords"
+    agenda-group "LOCAL_DICTIONARY_ADDS"
+
+    when
+        FileAttribute(label == "Vertebrate Study" , value.toLowerCase() != "yes")
+        $section: SectionNode((containsString("No:") && containsString("Fax")) || (containsString("Contact:") && containsString("Tel")))
+    then
+        Set<EntityNode> entities = new HashSet<>();
+        entities.addAll(entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section));
+        entities.addAll(entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section));
+        setFields(entities, 16, "Found between contacts keyword", null, Engine.RULE);
+        entities.forEach(entity -> insert(entity));
+        entities.forEach(entity -> dictionary.addLocalDictionaryEntry("PII", entity.getValue(), false));
+    end
+
+rule "17: redact line between contact keywords"
+    agenda-group "LOCAL_DICTIONARY_ADDS"
+
+    when
+        FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "yes")
+        $section: SectionNode((containsString("No:") && containsString("Fax")) || (containsString("Contact:") && containsString("Tel")))
+    then
+        Set<EntityNode> entities = new HashSet<>();
+        entities.addAll(entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section));
+        entities.addAll(entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section));
+        setFields(entities, 17, "Found between contacts keyword", null, Engine.RULE);
+        entities.forEach(entity -> insert(entity));
+        entities.forEach(entity -> dictionary.addLocalDictionaryEntry("PII", entity.getValue(), false));
+    end
+
+rule "18: Redact Phone and Fax by RegEx"
 
     when
         $section: SectionNode(containsString("Contact") ||
@@ -232,6 +307,6 @@ rule "16: Redact Phone and Fax by RegEx"
             containsString("Fer"))
     then
         Set<EntityNode> entities = entityCreationService.byRegex("\\b(contact|telephone|phone|fax|tel|ter|mobile|fel|fer)[a-zA-Z\\s]{0,10}[:.\\s]{0,3}([\\+\\d\\(][\\s\\d\\(\\)\\-\\/\\.]{4,100}\\d)\\b", "PII", EntityType.ENTITY, $section);
-        setFields(entities, 16, "Found by Phone and Fax regex", null, Engine.RULE);
+        setFields(entities, 18, "Found by Phone and Fax regex", null, Engine.RULE);
         entities.forEach(entity -> insert(entity));
     end

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/merge_entity_rules.drl

@@ -22,6 +22,9 @@ global DocumentGraph document
 global EntityCreationService entityCreationService
 global Dictionary dictionary
 
+query "getFileAttributes"
+    $fileAttribute: FileAttribute()
+    end
 
 rule "merge intersecting Entities of same type"
     salience 100
@@ -32,7 +35,7 @@ rule "merge intersecting Entities of same type"
     then
         $first.removeFromGraph();
         $second.removeFromGraph();
-        EntityNode mergedEntity = entityCreationService.createMergedEntity(List.of($first, $second), $type, $entityType, document);
+        EntityNode mergedEntity = entityCreationService.byEntities(List.of($first, $second), $type, $entityType, document);
         insert(mergedEntity);
         retract($first);
         retract($second);

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl

@@ -1,367 +1,390 @@
 package drools
 
-import com.iqser.red.service.redaction.v1.server.redaction.model.Section
+import static java.lang.String.format;
-
+import static com.iqser.red.service.redaction.v1.server.document.services.CharSequenceSearchUtils.anyMatch;
-global Section section
+import static com.iqser.red.service.redaction.v1.server.document.services.EntityEnrichmentService.setFields;
 
 
-// --------------------------------------- AI rules -------------------------------------------------------------------
+import java.util.List;
+import java.util.LinkedList;
+import java.util.HashSet;
 
-rule "0: Add CBI_author from ai"
+import com.iqser.red.service.redaction.v1.server.document.graph.*
-    when
+import com.iqser.red.service.redaction.v1.server.document.graph.nodes.*
-        Section(aiMatchesType("CBI_author"))
+import com.iqser.red.service.redaction.v1.server.document.graph.entity.*
-    then
+import com.iqser.red.service.redaction.v1.server.document.graph.textblock.*
-        section.addAiEntities("CBI_author", "CBI_author");
+import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;
+import java.util.Set
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine
+import com.iqser.red.service.redaction.v1.server.document.services.EntityCreationService;
+import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
+import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryModel;
+
+global DocumentGraph document
+global EntityCreationService entityCreationService
+global Dictionary dictionary
+
+
+query "getFileAttributes"
+    $fileAttribute: FileAttribute()
     end
 
-rule "0: Combine address parts from ai to CBI_address (org is mandatory)"
+rule "merge intersecting Entities of same type"
+    salience 100
+
     when
-        Section(aiMatchesType("ORG"))
+        $first: EntityNode($type: type, $entityType: entityType, !resized, !skipRemoveEntitiesContainedInLarger)
+        $second: EntityNode(intersects($first), type == $type, entityType == $entityType, this != $first, !resized, !skipRemoveEntitiesContainedInLarger)
     then
-        section.combineAiTypes("ORG", "STREET,POSTAL,COUNTRY,CARDINAL,CITY,STATE", 20, "CBI_address", 3, false);
+        $first.removeFromGraph();
+        $second.removeFromGraph();
+        EntityNode mergedEntity = entityCreationService.byEntities(List.of($first, $second), $type, $entityType, document);
+        insert(mergedEntity);
+        retract($first);
+        retract($second);
     end
 
-rule "0: Combine address parts from ai to CBI_address (street is mandatory)"
+rule "remove Entity of type ENTITY when contained by FALSE_POSITIVE"
+    salience 100
+
     when
-        Section(aiMatchesType("STREET"))
+        $falsePositive: EntityNode($type: type, entityType == EntityType.FALSE_POSITIVE)
+        entity: EntityNode(containedBy($falsePositive), type == $type, entityType == EntityType.ENTITY, !resized, !skipRemoveEntitiesContainedInLarger)
     then
-        section.combineAiTypes("STREET", "ORG,POSTAL,COUNTRY,CARDINAL,CITY,STATE", 20, "CBI_address", 3, false);
+        entity.removeFromGraph();
+        retract(entity);
     end
 
-rule "0: Combine address parts from ai to CBI_address (city is mandatory)"
+rule "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"
+    salience 100
+
     when
-        Section(aiMatchesType("CITY"))
+        $falseRecommendation: EntityNode($type: type, entityType == EntityType.FALSE_RECOMMENDATION)
+        $recommendation: EntityNode(containedBy($falseRecommendation), type == $type, entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger)
     then
-        section.combineAiTypes("CITY", "ORG,STREET,POSTAL,COUNTRY,CARDINAL,STATE", 20, "CBI_address", 3, false);
+        $recommendation.removeFromGraph();
+        retract($recommendation);
+    end
+
+rule "remove Entity of type RECOMMENDATION when contained by ENTITY"
+    salience 100
+
+    when
+        $entity: EntityNode($type: type, entityType == EntityType.ENTITY)
+        $recommendation: EntityNode(containedBy($entity), type == $type, entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger)
+    then
+        $recommendation.removeFromGraph();
+        retract($recommendation);
+    end
+
+rule "remove Entity of lower rank, when equal boundaries and entityType"
+    salience 100
+
+    when
+        $higherRank: EntityNode($type: type, $entityType: entityType, $boundary: boundary)
+        $lowerRank: EntityNode($boundary == boundary, type != $type, entityType == $entityType, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type))
+    then
+        $lowerRank.removeFromGraph();
+        retract($lowerRank);
+    end
+
+rule "run local dictionary search"
+    agenda-group "LOCAL_DICTIONARY_ADDS"
+    salience -999
+
+    when
+        DictionaryModel(!localEntries.isEmpty(), $type: type, $searchImplementation: localSearch) from dictionary.getDictionaryModels()
+    then
+        Set<EntityNode> entityNodes = entityCreationService.bySearchImplementation($searchImplementation, $type, EntityType.RECOMMENDATION, document);
+        System.out.printf("local dictionary search found %d entities\n", entityNodes.size());
+        entityNodes.forEach(entityNode -> insert(entityNode));
     end
 
 
 // --------------------------------------- CBI rules -------------------------------------------------------------------
 
 rule "1: Redact CBI Authors (Non vertebrate study)"
+    no-loop true
+
     when
-        Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesType("CBI_author"))
+        FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "yes")
+        $entity: EntityNode(type == "CBI_author", entityType == EntityType.ENTITY)
     then
-        section.redact("CBI_author", 1, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        $entity.setRedaction(true);
+        setFields($entity, 1, "Address found", "Article 39(e)(2) of Regulation (EC) No 178/2002", null);
+        update($entity)
     end
 
 rule "2: Redact CBI Authors (Vertebrate study)"
+    no-loop true
+
     when
-        Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesType("CBI_author"))
+        FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "no")
+        $entity: EntityNode(type == "CBI_author", entityType == EntityType.ENTITY)
     then
-        section.redact("CBI_author", 2, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        $entity.setRedaction(true);
+        setFields($entity, 2, "Address found", "Article 39(e)(2) of Regulation (EC) No 178/2002", null);
+        update($entity)
     end
 
+rule "3: Don't redact CBI Address (Non vertebrate study)"
+    no-loop true
 
-rule "3: Redact not CBI Address (Non vertebrate study)"
     when
-        Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesType("CBI_address"))
+        FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "no")
+        $entity: EntityNode(type == "CBI_address", entityType == EntityType.ENTITY)
     then
-        section.redactNot("CBI_address", 3, "Address found for non vertebrate study");
+        setFields($entity, 3, "Address found", "Article 39(e)(2) of Regulation (EC) No 178/2002", null);
-        section.ignoreRecommendations("CBI_address");
+        update($entity)
     end
 
 rule "4: Redact CBI Address (Vertebrate study)"
+    no-loop true
+
     when
-        Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesType("CBI_address"))
+        FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "yes")
+        $entity: EntityNode(type == "CBI_address", entityType == EntityType.ENTITY)
     then
-        section.redact("CBI_address", 4, "Address found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        $entity.setRedaction(true);
+        setFields($entity, 4, "Address found", "Article 39(e)(2) of Regulation (EC) No 178/2002", null);
+        update($entity)
+    end
+
+rule "5: Add FALSE_POSITIVE Entity for genitive CBI_author"
+
+    when
+        $entity: EntityNode(type == "CBI_author", anyMatch(textAfter, "['’’'ʼˈ´`‘′ʻ’']s"), redaction)
+    then
+        EntityNode entity = entityCreationService.byBoundary($entity.getBoundary(), "CBI_author", EntityType.FALSE_POSITIVE, document);
+        setFields($entity, 5, "Genitive Author", null, Engine.RULE);
+        insert(entity);
     end
 
 
-rule "5: Do not redact genitive CBI_author"
+rule "6: Create Entity from Author(s) cells in Tables with Author(s) header and add Authorname as Recommendation"
+    agenda-group "LOCAL_DICTIONARY_ADDS"
+
     when
-        Section(matchesType("CBI_author"))
+        authorCell: TableCellNode(!header, (hasHeader("Author(s)") || hasHeader("Author")), hasText())
     then
-        section.expandToFalsePositiveByRegEx("CBI_author", "['’’'ʼˈ´`‘′ʻ’']s", false, 0);
+        EntityNode entity = entityCreationService.bySemanticNode(authorCell, "CBI_author", EntityType.ENTITY);
+        setFields(entity, 6, "Header Author(s) found", null, Engine.RULE);
+        insert(entity);
+        dictionary.addLocalDictionaryEntry("CBI_author", entity.getValue(), true);
+    end
+
+rule "7: Add CBI_author with \"et al.\" Regex"
+    agenda-group "LOCAL_DICTIONARY_ADDS"
+
+    when
+         $section: SectionNode(containsString("et al."))
+    then
+         Set<EntityNode> entities = entityCreationService.byRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, $section);
+         setFields(entities, 7, "Found by \"et al.\" regex", null, Engine.RULE);
+         entities.forEach(entity -> insert(entity));
+         entities.forEach(entity -> dictionary.addLocalDictionaryEntry("CBI_author", entity.getValue(), false));
+    end
+
+rule "8: Add recommendation for Addresses in Test Organism sections"
+
+    when
+        FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "yes")
+        $section: SectionNode(containsString("Species") && containsString("Source") && !containsString("Species:") && !containsString("Source:"))
+    then
+        Set<EntityNode> entities = entityCreationService.lineAfterString("Source",  "CBI_address", EntityType.RECOMMENDATION, $section);
+        setFields(entities, 8, "Line after \"Source\" in Test Organism Section", null, Engine.RULE);
+        entities.forEach(entity -> insert(entity));
     end
 
 
-rule "6: Redact Author(s) cells in Tables with Author(s) header (Non vertebrate study)"
+rule "9: Add recommendation for Addresses in Test Animals sections"
+
     when
-        Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && hasTableHeader("Author(s)") && !hasTableHeader("Vertebrate study Y/N"))
+        FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "yes")
+        $section: SectionNode(containsString("Species:") && containsString("Source:"))
     then
-        section.redactCell("Author(s)", 6, "CBI_author", false, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        Set<EntityNode> entities = entityCreationService.lineAfterString("Source:", "CBI_address", EntityType.RECOMMENDATION, $section);
+        setFields(entities, 9, "Line after \"Source:\" in Test Organism Section", null, Engine.RULE);
+        entities.forEach(entity -> insert(entity));
     end
 
-rule "7: Redact Author(s) cells in Tables with Author(s) header (Vertebrate study)"
+rule "12: Recommend CTL/BL laboratory that start with BL or CTL"
+
     when
-        Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && hasTableHeader("Author(s)") && !hasTableHeader("Vertebrate study Y/N"))
+        $section : SectionNode(containsString("BL") || containsString("CT"))
     then
-        section.redactCell("Author(s)", 7, "CBI_author", false, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        Set<EntityNode> entities = entityCreationService.byRegex("((\\b((([Cc]T(([1ILli\\/])| L|~P))|(BL))[\\. ]?([\\dA-Ziltphz~\\/.:!]| ?[\\(',][Ppi](\\(e)?|([\\(-?']\\/))+( ?[\\(\\/\\dA-Znasieg]+)?)\\b( ?\\/? ?\\d+)?)|(\\bCT[L1i]\\b))", "CBI_address", EntityType.RECOMMENDATION, $section);
-    end
+        setFields(entities, 12, "Syngenta Specific Laboratory keyword", null, Engine.RULE);
-
+        entities.forEach(entity -> insert(entity));
-
+        entities.forEach(entity -> dictionary.addLocalDictionaryEntry("PII", entity.getValue(), false));
-rule "8: Redact Author cells in Tables with Author header (Non vertebrate study)"
-    when
-        Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && hasTableHeader("Author") && !hasTableHeader("Vertebrate study Y/N"))
-    then
-        section.redactCell("Author", 8, "CBI_author", false, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
-    end
-
-rule "9: Redact Author cells in Tables with Author header (Vertebrate study)"
-    when
-        Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && hasTableHeader("Author") && !hasTableHeader("Vertebrate study Y/N"))
-    then
-        section.redactCell("Author", 9, "CBI_author", false, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
-    end
-
-
-rule "10: Redact and recommand Authors in Tables with Vertebrate study Y/N header (Non vertebrate study)"
-    when
-        Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && (rowEquals("Vertebrate study Y/N", "Y") || rowEquals("Vertebrate study Y/N", "Yes") || rowEquals("Vertebrate study Y/N", "N") || rowEquals("Vertebrate study Y/N", "No")))
-    then
-        section.redactCell("Author(s)", 10, "CBI_author", true, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
-    end
-
-rule "11: Redact and recommand Authors in Tables with Vertebrate study Y/N header (Vertebrate study)"
-    when
-        Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && (rowEquals("Vertebrate study Y/N", "Y") || rowEquals("Vertebrate study Y/N", "Yes") || rowEquals("Vertebrate study Y/N", "N") || rowEquals("Vertebrate study Y/N", "No")))
-    then
-        section.redactCell("Author(s)", 11, "CBI_author", true, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
-    end
-
-rule "14: Redact and add recommendation for et al. author (Non vertebrate study)"
-    when
-        Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("et al"))
-    then
-        section.redactAndRecommendByRegEx("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", false, 1, "CBI_author", 14, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
-    end
-
-rule "15: Redact and add recommendation for et al. author (Vertebrate study)"
-    when
-        Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("et al"))
-    then
-        section.redactAndRecommendByRegEx("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", false, 1, "CBI_author", 15, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
-    end
-
-
-rule "16: Add recommendation for Addresses in Test Organism sections"
-    when
-        Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("Species:") && searchText.contains("Source:"))
-    then
-        section.recommendLineAfter("Source:", "CBI_address");
-    end
-
-rule "17: Add recommendation for Addresses in Test Animals sections"
-    when
-        Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("Species") && searchText.contains("Source"))
-    then
-        section.recommendLineAfter("Source", "CBI_address");
-    end
-
-
-rule "18: Do not redact Names and Addresses if Published Information found"
-    when
-        Section(matchesType("published_information"))
-    then
-        section.redactNotAndReference("CBI_author","published_information", 18, "Published Information found");
-        section.redactNotAndReference("CBI_address","published_information", 18, "Published Information found");
     end
 
 
 // --------------------------------------- PII rules -------------------------------------------------------------------
 
 
-rule "19: Redacted PII Personal Identification Information (Non vertebrate study)"
+rule "10: Redacted PII Personal Identification Information (Non vertebrate study)"
+    no-loop true
+
     when
-        Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesType("PII"))
+        FileAttribute(label == "Vertebrate Study" , value.toLowerCase() != "yes")
+        $entity: EntityNode(type == "PII", entityType == EntityType.ENTITY)
     then
-        section.redact("PII", 19, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        $entity.setRedaction(true);
-    end
+        setFields($entity, 10, "Personal Information found", "Article 39(e)(2) of Regulation (EC) No 178/2002", null);
-
+        update($entity)
-rule "20: Redacted PII Personal Identification Information (Vertebrate study)"
-    when
-        Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesType("PII"))
-    then
-        section.redact("PII", 20, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
-    end
-
-
-rule "21: Redact Emails by RegEx (Non vertebrate study)"
-    when
-        Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("@"))
-    then
-        section.redactByRegEx("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", true, 1, "PII", 21, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
-    end
-
-rule "22: Redact Emails by RegEx (Vertebrate study)"
-    when
-        Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("@"))
-    then
-        section.redactByRegEx("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", true, 1, "PII", 22, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
-    end
-
-
-rule "23: Redact contact information (Non vertebrate study)"
-    when
-       Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && (text.contains("Contact point:")
-            || text.contains("Contact:")
-            || text.contains("Alternative contact:")
-            || (text.contains("No:") && text.contains("Fax"))
-            || (text.contains("Contact:") && text.contains("Tel.:"))
-            || text.contains("European contact:")
-        ))
-    then
-        section.redactLineAfter("Contact point:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
-        section.redactLineAfter("Contact:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
-        section.redactLineAfter("Alternative contact:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
-        section.redactBetween("No:", "Fax", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
-        section.redactBetween("Contact:", "Tel.:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
-        section.redactLineAfter("European contact:",  "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
-     end
-
-rule "24: Redact contact information (Vertebrate study)"
-    when
-       Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && (text.contains("Contact point:")
-            || text.contains("Contact:")
-            || text.contains("Alternative contact:")
-            || (text.contains("No:") && text.contains("Fax"))
-            || (text.contains("Contact:") && text.contains("Tel.:"))
-            || text.contains("European contact:")
-        ))
-    then
-        section.redactLineAfter("Contact point:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
-        section.redactLineAfter("Contact:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
-        section.redactLineAfter("Alternative contact:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
-        section.redactBetween("No:", "Fax", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
-        section.redactBetween("Contact:", "Tel.:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
-        section.redactLineAfter("European contact:",  "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
-     end
-
-rule "25: Redact Phone and Fax by RegEx (Non vertebrate study)"
-    when
-        Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && (
-               text.contains("Contact")
-            || text.contains("Telephone")
-            || text.contains("Phone")
-            || text.contains("Fax")
-            || text.contains("Tel")
-            || text.contains("Ter")
-            || text.contains("Mobile")
-            || text.contains("Fel")
-            || text.contains("Fer")
-        ))
-    then
-        section.redactByRegEx("\\b(contact|telephone|phone|fax|tel|ter|mobile|fel|fer)[a-zA-Z\\s]{0,10}[:.\\s]{0,3}([\\+\\d\\(][\\s\\d\\(\\)\\-\\/\\.]{4,100}\\d)\\b", true, 2, "PII", 25, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
-    end
-
-rule "26: Redact Phone and Fax by RegEx (Vertebrate study)"
-    when
-        Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && (
-               text.contains("Contact")
-            || text.contains("Telephone")
-            || text.contains("Phone")
-            || text.contains("Fax")
-            || text.contains("Tel")
-            || text.contains("Ter")
-            || text.contains("Mobile")
-            || text.contains("Fel")
-            || text.contains("Fer")
-        ))
-    then
-        section.redactByRegEx("\\b(contact|telephone|phone|fax|tel|ter|mobile|fel|fer)[a-zA-Z\\s]{0,10}[:.\\s]{0,3}([\\+\\d\\(][\\s\\d\\(\\)\\-\\/\\.]{4,100}\\d)\\b", true, 2, "PII", 26, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
-    end
-
-rule "27: Redact AUTHOR(S) (Non vertebrate study)"
-    when
-        Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes")
-            && searchText.contains("AUTHOR(S):")
-            && searchText.contains("COMPLETION DATE:")
-            && !searchText.contains("STUDY COMPLETION DATE:")
-        )
-    then
-        section.redactLinesBetween("AUTHOR(S):", "COMPLETION DATE:", "PII", 27, true, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
-    end
-
-rule "28: Redact AUTHOR(S) (Vertebrate study)"
-    when
-        Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes")
-            && searchText.contains("AUTHOR(S):")
-            && searchText.contains("COMPLETION DATE:")
-            && !searchText.contains("STUDY COMPLETION DATE:")
-        )
-    then
-        section.redactLinesBetween("AUTHOR(S):", "COMPLETION DATE:", "PII", 28, true, "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
-    end
-
-
-rule "29: Redact AUTHOR(S) (Non vertebrate study)"
-    when
-        Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes")
-            && searchText.contains("AUTHOR(S):")
-            && searchText.contains("STUDY COMPLETION DATE:")
-        )
-    then
-        section.redactLinesBetween("AUTHOR(S):", "STUDY COMPLETION DATE:", "PII", 29, true, "AUTHOR(S) was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
-    end
-
-rule "30: Redact AUTHOR(S) (Vertebrate study)"
-    when
-        Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes")
-            && searchText.contains("AUTHOR(S):")
-            && searchText.contains("STUDY COMPLETION DATE:")
-        )
-    then
-        section.redactLinesBetween("AUTHOR(S):", "STUDY COMPLETION DATE:", "PII", 30, true, "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
-    end
-
-
-rule "31: Redact PERFORMING LABORATORY (Non vertebrate study)"
-    when
-        Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes")
-            && searchText.contains("PERFORMING LABORATORY:")
-        )
-    then
-        section.redactBetween("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", 31, true, "PERFORMING LABORATORY was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
-        section.redactNot("CBI_address", 31, "Performing laboratory found for non vertebrate study");
-    end
-
-rule "32: Redact PERFORMING LABORATORY (Vertebrate study)"
-    when
-        Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes")
-            && searchText.contains("PERFORMING LABORATORY:"))
-    then
-        section.redactBetween("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", 32, true, "PERFORMING LABORATORY was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
-    end
-
-rule "33: Redact study director abbreviation"
-    when
-        Section((searchText.contains("KATH") || searchText.contains("BECH") || searchText.contains("KML")))
-    then
-        section.redactWordPartByRegEx("((KATH)|(BECH)|(KML)) ?(\\d{4})", true, 0, 1, "PII", 34, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
-     end
-
-
-// --------------------------------------- other rules -------------------------------------------------------------------
-
-rule "34: Purity Hint"
-    when
-        Section(searchText.toLowerCase().contains("purity"))
-    then
-	    section.addHintAnnotationByRegEx("(purity ?( of|\\(.{1,20}\\))?( ?:)?) .{0,5}[\\d\\.]+( .{0,4}\\.)? ?%", true, 1, "hint_only");
     end
 
 
 
-rule "35: Redact signatures (Non vertebrate study)"
+rule "11: Redacted PII Personal Identification Information (Vertebrate study)"
+    no-loop true
+
     when
-        Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesImageType("signature"))
+        FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "yes")
+        $entity: EntityNode(type == "PII", entityType == EntityType.ENTITY)
     then
-        section.redactImage("signature", 35, "Signature found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        $entity.setRedaction(true);
+        setFields($entity, 11, "Personal Information found", "Article 39(e)(2) of Regulation (EC) No 178/2002", null);
+        update($entity)
     end
 
-rule "36: Redact signatures (Vertebrate study)"
+rule "12: Redact Emails by RegEx (Non vertebrate study)"
+
     when
-        Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesImageType("signature"))
+        FileAttribute(label == "Vertebrate Study" , value.toLowerCase() != "yes")
+        $section: SectionNode(containsString("@"))
     then
-        section.redactImage("signature", 36, "Signature found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        Set<EntityNode> entities = entityCreationService.byRegex("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", "PII", EntityType.ENTITY, $section);
+        setFields(entities, 12, "Found by email regex", null, Engine.RULE);
+        entities.forEach(entity -> insert(entity));
+    end
+
+rule "13: Redact Emails by RegEx (Vertebrate study)"
+
+    when
+        FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "yes")
+        $section: SectionNode(containsString("@"))
+    then
+        Set<EntityNode> entities = entityCreationService.byRegex("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", "PII", EntityType.ENTITY, $section);
+        setFields(entities, 13, "Found by email regex", null, Engine.RULE);
+        entities.forEach(entity -> insert(entity));
     end
 
 
-rule "43: Redact Logos (Vertebrate study)"
+rule "14: Redact line after contact information (Non vertebrate study)"
+    agenda-group "LOCAL_DICTIONARY_ADDS"
+
     when
-        Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesImageType("logo"))
+        FileAttribute(label == "Vertebrate Study" , value.toLowerCase() != "yes")
+        $string: String() from List.of("Contact point:",
+                                         "Contact:",
+                                         "Alternative contact:",
+                                         "European contact:",
+                                         "No:",
+                                         "Contact:",
+                                         "Tel.:",
+                                         "Tel:",
+                                         "Telephone number:",
+                                         "Telephone No:",
+                                         "Telephone:",
+                                         "Phone No:",
+                                         "Phone:",
+                                         "Fax number:",
+                                         "Fax:",
+                                         "E-mail:",
+                                         "Email:",
+                                         "e-mail:",
+                                         "E-mail address:")
+        $section: SectionNode(containsString($string))
     then
-        section.redactImage("logo", 43, "Logo found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+
+        Set<EntityNode> entities = new HashSet<>();
+        entities.addAll(entityCreationService.lineAfterString($string, "PII", EntityType.ENTITY, $section));
+        setFields(entities, 14, "Found by contacts keyword", null, Engine.RULE);
+        entities.forEach(entity -> insert(entity));
+        entities.forEach(entity -> dictionary.addLocalDictionaryEntry("PII", entity.getValue(), false));
+    end
+
+
+rule "15: Redact line after contact information (Vertebrate study)"
+    agenda-group "LOCAL_DICTIONARY_ADDS"
+
+    when
+        FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "yes")
+        $string: String() from List.of("Contact point:",
+                                         "Contact:",
+                                         "Alternative contact:",
+                                         "European contact:",
+                                         "No:",
+                                         "Contact:",
+                                         "Tel.:",
+                                         "Tel:",
+                                         "Telephone number:",
+                                         "Telephone No:",
+                                         "Telephone:",
+                                         "Phone No:",
+                                         "Phone:",
+                                         "Fax number:",
+                                         "Fax:",
+                                         "E-mail:",
+                                         "Email:",
+                                         "e-mail:",
+                                         "E-mail address:")
+        $section: SectionNode(containsString($string))
+    then
+        Set<EntityNode> entities = entityCreationService.lineAfterString($string, "PII", EntityType.ENTITY, $section);
+        setFields(entities, 15, "Found by contacts keyword", null, Engine.RULE);
+        entities.forEach(entity -> insert(entity));
+        entities.forEach(entity -> dictionary.addLocalDictionaryEntry("PII", entity.getValue(), false));
+    end
+
+
+rule "16: redact line between contact keywords"
+    agenda-group "LOCAL_DICTIONARY_ADDS"
+
+    when
+        FileAttribute(label == "Vertebrate Study" , value.toLowerCase() != "yes")
+        $section: SectionNode((containsString("No:") && containsString("Fax")) || (containsString("Contact:") && containsString("Tel")))
+    then
+        Set<EntityNode> entities = new HashSet<>();
+        entities.addAll(entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section));
+        entities.addAll(entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section));
+        setFields(entities, 16, "Found between contacts keyword", null, Engine.RULE);
+        entities.forEach(entity -> insert(entity));
+        entities.forEach(entity -> dictionary.addLocalDictionaryEntry("PII", entity.getValue(), false));
+    end
+
+rule "17: redact line between contact keywords"
+    agenda-group "LOCAL_DICTIONARY_ADDS"
+
+    when
+        FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "yes")
+        $section: SectionNode((containsString("No:") && containsString("Fax")) || (containsString("Contact:") && containsString("Tel")))
+    then
+        Set<EntityNode> entities = new HashSet<>();
+        entities.addAll(entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section));
+        entities.addAll(entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section));
+        setFields(entities, 17, "Found between contacts keyword", null, Engine.RULE);
+        entities.forEach(entity -> insert(entity));
+        entities.forEach(entity -> dictionary.addLocalDictionaryEntry("PII", entity.getValue(), false));
+    end
+
+rule "18: Redact Phone and Fax by RegEx"
+
+    when
+        $section: SectionNode(containsString("Contact") ||
+            containsString("Telephone") ||
+            containsString("Phone") ||
+            containsString("Fax") ||
+            containsString("Tel") ||
+            containsString("Ter") ||
+            containsString("Mobile") ||
+            containsString("Fel") ||
+            containsString("Fer"))
+    then
+        Set<EntityNode> entities = entityCreationService.byRegex("\\b(contact|telephone|phone|fax|tel|ter|mobile|fel|fer)[a-zA-Z\\s]{0,10}[:.\\s]{0,3}([\\+\\d\\(][\\s\\d\\(\\)\\-\\/\\.]{4,100}\\d)\\b", "PII", EntityType.ENTITY, $section);
+        setFields(entities, 18, "Found by Phone and Fax regex", null, Engine.RULE);
+        entities.forEach(entity -> insert(entity));
     end

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules2.drl

@@ -1 +0,0 @@
-rules.drl

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/testRules.drl

@@ -1,431 +1,390 @@
 package drools
 
-import com.iqser.red.service.redaction.v1.server.redaction.model.Section
+import static java.lang.String.format;
-
+import static com.iqser.red.service.redaction.v1.server.document.services.CharSequenceSearchUtils.anyMatch;
-global Section section
+import static com.iqser.red.service.redaction.v1.server.document.services.EntityEnrichmentService.setFields;
 
 
-// --------------------------------------- AI rules -------------------------------------------------------------------
+import java.util.List;
+import java.util.LinkedList;
+import java.util.HashSet;
 
-rule "0: Add CBI_author from ai"
+import com.iqser.red.service.redaction.v1.server.document.graph.*
-    when
+import com.iqser.red.service.redaction.v1.server.document.graph.nodes.*
-        Section(aiMatchesType("CBI_author"))
+import com.iqser.red.service.redaction.v1.server.document.graph.entity.*
-    then
+import com.iqser.red.service.redaction.v1.server.document.graph.textblock.*
-        section.addAiEntities("CBI_author", "CBI_author",dictionary);
+import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;
+import java.util.Set
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine
+import com.iqser.red.service.redaction.v1.server.document.services.EntityCreationService;
+import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
+import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryModel;
+
+global DocumentGraph document
+global EntityCreationService entityCreationService
+global Dictionary dictionary
+
+
+query "getFileAttributes"
+    $fileAttribute: FileAttribute()
     end
 
-rule "0: Combine address parts from ai to CBI_address (org is mandatory)"
+rule "merge intersecting Entities of same type"
+    salience 100
+
     when
-        Section(aiMatchesType("ORG"))
+        $first: EntityNode($type: type, $entityType: entityType, !resized, !skipRemoveEntitiesContainedInLarger)
+        $second: EntityNode(intersects($first), type == $type, entityType == $entityType, this != $first, !resized, !skipRemoveEntitiesContainedInLarger)
     then
-        section.combineAiTypes("ORG", "STREET,POSTAL,COUNTRY,CARDINAL,CITY,STATE", 20, "CBI_address", 3, false, dictionary);
+        $first.removeFromGraph();
+        $second.removeFromGraph();
+        EntityNode mergedEntity = entityCreationService.byEntities(List.of($first, $second), $type, $entityType, document);
+        insert(mergedEntity);
+        retract($first);
+        retract($second);
     end
 
-rule "0: Combine address parts from ai to CBI_address (street is mandatory)"
+rule "remove Entity of type ENTITY when contained by FALSE_POSITIVE"
+    salience 100
+
     when
-        Section(aiMatchesType("STREET"))
+        $falsePositive: EntityNode($type: type, entityType == EntityType.FALSE_POSITIVE)
+        $entity: EntityNode(containedBy($falsePositive), type == $type, entityType == EntityType.ENTITY, !resized, !skipRemoveEntitiesContainedInLarger)
     then
-        section.combineAiTypes("STREET", "ORG,POSTAL,COUNTRY,CARDINAL,CITY,STATE", 20, "CBI_address", 3, false, dictionary);
+        $entity.removeFromGraph();
+        retract($entity);
     end
 
-rule "0: Combine address parts from ai to CBI_address (city is mandatory)"
+rule "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"
+    salience 100
+
     when
-        Section(aiMatchesType("CITY"))
+        $falseRecommendation: EntityNode($type: type, entityType == EntityType.FALSE_RECOMMENDATION)
+        $recommendation: EntityNode(containedBy($falseRecommendation), type == $type, entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger)
     then
-        section.combineAiTypes("CITY", "ORG,STREET,POSTAL,COUNTRY,CARDINAL,STATE", 20, "CBI_address", 3, false, dictionary);
+        $recommendation.removeFromGraph();
+        retract($recommendation);
+    end
+
+rule "remove Entity of type RECOMMENDATION when contained by ENTITY"
+    salience 100
+
+    when
+        $entity: EntityNode($type: type, entityType == EntityType.ENTITY)
+        $recommendation: EntityNode(containedBy($entity), type == $type, entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger)
+    then
+        $recommendation.removeFromGraph();
+        retract($recommendation);
+    end
+
+rule "remove Entity of lower rank, when equal boundaries and entityType"
+    salience 100
+
+    when
+        $higherRank: EntityNode($type: type, $entityType: entityType, $boundary: boundary)
+        $lowerRank: EntityNode($boundary == boundary, type != $type, entityType == $entityType, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type))
+    then
+        $lowerRank.removeFromGraph();
+        retract($lowerRank);
+    end
+
+rule "run local dictionary search"
+    agenda-group "LOCAL_DICTIONARY_ADDS"
+    salience -999
+
+    when
+        DictionaryModel(!localEntries.isEmpty(), $type: type, $searchImplementation: localSearch) from dictionary.getDictionaryModels()
+    then
+        Set<EntityNode> entityNodes = entityCreationService.bySearchImplementation($searchImplementation, $type, EntityType.RECOMMENDATION, document);
+        System.out.printf("local dictionary search found %d entities\n", entityNodes.size());
+        entityNodes.forEach(entityNode -> insert(entityNode));
     end
 
 
 // --------------------------------------- CBI rules -------------------------------------------------------------------
 
 rule "1: Redact CBI Authors (Non vertebrate study)"
+    no-loop true
+
     when
-        Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesType("CBI_author"))
+        FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "yes")
+        $entity: EntityNode(type == "CBI_author", entityType == EntityType.ENTITY)
     then
-        section.redact("CBI_author", 1, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        $entity.setRedaction(true);
+        setFields($entity, 1, "Address found", "Article 39(e)(2) of Regulation (EC) No 178/2002", null);
+        update($entity)
     end
 
 rule "2: Redact CBI Authors (Vertebrate study)"
+    no-loop true
+
     when
-        Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesType("CBI_author"))
+        FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "no")
+        $entity: EntityNode(type == "CBI_author", entityType == EntityType.ENTITY)
     then
-        section.redact("CBI_author", 2, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        $entity.setRedaction(true);
+        setFields($entity, 2, "Address found", "Article 39(e)(2) of Regulation (EC) No 178/2002", null);
+        update($entity)
     end
 
+rule "3: Don't redact CBI Address (Non vertebrate study)"
+    no-loop true
 
-rule "3: Redact not CBI Address (Non vertebrate study)"
     when
-        Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesType("CBI_address"))
+        FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "no")
+        $entity: EntityNode(type == "CBI_address", entityType == EntityType.ENTITY)
     then
-        section.redactNot("CBI_address", 3, "Address found for non vertebrate study");
+        setFields($entity, 3, "Address found", "Article 39(e)(2) of Regulation (EC) No 178/2002", null);
-        section.ignoreRecommendations("CBI_address");
+        update($entity)
     end
 
 rule "4: Redact CBI Address (Vertebrate study)"
+    no-loop true
+
     when
-        Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesType("CBI_address"))
+        FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "yes")
+        $entity: EntityNode(type == "CBI_address", entityType == EntityType.ENTITY)
     then
-        section.redact("CBI_address", 4, "Address found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        $entity.setRedaction(true);
+        setFields($entity, 4, "Address found", "Article 39(e)(2) of Regulation (EC) No 178/2002", null);
+        update($entity)
+    end
+
+rule "5: Add FALSE_POSITIVE Entity for genitive CBI_author"
+
+    when
+        $entity: EntityNode(type == "CBI_author", anyMatch(textAfter, "['’’'ʼˈ´`‘′ʻ’']s"), redaction)
+    then
+        EntityNode entity = entityCreationService.byBoundary($entity.getBoundary(), "CBI_author", EntityType.FALSE_POSITIVE, document);
+        setFields($entity, 5, "Genitive Author", null, Engine.RULE);
+        insert(entity);
     end
 
 
-rule "5: Do not redact genitive CBI_author"
+rule "6: Create Entity from Author(s) cells in Tables with Author(s) header and add Authorname as Recommendation"
+    agenda-group "LOCAL_DICTIONARY_ADDS"
+
     when
-        Section(matchesType("CBI_author"))
+        authorCell: TableCellNode(!header, (hasHeader("Author(s)") || hasHeader("Author")), hasText())
     then
-        section.expandToFalsePositiveByRegEx("CBI_author", "['’’'ʼˈ´`‘′ʻ’']s", false, 0, dictionary);
+        EntityNode entity = entityCreationService.bySemanticNode(authorCell, "CBI_author", EntityType.ENTITY);
+        setFields(entity, 6, "Header Author(s) found", null, Engine.RULE);
+        insert(entity);
+        dictionary.addLocalDictionaryEntry("CBI_author", entity.getValue(), true);
+    end
+
+rule "7: Add CBI_author with \"et al.\" Regex"
+    agenda-group "LOCAL_DICTIONARY_ADDS"
+
+    when
+         $section: SectionNode(containsString("et al."))
+    then
+         Set<EntityNode> entities = entityCreationService.byRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, $section);
+         setFields(entities, 7, "Found by \"et al.\" regex", null, Engine.RULE);
+         entities.forEach(entity -> insert(entity));
+         entities.forEach(entity -> dictionary.addLocalDictionaryEntry("CBI_author", entity.getValue(), false));
+    end
+
+rule "8: Add recommendation for Addresses in Test Organism sections"
+
+    when
+        FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "yes")
+        $section: SectionNode(containsString("Species") && containsString("Source") && !containsString("Species:") && !containsString("Source:"))
+    then
+        Set<EntityNode> entities = entityCreationService.lineAfterString("Source",  "CBI_address", EntityType.RECOMMENDATION, $section);
+        setFields(entities, 8, "Line after \"Source\" in Test Organism Section", null, Engine.RULE);
+        entities.forEach(entity -> insert(entity));
     end
 
 
-rule "6: Redact Author(s) cells in Tables with Author(s) header (Non vertebrate study)"
+rule "9: Add recommendation for Addresses in Test Animals sections"
+
     when
-        Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && hasTableHeader("Author(s)") && !hasTableHeader("Vertebrate study Y/N"))
+        FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "yes")
+        $section: SectionNode(containsString("Species:") && containsString("Source:"))
     then
-        section.redactCell("Author(s)", 6, "CBI_author", false, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
+        Set<EntityNode> entities = entityCreationService.lineAfterString("Source:", "CBI_address", EntityType.RECOMMENDATION, $section);
+        setFields(entities, 9, "Line after \"Source:\" in Test Organism Section", null, Engine.RULE);
+        entities.forEach(entity -> insert(entity));
     end
 
-rule "7: Redact Author(s) cells in Tables with Author(s) header (Vertebrate study)"
-    when
-        Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && hasTableHeader("Author(s)") && !hasTableHeader("Vertebrate study Y/N"))
-    then
-        section.redactCell("Author(s)", 7, "CBI_author", false, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
-    end
-
-
-rule "8: Redact Author cells in Tables with Author header (Non vertebrate study)"
-    when
-        Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && hasTableHeader("Author") && !hasTableHeader("Vertebrate study Y/N"))
-    then
-        section.redactCell("Author", 8, "CBI_author", false, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
-    end
-
-rule "9: Redact Author cells in Tables with Author header (Vertebrate study)"
-    when
-        Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && hasTableHeader("Author") && !hasTableHeader("Vertebrate study Y/N"))
-    then
-        section.redactCell("Author", 9, "CBI_author", false, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
-    end
-
-
-rule "10: Redact and recommand Authors in Tables with Vertebrate study Y/N header (Non vertebrate study)"
-    when
-        Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && (rowEquals("Vertebrate study Y/N", "Y") || rowEquals("Vertebrate study Y/N", "Yes") || rowEquals("Vertebrate study Y/N", "N") || rowEquals("Vertebrate study Y/N", "No")))
-    then
-        section.redactCell("Author(s)", 10, "CBI_author", true, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
-    end
-
-rule "11: Redact and recommand Authors in Tables with Vertebrate study Y/N header (Vertebrate study)"
-    when
-        Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && (rowEquals("Vertebrate study Y/N", "Y") || rowEquals("Vertebrate study Y/N", "Yes") || rowEquals("Vertebrate study Y/N", "N") || rowEquals("Vertebrate study Y/N", "No")))
-    then
-        section.redactCell("Author(s)", 11, "CBI_author", true, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
-    end
-
-/* Syngenta specific laboratory rule */
 rule "12: Recommend CTL/BL laboratory that start with BL or CTL"
+
     when
-        Section(searchText.contains("CT") || searchText.contains("BL"))
+        $section : SectionNode(containsString("BL") || containsString("CT"))
     then
-        section.addRecommendationByRegEx("((\\b((([Cc]T(([1ILli\\/])| L|~P))|(BL))[\\. ]?([\\dA-Ziltphz~\\/.:!]| ?[\\(',][Ppi](\\(e)?|([\\(-?']\\/))+( ?[\\(\\/\\dA-Znasieg]+)?)\\b( ?\\/? ?\\d+)?)|(\\bCT[L1i]\\b))", true, 0, "CBI_address");
+        Set<EntityNode> entities = entityCreationService.byRegex("((\\b((([Cc]T(([1ILli\\/])| L|~P))|(BL))[\\. ]?([\\dA-Ziltphz~\\/.:!]| ?[\\(',][Ppi](\\(e)?|([\\(-?']\\/))+( ?[\\(\\/\\dA-Znasieg]+)?)\\b( ?\\/? ?\\d+)?)|(\\bCT[L1i]\\b))", "CBI_address", EntityType.RECOMMENDATION, $section);
-    end
+        setFields(entities, 12, "Syngenta Specific Laboratory keyword", null, Engine.RULE);
-
+        entities.forEach(entity -> insert(entity));
-rule "14: Redact and add recommendation for et al. author (Non vertebrate study)"
+        entities.forEach(entity -> dictionary.addLocalDictionaryEntry("PII", entity.getValue(), false));
-    when
-        Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("et al"))
-    then
-        section.redactAndRecommendByRegEx("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", false, 1, "CBI_author", 14, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
-    end
-
-rule "15: Redact and add recommendation for et al. author (Vertebrate study)"
-    when
-        Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("et al"))
-    then
-        section.redactAndRecommendByRegEx("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", false, 1, "CBI_author", 15, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
-    end
-
-
-rule "16: Add recommendation for Addresses in Test Organism sections"
-    when
-        Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("Species:") && searchText.contains("Source:"))
-    then
-        section.recommendLineAfter("Source:", "CBI_address");
-    end
-
-rule "17: Add recommendation for Addresses in Test Animals sections"
-    when
-        Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("Species") && searchText.contains("Source"))
-    then
-        section.recommendLineAfter("Source", "CBI_address");
-    end
-
-
-rule "18: Do not redact Names and Addresses if Published Information found"
-    when
-        Section(matchesType("published_information"))
-    then
-        section.redactNotAndReference("CBI_author","published_information", 18, "Published Information found");
-        section.redactNotAndReference("CBI_address","published_information", 18, "Published Information found");
     end
 
 
 // --------------------------------------- PII rules -------------------------------------------------------------------
 
 
-rule "19: Redacted PII Personal Identification Information (Non vertebrate study)"
+rule "10: Redacted PII Personal Identification Information (Non vertebrate study)"
-    when
+    no-loop true
-        Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesType("PII"))
-    then
-        section.redact("PII", 19, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
-    end
 
-rule "20: Redacted PII Personal Identification Information (Vertebrate study)"
     when
-        Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesType("PII"))
+        FileAttribute(label == "Vertebrate Study" , value.toLowerCase() != "yes")
+        $entity: EntityNode(type == "PII", entityType == EntityType.ENTITY)
     then
-        section.redact("PII", 20, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        $entity.setRedaction(true);
+        setFields($entity, 10, "Personal Information found", "Article 39(e)(2) of Regulation (EC) No 178/2002", null);
+        update($entity)
     end
 
 
-rule "21: Redact Emails by RegEx (Non vertebrate study)"
+
+rule "11: Redacted PII Personal Identification Information (Vertebrate study)"
+    no-loop true
+
     when
-        Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("@"))
+        FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "yes")
+        $entity: EntityNode(type == "PII", entityType == EntityType.ENTITY)
     then
-        section.redactByRegEx("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", true, 1, "PII", 21, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
+        $entity.setRedaction(true);
+        setFields($entity, 11, "Personal Information found", "Article 39(e)(2) of Regulation (EC) No 178/2002", null);
+        update($entity)
     end
 
-rule "22: Redact Emails by RegEx (Vertebrate study)"
+rule "12: Redact Emails by RegEx (Non vertebrate study)"
+
     when
-        Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("@"))
+        FileAttribute(label == "Vertebrate Study" , value.toLowerCase() != "yes")
+        $section: SectionNode(containsString("@"))
     then
-        section.redactByRegEx("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", true, 1, "PII", 22, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
+        Set<EntityNode> entities = entityCreationService.byRegex("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", "PII", EntityType.ENTITY, $section);
+        setFields(entities, 12, "Found by email regex", null, Engine.RULE);
+        entities.forEach(entity -> insert(entity));
+    end
+
+rule "13: Redact Emails by RegEx (Vertebrate study)"
+
+    when
+        FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "yes")
+        $section: SectionNode(containsString("@"))
+    then
+        Set<EntityNode> entities = entityCreationService.byRegex("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", "PII", EntityType.ENTITY, $section);
+        setFields(entities, 13, "Found by email regex", null, Engine.RULE);
+        entities.forEach(entity -> insert(entity));
     end
 
 
-rule "23: Redact contact information (Non vertebrate study)"
+rule "14: Redact line after contact information (Non vertebrate study)"
-    when
+    agenda-group "LOCAL_DICTIONARY_ADDS"
-       Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && (text.contains("Contact point:")
-            || text.contains("Phone:")
-            || text.contains("Fax:")
-            || text.contains("Tel.:")
-            || text.contains("Tel:")
-            || text.contains("E-mail:")
-            || text.contains("Email:")
-            || text.contains("e-mail:")
-            || text.contains("E-mail address:")
-            || text.contains("Contact:")
-            || text.contains("Alternative contact:")
-            || text.contains("Telephone number:")
-            || text.contains("Telephone No:")
-            || text.contains("Fax number:")
-            || text.contains("Telephone:")
-            || text.contains("Phone No.")
-            || (text.contains("No:") && text.contains("Fax"))
-            || (text.contains("Contact:") && text.contains("Tel.:"))
-            || text.contains("European contact:")
-        ))
-    then
-        section.redactLineAfter("Contact point:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
-        section.redactLineAfter("Phone:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
-        section.redactLineAfter("Fax:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
-        section.redactLineAfter("Tel.:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
-        section.redactLineAfter("Tel:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
-        section.redactLineAfter("E-mail:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
-        section.redactLineAfter("Email:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
-        section.redactLineAfter("e-mail:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
-        section.redactLineAfter("E-mail address:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
-        section.redactLineAfter("Contact:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
-        section.redactLineAfter("Alternative contact:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
-        section.redactLineAfter("Telephone number:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
-        section.redactLineAfter("Telephone No:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
-        section.redactLineAfter("Fax number:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
-        section.redactLineAfter("Telephone:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
-        section.redactLineAfter("Phone No.", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
-        section.redactBetween("No:", "Fax", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
-        section.redactBetween("Contact:", "Tel.:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
-        section.redactLineAfter("European contact:",  "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
-     end
 
-rule "24: Redact contact information (Vertebrate study)"
     when
-       Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && (text.contains("Contact point:")
+        FileAttribute(label == "Vertebrate Study" , value.toLowerCase() != "yes")
-            || text.contains("Phone:")
+        $string: String() from List.of("Contact point:",
-            || text.contains("Fax:")
+                                         "Contact:",
-            || text.contains("Tel.:")
+                                         "Alternative contact:",
-            || text.contains("Tel:")
+                                         "European contact:",
-            || text.contains("E-mail:")
+                                         "No:",
-            || text.contains("Email:")
+                                         "Contact:",
-            || text.contains("e-mail:")
+                                         "Tel.:",
-            || text.contains("E-mail address:")
+                                         "Tel:",
-            || text.contains("Contact:")
+                                         "Telephone number:",
-            || text.contains("Alternative contact:")
+                                         "Telephone No:",
-            || text.contains("Telephone number:")
+                                         "Telephone:",
-            || text.contains("Telephone No:")
+                                         "Phone No:",
-            || text.contains("Fax number:")
+                                         "Phone:",
-            || text.contains("Telephone:")
+                                         "Fax number:",
-            || text.contains("Phone No.")
+                                         "Fax:",
-            || (text.contains("No:") && text.contains("Fax"))
+                                         "E-mail:",
-            || (text.contains("Contact:") && text.contains("Tel.:"))
+                                         "Email:",
-            || text.contains("European contact:")
+                                         "e-mail:",
-        ))
+                                         "E-mail address:")
+        $section: SectionNode(containsString($string))
     then
-        section.redactLineAfter("Contact point:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
-        section.redactLineAfter("Phone:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
-        section.redactLineAfter("Fax:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
-        section.redactLineAfter("Tel.:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
-        section.redactLineAfter("Tel:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
-        section.redactLineAfter("E-mail:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
-        section.redactLineAfter("Email:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
-        section.redactLineAfter("e-mail:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
-        section.redactLineAfter("E-mail address:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
-        section.redactLineAfter("Contact:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
-        section.redactLineAfter("Alternative contact:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
-        section.redactLineAfter("Telephone number:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
-        section.redactLineAfter("Telephone No:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
-        section.redactLineAfter("Fax number:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
-        section.redactLineAfter("Telephone:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
-        section.redactLineAfter("Phone No.", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
-        section.redactBetween("No:", "Fax", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
-        section.redactBetween("Contact:", "Tel.:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
-        section.redactLineAfter("European contact:",  "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
-     end
 
-
+        Set<EntityNode> entities = new HashSet<>();
-rule "25: Redact Phone and Fax by RegEx (Non vertebrate study)"
+        entities.addAll(entityCreationService.lineAfterString($string, "PII", EntityType.ENTITY, $section));
-    when
+        setFields(entities, 14, "Found by contacts keyword", null, Engine.RULE);
-        Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && (
+        entities.forEach(entity -> insert(entity));
-               text.contains("Telephone")
+        entities.forEach(entity -> dictionary.addLocalDictionaryEntry("PII", entity.getValue(), false));
-            || text.contains("Phone")
-            || text.contains("Ph.")
-            || text.contains("Fax")
-            || text.contains("Tel")
-            || text.contains("Ter")
-            || text.contains("Cell")
-            || text.contains("Mobile")
-            || text.contains("Fel")
-            || text.contains("Fer")
-        ))
-    then
-        section.redactByRegEx("\\b(telephone|phone|fax|tel|ter|cell|mobile|fel|fer)[:.\\s]{0,3}((\\(?\\+?[0-9])(\\(?[0-9\\/.\\-\\s]+\\)?)*([0-9]+\\)?))\\b", true, 2, "PII", 25, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
-    end
-
-rule "26: Redact Phone and Fax by RegEx (Vertebrate study)"
-    when
-        Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && (
-               text.contains("Telephone")
-            || text.contains("Phone")
-            || text.contains("Ph.")
-            || text.contains("Fax")
-            || text.contains("Tel")
-            || text.contains("Ter")
-            || text.contains("Cell")
-            || text.contains("Mobile")
-            || text.contains("Fel")
-            || text.contains("Fer")
-        ))
-    then
-        section.redactByRegEx("\\b(telephone|phone|fax|tel|ter|cell|mobile|fel|fer)[:.\\s]{0,3}((\\(?\\+?[0-9])(\\(?[0-9\\/.\\-\\s]+\\)?)*([0-9]+\\)?))\\b", true, 2, "PII", 26, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
     end
 
 
-rule "27: Redact AUTHOR(S) (Non vertebrate study)"
+rule "15: Redact line after contact information (Vertebrate study)"
-    when
+    agenda-group "LOCAL_DICTIONARY_ADDS"
-        Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes")
-            && searchText.contains("AUTHOR(S):")
-            && searchText.contains("COMPLETION DATE:")
-            && !searchText.contains("STUDY COMPLETION DATE:")
-        )
-    then
-        section.redactLinesBetween("AUTHOR(S):", "COMPLETION DATE:", "PII", 27, true, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
-    end
 
-rule "28: Redact AUTHOR(S) (Vertebrate study)"
     when
-        Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes")
+        FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "yes")
-            && searchText.contains("AUTHOR(S):")
+        $string: String() from List.of("Contact point:",
-            && searchText.contains("COMPLETION DATE:")
+                                         "Contact:",
-            && !searchText.contains("STUDY COMPLETION DATE:")
+                                         "Alternative contact:",
-        )
+                                         "European contact:",
+                                         "No:",
+                                         "Contact:",
+                                         "Tel.:",
+                                         "Tel:",
+                                         "Telephone number:",
+                                         "Telephone No:",
+                                         "Telephone:",
+                                         "Phone No:",
+                                         "Phone:",
+                                         "Fax number:",
+                                         "Fax:",
+                                         "E-mail:",
+                                         "Email:",
+                                         "e-mail:",
+                                         "E-mail address:")
+        $section: SectionNode(containsString($string))
     then
-        section.redactLinesBetween("AUTHOR(S):", "COMPLETION DATE:", "PII", 28, true, "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
+        Set<EntityNode> entities = entityCreationService.lineAfterString($string, "PII", EntityType.ENTITY, $section);
+        setFields(entities, 15, "Found by contacts keyword", null, Engine.RULE);
+        entities.forEach(entity -> insert(entity));
+        entities.forEach(entity -> dictionary.addLocalDictionaryEntry("PII", entity.getValue(), false));
     end
 
 
-rule "29: Redact AUTHOR(S) (Non vertebrate study)"
+rule "16: redact line between contact keywords"
+    agenda-group "LOCAL_DICTIONARY_ADDS"
+
     when
-        Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes")
+        FileAttribute(label == "Vertebrate Study" , value.toLowerCase() != "yes")
-            && searchText.contains("AUTHOR(S):")
+        $section: SectionNode((containsString("No:") && containsString("Fax")) || (containsString("Contact:") && containsString("Tel")))
-            && searchText.contains("STUDY COMPLETION DATE:")
-        )
     then
-        section.redactLinesBetween("AUTHOR(S):", "STUDY COMPLETION DATE:", "PII", 29, true, "AUTHOR(S) was found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
+        Set<EntityNode> entities = new HashSet<>();
+        entities.addAll(entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section));
+        entities.addAll(entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section));
+        setFields(entities, 16, "Found between contacts keyword", null, Engine.RULE);
+        entities.forEach(entity -> insert(entity));
+        entities.forEach(entity -> dictionary.addLocalDictionaryEntry("PII", entity.getValue(), false));
     end
 
-rule "30: Redact AUTHOR(S) (Vertebrate study)"
+rule "17: redact line between contact keywords"
+    agenda-group "LOCAL_DICTIONARY_ADDS"
+
     when
-        Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes")
+        FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "yes")
-            && searchText.contains("AUTHOR(S):")
+        $section: SectionNode((containsString("No:") && containsString("Fax")) || (containsString("Contact:") && containsString("Tel")))
-            && searchText.contains("STUDY COMPLETION DATE:")
-        )
     then
-        section.redactLinesBetween("AUTHOR(S):", "STUDY COMPLETION DATE:", "PII", 30, true, "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
+        Set<EntityNode> entities = new HashSet<>();
+        entities.addAll(entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section));
+        entities.addAll(entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section));
+        setFields(entities, 17, "Found between contacts keyword", null, Engine.RULE);
+        entities.forEach(entity -> insert(entity));
+        entities.forEach(entity -> dictionary.addLocalDictionaryEntry("PII", entity.getValue(), false));
     end
 
+rule "18: Redact Phone and Fax by RegEx"
 
-rule "31: Redact PERFORMING LABORATORY (Non vertebrate study)"
     when
-        Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes")
+        $section: SectionNode(containsString("Contact") ||
-            && searchText.contains("PERFORMING LABORATORY:")
+            containsString("Telephone") ||
-        )
+            containsString("Phone") ||
+            containsString("Fax") ||
+            containsString("Tel") ||
+            containsString("Ter") ||
+            containsString("Mobile") ||
+            containsString("Fel") ||
+            containsString("Fer"))
     then
-        section.redactBetween("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", 31, true, "PERFORMING LABORATORY was found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
+        Set<EntityNode> entities = entityCreationService.byRegex("\\b(contact|telephone|phone|fax|tel|ter|mobile|fel|fer)[a-zA-Z\\s]{0,10}[:.\\s]{0,3}([\\+\\d\\(][\\s\\d\\(\\)\\-\\/\\.]{4,100}\\d)\\b", "PII", EntityType.ENTITY, $section);
-        section.redactNot("CBI_address", 31, "Performing laboratory found for non vertebrate study");
+        setFields(entities, 18, "Found by Phone and Fax regex", null, Engine.RULE);
-    end
+        entities.forEach(entity -> insert(entity));
-
-rule "32: Redact PERFORMING LABORATORY (Vertebrate study)"
-    when
-        Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes")
-            && searchText.contains("PERFORMING LABORATORY:"))
-    then
-        section.redactBetween("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", 32, true, "PERFORMING LABORATORY was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
-    end
-
-
-// --------------------------------------- other rules -------------------------------------------------------------------
-
-rule "33: Purity Hint"
-    when
-        Section(searchText.toLowerCase().contains("purity"))
-    then
-	    section.addHintAnnotationByRegEx("(purity ?( of|\\(.{1,20}\\))?( ?:)?) .{0,5}[\\d\\.]+( .{0,4}\\.)? ?%", true, 1, "hint_only",dictionary);
-    end
-
-
-rule "34: Ignore dossier_redaction entries if confidentiality is not 'confidential'"
-    when
-        Section(!fileAttributeByLabelEqualsIgnoreCase("Confidentiality","confidential") && matchesType("dossier_redaction"));
-    then
-        section.ignore("dossier_redaction");
-    end
-
-
-rule "35: Redact signatures (Non vertebrate study)"
-    when
-        Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesImageType("signature"))
-    then
-        section.redactImage("signature", 35, "Signature found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
-    end
-
-rule "36: Redact signatures (Vertebrate study)"
-    when
-        Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesImageType("signature"))
-    then
-        section.redactImage("signature", 36, "Signature found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
-    end
-
-
-rule "43: Redact Logos (Vertebrate study)"
-    when
-        Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesImageType("logo"))
-    then
-        section.redactImage("logo", 43, "Logo found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
     end

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/test_rules.drl

@@ -1,35 +0,0 @@
-package drools
-
-import static java.lang.String.format;
-
-import java.util.List;
-import java.util.LinkedList;
-import java.util.HashSet;
-
-import com.iqser.red.service.redaction.v1.server.document.graph.*
-import com.iqser.red.service.redaction.v1.server.redaction.model.*;
-import com.iqser.red.service.redaction.v1.model.FileAttribute;
-import com.iqser.red.service.redaction.v1.model.Engine;
-import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntry;
-import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUtils;
-import java.util.Set;
-
-global DocumentGraph document
-
-rule "dummy"
-    when
-
-    then
-        System.out.println("This actually works");
-    end
-
-rule "1: Redact CBI_author"
-    when
-        //FileAttribute(label == "Vertebrate Study" && (value == "Yes"))
-        $entity: EntityNode(type == "CBI_author")
-    then
-        System.out.println("Entity found");
-        modify($entity){
-            $entity.setRedact(true)
-        }
-    end