RED-6093: Prototype document structure

*refactored File Structure
*started refactor of original rules

redaction-service-v1/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <artifactId>platform-dependency</artifactId>
         <groupId>com.iqser.red</groupId>
-        <version>1.13.0</version>
+        <version>1.17.0</version>
         <relativePath/>
     </parent>
     <modelVersion>4.0.0</modelVersion>
@@ -32,7 +32,7 @@
             <dependency>
                 <groupId>com.iqser.red</groupId>
                 <artifactId>platform-commons-dependency</artifactId>
-                <version>1.20.0</version>
+                <version>1.21.0</version>
                 <scope>import</scope>
                 <type>pom</type>
             </dependency>

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/model/Document.java

@@ -14,7 +14,7 @@ import lombok.NoArgsConstructor;
 public class Document {
 
     private List<Page> pages = new ArrayList<>();
-    private List<Paragraph> paragraphs = new ArrayList<>();
+    private List<Section> sections = new ArrayList<>();
     private List<Header> headers = new ArrayList<>();
     private List<Footer> footers = new ArrayList<>();
     private List<UnclassifiedText> unclassifiedTexts = new ArrayList<>();

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/model/Section.java

@@ -1,7 +1,13 @@
 package com.iqser.red.service.redaction.v1.server.classification.model;
 
+import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus;
+import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.IdRemoval;
+import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualImageRecategorization;
+import com.iqser.red.service.redaction.v1.server.redaction.model.Entities;
+import com.iqser.red.service.redaction.v1.server.redaction.model.Image;
 import com.iqser.red.service.redaction.v1.server.redaction.model.PdfImage;
 import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText;
+import com.iqser.red.service.redaction.v1.server.redaction.utils.IdBuilder;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;
 
@@ -10,10 +16,11 @@ import lombok.NoArgsConstructor;
 
 import java.util.ArrayList;
 import java.util.List;
+import java.util.stream.Collectors;
 
 @Data
 @NoArgsConstructor
-public class Paragraph implements Comparable {
+public class Section implements Comparable {
 
     private List<AbstractTextContainer> pageBlocks = new ArrayList<>();
     private List<PdfImage> images = new ArrayList<>();
@@ -62,4 +69,9 @@ public class Paragraph implements Comparable {
         return 0;
     }
 
+
+
+
+
+
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/model/SectionText.java

@@ -1,11 +1,19 @@
 package com.iqser.red.service.redaction.v1.server.classification.model;
 
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
 import com.dslplatform.json.CompiledJson;
 import com.dslplatform.json.JsonAttribute;
 import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.iqser.red.service.redaction.v1.model.SectionArea;
 import com.iqser.red.service.redaction.v1.server.redaction.model.CellValue;
 import com.iqser.red.service.redaction.v1.server.redaction.model.Image;
+import com.iqser.red.service.redaction.v1.server.redaction.model.Paragraph;
 import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText;
 
 import lombok.AllArgsConstructor;
@@ -13,8 +21,6 @@ import lombok.Builder;
 import lombok.Data;
 import lombok.NoArgsConstructor;
 
-import java.util.*;
-
 @Data
 @Builder
 @CompiledJson
@@ -27,6 +33,7 @@ public class SectionText {
 
     private boolean isTable;
     private String headline;
+    List<Paragraph> paragraphs;
 
     @Builder.Default
     private List<SectionArea> sectionAreas = new ArrayList<>();

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/model/TextBlock.java

@@ -29,6 +29,8 @@ public class TextBlock extends AbstractTextContainer {
     @JsonIgnore
     private int rotation;
 
+    private int indexOnPage;
+
     @JsonIgnore
     private String mostPopularWordFont;
 
@@ -184,8 +186,8 @@ public class TextBlock extends AbstractTextContainer {
     }
 
 
-    public TextBlock(float minX, float maxX, float minY, float maxY, List<TextPositionSequence> sequences, int rotation) {
+    public TextBlock(float minX, float maxX, float minY, float maxY, List<TextPositionSequence> sequences, int rotation, int indexOnPage) {
-
+        this.indexOnPage = indexOnPage;
         this.minX = minX;
         this.maxX = maxX;
         this.minY = minY;
@@ -248,7 +250,7 @@ public class TextBlock extends AbstractTextContainer {
 
     public TextBlock copy() {
 
-        return new TextBlock(minX, maxX, minY, maxY, sequences, rotation);
+        return new TextBlock(minX, maxX, minY, maxY, sequences, rotation, indexOnPage);
     }
 
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/service/BlockificationService.java

@@ -30,13 +30,15 @@ public class BlockificationService {
      * This method is building blocks by expanding the minX/maxX and minY/maxY value on each word that is not split by the conditions.
      * This method must use text direction adjusted postions (DirAdj). Where {0,0} is on the upper left. Never try to change this!
      * Rulings (Table lines) must be adjusted to the text directions as well, when checking if a block is split by a ruling.
-     * @param textPositions The words of a page.
+     *
+     * @param textPositions         The words of a page.
      * @param horizontalRulingLines Horizontal table lines.
-     * @param verticalRulingLines Vertical table lines.
+     * @param verticalRulingLines   Vertical table lines.
      * @return Page object that contains the Textblock and text statistics.
      */
     public Page blockify(List<TextPositionSequence> textPositions, List<Ruling> horizontalRulingLines, List<Ruling> verticalRulingLines) {
 
+        int indexOnPage = 0;
         List<TextPositionSequence> chunkWords = new ArrayList<>();
         List<AbstractTextContainer> chunkBlockList1 = new ArrayList<>();
 
@@ -62,7 +64,9 @@ public class BlockificationService {
                     prevOrientation = chunkBlockList1.get(chunkBlockList1.size() - 1).getOrientation();
                 }
 
-                TextBlock cb1 = buildTextBlock(chunkWords);
+                TextBlock cb1 = buildTextBlock(chunkWords, indexOnPage);
+                indexOnPage++;
+
                 chunkBlockList1.add(cb1);
                 chunkWords = new ArrayList<>();
 
@@ -102,7 +106,7 @@ public class BlockificationService {
             }
         }
 
-        TextBlock cb1 = buildTextBlock(chunkWords);
+        TextBlock cb1 = buildTextBlock(chunkWords, indexOnPage);
         if (cb1 != null) {
             chunkBlockList1.add(cb1);
         }
@@ -163,7 +167,7 @@ public class BlockificationService {
     }
 
 
-    private TextBlock buildTextBlock(List<TextPositionSequence> wordBlockList) {
+    private TextBlock buildTextBlock(List<TextPositionSequence> wordBlockList, int indexOnPage) {
 
         TextBlock textBlock = null;
 
@@ -182,7 +186,13 @@ public class BlockificationService {
             styleFrequencyCounter.add(wordBlock.getFontStyle());
 
             if (textBlock == null) {
-                textBlock = new TextBlock(wordBlock.getMinXDirAdj(), wordBlock.getMaxXDirAdj(), wordBlock.getMinYDirAdj(), wordBlock.getMaxYDirAdj(), wordBlockList, wordBlock.getRotation());
+                textBlock = new TextBlock(wordBlock.getMinXDirAdj(),
+                        wordBlock.getMaxXDirAdj(),
+                        wordBlock.getMinYDirAdj(),
+                        wordBlock.getMaxYDirAdj(),
+                        wordBlockList,
+                        wordBlock.getRotation(),
+                        indexOnPage);
             } else {
                 TextBlock spatialEntity = textBlock.union(wordBlock);
                 textBlock.resize(spatialEntity.getMinX(), spatialEntity.getMinY(), spatialEntity.getWidth(), spatialEntity.getHeight());
@@ -213,10 +223,38 @@ public class BlockificationService {
                                     List<Ruling> horizontalRulingLines,
                                     List<Ruling> verticalRulingLines) {
 
-        return isSplitByRuling(maxX, minY, word.getMinXDirAdj(), word.getMinYDirAdj(), verticalRulingLines, word.getDir().getDegrees(), word.getPageWidth(), word.getPageHeight()) //
+        return isSplitByRuling(maxX,
-                || isSplitByRuling(minX, minY, word.getMinXDirAdj(), word.getMaxYDirAdj(), horizontalRulingLines, word.getDir().getDegrees(), word.getPageWidth(), word.getPageHeight()) //
+                minY,
-                || isSplitByRuling(maxX, minY, word.getMinXDirAdj(), word.getMinYDirAdj(), horizontalRulingLines, word.getDir().getDegrees(), word.getPageWidth(), word.getPageHeight()) //
+                word.getMinXDirAdj(),
-                || isSplitByRuling(minX, minY, word.getMinXDirAdj(), word.getMaxYDirAdj(), verticalRulingLines, word.getDir().getDegrees(), word.getPageWidth(), word.getPageHeight()); //
+                word.getMinYDirAdj(),
+                verticalRulingLines,
+                word.getDir().getDegrees(),
+                word.getPageWidth(),
+                word.getPageHeight()) //
+                || isSplitByRuling(minX,
+                minY,
+                word.getMinXDirAdj(),
+                word.getMaxYDirAdj(),
+                horizontalRulingLines,
+                word.getDir().getDegrees(),
+                word.getPageWidth(),
+                word.getPageHeight()) //
+                || isSplitByRuling(maxX,
+                minY,
+                word.getMinXDirAdj(),
+                word.getMinYDirAdj(),
+                horizontalRulingLines,
+                word.getDir().getDegrees(),
+                word.getPageWidth(),
+                word.getPageHeight()) //
+                || isSplitByRuling(minX,
+                minY,
+                word.getMinXDirAdj(),
+                word.getMaxYDirAdj(),
+                verticalRulingLines,
+                word.getDir().getDegrees(),
+                word.getPageWidth(),
+                word.getPageHeight()); //
     }
 
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/data/AtomicTextBlockData.java

@@ -0,0 +1,21 @@
+package com.iqser.red.service.redaction.v1.server.document.data;
+
+import lombok.AccessLevel;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.experimental.FieldDefaults;
+
+@Data
+@Builder
+@AllArgsConstructor
+@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
+public class AtomicTextBlockData {
+    Long id;
+    String searchText;
+    int start;
+    int end;
+    int[] lineBreaks;
+    int[] stringIdxToPositionIdx;
+    float[][] positions;
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/data/DocumentData.java

@@ -0,0 +1,19 @@
+package com.iqser.red.service.redaction.v1.server.document.data;
+
+import java.util.List;
+
+import lombok.AccessLevel;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.experimental.FieldDefaults;
+
+@Data
+@Builder
+@AllArgsConstructor
+@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
+public class DocumentData {
+    List<PageData> pages;
+    List<AtomicTextBlockData> atomicTextBlocks;
+    TableOfContentsData tableOfContents;
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/data/PageData.java

@@ -0,0 +1,20 @@
+package com.iqser.red.service.redaction.v1.server.document.data;
+
+import lombok.AccessLevel;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.experimental.FieldDefaults;
+
+@Data
+@Builder
+@AllArgsConstructor
+@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
+public class PageData {
+    int number;
+    int height;
+    int width;
+
+    Long header;
+    Long footer;
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/data/TableOfContentsData.java

@@ -0,0 +1,52 @@
+package com.iqser.red.service.redaction.v1.server.document.data;
+
+import static java.lang.String.format;
+
+import java.util.Arrays;
+import java.util.List;
+
+import javax.management.openmbean.InvalidKeyException;
+
+import com.iqser.red.service.redaction.v1.server.document.graph.nodes.NodeType;
+
+import lombok.AccessLevel;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.experimental.FieldDefaults;
+
+@Data
+@Builder
+@AllArgsConstructor
+@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
+public class TableOfContentsData {
+
+    List<EntryData> entries;
+
+
+    public EntryData get(String tocId) {
+
+        List<Integer> ids = getIds(tocId);
+        if (ids.size() < 1) {
+            throw new InvalidKeyException(format("Section Identifier: \"%s\" is not valid.", tocId));
+        }
+        EntryData entry = entries.get(ids.get(0));
+        for (int id : ids.subList(1, ids.size())) {
+            entry = entry.subEntries().get(id);
+        }
+        return entry;
+    }
+
+
+    private static List<Integer> getIds(String idsAsString) {
+
+        return Arrays.stream(idsAsString.split("\\.")).map(Integer::valueOf).toList();
+    }
+
+
+    @Builder
+    public record EntryData(String tocId, List<EntryData> subEntries, NodeType type, Long atomicTextBlock, Long page, int numberOnPage) {
+
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/Boundary.java

@@ -0,0 +1,88 @@
+package com.iqser.red.service.redaction.v1.server.document.graph;
+
+import static java.lang.String.format;
+
+import lombok.Setter;
+
+@Setter
+public class Boundary {
+
+    private int start;
+    private int end;
+
+
+    public Boundary(int start, int end) {
+
+        assert start <= end;
+        this.start = start;
+        this.end = end;
+    }
+
+
+    public int length() {
+
+        return end - start;
+    }
+
+
+    public int start() {
+
+        return start;
+    }
+
+
+    public int end() {
+
+        return end;
+    }
+
+
+    public boolean contains(Boundary boundary) {
+
+        return start <= boundary.start() && boundary.end() <= end;
+    }
+
+
+    public boolean containedBy(Boundary boundary) {
+
+        return boundary.start() <= start && end <= boundary.end();
+    }
+
+
+    public boolean contains(int start, int end) {
+
+        if (start > end) {
+            throw new UnsupportedOperationException("start > end");
+        }
+        return this.start <= start && end <= this.end;
+    }
+
+
+    public boolean containedBy(int start, int end) {
+
+        if (start > end) {
+            throw new UnsupportedOperationException("start > end");
+        }
+        return start <= this.start && this.end <= end;
+    }
+
+
+    public boolean contains(int index) {
+
+        return start <= index && index < end;
+    }
+
+
+    public boolean intersects(Boundary boundary) {
+
+        return contains(boundary.start()) || contains(boundary.end());
+    }
+
+
+    @Override
+    public String toString() {
+
+        return format("Boundary [%d|%d)", start, end);
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentGraph.java

@@ -0,0 +1,96 @@
+package com.iqser.red.service.redaction.v1.server.document.graph;
+
+import static com.iqser.red.service.redaction.v1.server.document.services.EntityEnrichmentUtility.enrichEntity;
+
+import java.util.List;
+import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import com.iqser.red.service.redaction.v1.server.document.graph.nodes.DocumentGraphNode;
+import com.iqser.red.service.redaction.v1.server.document.graph.nodes.EntityNode;
+import com.iqser.red.service.redaction.v1.server.document.graph.nodes.PageNode;
+import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SectionNode;
+import com.iqser.red.service.redaction.v1.server.document.graph.textblock.AtomicTextBlock;
+import com.iqser.red.service.redaction.v1.server.document.graph.textblock.ConcatenatedTextBlock;
+import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock;
+import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlockCollector;
+import com.iqser.red.service.redaction.v1.server.exception.NotFoundException;
+import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType;
+
+import lombok.AccessLevel;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.experimental.FieldDefaults;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+@Data
+@Builder
+@AllArgsConstructor
+@FieldDefaults(level = AccessLevel.PRIVATE)
+public class DocumentGraph {
+
+    List<SectionNode> sections;
+    List<PageNode> pages;
+    TableOfContents tableOfContents;
+    Integer numberOfPages;
+    TextBlock text;
+
+
+    public ConcatenatedTextBlock buildTextBlock() {
+
+        return streamAtomicTextBlocksInOrder().collect(new TextBlockCollector());
+    }
+
+
+    public Stream<AtomicTextBlock> streamAtomicTextBlocksInOrder() {
+
+        return Stream.concat(//
+                streamAllNodes().filter(DocumentGraphNode::isTerminal).map(DocumentGraphNode::getAtomicTextBlock),//
+                Stream.concat(//
+                        pages.stream().map(PageNode::getHeader),//
+                        pages.stream().map(PageNode::getFooter)));
+    }
+
+
+    public EntityNode createAndAddEntity(Boundary boundary, String type, EntityType entityType) {
+
+        EntityNode entity = EntityNode.initialEntityNode(boundary, type, entityType);
+        addEntityToGraphAndSetFields(entity);
+        return entity;
+    }
+
+
+    public void addEntityToGraphAndSetFields(EntityNode entity) {
+
+        try {
+            boolean inserted = streamAllNodes().anyMatch(node -> node.addEntityAndSetFieldsIfStartIndexContained(entity));
+        } catch (NotFoundException e) {
+            enrichEntity(entity, text);
+            log.warn("Entity \"{}\" with {} is in between two main sections and will be removed!", entity.getValue(), entity.getBoundary());
+            entity.removeFromGraph();
+        }
+    }
+
+
+    public Set<EntityNode> getEntities() {
+
+        return streamAllNodes().filter(DocumentGraphNode::isTerminal).map(DocumentGraphNode::getEntities).flatMap(List::stream).collect(Collectors.toSet());
+    }
+
+
+    private Stream<DocumentGraphNode> streamAllNodes() {
+
+        return tableOfContents.streamEntriesInOrder().map(TableOfContents.Entry::node);
+    }
+
+
+    @Override
+    public String toString() {
+
+        return text.toString();
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/TableOfContents.java

@@ -0,0 +1,113 @@
+package com.iqser.red.service.redaction.v1.server.document.graph;
+
+import static java.lang.String.format;
+
+import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.stream.Stream;
+
+import javax.management.openmbean.InvalidKeyException;
+
+import com.google.common.hash.Hashing;
+import com.iqser.red.service.redaction.v1.server.document.graph.nodes.DocumentGraphNode;
+import com.iqser.red.service.redaction.v1.server.document.graph.nodes.NodeType;
+
+import lombok.Data;
+
+@Data
+public class TableOfContents {
+
+    List<Entry> entries;
+
+
+    public TableOfContents() {
+
+        entries = new LinkedList<>();
+    }
+
+
+    public String createNewEntryAndReturnId(NodeType nodeType, String summary, DocumentGraphNode node) {
+
+        String id = String.format("%d", entries.size());
+        entries.add(new Entry(nodeType, id, summary, new LinkedList<>(), node));
+        return id;
+    }
+
+
+    public String createNewChildEntryAndReturnId(String parentId, NodeType nodeType, String summary, DocumentGraphNode node) {
+
+        Entry parent = getEntryById(parentId);
+        String childId = parentId + String.format(".%d", parent.children().size());
+        parent.children().add(new Entry(nodeType, childId, summary, new LinkedList<>(), node));
+        return childId;
+    }
+
+
+    public Entry getEntryById(String parentId) {
+
+        List<Integer> ids = getIds(parentId);
+        if (ids.size() < 1) {
+            throw new InvalidKeyException(format("Section Identifier: \"%s\" is not valid.", parentId));
+        }
+        Entry entry = entries.get(ids.get(0));
+        for (int id : ids.subList(1, ids.size())) {
+            entry = entry.children().get(id);
+        }
+        return entry;
+    }
+
+
+    @Override
+    public String toString() {
+
+        return String.join("\n", streamEntriesInOrder().map(Entry::toString).toList());
+    }
+
+
+    public String toString(String id) {
+
+        return String.join("\n", streamSubEntriesInOrder(id).map(Entry::toString).toList());
+    }
+
+
+    public Stream<Entry> streamEntriesInOrder() {
+
+        return entries.stream().flatMap(TableOfContents::flatten);
+    }
+
+
+    public Stream<Entry> streamSubEntriesInOrder(String parentId) {
+
+        return Stream.of(getEntryById(parentId)).flatMap(TableOfContents::flatten);
+    }
+
+
+    private static List<Integer> getIds(String idsAsString) {
+
+        return Arrays.stream(idsAsString.split("\\.")).map(Integer::valueOf).toList();
+    }
+
+
+    private static Stream<Entry> flatten(Entry entry) {
+
+        return Stream.concat(Stream.of(entry), entry.children().stream().flatMap(TableOfContents::flatten));
+    }
+
+
+    public record Entry(NodeType type, String id, String summary, List<Entry> children, DocumentGraphNode node) {
+
+        @Override
+        public String toString() {
+
+            return id + ": " + type + ".: " + summary;
+        }
+
+
+        @Override
+        public int hashCode() {
+            return Hashing.murmur3_32_fixed().hashString(type + id + summary + children.hashCode(), StandardCharsets.UTF_8).hashCode();
+        }
+    }
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/DocumentGraphNode.java

@@ -0,0 +1,219 @@
+package com.iqser.red.service.redaction.v1.server.document.graph.nodes;
+
+import static com.iqser.red.service.redaction.v1.server.document.services.EntityEnrichmentUtility.enrichEntity;
+import static java.lang.String.format;
+
+import java.util.Set;
+import java.util.stream.Stream;
+
+import com.iqser.red.service.redaction.v1.server.document.graph.Boundary;
+import com.iqser.red.service.redaction.v1.server.document.graph.textblock.AtomicTextBlock;
+import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock;
+import com.iqser.red.service.redaction.v1.server.exception.NotFoundException;
+import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType;
+
+public interface DocumentGraphNode {
+
+    /**
+     * Searches all Nodes located underneath this Node in the TableOfContents and concatenates their AtomicTextBlocks into a single TextBlockEntity.
+     * So, for a Section all AtomicTextBlocks of Subsections, Paragraphs, and Tables are concatenated into a single TextBlockEntity
+     *
+     * @return TextBlock containing all AtomicTextBlocks that are located under this Node.
+     */
+    TextBlock buildTextBlock();
+
+
+    /**
+     * Any Node maintains its own Set of Entities.
+     * This Set contains all Entities, whose first index is located in any of the AtomicTextBlocks underneath this Node.
+     *
+     * @return Set of all Entities associated with this Node
+     */
+    Set<EntityNode> getEntities();
+
+
+    /**
+     * Returns the PageNode associated with this Node.
+     * If the node has more than one PageNode associated, it returns the PageNode with the lowest number.
+     * For example a section might span multiple pages, it then returns the page where the section starts.
+     *
+     * @return PageNode representing the first page on which the Node is located in the document
+     */
+    PageNode getPage();
+
+
+    /**
+     * Any Node except the First level of Sections, Header, Footer, and Pages have a direct Parent.
+     * For example a Paragraph has a parent Section, a Table Cell has a parent Table, etc...
+     * hasParent() may be used to check whether a parent is present.
+     *
+     * @return Node that represents the Parent or null, if no parent is present.
+     */
+    DocumentGraphNode getParent();
+
+
+    Stream<DocumentGraphNode> streamAllSubNodes();
+
+
+    /**
+     * Each AtomicTextBlock has a number assigned per page, this returns the number of the first AtomicTextBlock underneath this node
+     *
+     * @return Integer representing the number on the page
+     */
+    Integer getNumberOnPage();
+
+
+    /**
+     *
+     * @return the fist headline whent traversing the tree upwards
+     */
+    default CharSequence getHeadline() {
+        return getParent().getHeadline();
+    }
+
+
+    /**
+     * By default, a parent is always present, this needs to be overwritten for Headers, Footers, Pages, and Sections.
+     *
+     * @return boolean, indicating whether a parent is present.
+     */
+    default boolean hasParent() {
+
+        return true;
+    }
+
+
+    /**
+     * by default a Node does not have direct access to an AtomicTextBlock
+     *
+     * @return boolean, indicating if a Node has direct access to an AtomicTextBlock
+     */
+    default boolean isTerminal() {
+
+        return false;
+    }
+
+
+    /**
+     * by default a Node does not have direct access to an AtomicTextBlock, this method throws a UnsupportedOperationException if not overridden.
+     *
+     * @return AtomicTextBlock
+     */
+    default AtomicTextBlock getAtomicTextBlock() {
+
+        throw new UnsupportedOperationException("Only terminal Nodes have access to AtomicTextBlocks!");
+    }
+
+
+    /**
+     * creates an EntityNode with only initial values set, inserts it into the subgraph contained by this node and sets the inferrable fields.
+     * Throws NotFoundException and removes the entity if the provided boundary could not be found in the subgraph.
+     *
+     * @param boundary      start and end indices in String coordinates of the entity to be created
+     * @param type       type of the entity to be created
+     * @param entityType entityType of the entity to be created
+     * @return the newly created and inserted EntityNode with all fields set.
+     */
+    default EntityNode createAndAddEntity(Boundary boundary, String type, EntityType entityType) {
+
+        EntityNode entity = EntityNode.initialEntityNode(boundary, type, entityType);
+        addEntityToNodeAndSetFields(entity);
+        return entity;
+    }
+
+
+    /**
+     * searches for the first terminal Node containing the start index of the entity to be inserted.
+     * Catches NotFoundException to remove the EntityNode from the graph, then rethrows it
+     *
+     * @param entity newly created EntityNode with only initial values set
+     */
+    default void addEntityToNodeAndSetFields(EntityNode entity) {
+
+        try {
+            streamAllSubNodes().anyMatch(node -> node.addEntityAndSetFieldsIfStartIndexContained(entity));
+        } catch (NotFoundException e) {
+            entity.removeFromGraph();
+            throw new RuntimeException(e);
+        }
+    }
+
+
+    /**
+     * If this Node's AtomicTextBlock contains the start index of the entity, the entity's position is read from the AtomicTextBlock.
+     * If the position can not be fully read from the AtomicTextBlock, it recursively looks in the TextBlocks of the parents until all positions are found.
+     * Further, the function throws NotFoundException if no parent contains all positions.
+     * This occurs, when the Entity is in between Nodes that do not share a parent, e.g. main sections.
+     * Finally, the function adds the Entity to its own list of Entities and to every parents' list recursively.
+     *
+     * @param entity The entity to be added to the graph
+     * @return true, if the entity has been added successfully.
+     * false, if the entity's start index is not contained or the Node doesn't have an AtomicTextBlock
+     */
+    default boolean addEntityAndSetFieldsIfStartIndexContained(EntityNode entity) {
+
+        if (!isTerminal()) {
+            return false;
+        }
+
+        AtomicTextBlock atomicTextBlock = getAtomicTextBlock();
+        if (atomicTextBlock.containsIndex(entity.getBoundary().start())) {
+
+            entity.addContainingNode(this);
+
+            getEntities().add(entity);
+
+            addEntityToPage(entity);
+            addEntityToParents(entity);
+            setFields(entity, atomicTextBlock);
+            return true;
+        }
+
+        return false;
+    }
+
+
+    private void addEntityToPage(EntityNode entity) {
+
+        getPage().getEntities().add(entity);
+        entity.setPage(getPage());
+    }
+
+
+    private void setFields(EntityNode entity, AtomicTextBlock atomicTextBlock) {
+
+        if (atomicTextBlock.containsBoundary(entity.getBoundary())) {
+            enrichEntity(entity, atomicTextBlock);
+        } else {
+            this.setFieldsFromParents(this, entity);
+        }
+    }
+
+
+    private void addEntityToParents(EntityNode entity) {
+
+        DocumentGraphNode node = this;
+        while (node.hasParent()) {
+            node = node.getParent();
+            node.getEntities().add(entity);
+            entity.addContainingNode(node);
+        }
+    }
+
+
+    private void setFieldsFromParents(DocumentGraphNode node, EntityNode entity) {
+
+        if (node.hasParent()) {
+            DocumentGraphNode parent = node.getParent();
+            TextBlock textBlock = parent.buildTextBlock();
+            if (textBlock.containsBoundary(entity.getBoundary())) {
+                enrichEntity(entity, textBlock);
+                return;
+            } else {
+                setFieldsFromParents(parent, entity);
+            }
+        }
+        throw new NotFoundException(format("Position could not be found for Entity %s", entity.toString()));
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/EntityNode.java

@@ -0,0 +1,103 @@
+package com.iqser.red.service.redaction.v1.server.document.graph.nodes;
+
+import java.awt.geom.Rectangle2D;
+import java.nio.charset.StandardCharsets;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+import com.google.common.hash.Hashing;
+import com.iqser.red.service.redaction.v1.model.Engine;
+import com.iqser.red.service.redaction.v1.server.document.graph.Boundary;
+import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
+import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType;
+
+import lombok.AccessLevel;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.experimental.FieldDefaults;
+
+@Data
+@Builder
+@AllArgsConstructor
+@FieldDefaults(level = AccessLevel.PRIVATE)
+public class EntityNode {
+
+    public static EntityNode initialEntityNode(Boundary boundary, String type, EntityType entityType) {
+
+        return EntityNode.builder().type(type).entityType(entityType).boundary(boundary).build();
+    }
+
+
+    // initial values
+    Boundary boundary;
+    String type;
+    EntityType entityType;
+
+    @Builder.Default
+    boolean redaction = false;
+    @Builder.Default
+    boolean falsePositive = false;
+    @Builder.Default
+    boolean removed = false;
+    @Builder.Default
+    boolean ignored = false;
+    @Builder.Default
+    boolean resized = false;
+    @Builder.Default
+    boolean skipRemoveEntitiesContainedInLarger = false;
+    @Builder.Default
+    boolean isDictionaryEntry = false;
+    @Builder.Default
+    Set<Engine> engines = new HashSet<>();
+    @Builder.Default
+    Set<Entity> references = new HashSet<>();
+    @Builder.Default
+    int matchedRule = -1;
+    @Builder.Default
+    String redactionReason = "";
+    @Builder.Default
+    String legalBasis = "";
+
+    // inferrable from graph
+    String value;
+    CharSequence textBefore;
+    CharSequence textAfter;
+    PageNode page;
+    List<Rectangle2D> positions;
+    @Builder.Default
+    Set<DocumentGraphNode> containingNodes = new HashSet<>();
+
+
+    public void addContainingNode(DocumentGraphNode containingNode) {
+
+        containingNodes.add(containingNode);
+    }
+
+
+    public void removeFromGraph() {
+
+        getContainingNodes().forEach(node -> node.getEntities().remove(this));
+        getPage().getEntities().remove(this);
+        setRemoved(true);
+    }
+
+
+    @Override
+    public int hashCode() {
+
+        var sb = new StringBuilder();
+        sb.append(value);
+        sb.append(boundary.start());
+        sb.append(page.getNumber());
+        positions.forEach(r -> {
+            sb.append(r.getMinX());
+            sb.append(r.getMinY());
+            sb.append(r.getWidth());
+            sb.append(r.getHeight());
+        });
+        return Hashing.murmur3_128().hashString(sb.toString(), StandardCharsets.UTF_8).hashCode();
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/NodeType.java

@@ -0,0 +1,28 @@
+package com.iqser.red.service.redaction.v1.server.document.graph.nodes;
+
+public enum NodeType {
+    SECTION {
+        public String toString() {
+
+            return "Section";
+        }
+    },
+    PARAGRAPH {
+        public String toString() {
+
+            return "Paragraph";
+        }
+    },
+    TABLE {
+        public String toString() {
+
+            return "Table";
+        }
+    },
+    TABLE_CELL {
+        public String toString() {
+
+            return "Cell";
+        }
+    }
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/PageNode.java

@@ -0,0 +1,85 @@
+package com.iqser.red.service.redaction.v1.server.document.graph.nodes;
+
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.stream.Stream;
+
+import com.iqser.red.service.redaction.v1.server.document.graph.textblock.AtomicTextBlock;
+import com.iqser.red.service.redaction.v1.server.document.graph.textblock.ConcatenatedTextBlock;
+import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlockCollector;
+
+import lombok.AccessLevel;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.EqualsAndHashCode;
+import lombok.experimental.FieldDefaults;
+
+@Data
+@Builder
+@AllArgsConstructor
+@FieldDefaults(level = AccessLevel.PRIVATE)
+public class PageNode implements DocumentGraphNode{
+
+    Integer number;
+    Integer height;
+    Integer width;
+    List<DocumentGraphNode> mainBody;
+    AtomicTextBlock header;
+    AtomicTextBlock footer;
+
+    @Builder.Default
+    @EqualsAndHashCode.Exclude
+    Set<EntityNode> entities = new HashSet<>();
+
+
+
+    public ConcatenatedTextBlock buildTextBlock() {
+
+        return mainBody.stream().filter(DocumentGraphNode::isTerminal).map(DocumentGraphNode::getAtomicTextBlock).collect(new TextBlockCollector());
+    }
+
+
+    @Override
+    public PageNode getPage() {
+
+        return this;
+    }
+
+
+    @Override
+    public DocumentGraphNode getParent() {
+
+        return null;
+    }
+
+
+    @Override
+    public boolean hasParent() {
+
+        return false;
+    }
+
+
+    @Override
+    public Stream<DocumentGraphNode> streamAllSubNodes() {
+
+        return mainBody.stream();
+    }
+
+
+    @Override
+    public Integer getNumberOnPage() {
+
+        return 0;
+    }
+
+
+    @Override
+    public String toString() {
+
+        return header.getSearchText() + buildTextBlock().toString() + footer.getSearchText();
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/ParagraphNode.java

@@ -0,0 +1,70 @@
+package com.iqser.red.service.redaction.v1.server.document.graph.nodes;
+
+import java.util.HashSet;
+import java.util.Set;
+import java.util.stream.Stream;
+
+import com.iqser.red.service.redaction.v1.server.document.graph.textblock.AtomicTextBlock;
+
+import lombok.AccessLevel;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.EqualsAndHashCode;
+import lombok.experimental.FieldDefaults;
+
+@Data
+@Builder
+@AllArgsConstructor
+@FieldDefaults(level = AccessLevel.PRIVATE)
+public class ParagraphNode implements DocumentGraphNode {
+
+    String tocId;
+    Integer numberOnPage;
+    Integer numberInSection;
+
+    @EqualsAndHashCode.Exclude
+    SectionNode parentSection;
+    @EqualsAndHashCode.Exclude
+    PageNode page;
+    AtomicTextBlock atomicTextBlock;
+
+    @Builder.Default
+    @EqualsAndHashCode.Exclude
+    Set<EntityNode> entities = new HashSet<>();
+
+
+    @Override
+    public AtomicTextBlock buildTextBlock() {
+
+        return atomicTextBlock;
+    }
+
+
+    @Override
+    public DocumentGraphNode getParent() {
+
+        return parentSection;
+    }
+
+
+    @Override
+    public boolean isTerminal() {
+
+        return true;
+    }
+
+
+    @Override
+    public String toString() {
+
+        return tocId + ": " + atomicTextBlock.toString();
+    }
+
+    @Override
+    public Stream<DocumentGraphNode> streamAllSubNodes() {
+
+        return Stream.of(this);
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/SectionNode.java

@@ -0,0 +1,108 @@
+package com.iqser.red.service.redaction.v1.server.document.graph.nodes;
+
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.stream.Stream;
+
+import com.iqser.red.service.redaction.v1.server.document.graph.TableOfContents;
+import com.iqser.red.service.redaction.v1.server.document.graph.textblock.AtomicTextBlock;
+import com.iqser.red.service.redaction.v1.server.document.graph.textblock.ConcatenatedTextBlock;
+import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlockCollector;
+
+import lombok.AccessLevel;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.EqualsAndHashCode;
+import lombok.experimental.FieldDefaults;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+@Data
+@Builder
+@AllArgsConstructor
+@FieldDefaults(level = AccessLevel.PRIVATE)
+public class SectionNode implements DocumentGraphNode {
+
+    String tocId;
+    Integer numberOnPage;
+    @EqualsAndHashCode.Exclude
+    TableOfContents tableOfContents;
+    @EqualsAndHashCode.Exclude
+    DocumentGraphNode parentSection;
+
+
+    AtomicTextBlock headline;
+
+    List<SectionNode> subSections;
+    List<ParagraphNode> paragraphs;
+    List<TableNode> tables;
+    @EqualsAndHashCode.Exclude
+    List<PageNode> pages;
+
+    @Builder.Default
+    @EqualsAndHashCode.Exclude
+    Set<EntityNode> entities = new HashSet<>();
+
+
+    @Override
+    public String toString() {
+
+        return tocId + ": " + headline.toString();
+    }
+
+
+    @Override
+    public ConcatenatedTextBlock buildTextBlock() {
+
+        return streamAllSubNodes().map(DocumentGraphNode::getAtomicTextBlock).collect(new TextBlockCollector());
+    }
+
+
+    @Override
+    public DocumentGraphNode getParent() {
+
+        if (hasParent()) {
+            return parentSection;
+        } else {
+            throw new UnsupportedOperationException("This section has no parent Section!");
+        }
+    }
+
+
+    @Override
+    public Stream<DocumentGraphNode> streamAllSubNodes() {
+
+        return tableOfContents.streamSubEntriesInOrder(tocId).map(TableOfContents.Entry::node);
+    }
+
+
+    @Override
+    public boolean hasParent() {
+
+        return parentSection != null;
+    }
+
+
+    @Override
+    public boolean isTerminal() {
+
+        return true;
+    }
+
+
+    @Override
+    public AtomicTextBlock getAtomicTextBlock() {
+
+        return headline;
+    }
+
+
+    @Override
+    public PageNode getPage() {
+
+        return pages.get(0);
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/TableCellNode.java

@@ -0,0 +1,59 @@
+package com.iqser.red.service.redaction.v1.server.document.graph.nodes;
+
+import java.util.LinkedList;
+import java.util.List;
+import java.util.stream.Stream;
+
+import com.iqser.red.service.redaction.v1.server.document.graph.textblock.AtomicTextBlock;
+
+import lombok.AccessLevel;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.EqualsAndHashCode;
+import lombok.experimental.FieldDefaults;
+
+@Data
+@Builder
+@AllArgsConstructor
+@FieldDefaults(level = AccessLevel.PRIVATE)
+public class TableCellNode implements DocumentGraphNode {
+
+    @EqualsAndHashCode.Exclude
+    TableNode parentTable;
+    Integer numberOnPage;
+    AtomicTextBlock atomicTextBlock;
+    PageNode page;
+
+    @Builder.Default
+    @EqualsAndHashCode.Exclude
+    List<EntityNode> entities = new LinkedList<>();
+
+
+    @Override
+    public AtomicTextBlock buildTextBlock() {
+
+        return atomicTextBlock;
+    }
+
+
+    @Override
+    public DocumentGraphNode getParent() {
+
+        return parentTable;
+    }
+
+
+    @Override
+    public boolean isTerminal() {
+
+        return true;
+    }
+
+    @Override
+    public Stream<DocumentGraphNode> streamAllSubNodes() {
+
+        return Stream.of(this);
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/TableNode.java

@@ -0,0 +1,88 @@
+package com.iqser.red.service.redaction.v1.server.document.graph.nodes;
+
+import java.util.LinkedList;
+import java.util.List;
+import java.util.function.Function;
+import java.util.stream.Stream;
+
+import com.iqser.red.service.redaction.v1.server.document.graph.TableOfContents;
+import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlockCollector;
+import com.iqser.red.service.redaction.v1.server.document.graph.textblock.ConcatenatedTextBlock;
+
+import lombok.AccessLevel;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.EqualsAndHashCode;
+import lombok.experimental.FieldDefaults;
+
+@Data
+@Builder
+@AllArgsConstructor
+@FieldDefaults(level = AccessLevel.PRIVATE)
+public class TableNode implements DocumentGraphNode {
+
+    Integer id;
+    String tocId;
+    Integer numberOfRows;
+    Integer numberOfCols;
+    Integer numberOnPage;
+    List<TableCellNode> tableHeaders;
+    List<List<TableCellNode>> tableCells;
+    TableOfContents tableOfContents;
+
+    @EqualsAndHashCode.Exclude
+    SectionNode parentSection;
+    @EqualsAndHashCode.Exclude
+    List<PageNode> pages;
+
+    @Builder.Default
+    @EqualsAndHashCode.Exclude
+    List<EntityNode> entities = new LinkedList<>();
+
+
+    private Stream<TableCellNode> streamTableCells() {
+
+        return tableCells.stream().flatMap(List::stream);
+    }
+
+
+    private Stream<TableCellNode> streamTableRow(int row) {
+
+        return tableCells.get(row).stream();
+    }
+
+
+    private Stream<TableCellNode> streamTableCol(int col) {
+
+        return tableCells.stream().map(row -> row.get(col));
+    }
+
+
+    @Override
+    public ConcatenatedTextBlock buildTextBlock() {
+
+        return streamTableCells().map(TableCellNode::getAtomicTextBlock).collect(new TextBlockCollector());
+    }
+
+    @Override
+    public Stream<DocumentGraphNode> streamAllSubNodes() {
+
+        return streamTableCells().map(Function.identity());
+    }
+
+
+    @Override
+    public DocumentGraphNode getParent() {
+
+        return parentSection;
+    }
+
+
+    @Override
+    public PageNode getPage() {
+
+        return pages.get(0);
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/textblock/AtomicTextBlock.java

@@ -0,0 +1,126 @@
+package com.iqser.red.service.redaction.v1.server.document.graph.textblock;
+
+import static java.lang.String.format;
+
+import java.awt.geom.Rectangle2D;
+import java.util.List;
+
+import com.iqser.red.service.redaction.v1.server.document.graph.Boundary;
+import com.iqser.red.service.redaction.v1.server.document.graph.nodes.DocumentGraphNode;
+
+import lombok.AccessLevel;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.EqualsAndHashCode;
+import lombok.experimental.FieldDefaults;
+
+@Data
+@Builder
+@AllArgsConstructor
+@FieldDefaults(level = AccessLevel.PRIVATE)
+public class AtomicTextBlock implements TextBlock {
+
+    Long id;
+
+    //string coordinates
+    Boundary boundary;
+    String searchText;
+    List<Integer> lineBreaks;
+
+    //position coordinates
+    List<Integer> stringIdxToPositionIdx;
+    List<Rectangle2D> positions;
+
+    @EqualsAndHashCode.Exclude
+    DocumentGraphNode parent;
+
+
+    public int indexOf(String searchTerm) {
+
+        int pos = searchText.indexOf(searchTerm);
+        return pos == -1 ? -1 : pos + boundary.start();
+    }
+
+
+    public int numberOfLines() {
+
+        return lineBreaks.size();
+    }
+
+
+    @Override
+    public List<AtomicTextBlock> getAtomicTextBlocks() {
+
+        return List.of(this);
+    }
+
+
+    public int getNextLinebreak(int fromIndex) {
+
+        return lineBreaks.stream()//
+                .filter(linebreak -> linebreak > fromIndex) //
+                .findFirst() //
+                .orElse(searchText.length()) + boundary.start();
+    }
+
+
+    public int getPreviousLinebreak(int fromIndex) {
+
+        return lineBreaks.stream()//
+                .filter(linebreak -> linebreak <= fromIndex)//
+                .reduce((a, b) -> b)//
+                .orElse(0) + boundary.start();
+    }
+
+
+    public Rectangle2D getPosition(int stringIdx) {
+
+        return positions.get(stringIdxToPositionIdx.get(stringIdx - boundary.start()));
+    }
+
+
+    public List<Rectangle2D> getPositions(Boundary boundary) {
+
+        if (!containsBoundary(boundary)) {
+            throw new IndexOutOfBoundsException(format("%s is out of bounds for %s",
+                    boundary,
+                    this.boundary));
+        }
+
+        if (boundary.end() == this.boundary.end()) {
+            return positions.subList(stringIdxToPositionIdx.get(boundary.start() - this.boundary.start()), positions.size());
+        }
+
+        return positions.subList(stringIdxToPositionIdx.get(boundary.start() - this.boundary.start()), stringIdxToPositionIdx.get(boundary.end() - this.boundary.start()));
+    }
+
+
+    @Override
+    public int length() {
+
+        return searchText.length();
+    }
+
+
+    @Override
+    public char charAt(int index) {
+
+        return searchText.charAt(index - boundary.start());
+    }
+
+
+    @Override
+    public CharSequence subSequence(int start, int end) {
+
+        return searchText.substring(start - boundary.start(), end - boundary.start());
+    }
+
+
+    @Override
+    public String toString() {
+
+        return searchText;
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/textblock/ConcatenatedTextBlock.java

@@ -0,0 +1,155 @@
+package com.iqser.red.service.redaction.v1.server.document.graph.textblock;
+
+import static java.lang.String.format;
+
+import java.awt.geom.Rectangle2D;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.function.Supplier;
+
+import com.iqser.red.service.redaction.v1.server.document.graph.Boundary;
+
+import lombok.AccessLevel;
+import lombok.Data;
+import lombok.experimental.FieldDefaults;
+
+@Data
+@FieldDefaults(level = AccessLevel.PRIVATE)
+public class ConcatenatedTextBlock implements TextBlock, Supplier<ConcatenatedTextBlock> {
+
+    List<AtomicTextBlock> atomicTextBlocks;
+    StringBuilder searchText;
+    Boundary boundary;
+
+
+    public ConcatenatedTextBlock(List<AtomicTextBlock> atomicTextBlocks) {
+
+        this.atomicTextBlocks = new LinkedList<>();
+        this.searchText = new StringBuilder();
+        if (atomicTextBlocks.isEmpty()) {
+            boundary = new Boundary(-1, -1);
+            return;
+        }
+        var firstTextBlock = atomicTextBlocks.get(0);
+        this.atomicTextBlocks.add(firstTextBlock);
+        this.searchText.append(firstTextBlock.getSearchText());
+        boundary = new Boundary(firstTextBlock.getBoundary().start(), firstTextBlock.getBoundary().end());
+
+        atomicTextBlocks.subList(1, atomicTextBlocks.size()).forEach(this::concat);
+    }
+
+
+    public ConcatenatedTextBlock(AtomicTextBlock atomicTextBlocks) {
+
+        new ConcatenatedTextBlock(List.of(atomicTextBlocks));
+    }
+
+
+    public ConcatenatedTextBlock concat(TextBlock textBlock) {
+
+        if (this.atomicTextBlocks.isEmpty()) {
+            boundary.setStart(textBlock.getBoundary().start());
+            boundary.setEnd(textBlock.getBoundary().end());
+        } else if (boundary.end() != textBlock.getBoundary().start()) {
+            throw new UnsupportedOperationException(format("Can only concat consecutive TextBlocks, trying to concat %s to %s", textBlock.getBoundary(), boundary));
+        }
+        this.searchText.append(textBlock.getSearchText());
+        this.atomicTextBlocks.addAll(textBlock.getAtomicTextBlocks());
+        boundary.setEnd(textBlock.getBoundary().end());
+        return this;
+    }
+
+
+    public int indexOf(String searchTerm) {
+
+        int pos = this.searchText.indexOf(searchTerm);
+        return pos == -1 ? -1 : pos + boundary.start();
+    }
+
+
+    public int numberOfLines() {
+
+        return atomicTextBlocks.stream().map(AtomicTextBlock::getLineBreaks).mapToInt(List::size).sum();
+    }
+
+
+    public int getNextLinebreak(int fromIndex) {
+
+        return getAtomicTextBlockByStringIndex(fromIndex).getNextLinebreak(fromIndex);
+    }
+
+
+    public int getPreviousLinebreak(int fromIndex) {
+
+        return getAtomicTextBlockByStringIndex(fromIndex).getPreviousLinebreak(fromIndex);
+    }
+
+
+    public Rectangle2D getPosition(int stringIdx) {
+
+        return getAtomicTextBlockByStringIndex(stringIdx).getPosition(stringIdx);
+    }
+
+
+    public List<Rectangle2D> getPositions(Boundary boundary) {
+
+        List<AtomicTextBlock> textBlocks = getAllAtomicTextBlocksPartiallyInStringIdxRange(boundary);
+
+        if (textBlocks.size() == 1) {
+            return textBlocks.get(0).getPositions(boundary);
+        }
+
+        AtomicTextBlock firstTextBlock = textBlocks.get(0);
+        List<Rectangle2D> positions = new LinkedList<>(firstTextBlock.getPositions(new Boundary(boundary.start(), firstTextBlock.getBoundary().end())));
+
+        for (AtomicTextBlock textBlock : textBlocks.subList(1, textBlocks.size() - 1)) {
+            positions.addAll(textBlock.getPositions());
+        }
+
+        var lastTextBlock = textBlocks.get(textBlocks.size() - 1);
+        positions.addAll(lastTextBlock.getPositions(new Boundary(lastTextBlock.getBoundary().start(), boundary.end())));
+
+        return positions;
+    }
+
+
+    private AtomicTextBlock getAtomicTextBlockByStringIndex(int stringIdx) {
+
+        return atomicTextBlocks.stream().filter(textBlock -> (textBlock.getBoundary().end()) > stringIdx).findFirst().orElseThrow(IndexOutOfBoundsException::new);
+    }
+
+
+    private List<AtomicTextBlock> getAllAtomicTextBlocksPartiallyInStringIdxRange(Boundary boundary) {
+
+        return atomicTextBlocks.stream().filter(tb -> tb.getBoundary().intersects(boundary)).toList();
+    }
+
+
+    @Override
+    public int length() {
+
+        return this.searchText.length();
+    }
+
+
+    @Override
+    public char charAt(int index) {
+
+        return searchText.charAt(index - boundary.start());
+    }
+
+
+    @Override
+    public CharSequence subSequence(int start, int end) {
+
+        return searchText.subSequence(start - boundary.start(), end - boundary.start());
+    }
+
+
+    @Override
+    public ConcatenatedTextBlock get() {
+
+        return this;
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/textblock/TextBlock.java

@@ -0,0 +1,65 @@
+package com.iqser.red.service.redaction.v1.server.document.graph.textblock;
+
+import static java.lang.String.format;
+
+import java.awt.geom.Rectangle2D;
+import java.util.List;
+
+import com.iqser.red.service.redaction.v1.server.document.graph.Boundary;
+
+public interface TextBlock extends CharSequence {
+
+    CharSequence getSearchText();
+
+
+    List<AtomicTextBlock> getAtomicTextBlocks();
+
+
+    Boundary getBoundary();
+
+
+    int getNextLinebreak(int fromIndex);
+
+
+    int getPreviousLinebreak(int fromIndex);
+
+
+    Rectangle2D getPosition(int stringIdx);
+
+
+    List<Rectangle2D> getPositions(Boundary range);
+
+
+    int numberOfLines();
+
+
+    int indexOf(String searchTerm);
+
+
+    default CharSequence getFirstLine() {
+
+        return subSequence(getBoundary().start(), getNextLinebreak(getBoundary().start()));
+    }
+
+
+    default boolean containsBoundary(Boundary boundary) {
+
+        if (boundary.end() < boundary.start()) {
+            throw new IllegalArgumentException(format("Invalid %s, StartIndex must be smaller than EndIndex", boundary));
+        }
+        return getBoundary().contains(boundary);
+    }
+
+
+    default boolean containsIndex(int stringIndex) {
+
+        return getBoundary().contains(stringIndex);
+    }
+
+
+    default CharSequence subSequence(Boundary boundary) {
+
+        return subSequence(boundary.start(), boundary.end());
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/textblock/TextBlockCollector.java

@@ -0,0 +1,51 @@
+package com.iqser.red.service.redaction.v1.server.document.graph.textblock;
+
+import java.util.Collections;
+import java.util.Set;
+import java.util.function.BiConsumer;
+import java.util.function.BinaryOperator;
+import java.util.function.Function;
+import java.util.function.Supplier;
+import java.util.stream.Collector;
+
+import lombok.NoArgsConstructor;
+
+@NoArgsConstructor
+public class TextBlockCollector implements Collector<AtomicTextBlock, ConcatenatedTextBlock, ConcatenatedTextBlock> {
+
+
+    @Override
+    public Supplier<ConcatenatedTextBlock> supplier() {
+
+        return new ConcatenatedTextBlock(Collections.emptyList());
+    }
+
+
+    @Override
+    public BiConsumer<ConcatenatedTextBlock, AtomicTextBlock> accumulator() {
+
+        return ConcatenatedTextBlock::concat;
+    }
+
+
+    @Override
+    public BinaryOperator<ConcatenatedTextBlock> combiner() {
+
+        return ConcatenatedTextBlock::concat;
+    }
+
+
+    @Override
+    public Function<ConcatenatedTextBlock, ConcatenatedTextBlock> finisher() {
+
+        return Function.identity();
+    }
+
+
+    @Override
+    public Set<Characteristics> characteristics() {
+
+        return Set.of(Characteristics.IDENTITY_FINISH, Characteristics.CONCURRENT);
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/DocumentDataMapper.java

@@ -0,0 +1,98 @@
+package com.iqser.red.service.redaction.v1.server.document.services;
+
+import java.awt.geom.Rectangle2D;
+import java.util.List;
+
+import org.springframework.stereotype.Service;
+
+import com.iqser.red.service.redaction.v1.server.document.data.AtomicTextBlockData;
+import com.iqser.red.service.redaction.v1.server.document.data.DocumentData;
+import com.iqser.red.service.redaction.v1.server.document.data.PageData;
+import com.iqser.red.service.redaction.v1.server.document.data.TableOfContentsData;
+import com.iqser.red.service.redaction.v1.server.document.graph.DocumentGraph;
+import com.iqser.red.service.redaction.v1.server.document.graph.TableOfContents;
+import com.iqser.red.service.redaction.v1.server.document.graph.nodes.PageNode;
+import com.iqser.red.service.redaction.v1.server.document.graph.textblock.AtomicTextBlock;
+
+@Service
+public class DocumentDataMapper {
+
+    public DocumentData toDocumentData(DocumentGraph documentGraph) {
+
+        List<AtomicTextBlockData> atomicTextBlockData = documentGraph.streamAtomicTextBlocksInOrder().map(this::toAtomicTextBlockData).toList();
+        List<PageData> pageData = documentGraph.getPages().stream().map(this::toPageData).toList();
+        TableOfContentsData tableOfContentsData = toTableOfContentsData(documentGraph.getTableOfContents());
+        return DocumentData.builder().atomicTextBlocks(atomicTextBlockData).pages(pageData).tableOfContents(tableOfContentsData).build();
+    }
+
+
+    private TableOfContentsData toTableOfContentsData(TableOfContents tableOfContents) {
+
+        return new TableOfContentsData(tableOfContents.getEntries().stream().map(this::toEntryData).toList());
+    }
+
+
+    private TableOfContentsData.EntryData toEntryData(TableOfContents.Entry entry) {
+
+        return TableOfContentsData.EntryData.builder()
+                .tocId(entry.id())
+                .subEntries(entry.children().stream().map(this::toEntryData).toList())
+                .type(entry.type())
+                .atomicTextBlock(entry.node().isTerminal() ? entry.node().getAtomicTextBlock().getId() : -1L)
+                .page(Long.valueOf(entry.node().getPage().getNumber()))
+                .numberOnPage(entry.node().getNumberOnPage())
+                .build();
+    }
+
+
+    private PageData toPageData(PageNode pageNode) {
+
+        return PageData.builder()
+                .height(pageNode.getHeight())
+                .width(pageNode.getWidth())
+                .number(pageNode.getNumber())
+                .footer(pageNode.getFooter().getId())
+                .header(pageNode.getHeader().getId())
+                .build();
+    }
+
+
+    private AtomicTextBlockData toAtomicTextBlockData(AtomicTextBlock atomicTextBlock) {
+
+        return AtomicTextBlockData.builder()
+                .id(atomicTextBlock.getId())
+                .searchText(atomicTextBlock.getSearchText())
+                .start(atomicTextBlock.getBoundary().start())
+                .end(atomicTextBlock.getBoundary().end())
+                .lineBreaks(toPrimitiveIntArray(atomicTextBlock.getLineBreaks()))
+                .stringIdxToPositionIdx(toPrimitiveIntArray(atomicTextBlock.getStringIdxToPositionIdx()))
+                .positions(toPrimitiveFloatMatrix(atomicTextBlock.getPositions()))
+                .build();
+    }
+
+
+    private float[][] toPrimitiveFloatMatrix(List<Rectangle2D> positions) {
+
+        float[][] positionMatrix = new float[positions.size()][];
+        for (int i = 0; i < positions.size(); i++) {
+            float[] singlePositions = new float[4];
+            singlePositions[0] = (float) positions.get(i).getMinX();
+            singlePositions[1] = (float) positions.get(i).getMinY();
+            singlePositions[2] = (float) positions.get(i).getWidth();
+            singlePositions[3] = (float) positions.get(i).getHeight();
+            positionMatrix[i] = singlePositions;
+        }
+        return positionMatrix;
+    }
+
+
+    private int[] toPrimitiveIntArray(List<Integer> list) {
+
+        int[] array = new int[list.size()];
+        for (int i = 0; i < list.size(); i++) {
+            array[i] = list.get(i);
+        }
+        return array;
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/DocumentGraphFactory.java

@@ -0,0 +1,304 @@
+package com.iqser.red.service.redaction.v1.server.document.services;
+
+import static java.lang.String.format;
+
+import java.awt.geom.Rectangle2D;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.stream.Collectors;
+
+import org.springframework.stereotype.Service;
+
+import com.iqser.red.service.redaction.v1.server.classification.model.Document;
+import com.iqser.red.service.redaction.v1.server.classification.model.Footer;
+import com.iqser.red.service.redaction.v1.server.classification.model.Header;
+import com.iqser.red.service.redaction.v1.server.classification.model.Page;
+import com.iqser.red.service.redaction.v1.server.classification.model.Section;
+import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
+import com.iqser.red.service.redaction.v1.server.document.graph.Boundary;
+import com.iqser.red.service.redaction.v1.server.document.graph.DocumentGraph;
+import com.iqser.red.service.redaction.v1.server.document.graph.TableOfContents;
+import com.iqser.red.service.redaction.v1.server.document.graph.nodes.DocumentGraphNode;
+import com.iqser.red.service.redaction.v1.server.document.graph.nodes.NodeType;
+import com.iqser.red.service.redaction.v1.server.document.graph.nodes.PageNode;
+import com.iqser.red.service.redaction.v1.server.document.graph.nodes.ParagraphNode;
+import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SectionNode;
+import com.iqser.red.service.redaction.v1.server.document.graph.nodes.TableNode;
+import com.iqser.red.service.redaction.v1.server.document.graph.textblock.AtomicTextBlock;
+import com.iqser.red.service.redaction.v1.server.exception.NotFoundException;
+import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
+import com.iqser.red.service.redaction.v1.server.redaction.model.RedRectangle2D;
+import com.iqser.red.service.redaction.v1.server.redaction.model.SearchTextWithTextPositionModel;
+import com.iqser.red.service.redaction.v1.server.redaction.service.SearchTextWithTextPositionFactory;
+import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
+import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;
+
+import lombok.RequiredArgsConstructor;
+
+@Service
+@RequiredArgsConstructor
+public class DocumentGraphFactory {
+
+    private final SearchTextWithTextPositionFactory searchTextWithTextPositionFactory;
+
+
+    public DocumentGraph buildDocumentGraph(Document document) {
+
+        Context context = new Context(new TableOfContents(), new LinkedList<>(), new LinkedList<>(), new AtomicInteger(0), new AtomicLong(0));
+
+        context.pages.addAll(document.getPages().stream().map(this::buildPage).toList());
+
+        // is tracked by Table of Contents
+        addSections(document, context);
+
+        // not tracked by Table of Contents
+        addHeaderAndFooterToEachPage(document, context);
+        DocumentGraph documentGraph = DocumentGraph.builder()
+                .numberOfPages(context.pages.size())
+                .pages(context.pages)
+                .sections(context.sections)
+                .tableOfContents(context.tableOfContents)
+                .build();
+        documentGraph.setText(documentGraph.buildTextBlock());
+        return documentGraph;
+    }
+
+
+    private void addSections(Document document, Context context) {
+
+        for (var section : document.getSections()) {
+            addSection(section, context);
+        }
+    }
+
+
+    private void addSection(Section section, Context context) {
+
+        SectionNode sectionEntity = SectionNode.builder()
+                .entities(new LinkedList<>())
+                .pages(new LinkedList<>())
+                .paragraphs(new LinkedList<>())
+                .tables(new LinkedList<>())
+                .subSections(new LinkedList<>())
+                .tableOfContents(context.tableOfContents())
+                .build();
+
+        context.sections().add(sectionEntity);
+        List<AbstractTextContainer> pageBlocks = new ArrayList<>(section.getPageBlocks());
+        PageNode page = getPage(section.getPageBlocks().get(0).getPage(), context);
+        sectionEntity.getPages().add(page);
+        page.getMainBody().add(sectionEntity);
+        if (pageBlocks.get(0) instanceof TextBlock) {
+            sectionEntity.setHeadline(buildAtomicTextBlock(((TextBlock) pageBlocks.get(0)).getSequences(), sectionEntity, context));
+            sectionEntity.setNumberOnPage(((TextBlock) pageBlocks.get(0)).getIndexOnPage());
+            pageBlocks.remove(0);
+        } else {
+            sectionEntity.setNumberOnPage(1);
+            sectionEntity.setHeadline(emptyTextBlock(sectionEntity, context));
+        }
+
+        String sectionId = context.tableOfContents.createNewEntryAndReturnId(NodeType.SECTION, buildSummary(sectionEntity.getHeadline()), sectionEntity);
+        sectionEntity.setTocId(sectionId);
+
+        int paragraphIdx = 0;
+        int tableIdx = 0;
+        for (AbstractTextContainer abstractTextContainer : pageBlocks) {
+            if (abstractTextContainer instanceof TextBlock) {
+                addParagraph(sectionEntity, (TextBlock) abstractTextContainer, paragraphIdx, context);
+                paragraphIdx++;
+            } else if (abstractTextContainer instanceof Table) {
+                //addTable(sectionEntity, (Table) abstractTextContainer, tableIdx, context);
+                tableIdx++;
+
+            }
+        }
+    }
+
+
+    private void addTable(SectionNode sectionEntity, Table table, int tableIdx, Context context) {
+
+        PageNode page = getPage(table.getPage(), context);
+        TableNode tableEntity = TableNode.builder().id(tableIdx).tableOfContents(context.tableOfContents()).pages(new LinkedList<>()).parentSection(sectionEntity).build();
+        sectionEntity.getTables().add(tableEntity);
+
+        if (!page.getMainBody().contains(sectionEntity)) {
+            sectionEntity.getPages().add(page);
+        }
+        page.getMainBody().add(tableEntity);
+
+    }
+
+
+    private void addParagraph(SectionNode sectionEntity, TextBlock originalTextBlock, int paragraphIdx, Context context) {
+
+        PageNode page = getPage(originalTextBlock.getPage(), context);
+        ParagraphNode paragraph = ParagraphNode.builder().numberOnPage(originalTextBlock.getIndexOnPage()).page(page).parentSection(sectionEntity).build();
+        sectionEntity.getParagraphs().add(paragraph);
+
+        if (!page.getMainBody().contains(sectionEntity)) {
+            sectionEntity.getPages().add(page);
+        }
+        page.getMainBody().add(paragraph);
+
+        var textBlock = buildAtomicTextBlock(originalTextBlock.getSequences(), paragraph, context);
+        paragraph.setAtomicTextBlock(textBlock);
+
+        String tocId = context.tableOfContents.createNewChildEntryAndReturnId(sectionEntity.getTocId(), NodeType.PARAGRAPH, buildSummary(textBlock), paragraph);
+        paragraph.setTocId(tocId);
+    }
+
+
+    private void addHeaderAndFooterToEachPage(Document document, Context context) {
+
+        Map<Integer, List<TextBlock>> headers = document.getHeaders()
+                .stream()
+                .map(Header::getTextBlocks)
+                .flatMap(List::stream)
+                .collect(Collectors.groupingBy(AbstractTextContainer::getPage, Collectors.toList()));
+
+        Map<Integer, List<TextBlock>> footers = document.getFooters()
+                .stream()
+                .map(Footer::getTextBlocks)
+                .flatMap(List::stream)
+                .collect(Collectors.groupingBy(AbstractTextContainer::getPage, Collectors.toList()));
+
+        for (int pageIndex = 1; pageIndex <= document.getPages().size(); pageIndex++) {
+            if (headers.containsKey(pageIndex)) {
+                addHeader(headers.get(pageIndex), context);
+            } else {
+                addEmptyHeader(pageIndex, context);
+            }
+        }
+
+        for (int pageIndex = 1; pageIndex <= document.getPages().size(); pageIndex++) {
+            if (footers.containsKey(pageIndex)) {
+                addFooter(footers.get(pageIndex), context);
+            } else {
+                addEmptyFooter(pageIndex, context);
+            }
+        }
+    }
+
+
+    private void addFooter(List<TextBlock> textBlocks, Context context) {
+
+        PageNode page = getPage(textBlocks.get(0).getPage(), context);
+        AtomicTextBlock footer = buildAtomicTextBlock(mergeAndSortTextPositionSequences(textBlocks), page, context);
+        page.setFooter(footer);
+    }
+
+
+    public void addHeader(List<TextBlock> textBlocks, Context context) {
+
+        PageNode page = getPage(textBlocks.get(0).getPage(), context);
+        AtomicTextBlock header = buildAtomicTextBlock(mergeAndSortTextPositionSequences(textBlocks), page, context);
+        page.setHeader(header);
+    }
+
+
+    private void addEmptyFooter(int pageIndex, Context context) {
+
+        PageNode page = getPage(pageIndex, context);
+        page.setFooter(emptyTextBlock(page, context));
+    }
+
+
+    private void addEmptyHeader(int pageIndex, Context context) {
+
+        PageNode page = getPage(pageIndex, context);
+        page.setHeader(emptyTextBlock(page, context));
+    }
+
+
+    private AtomicTextBlock emptyTextBlock(DocumentGraphNode parent, Context context) {
+
+        return AtomicTextBlock.builder()
+                .id(context.textBlockIdx.getAndIncrement())
+                .boundary(new Boundary(context.stringOffset.get(), context.stringOffset.get()))
+                .searchText("")
+                .lineBreaks(Collections.emptyList())
+                .stringIdxToPositionIdx(Collections.emptyList())
+                .positions(Collections.emptyList())
+                .parent(parent)
+                .build();
+    }
+
+
+    private static String buildSummary(AtomicTextBlock textBlock) {
+
+        if (textBlock == null) {
+            return " probably a table";
+        }
+
+        String[] words = textBlock.getFirstLine().toString().split(" ");
+        int bound = Math.min(words.length, 4);
+        List<String> list = new ArrayList<>(Arrays.asList(words).subList(0, bound));
+
+        return String.join(" ", list);
+    }
+
+
+    private PageNode buildPage(Page p) {
+
+        return PageNode.builder().height((int) p.getPageHeight()).width((int) p.getPageWidth()).number(p.getPageNumber()).mainBody(new LinkedList<>()).build();
+    }
+
+
+    private List<TextPositionSequence> mergeAndSortTextPositionSequences(List<TextBlock> textBlocks) {
+
+        Comparator<TextPositionSequence> sortByX = (sequence1, sequence2) -> (int) (sequence1.getTextPositions().get(0).getPosition()[0] - sequence2.getTextPositions()
+                .get(0)
+                .getPosition()[0]);
+        Comparator<TextPositionSequence> sortByY = (sequence1, sequence2) -> (int) (sequence1.getTextPositions().get(0).getPosition()[1] - sequence2.getTextPositions()
+                .get(0)
+                .getPosition()[1]);
+
+        return textBlocks.stream().map(TextBlock::getSequences).flatMap(List::stream).sorted(sortByX.thenComparing(sortByY)).toList();
+    }
+
+
+    private AtomicTextBlock buildAtomicTextBlock(List<TextPositionSequence> sequences, DocumentGraphNode parent, Context context) {
+
+        SearchTextWithTextPositionModel searchTextWithTextPositionModel = searchTextWithTextPositionFactory.buildSearchTextToTextPositionModel(sequences);
+        int offset = context.stringOffset().getAndAdd(searchTextWithTextPositionModel.getSearchText().length());
+
+        return AtomicTextBlock.builder()
+                .id(context.textBlockIdx.getAndIncrement())
+                .parent(parent)
+                .searchText(searchTextWithTextPositionModel.getSearchText())
+                .lineBreaks(searchTextWithTextPositionModel.getLineBreaks())
+                .positions(toRectangle2D(searchTextWithTextPositionModel.getPositions()))
+                .stringIdxToPositionIdx(searchTextWithTextPositionModel.getStringCoordsToPositionCoords())
+                .boundary(new Boundary(offset, offset + searchTextWithTextPositionModel.getSearchText().length()))
+                .build();
+    }
+
+
+    private List<Rectangle2D> toRectangle2D(List<RedRectangle2D> positions) {
+
+        return positions.stream().map(r -> (Rectangle2D) new Rectangle2D.Double(r.getX(), r.getY(), r.getWidth(), r.getHeight())).toList();
+    }
+
+
+    private PageNode getPage(int pageIndex, Context context) {
+
+        return context.pages.stream()
+                .filter(page -> page.getNumber() == pageIndex)
+                .findFirst()
+                .orElseThrow(() -> new NotFoundException(format("Page with number %d not found", pageIndex)));
+    }
+
+
+    record Context(
+            TableOfContents tableOfContents, List<PageNode> pages, List<SectionNode> sections, AtomicInteger stringOffset, AtomicLong textBlockIdx) {
+
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/DocumentGraphMapper.java

@@ -0,0 +1,171 @@
+package com.iqser.red.service.redaction.v1.server.document.services;
+import static java.lang.Math.toIntExact;
+import static java.lang.String.format;
+
+import java.awt.geom.Rectangle2D;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.LinkedList;
+import java.util.List;
+
+import org.apache.commons.lang3.NotImplementedException;
+import org.springframework.stereotype.Service;
+
+import com.google.common.primitives.Ints;
+import com.iqser.red.service.redaction.v1.server.document.data.AtomicTextBlockData;
+import com.iqser.red.service.redaction.v1.server.document.data.DocumentData;
+import com.iqser.red.service.redaction.v1.server.document.data.PageData;
+import com.iqser.red.service.redaction.v1.server.document.data.TableOfContentsData;
+import com.iqser.red.service.redaction.v1.server.document.graph.Boundary;
+import com.iqser.red.service.redaction.v1.server.document.graph.DocumentGraph;
+import com.iqser.red.service.redaction.v1.server.document.graph.TableOfContents;
+import com.iqser.red.service.redaction.v1.server.document.graph.nodes.DocumentGraphNode;
+import com.iqser.red.service.redaction.v1.server.document.graph.nodes.NodeType;
+import com.iqser.red.service.redaction.v1.server.document.graph.nodes.PageNode;
+import com.iqser.red.service.redaction.v1.server.document.graph.nodes.ParagraphNode;
+import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SectionNode;
+import com.iqser.red.service.redaction.v1.server.document.graph.textblock.AtomicTextBlock;
+import com.iqser.red.service.redaction.v1.server.exception.NotFoundException;
+
+@Service
+public class DocumentGraphMapper {
+
+    public DocumentGraph toDocumentGraph(DocumentData documentData) {
+
+        Context context = new Context(documentData, new TableOfContents(), new LinkedList<>(), new LinkedList<>(), documentData.getAtomicTextBlocks());
+
+        context.pages.addAll(documentData.getPages().stream().map(pageData -> buildPage(pageData, context)).toList());
+        buildNodesFromTableOfContents("", context);
+        DocumentGraph documentGraph= DocumentGraph.builder()
+                .numberOfPages(documentData.getPages().size())
+                .pages(context.pages)
+                .sections(context.sections)
+                .tableOfContents(context.tableOfContents)
+                .build();
+        documentGraph.setText(documentGraph.buildTextBlock());
+        return documentGraph;
+    }
+
+
+    private void buildNodesFromTableOfContents(String currentTocId, Context context) {
+        List <TableOfContentsData.EntryData> entries;
+        if(currentTocId.equals("")) {
+            entries = context.documentData().getTableOfContents().getEntries();
+        } else {
+            entries = context.documentData().getTableOfContents().get(currentTocId).subEntries();
+        }
+        for (TableOfContentsData.EntryData entryData : entries) {
+
+            switch (entryData.type()) {
+                case SECTION -> buildSection(entryData, currentTocId, context);
+                case PARAGRAPH -> buildParagraph(entryData, currentTocId, context);
+                default -> throw new NotImplementedException("Not yet implemented for type " + entryData.type());
+            }
+        }
+    }
+
+
+    private void buildSection(TableOfContentsData.EntryData entryData, String currentTocId, Context context) {
+
+        SectionNode section = SectionNode.builder()
+                .entities(new LinkedList<>())
+                .pages(new LinkedList<>())
+                .paragraphs(new LinkedList<>())
+                .tables(new LinkedList<>())
+                .subSections(new LinkedList<>())
+                .tableOfContents(context.tableOfContents())
+                .numberOnPage(entryData.numberOnPage())
+                .build();
+
+        context.sections().add(section);
+        section.setHeadline(toAtomicTextBlock(context.atomicTextBlockData().get(toIntExact(entryData.atomicTextBlock())), section));
+
+        if (!currentTocId.equals("")) {
+            SectionNode parent = (SectionNode) context.tableOfContents().getEntryById(currentTocId).node();
+            section.setParentSection(parent);
+            parent.getSubSections().add(section);
+        }
+
+        PageNode page = getPage(entryData.page(), context);
+        page.getMainBody().add(section);
+        section.getPages().add(page);
+
+        String sectionId = context.tableOfContents.createNewEntryAndReturnId(NodeType.SECTION, buildSummary(section.getHeadline()), section);
+        section.setTocId(sectionId);
+        buildNodesFromTableOfContents(sectionId, context);
+    }
+
+
+    private void buildParagraph(TableOfContentsData.EntryData entryData, String currentTocId, Context context) {
+
+        PageNode page = getPage(entryData.page(), context);
+        SectionNode parentSection = (SectionNode) context.tableOfContents().getEntryById(currentTocId).node();
+        ParagraphNode paragraph = ParagraphNode.builder().numberOnPage(entryData.numberOnPage()).page(page).parentSection(parentSection).build();
+        AtomicTextBlock atomicTextBlock = toAtomicTextBlock(context.atomicTextBlockData.get(toIntExact(entryData.atomicTextBlock())), paragraph);
+        paragraph.setAtomicTextBlock(atomicTextBlock);
+
+        if (!page.getMainBody().contains(parentSection)) {
+            parentSection.getPages().add(page);
+        }
+        page.getMainBody().add(paragraph);
+
+        String tocId = context.tableOfContents.createNewChildEntryAndReturnId(currentTocId, NodeType.PARAGRAPH, buildSummary(atomicTextBlock), paragraph);
+        paragraph.setTocId(tocId);
+    }
+
+
+    private PageNode buildPage(PageData p, Context context) {
+
+        PageNode page = PageNode.builder().height(p.getHeight()).width(p.getWidth()).number(p.getNumber()).mainBody(new LinkedList<>()).build();
+        AtomicTextBlock header = toAtomicTextBlock(context.atomicTextBlockData().get(toIntExact(p.getHeader())), page);
+        AtomicTextBlock footer = toAtomicTextBlock(context.atomicTextBlockData().get(toIntExact(p.getFooter())), page);
+        page.setHeader(header);
+        page.setFooter(footer);
+        return page;
+    }
+
+
+    private static String buildSummary(com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock textBlock) {
+
+        if (textBlock == null) {
+            return " probably a table";
+        }
+
+        String[] words = textBlock.getFirstLine().toString().split(" ");
+        int bound = Math.min(words.length, 4);
+        List<String> list = new ArrayList<>(Arrays.asList(words).subList(0, bound));
+
+        return String.join(" ", list);
+    }
+
+
+    private AtomicTextBlock toAtomicTextBlock(AtomicTextBlockData atomicTextBlockData, DocumentGraphNode parent) {
+
+        return AtomicTextBlock.builder()
+                .id(atomicTextBlockData.getId())
+                .searchText(atomicTextBlockData.getSearchText())
+                .boundary(new Boundary(atomicTextBlockData.getStart(), atomicTextBlockData.getEnd()))
+                .lineBreaks(Ints.asList(atomicTextBlockData.getLineBreaks()))
+                .positions(Arrays.stream(atomicTextBlockData.getPositions())
+                        .map(floatArr -> (Rectangle2D) new Rectangle2D.Float(floatArr[0], floatArr[1], floatArr[2], floatArr[3]))
+                        .toList())
+                .stringIdxToPositionIdx(Ints.asList(atomicTextBlockData.getStringIdxToPositionIdx()))
+                .parent(parent)
+                .build();
+    }
+
+
+    private PageNode getPage(Long pageIndex, Context context) {
+
+        return context.pages.stream()
+                .filter(page -> page.getNumber() == toIntExact(pageIndex))
+                .findFirst()
+                .orElseThrow(() -> new NotFoundException(format("Page with number %d not found", pageIndex)));
+    }
+
+
+    record Context(DocumentData documentData, TableOfContents tableOfContents, List<PageNode> pages, List<SectionNode> sections, List<AtomicTextBlockData> atomicTextBlockData) {
+
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/EntityEnrichmentUtility.java

@@ -0,0 +1,31 @@
+package com.iqser.red.service.redaction.v1.server.document.services;
+
+import com.iqser.red.service.redaction.v1.server.document.graph.nodes.EntityNode;
+import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock;
+
+public class EntityEnrichmentUtility {
+
+    public static EntityNode enrichEntity(EntityNode entity, TextBlock textBlock) {
+
+        entity.setPositions(textBlock.getPositions(entity.getBoundary()));
+        entity.setTextAfter(findTextAfter(entity.getBoundary().end(), textBlock));
+        entity.setTextBefore(findTextBefore(entity.getBoundary().start(), textBlock));
+        entity.setValue(textBlock.subSequence(entity.getBoundary()).toString());
+        return entity;
+    }
+
+
+    private static CharSequence findTextAfter(int index, TextBlock textBlock) {
+
+        int nextLineBreak = textBlock.getNextLinebreak(index);
+        return textBlock.subSequence(index, nextLineBreak);
+    }
+
+
+    private static CharSequence findTextBefore(int index, TextBlock textBlock) {
+
+        int previousLinebreak = textBlock.getPreviousLinebreak(index);
+        return textBlock.subSequence(previousLinebreak, index);
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/RangeComparators.java

@@ -0,0 +1,38 @@
+package com.iqser.red.service.redaction.v1.server.document.services;
+
+import java.util.Comparator;
+
+import com.iqser.red.service.redaction.v1.server.document.graph.Boundary;
+
+public class RangeComparators {
+
+    public static Comparator<Boundary> contained() {
+
+        return (range1, range2) -> {
+            if (contained(range1, range2)) {
+                return -1;
+            } else if (contained(range2, range1)) {
+                return 1;
+            } else {
+                return 0;
+            }
+        };
+    }
+
+
+    /**
+     * @param range1 A Range
+     * @param range2 Also Range
+     * @return true, if range1 contains range2
+     * false, otherwise
+     */
+    public static boolean contained(Boundary range1, Boundary range2) {
+
+        return range1.start() <= range2.start() && range2.end() <= range1.end();
+    }
+
+    public static boolean contained(Boundary range, int index) {
+
+        return range.start() <= index && index < range.end();
+    }
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/RegexMatcher.java

@@ -0,0 +1,37 @@
+package com.iqser.red.service.redaction.v1.server.document.services;
+
+import java.util.LinkedList;
+import java.util.List;
+import java.util.regex.Matcher;
+
+import com.iqser.red.service.redaction.v1.server.document.graph.Boundary;
+import com.iqser.red.service.redaction.v1.server.redaction.utils.Patterns;
+
+public class RegexMatcher {
+
+    public static boolean anyMatch(CharSequence searchText, String regexPattern) {
+
+        var pattern = Patterns.getCompiledPattern(regexPattern, false);
+        return pattern.matcher(searchText).find();
+    }
+
+
+    public static Boundary findFirstBoundary(String regexPattern, CharSequence searchText) {
+
+        var pattern = Patterns.getCompiledPattern(regexPattern, false);
+        Matcher matcher = pattern.matcher(searchText);
+        return new Boundary(matcher.start(), matcher.end());
+    }
+
+    public static List<Boundary> findBoundaries(String regexPattern, CharSequence searchText) {
+
+        var pattern = Patterns.getCompiledPattern(regexPattern, false);
+        Matcher matcher = pattern.matcher(searchText);
+        List<Boundary> boundaries = new LinkedList<>();
+        while (matcher.find()) {
+            boundaries.add(new Boundary(matcher.start(), matcher.end()));
+        }
+        return boundaries;
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Entity.java

@@ -1,22 +1,25 @@
 package com.iqser.red.service.redaction.v1.server.redaction.model;
 
-import com.iqser.red.service.redaction.v1.model.Engine;
-import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
-
-import lombok.*;
-
 import java.util.ArrayList;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
 
+import com.iqser.red.service.redaction.v1.model.Engine;
+import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.EqualsAndHashCode;
+import lombok.NoArgsConstructor;
+
 @Data
 @Builder
 @NoArgsConstructor
 @AllArgsConstructor
 @EqualsAndHashCode(onlyExplicitlyIncluded = true)
 public class Entity implements ReasonHolder {
-
     private String word;
     private String type;
     private boolean redaction;
@@ -26,6 +29,7 @@ public class Entity implements ReasonHolder {
     private List<EntityPositionSequence> positionSequences = new ArrayList<>();
     private List<TextPositionSequence> targetSequences;
 
+
     @EqualsAndHashCode.Include
     private Integer start;
     @EqualsAndHashCode.Include
@@ -38,6 +42,9 @@ public class Entity implements ReasonHolder {
     @EqualsAndHashCode.Include
     private int sectionNumber;
 
+    @EqualsAndHashCode.Include
+    private int paragraphNumber;
+
     private boolean isDictionaryEntry;
 
     private String textBefore;
@@ -66,6 +73,7 @@ public class Entity implements ReasonHolder {
                   String headline,
                   int matchedRule,
                   int sectionNumber,
+                  int paragraphNumber,
                   String legalBasis,
                   boolean isDictionaryEntry,
                   String textBefore,
@@ -85,6 +93,7 @@ public class Entity implements ReasonHolder {
         this.headline = headline;
         this.matchedRule = matchedRule;
         this.sectionNumber = sectionNumber;
+        this.paragraphNumber = paragraphNumber;
         this.legalBasis = legalBasis;
         this.isDictionaryEntry = isDictionaryEntry;
         this.textBefore = textBefore;
@@ -104,6 +113,7 @@ public class Entity implements ReasonHolder {
                   Integer end,
                   String headline,
                   int sectionNumber,
+                  int paragraphNumber,
                   boolean isDictionaryEntry,
                   boolean isDossierDictionaryEntry,
                   Engine engine,
@@ -115,6 +125,7 @@ public class Entity implements ReasonHolder {
         this.end = end;
         this.headline = headline;
         this.sectionNumber = sectionNumber;
+        this.paragraphNumber = paragraphNumber;
         this.isDictionaryEntry = isDictionaryEntry;
         this.isDossierDictionaryEntry = isDossierDictionaryEntry;
         this.engines.add(engine);

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Paragraph.java

@@ -0,0 +1,16 @@
+package com.iqser.red.service.redaction.v1.server.redaction.model;
+
+import lombok.AccessLevel;
+import lombok.Builder;
+import lombok.Getter;
+import lombok.experimental.FieldDefaults;
+
+@Getter
+@Builder
+@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
+public class Paragraph {
+
+    SearchTextWithTextPositionModel searchTextToTextPosition;
+    int sectionNumber;
+    int paragraphNumber;
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/SearchTextWithTextPositionModel.java

@@ -0,0 +1,19 @@
+package com.iqser.red.service.redaction.v1.server.redaction.model;
+
+import java.util.List;
+
+import lombok.AccessLevel;
+import lombok.Builder;
+import lombok.Getter;
+import lombok.experimental.FieldDefaults;
+
+@Builder
+@Getter
+@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
+public class SearchTextWithTextPositionModel {
+
+    String searchText;
+    List<Integer> lineBreaks;
+    List<Integer> stringCoordsToPositionCoords;
+    List<RedRectangle2D> positions;
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/SearchableText.java

@@ -1,5 +1,11 @@
 package com.iqser.red.service.redaction.v1.server.redaction.model;
 
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.List;
+import java.util.stream.Collectors;
+
 import com.dslplatform.json.JsonAttribute;
 import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
@@ -7,12 +13,6 @@ import com.iqser.red.service.redaction.v1.server.redaction.utils.IdBuilder;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.SeparatorUtils;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.TextNormalizationUtilities;
 
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.List;
-import java.util.stream.Collectors;
-
 import lombok.Getter;
 
 public class SearchableText {
@@ -186,25 +186,13 @@ public class SearchableText {
         return stringRepresentation;
     }
 
-
     public static String buildString(List<TextPositionSequence> sequences) {
 
         StringBuilder sb = new StringBuilder();
-
-        TextPositionSequence previous = null;
         for (TextPositionSequence word : sequences) {
-
-            if (previous != null) {
-                if (Math.abs(previous.getMaxYDirAdj() - word.getMaxYDirAdj()) > word.getTextHeight()) {
-                    sb.append('\n');
-                } else {
-                    sb.append(' ');
-                }
-            }
             sb.append(word.toString());
-            previous = word;
+            sb.append(' ');
         }
-
         return TextNormalizationUtilities.removeHyphenLineBreaks(sb.toString()).replaceAll("\n", " ").replaceAll(" {2}", " ");
     }
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java

@@ -18,7 +18,10 @@ import java.util.stream.Collectors;
 
 import org.apache.commons.lang3.StringUtils;
 
+import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus;
 import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
+import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.IdRemoval;
+import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualImageRecategorization;
 import com.iqser.red.service.redaction.v1.model.ArgumentType;
 import com.iqser.red.service.redaction.v1.model.Engine;
 import com.iqser.red.service.redaction.v1.model.FileAttribute;
@@ -26,12 +29,14 @@ import com.iqser.red.service.redaction.v1.model.SectionArea;
 import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
 import com.iqser.red.service.redaction.v1.server.parsing.model.RedTextPosition;
 import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
+import com.iqser.red.service.redaction.v1.server.redaction.service.SurroundingWordsService;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUtils;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.FindEntityDetails;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.IdBuilder;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.OffsetStringUtils;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.Patterns;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.SearchImplementation;
+import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;
 
 import lombok.Builder;
 import lombok.Data;
@@ -44,12 +49,11 @@ public class Section {
 
     private boolean isLocal;
 
-    private Set<String> dictionaryTypes;
-
     @Builder.Default
     private Map<String, Set<String>> localDictionaryAdds = new HashMap<>();
 
-    private Set<Entity> entities;
+    @Builder.Default
+    private Set<Entity> entities = new HashSet<>();
 
     private Set<Entity> nerEntities;
 
@@ -65,8 +69,6 @@ public class Section {
 
     private Map<String, CellValue> tabularData;
 
-    private Dictionary dictionary;
-
     private SearchableText searchableText;
 
     @Builder.Default
@@ -85,21 +87,24 @@ public class Section {
 
     private boolean isInTable;
 
+    @Builder.Default
+    private List<Integer> cellStarts = new ArrayList<>();
+
 
     @Deprecated
     @SuppressWarnings("unused")
     @ThenAction
-    public void addAiEntities(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.TYPE) String asType) {
+    public void addAiEntities(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.TYPE) String asType, Dictionary dictionary) {
 
-        redactOrRecommendAiEntities(type, asType, false, 0, null, null);
+        redactOrRecommendAiEntities(type, asType, false, 0, null, null, dictionary);
     }
 
 
     @SuppressWarnings("unused")
     @ThenAction
-    public void recommendAiEntities(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.TYPE) String asType) {
+    public void recommendAiEntities(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.TYPE) String asType, Dictionary dictionary) {
 
-        redactOrRecommendAiEntities(type, asType, false, 0, null, null);
+        redactOrRecommendAiEntities(type, asType, false, 0, null, null, dictionary);
     }
 
 
@@ -109,9 +114,10 @@ public class Section {
                                  @Argument(ArgumentType.TYPE) String asType,
                                  @Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
                                  @Argument(ArgumentType.STRING) String reason,
-                                 @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) {
+                                 @Argument(ArgumentType.LEGAL_BASIS) String legalBasis,
+                                 Dictionary dictionary) {
 
-        redactOrRecommendAiEntities(type, asType, true, ruleNumber, reason, legalBasis);
+        redactOrRecommendAiEntities(type, asType, true, ruleNumber, reason, legalBasis, dictionary);
     }
 
 
@@ -122,7 +128,8 @@ public class Section {
                                @Argument(ArgumentType.INTEGER) int maxDistanceBetween,
                                @Argument(ArgumentType.TYPE) String asType,
                                @Argument(ArgumentType.INTEGER) int minPartMatches,
-                               @Argument(ArgumentType.BOOLEAN) boolean allowDuplicateTypes) {
+                               @Argument(ArgumentType.BOOLEAN) boolean allowDuplicateTypes,
+                               Dictionary dictionary) {
 
         Set<String> combineSet = Set.of(combineTypes.split(","));
 
@@ -141,7 +148,7 @@ public class Section {
             } else if (!allowDuplicateTypes && foundParts.contains(entity.getType())) {
                 if (numberOfMatchParts >= minPartMatches) {
                     String value = searchText.substring(start, lastEnd);
-                    found.addAll(findEntities(value, asType, false, true, 0, null, null, Engine.NER, true));
+                    found.addAll(findEntities(value, asType, false, true, 0, null, null, Engine.NER, true, dictionary));
                 }
                 start = -1;
                 lastEnd = -1;
@@ -156,7 +163,7 @@ public class Section {
             } else if (entity.getType().equals(startType) && start != -1) {
                 if (numberOfMatchParts >= minPartMatches) {
                     String value = searchText.substring(start, lastEnd);
-                    found.addAll(findEntities(value, asType, false, true, 0, null, null, Engine.NER, true));
+                    found.addAll(findEntities(value, asType, false, true, 0, null, null, Engine.NER, true, dictionary));
                 }
                 start = entity.getStart();
                 lastEnd = entity.getEnd();
@@ -173,7 +180,7 @@ public class Section {
 
         if (numberOfMatchParts >= minPartMatches) {
             String value = searchText.substring(start, lastEnd);
-            found.addAll(findEntities(value, asType, false, true, 0, null, null, Engine.NER, true));
+            found.addAll(findEntities(value, asType, false, true, 0, null, null, Engine.NER, true, dictionary));
         }
 
         if (!found.isEmpty()) {
@@ -207,6 +214,7 @@ public class Section {
         return fileAttributes != null && fileAttributes.stream().anyMatch(attribute -> label.equals(attribute.getLabel()) && value.equals(attribute.getValue()));
     }
 
+
     @SuppressWarnings("unused")
     @WhenCondition
     public boolean fileAttributeContainsAnyOf(@Argument(ArgumentType.FILE_ATTRIBUTE) String label, @Argument(ArgumentType.STRING) Set<String> value) {
@@ -214,6 +222,7 @@ public class Section {
         return fileAttributes != null && fileAttributes.stream().anyMatch(attribute -> label.equals(attribute.getLabel()) && value.contains(attribute.getValue()));
     }
 
+
     @SuppressWarnings("unused")
     @WhenCondition
     public boolean fileAttributeByIdEqualsIgnoreCase(@Argument(ArgumentType.FILE_ATTRIBUTE) String id, @Argument(ArgumentType.STRING) String value) {
@@ -353,9 +362,10 @@ public class Section {
     public void expandByPrefixRegEx(@Argument(ArgumentType.TYPE) String type,
                                     @Argument(ArgumentType.REGEX) String prefixPattern,
                                     @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive,
-                                    @Argument(ArgumentType.INTEGER) int group) {
+                                    @Argument(ArgumentType.INTEGER) int group,
+                                    Dictionary dictionary) {
 
-        expandByPrefixRegEx(type, prefixPattern, patternCaseInsensitive, group, null);
+        expandByPrefixRegEx(type, prefixPattern, patternCaseInsensitive, group, null, dictionary);
     }
 
 
@@ -365,7 +375,8 @@ public class Section {
                                     @Argument(ArgumentType.REGEX) String prefixPattern,
                                     @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive,
                                     @Argument(ArgumentType.INTEGER) int group,
-                                    @Argument(ArgumentType.REGEX) String valuePattern) {
+                                    @Argument(ArgumentType.REGEX) String valuePattern,
+                                    Dictionary dictionary) {
 
         if (StringUtils.isEmpty(prefixPattern)) {
             return;
@@ -408,7 +419,8 @@ public class Section {
                             entity.getRedactionReason(),
                             entity.getLegalBasis(),
                             Engine.RULE,
-                            false);
+                            false,
+                            dictionary);
                     expanded.addAll(EntitySearchUtils.findNonOverlappingMatchEntities(entities, expandedEntities));
                 }
             }
@@ -424,9 +436,10 @@ public class Section {
     public void expandByRegEx(@Argument(ArgumentType.TYPE) String type,
                               @Argument(ArgumentType.REGEX) String suffixPattern,
                               @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive,
-                              @Argument(ArgumentType.INTEGER) int group) {
+                              @Argument(ArgumentType.INTEGER) int group,
+                              Dictionary dictionary) {
 
-        expandByRegEx(type, suffixPattern, patternCaseInsensitive, group, null);
+        expandByRegEx(type, suffixPattern, patternCaseInsensitive, group, null, dictionary);
     }
 
 
@@ -436,7 +449,8 @@ public class Section {
                               @Argument(ArgumentType.REGEX) String suffixPattern,
                               @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive,
                               @Argument(ArgumentType.INTEGER) int group,
-                              @Argument(ArgumentType.REGEX) String valuePattern) {
+                              @Argument(ArgumentType.REGEX) String valuePattern,
+                              Dictionary dictionary) {
 
         if (StringUtils.isEmpty(suffixPattern)) {
             return;
@@ -479,7 +493,8 @@ public class Section {
                             entity.getRedactionReason(),
                             entity.getLegalBasis(),
                             Engine.RULE,
-                            false);
+                            false,
+                            dictionary);
                     expanded.addAll(EntitySearchUtils.findNonOverlappingMatchEntities(entities, expandedEntities));
                 }
             }
@@ -535,9 +550,10 @@ public class Section {
                                 @Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
                                 @Argument(ArgumentType.BOOLEAN) boolean redactEverywhere,
                                 @Argument(ArgumentType.STRING) String reason,
-                                @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) {
+                                @Argument(ArgumentType.LEGAL_BASIS) String legalBasis,
+                                Dictionary dictionary) {
 
-        redactLineAfter(start, asType, ruleNumber, redactEverywhere, reason, legalBasis, true);
+        redactLineAfter(start, asType, ruleNumber, redactEverywhere, reason, legalBasis, true, dictionary);
     }
 
 
@@ -547,9 +563,10 @@ public class Section {
                                    @Argument(ArgumentType.TYPE) String asType,
                                    @Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
                                    @Argument(ArgumentType.BOOLEAN) boolean redactEverywhere,
-                                   @Argument(ArgumentType.STRING) String reason) {
+                                   @Argument(ArgumentType.STRING) String reason,
+                                   Dictionary dictionary) {
 
-        redactLineAfter(start, asType, ruleNumber, redactEverywhere, reason, null, false);
+        redactLineAfter(start, asType, ruleNumber, redactEverywhere, reason, null, false, dictionary);
 
     }
 
@@ -562,23 +579,25 @@ public class Section {
                               @Argument(ArgumentType.TYPE) String asType,
                               @Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
                               @Argument(ArgumentType.STRING) String reason,
-                              @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) {
+                              @Argument(ArgumentType.LEGAL_BASIS) String legalBasis,
+                              Dictionary dictionary) {
 
-        redactByRegEx(pattern, patternCaseInsensitive, group, asType, ruleNumber, reason, legalBasis, true, false);
+        redactByRegEx(pattern, patternCaseInsensitive, group, asType, ruleNumber, reason, legalBasis, true, false, dictionary);
     }
 
 
     @ThenAction
     @SuppressWarnings("unused")
     public void redactByRegExWithNewlines(@Argument(ArgumentType.REGEX) String pattern,
-                              @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive,
+                                          @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive,
-                              @Argument(ArgumentType.INTEGER) int group,
+                                          @Argument(ArgumentType.INTEGER) int group,
-                              @Argument(ArgumentType.TYPE) String asType,
+                                          @Argument(ArgumentType.TYPE) String asType,
-                              @Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
+                                          @Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
-                              @Argument(ArgumentType.STRING) String reason,
+                                          @Argument(ArgumentType.STRING) String reason,
-                              @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) {
+                                          @Argument(ArgumentType.LEGAL_BASIS) String legalBasis,
+                                          Dictionary dictionary) {
 
-        redactByRegExWithNewlines(pattern, patternCaseInsensitive, group, asType, ruleNumber, reason, legalBasis, true, false);
+        redactByRegExWithNewlines(pattern, patternCaseInsensitive, group, asType, ruleNumber, reason, legalBasis, true, false, dictionary);
     }
 
 
@@ -591,9 +610,10 @@ public class Section {
                               @Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
                               @Argument(ArgumentType.BOOLEAN) boolean skipRemoveEntitiesContainedInLarger,
                               @Argument(ArgumentType.STRING) String reason,
-                              @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) {
+                              @Argument(ArgumentType.LEGAL_BASIS) String legalBasis,
+                              Dictionary dictionary) {
 
-        redactByRegEx(pattern, patternCaseInsensitive, group, asType, ruleNumber, reason, legalBasis, true, skipRemoveEntitiesContainedInLarger);
+        redactByRegEx(pattern, patternCaseInsensitive, group, asType, ruleNumber, reason, legalBasis, true, skipRemoveEntitiesContainedInLarger, dictionary);
     }
 
 
@@ -604,9 +624,10 @@ public class Section {
                                  @Argument(ArgumentType.INTEGER) int group,
                                  @Argument(ArgumentType.TYPE) String asType,
                                  @Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
-                                 @Argument(ArgumentType.STRING) String reason) {
+                                 @Argument(ArgumentType.STRING) String reason,
+                                 Dictionary dictionary) {
 
-        redactByRegEx(pattern, patternCaseInsensitive, group, asType, ruleNumber, reason, null, false, false);
+        redactByRegEx(pattern, patternCaseInsensitive, group, asType, ruleNumber, reason, null, false, false, dictionary);
     }
 
 
@@ -619,9 +640,10 @@ public class Section {
                               @Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
                               @Argument(ArgumentType.BOOLEAN) boolean redactEverywhere,
                               @Argument(ArgumentType.STRING) String reason,
-                              @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) {
+                              @Argument(ArgumentType.LEGAL_BASIS) String legalBasis,
+                              Dictionary dictionary) {
 
-        redactBetween(start, stop, false, false, asType, ruleNumber, redactEverywhere, false, reason, legalBasis, true, false, false, false);
+        redactBetween(start, stop, false, false, asType, ruleNumber, redactEverywhere, false, reason, legalBasis, true, false, false, false, dictionary);
     }
 
 
@@ -634,9 +656,10 @@ public class Section {
                               @Argument(ArgumentType.BOOLEAN) boolean redactEverywhere,
                               @Argument(ArgumentType.BOOLEAN) boolean excludeHeadLine,
                               @Argument(ArgumentType.STRING) String reason,
-                              @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) {
+                              @Argument(ArgumentType.LEGAL_BASIS) String legalBasis,
+                              Dictionary dictionary) {
 
-        redactBetween(start, stop, false, false, asType, ruleNumber, redactEverywhere, excludeHeadLine, reason, legalBasis, true, false, false, false);
+        redactBetween(start, stop, false, false, asType, ruleNumber, redactEverywhere, excludeHeadLine, reason, legalBasis, true, false, false, false, dictionary);
     }
 
 
@@ -651,7 +674,8 @@ public class Section {
                               @Argument(ArgumentType.STRING) String reason,
                               @Argument(ArgumentType.LEGAL_BASIS) String legalBasis,
                               @Argument(ArgumentType.BOOLEAN) boolean skipRemoveEntitiesContainedInLarger,
-                              @Argument(ArgumentType.BOOLEAN) boolean sortedResult) {
+                              @Argument(ArgumentType.BOOLEAN) boolean sortedResult,
+                              Dictionary dictionary) {
 
         redactBetween(start,
                 stop,
@@ -665,7 +689,9 @@ public class Section {
                 legalBasis,
                 true,
                 skipRemoveEntitiesContainedInLarger,
-                sortedResult, false);
+                sortedResult,
+                false,
+                dictionary);
     }
 
 
@@ -683,7 +709,8 @@ public class Section {
                               @Argument(ArgumentType.LEGAL_BASIS) String legalBasis,
                               @Argument(ArgumentType.BOOLEAN) boolean skipRemoveEntitiesContainedInLarger,
                               @Argument(ArgumentType.BOOLEAN) boolean sortedResult,
-                              @Argument(ArgumentType.BOOLEAN) boolean ignoreTables) {
+                              @Argument(ArgumentType.BOOLEAN) boolean ignoreTables,
+                              Dictionary dictionary) {
 
         redactBetween(start,
                 stop,
@@ -697,11 +724,12 @@ public class Section {
                 legalBasis,
                 true,
                 skipRemoveEntitiesContainedInLarger,
-                sortedResult, ignoreTables);
+                sortedResult,
+                ignoreTables,
+                dictionary);
     }
 
 
-
     @ThenAction
     @SuppressWarnings("unused")
     public void redactBetween(@Argument(ArgumentType.STRING) String start,
@@ -715,7 +743,8 @@ public class Section {
                               @Argument(ArgumentType.STRING) String reason,
                               @Argument(ArgumentType.LEGAL_BASIS) String legalBasis,
                               @Argument(ArgumentType.BOOLEAN) boolean skipRemoveEntitiesContainedInLarger,
-                              @Argument(ArgumentType.BOOLEAN) boolean sortedResult) {
+                              @Argument(ArgumentType.BOOLEAN) boolean sortedResult,
+                              Dictionary dictionary) {
 
         redactBetween(start,
                 stop,
@@ -729,7 +758,9 @@ public class Section {
                 legalBasis,
                 true,
                 skipRemoveEntitiesContainedInLarger,
-                sortedResult, false);
+                sortedResult,
+                false,
+                dictionary);
     }
 
 
@@ -747,7 +778,8 @@ public class Section {
                                      @Argument(ArgumentType.STRING) String reason,
                                      @Argument(ArgumentType.STRING) String legalBasis,
                                      @Argument(ArgumentType.BOOLEAN) boolean skipRemoveEntitiesContainedInLarger,
-                                     @Argument(ArgumentType.BOOLEAN) boolean sortedResult) {
+                                     @Argument(ArgumentType.BOOLEAN) boolean sortedResult,
+                                     Dictionary dictionary) {
 
         String startValue = getFirstRexExMatch(searchText, startPattern, startPatternCaseInsensitive, startGroup);
 
@@ -772,7 +804,9 @@ public class Section {
                     legalBasis,
                     true,
                     skipRemoveEntitiesContainedInLarger,
-                    sortedResult, false);
+                    sortedResult,
+                    false,
+                    dictionary);
         }
     }
 
@@ -785,21 +819,10 @@ public class Section {
                                  @Argument(ArgumentType.TYPE) String asType,
                                  @Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
                                  @Argument(ArgumentType.BOOLEAN) boolean redactEverywhere,
-                                 @Argument(ArgumentType.STRING) String reason) {
+                                 @Argument(ArgumentType.STRING) String reason,
+                                 Dictionary dictionary) {
 
-        redactBetween(start,
+        redactBetween(start, stop, false, false, asType, ruleNumber, redactEverywhere, false, reason, null, false, false, false, false, dictionary);
-                stop,
-                false,
-                false,
-                asType,
-                ruleNumber,
-                redactEverywhere,
-                false,
-                reason,
-                null,
-                false,
-                false,
-                false, false);
     }
 
 
@@ -811,21 +834,10 @@ public class Section {
                                  @Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
                                  @Argument(ArgumentType.BOOLEAN) boolean redactEverywhere,
                                  @Argument(ArgumentType.BOOLEAN) boolean excludeHeadLine,
-                                 @Argument(ArgumentType.STRING) String reason) {
+                                 @Argument(ArgumentType.STRING) String reason,
+                                 Dictionary dictionary) {
 
-        redactBetween(start,
+        redactBetween(start, stop, false, false, asType, ruleNumber, redactEverywhere, excludeHeadLine, reason, null, false, false, false, false, dictionary);
-                stop,
-                false,
-                false,
-                asType,
-                ruleNumber,
-                redactEverywhere,
-                excludeHeadLine,
-                reason,
-                null,
-                false,
-                false,
-                false, false);
     }
 
 
@@ -837,9 +849,10 @@ public class Section {
                                    @Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
                                    @Argument(ArgumentType.BOOLEAN) boolean redactEverywhere,
                                    @Argument(ArgumentType.STRING) String reason,
-                                   @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) {
+                                   @Argument(ArgumentType.LEGAL_BASIS) String legalBasis,
+                                   Dictionary dictionary) {
 
-        redactLinesBetween(start, stop, asType, ruleNumber, redactEverywhere, reason, legalBasis, true);
+        redactLinesBetween(start, stop, asType, ruleNumber, redactEverywhere, reason, legalBasis, true, dictionary);
     }
 
 
@@ -850,9 +863,10 @@ public class Section {
                                       @Argument(ArgumentType.TYPE) String asType,
                                       @Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
                                       @Argument(ArgumentType.BOOLEAN) boolean redactEverywhere,
-                                      @Argument(ArgumentType.STRING) String reason) {
+                                      @Argument(ArgumentType.STRING) String reason,
+                                      Dictionary dictionary) {
 
-        redactLinesBetween(start, stop, asType, ruleNumber, redactEverywhere, reason, null, false);
+        redactLinesBetween(start, stop, asType, ruleNumber, redactEverywhere, reason, null, false, dictionary);
     }
 
 
@@ -863,9 +877,10 @@ public class Section {
                            @Argument(ArgumentType.TYPE) String type,
                            @Argument(ArgumentType.BOOLEAN) boolean addAsRecommendations,
                            @Argument(ArgumentType.STRING) String reason,
-                           @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) {
+                           @Argument(ArgumentType.LEGAL_BASIS) String legalBasis,
+                           Dictionary dictionary) {
 
-        annotateCell(cellHeader, ruleNumber, type, true, addAsRecommendations, reason, legalBasis);
+        annotateCell(cellHeader, ruleNumber, type, true, addAsRecommendations, reason, legalBasis, dictionary);
     }
 
 
@@ -875,9 +890,10 @@ public class Section {
                               @Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
                               @Argument(ArgumentType.TYPE) String type,
                               @Argument(ArgumentType.BOOLEAN) boolean addAsRecommendations,
-                              @Argument(ArgumentType.STRING) String reason) {
+                              @Argument(ArgumentType.STRING) String reason,
+                              Dictionary dictionary) {
 
-        annotateCell(cellHeader, ruleNumber, type, false, addAsRecommendations, reason, null);
+        annotateCell(cellHeader, ruleNumber, type, false, addAsRecommendations, reason, null, dictionary);
     }
 
 
@@ -889,9 +905,10 @@ public class Section {
                                           @Argument(ArgumentType.TYPE) String asType,
                                           @Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
                                           @Argument(ArgumentType.STRING) String reason,
-                                          @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) {
+                                          @Argument(ArgumentType.LEGAL_BASIS) String legalBasis,
+                                          Dictionary dictionary) {
 
-        redactAndRecommendByRegEx(pattern, patternCaseInsensitive, group, asType, ruleNumber, reason, legalBasis, true);
+        redactAndRecommendByRegEx(pattern, patternCaseInsensitive, group, asType, ruleNumber, reason, legalBasis, true, dictionary);
     }
 
 
@@ -902,9 +919,10 @@ public class Section {
                                              @Argument(ArgumentType.INTEGER) int group,
                                              @Argument(ArgumentType.TYPE) String asType,
                                              @Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
-                                             @Argument(ArgumentType.STRING) String reason) {
+                                             @Argument(ArgumentType.STRING) String reason,
+                                             Dictionary dictionary) {
 
-        redactAndRecommendByRegEx(pattern, patternCaseInsensitive, group, asType, ruleNumber, reason, null, false);
+        redactAndRecommendByRegEx(pattern, patternCaseInsensitive, group, asType, ruleNumber, reason, null, false, dictionary);
     }
 
 
@@ -973,9 +991,10 @@ public class Section {
                              @Argument(ArgumentType.TYPE) String asType,
                              @Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
                              @Argument(ArgumentType.STRING) String reason,
-                             @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) {
+                             @Argument(ArgumentType.LEGAL_BASIS) String legalBasis,
+                             Dictionary dictionary) {
 
-        Set<Entity> found = findEntities(value.trim(), asType, true, true, ruleNumber, reason, legalBasis, Engine.RULE, false);
+        Set<Entity> found = findEntities(value.trim(), asType, true, true, ruleNumber, reason, legalBasis, Engine.RULE, false, dictionary);
         EntitySearchUtils.addEntitiesIgnoreRank(entities, found);
     }
 
@@ -1001,7 +1020,8 @@ public class Section {
     public void expandToFalsePositiveByRegEx(@Argument(ArgumentType.TYPE) String type,
                                              @Argument(ArgumentType.STRING) String pattern,
                                              @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive,
-                                             @Argument(ArgumentType.INTEGER) int group) {
+                                             @Argument(ArgumentType.INTEGER) int group,
+                                             Dictionary dictionary) {
 
         Pattern compiledPattern = Patterns.getCompiledPattern(pattern, patternCaseInsensitive);
 
@@ -1017,7 +1037,7 @@ public class Section {
             while (matcher.find()) {
                 String match = matcher.group(group);
                 if (StringUtils.isNotBlank(match)) {
-                    expanded.addAll(findEntities(entity.getWord() + match, type, false, false, 0, null, null, Engine.RULE, false));
+                    expanded.addAll(findEntities(entity.getWord() + match, type, false, false, 0, null, null, Engine.RULE, false, dictionary));
                 }
             }
         }
@@ -1033,7 +1053,8 @@ public class Section {
     public void addHintAnnotationByRegEx(@Argument(ArgumentType.REGEX) String pattern,
                                          @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive,
                                          @Argument(ArgumentType.INTEGER) int group,
-                                         @Argument(ArgumentType.TYPE) String asType) {
+                                         @Argument(ArgumentType.TYPE) String asType,
+                                         Dictionary dictionary) {
 
         Pattern compiledPattern = Patterns.getCompiledPattern(pattern, patternCaseInsensitive);
 
@@ -1042,7 +1063,7 @@ public class Section {
         while (matcher.find()) {
             String match = matcher.group(group);
             if (StringUtils.isNotBlank(match)) {
-                Set<Entity> found = findEntities(match.trim(), asType, false, false, 0, null, null, Engine.RULE, false);
+                Set<Entity> found = findEntities(match.trim(), asType, false, false, 0, null, null, Engine.RULE, false, dictionary);
                 EntitySearchUtils.addEntitiesWithHigherRank(entities, found, dictionary);
             }
         }
@@ -1051,9 +1072,9 @@ public class Section {
 
     @ThenAction
     @SuppressWarnings("unused")
-    public void addHintAnnotation(@Argument(ArgumentType.STRING) String value, @Argument(ArgumentType.TYPE) String asType) {
+    public void addHintAnnotation(@Argument(ArgumentType.STRING) String value, @Argument(ArgumentType.TYPE) String asType, Dictionary dictionary) {
 
-        Set<Entity> found = findEntities(value.trim(), asType, true, false, 0, null, null, Engine.RULE, false);
+        Set<Entity> found = findEntities(value.trim(), asType, true, false, 0, null, null, Engine.RULE, false, dictionary);
         EntitySearchUtils.addEntitiesIgnoreRank(entities, found);
     }
 
@@ -1085,9 +1106,12 @@ public class Section {
 
     @ThenAction
     @SuppressWarnings("unused")
-    public void highlightCell(@Argument(ArgumentType.STRING) String cellHeader, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.TYPE) String type) {
+    public void highlightCell(@Argument(ArgumentType.STRING) String cellHeader,
+                              @Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
+                              @Argument(ArgumentType.TYPE) String type,
+                              Dictionary dictionary) {
 
-        annotateCell(cellHeader, ruleNumber, type, false, false, null, null);
+        annotateCell(cellHeader, ruleNumber, type, false, false, null, null, dictionary);
     }
 
 
@@ -1096,9 +1120,10 @@ public class Section {
     public void redactSectionText(@Argument(ArgumentType.TYPE) String type,
                                   @Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
                                   @Argument(ArgumentType.STRING) String reason,
-                                  @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) {
+                                  @Argument(ArgumentType.LEGAL_BASIS) String legalBasis,
+                                  Dictionary dictionary) {
 
-        redactBetween("", "", type, ruleNumber, false, false, reason, legalBasis, true, false);
+        redactBetween("", "", type, ruleNumber, false, false, reason, legalBasis, true, false, dictionary);
     }
 
 
@@ -1107,9 +1132,10 @@ public class Section {
     public void redactSectionTextWithoutHeadLine(@Argument(ArgumentType.TYPE) String type,
                                                  @Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
                                                  @Argument(ArgumentType.STRING) String reason,
-                                                 @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) {
+                                                 @Argument(ArgumentType.LEGAL_BASIS) String legalBasis,
+                                                 Dictionary dictionary) {
 
-        redactBetween("", "", type, ruleNumber, false, true, reason, legalBasis, true, false);
+        redactBetween("", "", type, ruleNumber, false, true, reason, legalBasis, true, false, dictionary);
     }
 
 
@@ -1117,13 +1143,14 @@ public class Section {
     public void redactHeadline(@Argument(ArgumentType.TYPE) String type,
                                @Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
                                @Argument(ArgumentType.STRING) String reason,
-                               @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) {
+                               @Argument(ArgumentType.LEGAL_BASIS) String legalBasis,
+                               Dictionary dictionary) {
 
         if (!headline.isBlank()) {
 
             String cleanHeadline = headline.replaceAll("\\n", " ").replaceAll("  ", " ").trim();
             if (searchText.contains(cleanHeadline)) {
-                Set<Entity> found = findEntities(cleanHeadline, type, false, true, ruleNumber, reason, legalBasis, Engine.RULE, false);
+                Set<Entity> found = findEntities(cleanHeadline, type, false, true, ruleNumber, reason, legalBasis, Engine.RULE, false, dictionary);
                 EntitySearchUtils.addEntitiesWithHigherRank(entities, found, dictionary);
             }
         }
@@ -1139,7 +1166,8 @@ public class Section {
                                       @Argument(ArgumentType.TYPE) String asType,
                                       @Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
                                       @Argument(ArgumentType.STRING) String reason,
-                                      @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) {
+                                      @Argument(ArgumentType.LEGAL_BASIS) String legalBasis,
+                                      Dictionary dictionary) {
 
         Pattern compiledPattern = Patterns.getCompiledPattern(pattern, patternCaseInsensitive);
 
@@ -1148,7 +1176,7 @@ public class Section {
         while (findMatcher.find()) {
             String findMatch = findMatcher.group(group);
             if (StringUtils.isNotBlank(findMatch)) {
-                Set<Entity> found = findEntities(findMatch.trim(), asType, false, true, ruleNumber, reason, legalBasis, Engine.RULE, false);
+                Set<Entity> found = findEntities(findMatch.trim(), asType, false, true, ruleNumber, reason, legalBasis, Engine.RULE, false, dictionary);
 
                 for (Entity entity : found) {
 
@@ -1235,9 +1263,10 @@ public class Section {
                                              @Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
                                              @Argument(ArgumentType.BOOLEAN) boolean redactEverywhere,
                                              @Argument(ArgumentType.STRING) String reason,
-                                             @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) {
+                                             @Argument(ArgumentType.LEGAL_BASIS) String legalBasis,
+                                             Dictionary dictionary) {
 
-        redactLineAfterAcrossColumns(start, asType, ruleNumber, redactEverywhere, reason, legalBasis, false, false);
+        redactLineAfterAcrossColumns(start, asType, ruleNumber, redactEverywhere, reason, legalBasis, false, false, dictionary);
     }
 
 
@@ -1249,9 +1278,10 @@ public class Section {
                                              @Argument(ArgumentType.BOOLEAN) boolean redactEverywhere,
                                              @Argument(ArgumentType.BOOLEAN) boolean skipRemoveEntitiesContainedInLarger,
                                              @Argument(ArgumentType.STRING) String reason,
-                                             @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) {
+                                             @Argument(ArgumentType.LEGAL_BASIS) String legalBasis,
+                                             Dictionary dictionary) {
 
-        redactLineAfterAcrossColumns(start, asType, ruleNumber, redactEverywhere, reason, legalBasis, skipRemoveEntitiesContainedInLarger, false);
+        redactLineAfterAcrossColumns(start, asType, ruleNumber, redactEverywhere, reason, legalBasis, skipRemoveEntitiesContainedInLarger, false, dictionary);
     }
 
 
@@ -1264,9 +1294,10 @@ public class Section {
                                              @Argument(ArgumentType.BOOLEAN) boolean skipRemoveEntitiesContainedInLarger,
                                              @Argument(ArgumentType.BOOLEAN) boolean onlyExactMatch,
                                              @Argument(ArgumentType.STRING) String reason,
-                                             @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) {
+                                             @Argument(ArgumentType.LEGAL_BASIS) String legalBasis,
+                                             Dictionary dictionary) {
 
-        redactLineAfterAcrossColumns(start, asType, ruleNumber, redactEverywhere, reason, legalBasis, skipRemoveEntitiesContainedInLarger, onlyExactMatch);
+        redactLineAfterAcrossColumns(start, asType, ruleNumber, redactEverywhere, reason, legalBasis, skipRemoveEntitiesContainedInLarger, onlyExactMatch, dictionary);
     }
 
 
@@ -1277,14 +1308,15 @@ public class Section {
                                            int ruleNumber,
                                            String reason,
                                            String legalBasis,
-                                           boolean redaction) {
+                                           boolean redaction,
+                                           Dictionary dictionary) {
 
         Pattern compiledPattern = Patterns.getCompiledPattern(pattern, patternCaseInsensitive);
         Matcher matcher = compiledPattern.matcher(searchText);
         while (matcher.find()) {
             String match = matcher.group(group);
             if (StringUtils.isNotBlank(match) && match.length() >= 3) {
-                Set<Entity> found = findEntities(match.trim(), asType, false, redaction, ruleNumber, reason, legalBasis, Engine.RULE, false);
+                Set<Entity> found = findEntities(match.trim(), asType, false, redaction, ruleNumber, reason, legalBasis, Engine.RULE, false, dictionary);
                 EntitySearchUtils.addEntitiesWithHigherRank(entities, found, dictionary);
                 localDictionaryAdds.computeIfAbsent(asType, x -> new HashSet<>()).add(match);
             }
@@ -1292,15 +1324,16 @@ public class Section {
     }
 
 
-    private Set<Entity> findEntities(String value,
+    public Set<Entity> findEntities(String value,
-                                     String asType,
+                                    String asType,
-                                     boolean caseInsensitive,
+                                    boolean caseInsensitive,
-                                     boolean redacted,
+                                    boolean redacted,
-                                     int ruleNumber,
+                                    int ruleNumber,
-                                     String reason,
+                                    String reason,
-                                     String legalBasis,
+                                    String legalBasis,
-                                     Engine engine,
+                                    Engine engine,
-                                     boolean asRecommendation) {
+                                    boolean asRecommendation,
+                                    Dictionary dictionary) {
 
         String text = caseInsensitive ? searchText.toLowerCase() : searchText;
         Set<Entity> found = EntitySearchUtils.findEntities(text,
@@ -1347,7 +1380,14 @@ public class Section {
     }
 
 
-    private void annotateCell(String cellHeader, int ruleNumber, String type, boolean redact, boolean addAsRecommendations, String reason, String legalBasis) {
+    private void annotateCell(String cellHeader,
+                              int ruleNumber,
+                              String type,
+                              boolean redact,
+                              boolean addAsRecommendations,
+                              String reason,
+                              String legalBasis,
+                              Dictionary dictionary) {
 
         String cleanHeaderName = cellHeader.replaceAll("\n", "").replaceAll(" ", "").replaceAll("-", "");
 
@@ -1363,6 +1403,7 @@ public class Section {
                     value.getRowSpanStart() + word.length(),
                     headline,
                     sectionNumber,
+                    -1,
                     false,
                     false,
                     Engine.RULE,
@@ -1410,14 +1451,21 @@ public class Section {
     }
 
 
-    private void redactLineAfter(String start, String asType, int ruleNumber, boolean redactEverywhere, String reason, String legalBasis, boolean redaction) {
+    private void redactLineAfter(String start,
+                                 String asType,
+                                 int ruleNumber,
+                                 boolean redactEverywhere,
+                                 String reason,
+                                 String legalBasis,
+                                 boolean redaction,
+                                 Dictionary dictionary) {
 
         String[] values = StringUtils.substringsBetween(text, start, "\n");
 
         if (values != null) {
             for (String value : values) {
                 if (StringUtils.isNotBlank(value)) {
-                    Set<Entity> found = findEntities(value.trim(), asType, false, redaction, ruleNumber, reason, legalBasis, Engine.RULE, false);
+                    Set<Entity> found = findEntities(value.trim(), asType, false, redaction, ruleNumber, reason, legalBasis, Engine.RULE, false, dictionary);
                     EntitySearchUtils.addEntitiesWithHigherRank(entities, found, dictionary);
 
                     if (redactEverywhere && !isLocal()) {
@@ -1436,7 +1484,8 @@ public class Section {
                                               String reason,
                                               String legalBasis,
                                               boolean skipRemoveEntitiesContainedInLarger,
-                                              boolean onlyExactMatch) {
+                                              boolean onlyExactMatch,
+                                              Dictionary dictionary) {
 
         var stringOffsets = OffsetStringUtils.substringsBetween(searchableText.getAsStringWithLinebreaksSorted(), start, "\n");
 
@@ -1444,8 +1493,9 @@ public class Section {
             for (var stringOffset : stringOffsets) {
                 if (StringUtils.isNotBlank(stringOffset.getValue())) {
                     var trimmedOffsetString = stringOffset.trim();
-                    Set<Entity> found = findEntities(trimmedOffsetString.getValue(), asType, false, true, ruleNumber, reason, legalBasis, Engine.RULE, false).stream()
+                    Set<Entity> found = findEntities(trimmedOffsetString.getValue(), asType, false, true, ruleNumber, reason, legalBasis, Engine.RULE, false, dictionary).stream()
-                            .filter(f -> !onlyExactMatch || f.getStart() == trimmedOffsetString.getStart() && f.getEnd() == trimmedOffsetString.getEnd()).collect(Collectors.toSet());
+                            .filter(f -> !onlyExactMatch || f.getStart() == trimmedOffsetString.getStart() && f.getEnd() == trimmedOffsetString.getEnd())
+                            .collect(Collectors.toSet());
                     found.forEach(f -> f.setSkipRemoveEntitiesContainedInLarger(skipRemoveEntitiesContainedInLarger));
 
                     EntitySearchUtils.addEntitiesWithHigherRank(entities, found, dictionary);
@@ -1459,7 +1509,16 @@ public class Section {
     }
 
 
-    private void redactByRegExWithNewlines(String pattern, boolean patternCaseInsensitive, int group, String asType, int ruleNumber, String reason, String legalBasis, boolean redaction, boolean skipRemoveEntitiesContainedInLarger) {
+    private void redactByRegExWithNewlines(String pattern,
+                                           boolean patternCaseInsensitive,
+                                           int group,
+                                           String asType,
+                                           int ruleNumber,
+                                           String reason,
+                                           String legalBasis,
+                                           boolean redaction,
+                                           boolean skipRemoveEntitiesContainedInLarger,
+                                           Dictionary dictionary) {
 
         Pattern compiledPattern = Patterns.getCompiledMultilinePattern(pattern, patternCaseInsensitive);
 
@@ -1468,7 +1527,7 @@ public class Section {
         while (matcher.find()) {
             String match = matcher.group(group);
             if (StringUtils.isNotBlank(match)) {
-                Set<Entity> found = findEntities(match.replaceAll("\\n", " ").trim(), asType, false, redaction, ruleNumber, reason, legalBasis, Engine.RULE, false);
+                Set<Entity> found = findEntities(match.replaceAll("\\n", " ").trim(), asType, false, redaction, ruleNumber, reason, legalBasis, Engine.RULE, false, dictionary);
                 found.forEach(f -> f.setSkipRemoveEntitiesContainedInLarger(skipRemoveEntitiesContainedInLarger));
                 EntitySearchUtils.addEntitiesWithHigherRank(entities, found, dictionary);
             }
@@ -1476,7 +1535,16 @@ public class Section {
     }
 
 
-    private void redactByRegEx(String pattern, boolean patternCaseInsensitive, int group, String asType, int ruleNumber, String reason, String legalBasis, boolean redaction, boolean skipRemoveEntitiesContainedInLarger) {
+    private void redactByRegEx(String pattern,
+                               boolean patternCaseInsensitive,
+                               int group,
+                               String asType,
+                               int ruleNumber,
+                               String reason,
+                               String legalBasis,
+                               boolean redaction,
+                               boolean skipRemoveEntitiesContainedInLarger,
+                               Dictionary dictionary) {
 
         Pattern compiledPattern = Patterns.getCompiledPattern(pattern, patternCaseInsensitive);
 
@@ -1485,7 +1553,7 @@ public class Section {
         while (matcher.find()) {
             String match = matcher.group(group);
             if (StringUtils.isNotBlank(match)) {
-                Set<Entity> found = findEntities(match.trim(), asType, false, redaction, ruleNumber, reason, legalBasis, Engine.RULE, false);
+                Set<Entity> found = findEntities(match.trim(), asType, false, redaction, ruleNumber, reason, legalBasis, Engine.RULE, false, dictionary);
                 found.forEach(f -> f.setSkipRemoveEntitiesContainedInLarger(skipRemoveEntitiesContainedInLarger));
                 EntitySearchUtils.addEntitiesWithHigherRank(entities, found, dictionary);
             }
@@ -1522,10 +1590,10 @@ public class Section {
                                boolean redaction,
                                boolean skipRemoveEntitiesContainedInLarger,
                                boolean sortedResult,
-                               boolean ignoreTables) {
+                               boolean ignoreTables,
+                               Dictionary dictionary) {
 
-
+        if (isInTable && ignoreTables) {
-        if(isInTable && ignoreTables){
             return;
         }
 
@@ -1559,7 +1627,7 @@ public class Section {
                         searchString = searchString + stop;
                     }
 
-                    Set<Entity> found = findEntities(searchString.trim(), asType, false, redaction, ruleNumber, reason, legalBasis, Engine.RULE, false);
+                    Set<Entity> found = findEntities(searchString.trim(), asType, false, redaction, ruleNumber, reason, legalBasis, Engine.RULE, false, dictionary);
                     found.forEach(f -> {
                         f.setSkipRemoveEntitiesContainedInLarger(skipRemoveEntitiesContainedInLarger);
                         if (sortedResult) {
@@ -1582,7 +1650,15 @@ public class Section {
     }
 
 
-    private void redactLinesBetween(String start, String stop, String asType, int ruleNumber, boolean redactEverywhere, String reason, String legalBasis, boolean redaction) {
+    private void redactLinesBetween(String start,
+                                    String stop,
+                                    String asType,
+                                    int ruleNumber,
+                                    boolean redactEverywhere,
+                                    String reason,
+                                    String legalBasis,
+                                    boolean redaction,
+                                    Dictionary dictionary) {
 
         String[] values = StringUtils.substringsBetween(text, start, stop);
 
@@ -1597,7 +1673,7 @@ public class Section {
                             return;
                         }
 
-                        Set<Entity> found = findEntities(line.trim(), asType, false, redaction, ruleNumber, reason, legalBasis, Engine.RULE, false);
+                        Set<Entity> found = findEntities(line.trim(), asType, false, redaction, ruleNumber, reason, legalBasis, Engine.RULE, false, dictionary);
                         EntitySearchUtils.addEntitiesWithHigherRank(entities, found, dictionary);
 
                         if (redactEverywhere && !isLocal()) {
@@ -1610,7 +1686,7 @@ public class Section {
     }
 
 
-    private void redactOrRecommendAiEntities(String type, String asType, boolean redact, int ruleNumber, String reason, String legalBasis) {
+    private void redactOrRecommendAiEntities(String type, String asType, boolean redact, int ruleNumber, String reason, String legalBasis, Dictionary dictionary) {
 
         Set<Entity> entitiesOfType = nerEntities.stream().filter(nerEntity -> nerEntity.getType().equals(type)).collect(Collectors.toSet());
         List<String> values = entitiesOfType.stream().map(Entity::getWord).collect(Collectors.toList());
@@ -1675,10 +1751,98 @@ public class Section {
 
     }
 
+
+    public boolean findDictionaryEntities(Dictionary dictionary, RedactionServiceSettings redactionServiceSettings) {
+
+        findEntities(dictionary);
+
+        if (cellStarts != null && !cellStarts.isEmpty()) {
+            SurroundingWordsService.addSurroundingText(entities,
+                    searchableText,
+                    dictionary,
+                    cellStarts,
+                    redactionServiceSettings.getSurroundingWordsOffsetWindow(),
+                    redactionServiceSettings.getNumberOfSurroundingWords());
+        } else {
+            SurroundingWordsService.addSurroundingText(entities,
+                    searchableText,
+                    dictionary,
+                    redactionServiceSettings.getSurroundingWordsOffsetWindow(),
+                    redactionServiceSettings.getNumberOfSurroundingWords());
+        }
+
+        if (!isLocal && manualRedactions != null) {
+
+            var approvedForceRedactions = manualRedactions.getForceRedactions()
+                    .stream()
+                    .filter(fr -> fr.getStatus() == AnnotationStatus.APPROVED)
+                    .filter(fr -> fr.getRequestDate() != null)
+                    .collect(Collectors.toList());
+            // only approved id removals, that haven't been forced back afterwards
+            var idsToRemove = manualRedactions.getIdsToRemove()
+                    .stream()
+                    .filter(idr -> idr.getStatus() == AnnotationStatus.APPROVED && !idr.isRemoveFromDictionary())
+                    .filter(idr -> idr.getRequestDate() != null)
+                    .filter(idr -> approvedForceRedactions.stream()
+                            .noneMatch(forceRedact -> forceRedact.getAnnotationId().equals(idr.getAnnotationId()) && forceRedact.getRequestDate().isAfter(idr.getRequestDate())))
+                    .map(IdRemoval::getAnnotationId)
+                    .collect(Collectors.toSet());
+
+            if (images != null && !images.isEmpty() && manualRedactions.getImageRecategorization() != null) {
+                for (Image image : images) {
+                    String imageId = IdBuilder.buildId(image.getPosition(), image.getPage());
+                    for (ManualImageRecategorization imageRecategorization : manualRedactions.getImageRecategorization()) {
+                        if (imageRecategorization.getStatus().equals(AnnotationStatus.APPROVED) && imageRecategorization.getAnnotationId().equals(imageId)) {
+                            image.setType(imageRecategorization.getType());
+                        }
+                    }
+                    if (idsToRemove.contains(imageId)) {
+                        image.setIgnored(true);
+                    }
+                }
+            }
+
+            entities.forEach(entity -> entity.getPositionSequences().forEach(ps -> {
+                if (idsToRemove.contains(ps.getId())) {
+                    entity.setIgnored(true);
+                }
+            }));
+        }
+
+        return false;
+    }
+
+
+    private void findEntities(Dictionary dictionary) {
+
+        Set<Entity> found = new HashSet<>();
+        String searchableString = searchableText.asString();
+
+        if (StringUtils.isEmpty(searchableString)) {
+            entities = new HashSet<>();
+            return;
+        }
+
+        String lowercaseInputString = searchableString.toLowerCase();
+        for (DictionaryModel model : dictionary.getDictionaryModels()) {
+
+            var searchImplementation = isLocal ? model.getLocalSearch() : model.getEntriesSearch();
+            var entities = EntitySearchUtils.findEntities(model.isCaseInsensitive() ? lowercaseInputString : searchableString,
+                    searchImplementation,
+                    model,
+                    new FindEntityDetails(model.getType(),
+                            headline,
+                            sectionNumber,
+                            !isLocal,
+                            model.isDossierDictionary(),
+                            isLocal ? Engine.RULE : Engine.DICTIONARY,
+                            isLocal ? EntityType.RECOMMENDATION : EntityType.ENTITY));
+
+            EntitySearchUtils.addOrAddEngine(found, entities);
+        }
+
+        entities = EntitySearchUtils.clearAndFindPositions(found, searchableText, dictionary, manualRedactions);
+        nerEntities = EntitySearchUtils.clearAndFindPositions(nerEntities, searchableText, dictionary, manualRedactions);
+    }
+
 }
-
-
-
-
-
-

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/SectionSearchableTextPair.java

@@ -1,16 +0,0 @@
-package com.iqser.red.service.redaction.v1.server.redaction.model;
-
-import java.util.List;
-
-import lombok.AllArgsConstructor;
-import lombok.Data;
-
-@Data
-@AllArgsConstructor
-public class SectionSearchableTextPair {
-
-    private Section section;
-    private SearchableText searchableText;
-    private List<Integer> cellStarts;
-
-}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/AnalyzeService.java

@@ -41,6 +41,7 @@ import com.iqser.red.service.redaction.v1.server.classification.model.Simplified
 import com.iqser.red.service.redaction.v1.server.classification.model.Text;
 import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient;
 import com.iqser.red.service.redaction.v1.server.client.model.NerEntities;
+import com.iqser.red.service.redaction.v1.server.document.services.DocumentGraphFactory;
 import com.iqser.red.service.redaction.v1.server.exception.RedactionException;
 import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
 import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryIncrement;
@@ -80,6 +81,7 @@ public class AnalyzeService {
     private final SectionGridCreatorService sectionGridCreatorService;
     private final ImageService imageService;
     private final ImportedRedactionService importedRedactionService;
+    private final DocumentGraphFactory documentGraphFactory;
 
 
     @Timed("redactmanager_analyzeDocumentStructure")
@@ -107,7 +109,6 @@ public class AnalyzeService {
         }
 
         List<SectionText> sectionTexts = sectionTextBuilderService.buildSectionText(classifiedDoc);
-
         sectionGridCreatorService.createSectionGrid(classifiedDoc, pageCount);
 
         Text text = new Text(pageCount, sectionTexts);

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DroolsExecutionService.java

@@ -1,11 +1,12 @@
 package com.iqser.red.service.redaction.v1.server.redaction.service;
 
-import com.iqser.red.service.redaction.v1.server.client.RulesClient;
+import java.io.ByteArrayInputStream;
-import com.iqser.red.service.redaction.v1.server.exception.RulesValidationException;
+import java.io.InputStream;
-import com.iqser.red.service.redaction.v1.server.redaction.model.Section;
+import java.nio.charset.StandardCharsets;
-
+import java.util.HashMap;
-import io.micrometer.core.annotation.Timed;
+import java.util.LinkedList;
-import lombok.RequiredArgsConstructor;
+import java.util.List;
+import java.util.Map;
 
 import org.apache.commons.lang3.StringUtils;
 import org.kie.api.KieServices;
@@ -14,13 +15,19 @@ import org.kie.api.builder.KieFileSystem;
 import org.kie.api.builder.KieModule;
 import org.kie.api.runtime.KieContainer;
 import org.kie.api.runtime.KieSession;
+import org.kie.api.runtime.rule.QueryResults;
+import org.kie.api.runtime.rule.QueryResultsRow;
 import org.springframework.stereotype.Service;
 
-import java.io.ByteArrayInputStream;
+import com.iqser.red.service.redaction.v1.server.client.RulesClient;
-import java.io.InputStream;
+import com.iqser.red.service.redaction.v1.server.exception.RulesValidationException;
-import java.nio.charset.StandardCharsets;
+import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
-import java.util.HashMap;
+import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
-import java.util.Map;
+import com.iqser.red.service.redaction.v1.server.redaction.model.Paragraph;
+import com.iqser.red.service.redaction.v1.server.redaction.model.Section;
+
+import io.micrometer.core.annotation.Timed;
+import lombok.RequiredArgsConstructor;
 
 @Service
 @RequiredArgsConstructor
@@ -45,18 +52,29 @@ public class DroolsExecutionService {
 
 
     @Timed("redactmanager_executeRules")
-    public Section executeRules(KieContainer kieContainer, Section section) {
+    public List<Entity> executeRules(KieContainer kieContainer, List<Section> sections, List<Paragraph> paragraphs, Dictionary dictionary) {
 
         KieSession kieSession = kieContainer.newKieSession();
-        kieSession.setGlobal("section", section);
+        kieSession.setGlobal("dictionary", dictionary);
-        kieSession.insert(section);
+        sections.forEach(kieSession::insert);
+        paragraphs.forEach(kieSession::insert);
         kieSession.fireAllRules();
+        List<Entity> entities = getEntities(kieSession);
         kieSession.dispose();
 
-        return section;
+        return entities;
 
     }
 
+    public List<Entity> getEntities(KieSession ks) {
+        List<Entity> entities = new LinkedList<>();
+        QueryResults entitiesResult = ks.getQueryResults("getEntities");
+        for (QueryResultsRow resultsRow : entitiesResult) {
+            entities.add((Entity) resultsRow.get("$result"));
+        }
+        return entities;
+    }
+
 
     public KieContainer updateRules(String dossierTemplateId) {
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionService.java

@@ -1,33 +1,43 @@
 package com.iqser.red.service.redaction.v1.server.redaction.service;
 
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import org.apache.commons.lang3.StringUtils;
+import org.kie.api.runtime.KieContainer;
+import org.springframework.stereotype.Service;
+
 import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.IdRemoval;
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualImageRecategorization;
 import com.iqser.red.service.redaction.v1.model.AnalyzeRequest;
 import com.iqser.red.service.redaction.v1.model.Engine;
 import com.iqser.red.service.redaction.v1.model.FileAttribute;
 import com.iqser.red.service.redaction.v1.server.classification.model.SectionText;
 import com.iqser.red.service.redaction.v1.server.client.model.NerEntities;
 import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
-import com.iqser.red.service.redaction.v1.server.redaction.model.*;
+import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryModel;
+import com.iqser.red.service.redaction.v1.server.redaction.model.Entities;
+import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
+import com.iqser.red.service.redaction.v1.server.redaction.model.EntityPositionSequence;
+import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType;
+import com.iqser.red.service.redaction.v1.server.redaction.model.FindEntitiesResult;
+import com.iqser.red.service.redaction.v1.server.redaction.model.Image;
+import com.iqser.red.service.redaction.v1.server.redaction.model.PageEntities;
+import com.iqser.red.service.redaction.v1.server.redaction.model.Paragraph;
+import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText;
+import com.iqser.red.service.redaction.v1.server.redaction.model.Section;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUtils;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.FindEntityDetails;
-import com.iqser.red.service.redaction.v1.server.redaction.utils.IdBuilder;
 import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;
 
 import io.micrometer.core.annotation.Timed;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
 
-import org.apache.commons.lang3.StringUtils;
-import org.kie.api.runtime.KieContainer;
-import org.springframework.stereotype.Service;
-
-import java.util.*;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
-
 @Slf4j
 @Service
 @RequiredArgsConstructor
@@ -35,7 +45,7 @@ public class EntityRedactionService {
 
     private final RedactionServiceSettings redactionServiceSettings;
     private final DroolsExecutionService droolsExecutionService;
-    private final SurroundingWordsService surroundingWordsService;
+    private final SearchTextWithTextPositionFactory searchTextWithTextPositionFactory;
 
 
     public PageEntities findEntities(Dictionary dictionary, List<SectionText> sectionTexts, KieContainer kieContainer, AnalyzeRequest analyzeRequest, NerEntities nerEntities) {
@@ -45,7 +55,7 @@ public class EntityRedactionService {
 
         if (dictionary.hasLocalEntries() || !findEntitiesResult.getAddedFileAttributes().isEmpty()) {
 
-            if(!findEntitiesResult.getAddedFileAttributes().isEmpty()) {
+            if (!findEntitiesResult.getAddedFileAttributes().isEmpty()) {
                 //AnalyzeRequest provides immutable list.
                 List<FileAttribute> mergedFileAttributes = new ArrayList<>();
                 mergedFileAttributes.addAll(analyzeRequest.getFileAttributes());
@@ -54,7 +64,14 @@ public class EntityRedactionService {
             }
 
             Map<Integer, Set<Entity>> hintsPerSectionNumber = getHintsPerSection(findEntitiesResult.getEntities(), dictionary);
-            FindEntitiesResult foundByLocalEntitiesResult = findEntities(sectionTexts, dictionary, kieContainer, analyzeRequest, true, hintsPerSectionNumber, imagesPerPage, nerEntities);
+            FindEntitiesResult foundByLocalEntitiesResult = findEntities(sectionTexts,
+                    dictionary,
+                    kieContainer,
+                    analyzeRequest,
+                    true,
+                    hintsPerSectionNumber,
+                    imagesPerPage,
+                    nerEntities);
             EntitySearchUtils.addEntitiesWithHigherRank(findEntitiesResult.getEntities(), foundByLocalEntitiesResult.getEntities(), dictionary);
             EntitySearchUtils.removeEntitiesContainedInLarger(findEntitiesResult.getEntities());
         }
@@ -67,81 +84,28 @@ public class EntityRedactionService {
 
 
     public FindEntitiesResult findEntities(List<SectionText> reanalysisSections,
-                                    Dictionary dictionary,
+                                           Dictionary dictionary,
-                                    KieContainer kieContainer,
+                                           KieContainer kieContainer,
-                                    AnalyzeRequest analyzeRequest,
+                                           AnalyzeRequest analyzeRequest,
-                                    boolean local,
+                                           boolean local,
-                                    Map<Integer, Set<Entity>> hintsPerSectionNumber,
+                                           Map<Integer, Set<Entity>> hintsPerSectionNumber,
-                                    Map<Integer, Set<Image>> imagesPerPage,
+                                           Map<Integer, Set<Image>> imagesPerPage,
-                                    NerEntities nerEntities) {
+                                           NerEntities nerEntities) {
+
+        List<Section> sections = new ArrayList<>(reanalysisSections.size());
 
-        List<SectionSearchableTextPair> sectionSearchableTextPairs = new ArrayList<>();
         for (SectionText reanalysisSection : reanalysisSections) {
 
-            Entities entities = findEntities(reanalysisSection.getSearchableText(),
+            Set<Entity> nerFound = new HashSet<>();
-                    reanalysisSection.getHeadline(),
+            if (!local) {
-                    reanalysisSection.getSectionNumber(),
+                nerFound.addAll(getNerValues(reanalysisSection.getSectionNumber(), nerEntities, reanalysisSection.getCellStarts(), reanalysisSection.getHeadline()));
-                    dictionary,
-                    local,
-                    nerEntities,
-                    reanalysisSection.getCellStarts(),
-                    analyzeRequest.getManualRedactions());
-
-            if (reanalysisSection.getCellStarts() != null && !reanalysisSection.getCellStarts().isEmpty()) {
-                surroundingWordsService.addSurroundingText(entities.getEntities(), reanalysisSection.getSearchableText(), dictionary, reanalysisSection.getCellStarts());
-            } else {
-                surroundingWordsService.addSurroundingText(entities.getEntities(), reanalysisSection.getSearchableText(), dictionary);
-            }
-
-            if (!local && analyzeRequest.getManualRedactions() != null) {
-
-                var approvedForceRedactions = analyzeRequest.getManualRedactions()
-                        .getForceRedactions()
-                        .stream()
-                        .filter(fr -> fr.getStatus() == AnnotationStatus.APPROVED)
-                        .filter(fr -> fr.getRequestDate() != null)
-                        .collect(Collectors.toList());
-                // only approved id removals, that haven't been forced back afterwards
-                var idsToRemove = analyzeRequest.getManualRedactions()
-                        .getIdsToRemove()
-                        .stream()
-                        .filter(idr -> idr.getStatus() == AnnotationStatus.APPROVED && !idr.isRemoveFromDictionary())
-                        .filter(idr -> idr.getRequestDate() != null)
-                        .filter(idr -> approvedForceRedactions.stream()
-                                .noneMatch(forceRedact -> forceRedact.getAnnotationId().equals(idr.getAnnotationId()) && forceRedact.getRequestDate()
-                                        .isAfter(idr.getRequestDate())))
-                        .map(IdRemoval::getAnnotationId)
-                        .collect(Collectors.toSet());
-
-                if (reanalysisSection.getImages() != null && !reanalysisSection.getImages().isEmpty() && analyzeRequest.getManualRedactions().getImageRecategorization() != null) {
-                    for (Image image : reanalysisSection.getImages()) {
-                        String imageId = IdBuilder.buildId(image.getPosition(), image.getPage());
-                        for (ManualImageRecategorization imageRecategorization : analyzeRequest.getManualRedactions().getImageRecategorization()) {
-                            if (imageRecategorization.getStatus().equals(AnnotationStatus.APPROVED) && imageRecategorization.getAnnotationId().equals(imageId)) {
-                                image.setType(imageRecategorization.getType());
-                            }
-                        }
-                        if (idsToRemove.contains(imageId)) {
-                            image.setIgnored(true);
-                        }
-                    }
-                }
-
-                entities.getEntities().forEach(entity -> entity.getPositionSequences().forEach(ps -> {
-                    if (idsToRemove.contains(ps.getId())) {
-                        entity.setIgnored(true);
-                    }
-                }));
             }
 
             log.debug("Section {}, Images: {}", reanalysisSection.getSectionNumber(), reanalysisSection.getImages());
-
+            sections.add(Section.builder()
-            sectionSearchableTextPairs.add(new SectionSearchableTextPair(Section.builder()
                     .isLocal(false)
                     .dictionaryTypes(dictionary.getTypes())
-                    .entities(hintsPerSectionNumber != null && hintsPerSectionNumber.containsKey(reanalysisSection.getSectionNumber()) ? Stream.concat(entities.getEntities()
+                    .nerEntities(nerFound)
-                            .stream(), hintsPerSectionNumber.get(reanalysisSection.getSectionNumber()).stream()).collect(Collectors.toSet()) : entities.getEntities())
-                    .nerEntities(entities.getNerEntities())
                     .text(reanalysisSection.getSearchableText().getAsStringWithLinebreaks())
                     .searchText(reanalysisSection.getSearchableText().toString())
                     .headline(reanalysisSection.getHeadline())
@@ -154,45 +118,63 @@ public class EntityRedactionService {
                     .fileAttributes(analyzeRequest.getFileAttributes())
                     .manualRedactions(analyzeRequest.getManualRedactions())
                     .isInTable(reanalysisSection.isTable())
-                    .build(), reanalysisSection.getSearchableText(), reanalysisSection.getCellStarts()));
+                    .redactionServiceSettings(redactionServiceSettings)
+                    .cellStarts(reanalysisSection.getCellStarts())
+                    .build());
 
         }
 
-
         Set<FileAttribute> addedFileAttributes = new HashSet<>();
-        Set<Entity> entities = new HashSet<>();
+        sections.forEach(section -> {
-        sectionSearchableTextPairs.forEach(sectionSearchableTextPair -> {
 
-            if(!addedFileAttributes.isEmpty()) {
+            if (!addedFileAttributes.isEmpty()) {
                 //Section.Builder provides immutable list.
                 List<FileAttribute> mergedFileAttributes = new ArrayList<>();
-                mergedFileAttributes.addAll(sectionSearchableTextPair.getSection().getAddedFileAttributes());
+                mergedFileAttributes.addAll(section.getAddedFileAttributes());
                 mergedFileAttributes.addAll(addedFileAttributes);
-                sectionSearchableTextPair.getSection().setFileAttributes(mergedFileAttributes);
+                section.setFileAttributes(mergedFileAttributes);
             }
+        });
 
-            Section analysedSection = droolsExecutionService.executeRules(kieContainer, sectionSearchableTextPair.getSection());
+        List<Paragraph> paragraphs = new ArrayList<>();
+        for (var reanalysisSection : reanalysisSections) {
+            for (int i = 0; i < reanalysisSection.getTextBlocks().size(); ++i) {
+                paragraphs.add(Paragraph.builder()
+                        .paragraphNumber(i)
+                        .sectionNumber(reanalysisSection.getSectionNumber())
+                        .searchTextToTextPosition(searchTextWithTextPositionFactory.buildSearchTextToTextPositionModel(reanalysisSection.getTextBlocks().get(i).getSequences()))
+                        .build());
+            }
+        }
 
+        List<Entity> entitiesList = droolsExecutionService.executeRules(kieContainer, sections, paragraphs, dictionary);
+        Set<Entity> entities = new HashSet<>(entitiesList);
+
+        sections.forEach(analysedSection -> {
             addedFileAttributes.addAll(analysedSection.getAddedFileAttributes());
 
-            EntitySearchUtils.removeEntitiesContainedInLarger(analysedSection.getEntities());
+            EntitySearchUtils.removeEntitiesContainedInLarger(entities);
 
-            var entriesWithoutSurroundingText = analysedSection.getEntities()
+            var entriesWithoutSurroundingText = entities.stream()
-                    .stream()
+                    .filter(e -> e.getSectionNumber() == analysedSection.getSectionNumber())
                     .filter(e -> e.getTextAfter() == null && e.getTextBefore() == null)
                     .collect(Collectors.toSet());
 
-            if (sectionSearchableTextPair.getCellStarts() != null && !sectionSearchableTextPair.getCellStarts().isEmpty()) {
+            if (analysedSection.getCellStarts() != null && !analysedSection.getCellStarts().isEmpty()) {
-                surroundingWordsService.addSurroundingText(entriesWithoutSurroundingText,
+                SurroundingWordsService.addSurroundingText(entriesWithoutSurroundingText,
-                        sectionSearchableTextPair.getSearchableText(),
+                        analysedSection.getSearchableText(),
                         dictionary,
-                        sectionSearchableTextPair.getCellStarts());
+                        analysedSection.getCellStarts(),
+                        redactionServiceSettings.getSurroundingWordsOffsetWindow(),
+                        redactionServiceSettings.getNumberOfSurroundingWords());
             } else {
-                surroundingWordsService.addSurroundingText(entriesWithoutSurroundingText, sectionSearchableTextPair.getSearchableText(), dictionary);
+                SurroundingWordsService.addSurroundingText(entriesWithoutSurroundingText,
+                        analysedSection.getSearchableText(),
+                        dictionary,
+                        redactionServiceSettings.getSurroundingWordsOffsetWindow(),
+                        redactionServiceSettings.getNumberOfSurroundingWords());
             }
 
-            entities.addAll(analysedSection.getEntities());
-
             if (!local) {
                 for (Image image : analysedSection.getImages()) {
                     imagesPerPage.computeIfAbsent(image.getPage(), (a) -> new HashSet<>()).add(image);
@@ -225,6 +207,7 @@ public class EntityRedactionService {
                                 entity.getHeadline(),
                                 entity.getMatchedRule(),
                                 entity.getSectionNumber(),
+                                -1,
                                 entity.getLegalBasis(),
                                 entity.isDictionaryEntry(),
                                 entity.getTextBefore(),
@@ -328,6 +311,7 @@ public class EntityRedactionService {
                             res.getEndOffset(),
                             headline,
                             sectionNumber,
+                            -1,
                             false,
                             false,
                             Engine.NER,
@@ -347,6 +331,7 @@ public class EntityRedactionService {
                                 res.getEndOffset(),
                                 headline,
                                 sectionNumber,
+                                -1,
                                 false,
                                 false,
                                 Engine.NER,

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ManualRedactionSurroundingTextService.java

@@ -22,6 +22,7 @@ import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUtils;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.FindEntityDetails;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.SearchImplementation;
+import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;
 import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
 
 import io.micrometer.core.annotation.Timed;
@@ -34,8 +35,7 @@ import lombok.extern.slf4j.Slf4j;
 public class ManualRedactionSurroundingTextService {
 
     private final RedactionStorageService redactionStorageService;
-    private final SurroundingWordsService surroundingWordsService;
+    private final RedactionServiceSettings redactionServiceSettings;
-
 
     @Timed("redactmanager_surroundingTextAnalysis")
     public AnalyzeResult addSurroundingText(String dossierId, String fileId, ManualRedactions manualRedactions) {
@@ -87,9 +87,9 @@ public class ManualRedactionSurroundingTextService {
         }
 
         if (sectionText.getCellStarts() != null && !sectionText.getCellStarts().isEmpty()) {
-            surroundingWordsService.addSurroundingText(Set.of(correctEntity), sectionText.getSearchableText(), null, sectionText.getCellStarts());
+            SurroundingWordsService.addSurroundingText(Set.of(correctEntity), sectionText.getSearchableText(), null, sectionText.getCellStarts(), redactionServiceSettings.getSurroundingWordsOffsetWindow(), redactionServiceSettings.getNumberOfSurroundingWords());
         } else {
-            surroundingWordsService.addSurroundingText(Set.of(correctEntity), sectionText.getSearchableText(), null);
+            SurroundingWordsService.addSurroundingText(Set.of(correctEntity), sectionText.getSearchableText(), null, redactionServiceSettings.getSurroundingWordsOffsetWindow(), redactionServiceSettings.getNumberOfSurroundingWords());
         }
 
         return Pair.of(correctEntity.getTextBefore(), correctEntity.getTextAfter());

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/SearchTextWithTextPositionFactory.java

@@ -0,0 +1,90 @@
+package com.iqser.red.service.redaction.v1.server.redaction.service;
+
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Objects;
+
+import org.springframework.stereotype.Service;
+
+import com.iqser.red.service.redaction.v1.server.parsing.model.RedTextPosition;
+import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
+import com.iqser.red.service.redaction.v1.server.redaction.model.RedRectangle2D;
+import com.iqser.red.service.redaction.v1.server.redaction.model.SearchTextWithTextPositionModel;
+
+@Service
+public class SearchTextWithTextPositionFactory {
+
+    public SearchTextWithTextPositionModel buildSearchTextToTextPositionModel(List<TextPositionSequence> sequences) {
+
+        List<Integer> stringIdxToPositionIdx = new LinkedList<>();
+        List<Integer> lineBreaksStringIdx = new LinkedList<>();
+        StringBuilder sb = new StringBuilder();
+
+        int stringIdx = 0;
+        int positionIdx = 0;
+
+        String currentUnicode;
+        String previousUnicode = " ";
+
+        for (TextPositionSequence word : sequences) {
+            for (int i = 0; i < word.getTextPositions().size(); ++i) {
+
+                currentUnicode = word.getTextPositions().get(i).getUnicode();
+
+                if (isLineBreak(currentUnicode)) {
+                    lineBreaksStringIdx.add(stringIdx + 1);
+                } else if (!isRepeatedWhitespace(currentUnicode, previousUnicode) && //
+                        !isHyphenLinebreak(currentUnicode)) {
+
+                    sb.append(currentUnicode);
+                    stringIdxToPositionIdx.add(positionIdx);
+                    ++stringIdx;
+                }
+
+                previousUnicode = currentUnicode;
+                ++positionIdx;
+            }
+
+            previousUnicode = " ";
+            sb.append(previousUnicode);
+            stringIdxToPositionIdx.add(positionIdx);
+            ++stringIdx;
+        }
+
+        assert sb.length() == stringIdxToPositionIdx.size();
+
+        List<RedRectangle2D> positions = sequences.stream()//
+                .map(TextPositionSequence::getTextPositions)//
+                .flatMap(List::stream)//
+                .map(RedTextPosition::getPosition)//
+                .map(a -> (RedRectangle2D) new RedRectangle2D(a[0], a[1], a[2], a[3]))
+                .toList();
+
+        return SearchTextWithTextPositionModel.builder()
+                .searchText(sb.toString())
+                .lineBreaks(lineBreaksStringIdx)
+                .stringCoordsToPositionCoords(stringIdxToPositionIdx)
+                .positions(positions)
+                .build();
+    }
+
+
+    private boolean isLineBreak(String currentUnicode) {
+
+        return Objects.equals(currentUnicode, "\n");
+    }
+
+
+    private boolean isRepeatedWhitespace(String currentUnicode, String previousUnicode) {
+
+        return Objects.equals(previousUnicode, " ") && Objects.equals(currentUnicode, " ");
+    }
+
+
+    private boolean isHyphenLinebreak(String unicodeCharacter) {
+
+        return unicodeCharacter.matches("([^\\s\\d\\-]{2,500})[\\-\\u00AD]\\R");
+
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/SectionGridCreatorService.java

@@ -4,7 +4,7 @@ import com.iqser.red.service.redaction.v1.model.CellRectangle;
 import com.iqser.red.service.redaction.v1.model.Point;
 import com.iqser.red.service.redaction.v1.model.SectionRectangle;
 import com.iqser.red.service.redaction.v1.server.classification.model.Document;
-import com.iqser.red.service.redaction.v1.server.classification.model.Paragraph;
+import com.iqser.red.service.redaction.v1.server.classification.model.Section;
 import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.Cell;
@@ -31,7 +31,7 @@ public class SectionGridCreatorService {
 
     private void addSectionGrid(Document classifiedDoc, int page) {
 
-        for (Paragraph paragraph : classifiedDoc.getParagraphs()) {
+        for (Section paragraph : classifiedDoc.getSections()) {
 
             for (int i = 0; i <= paragraph.getPageBlocks().size() - 1; i++) {
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/SectionTextBuilderService.java

@@ -16,7 +16,7 @@ import com.iqser.red.service.redaction.v1.model.SectionArea;
 import com.iqser.red.service.redaction.v1.server.classification.model.Document;
 import com.iqser.red.service.redaction.v1.server.classification.model.Footer;
 import com.iqser.red.service.redaction.v1.server.classification.model.Header;
-import com.iqser.red.service.redaction.v1.server.classification.model.Paragraph;
+import com.iqser.red.service.redaction.v1.server.classification.model.Section;
 import com.iqser.red.service.redaction.v1.server.classification.model.SectionText;
 import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
 import com.iqser.red.service.redaction.v1.server.classification.model.UnclassifiedText;
@@ -40,14 +40,14 @@ public class SectionTextBuilderService {
 
         List<SectionText> sectionTexts = new ArrayList<>();
         AtomicInteger sectionNumber = new AtomicInteger(1);
-        for (Paragraph paragraph : classifiedDoc.getParagraphs()) {
+        for (Section section : classifiedDoc.getSections()) {
 
-            List<Table> tables = paragraph.getTables();
+            List<Table> tables = section.getTables();
             for (Table table : tables) {
                 sectionTexts.addAll(processTablePerRow(table, sectionNumber));
                 sectionNumber.incrementAndGet();
             }
-            sectionTexts.add(processText(paragraph.getSearchableText(), paragraph.getTextBlocks(), paragraph.getHeadline(), sectionNumber, paragraph.getImages()));
+            sectionTexts.add(processText(section.getSearchableText(), section.getTextBlocks(), section.getHeadline(), sectionNumber, section.getImages()));
             sectionNumber.incrementAndGet();
         }
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/SurroundingWordsService.java

@@ -7,6 +7,7 @@ import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettin
 
 import io.micrometer.core.annotation.Timed;
 import lombok.RequiredArgsConstructor;
+import lombok.experimental.UtilityClass;
 import lombok.extern.slf4j.Slf4j;
 
 import org.springframework.stereotype.Service;
@@ -15,15 +16,12 @@ import java.util.List;
 import java.util.Set;
 
 @Slf4j
-@Service
+@UtilityClass
-@RequiredArgsConstructor
 public class SurroundingWordsService {
 
-    private final RedactionServiceSettings redactionServiceSettings;
-
 
     @Timed("redactmanager_addSurroundingText")
-    public void addSurroundingText(Set<Entity> entities, SearchableText searchableText, Dictionary dictionary) {
+    public void addSurroundingText(Set<Entity> entities, SearchableText searchableText, Dictionary dictionary, int surroundingWordsOffsetWindow, int numberOfSurroundingWords) {
 
         if (entities.isEmpty()) {
             return;
@@ -35,7 +33,7 @@ public class SurroundingWordsService {
                 if (dictionary != null && dictionary.isHint(entity.getType())) {
                     continue;
                 }
-                findSurroundingWords(entity, searchableText.asString(), entity.getStart(), entity.getEnd());
+                findSurroundingWords(entity, searchableText.asString(), entity.getStart(), entity.getEnd(), surroundingWordsOffsetWindow, numberOfSurroundingWords);
             }
         } catch (Exception e) {
             log.warn("Could not get surrounding text!");
@@ -44,7 +42,7 @@ public class SurroundingWordsService {
 
 
     @Timed("redactmanager_addSurroundingTextTables")
-    public void addSurroundingText(Set<Entity> entities, SearchableText searchableText, Dictionary dictionary, List<Integer> cellstarts) {
+    public void addSurroundingText(Set<Entity> entities, SearchableText searchableText, Dictionary dictionary, List<Integer> cellstarts, int surroundingWordsOffsetWindow, int numberOfSurroundingWords) {
 
         if (entities.isEmpty()) {
             return;
@@ -75,7 +73,7 @@ public class SurroundingWordsService {
                         if (entity.getStart() >= startOffset && entity.getEnd() <= endOffset) {
                             int entityStartOffset = entity.getStart() - startOffset;
                             int entityEndOffset = entity.getEnd() - startOffset;
-                            findSurroundingWords(entity, text, entityStartOffset, entityEndOffset);
+                            findSurroundingWords(entity, text, entityStartOffset, entityEndOffset, surroundingWordsOffsetWindow, numberOfSurroundingWords);
                         }
                     }
                 }
@@ -86,23 +84,23 @@ public class SurroundingWordsService {
     }
 
 
-    private void findSurroundingWords(Entity entity, String text, int entityStartOffset, int entityEndOffset) {
+    private void findSurroundingWords(Entity entity, String text, int entityStartOffset, int entityEndOffset, int surroundingWordsOffsetWindow, int numberOfSurroundingWords) {
 
-        int offsetBefore = entityStartOffset - redactionServiceSettings.getSurroundingWordsOffsetWindow() < 0 ? 0 : entityStartOffset - redactionServiceSettings.getSurroundingWordsOffsetWindow();
+        int offsetBefore = entityStartOffset - surroundingWordsOffsetWindow < 0 ? 0 : entityStartOffset - surroundingWordsOffsetWindow;
         String textBefore = text.substring(offsetBefore, entityStartOffset);
         if (!textBefore.isBlank()) {
             String[] wordsBefore = textBefore.split(" ");
-            int numberOfWordsBefore = wordsBefore.length > redactionServiceSettings.getNumberOfSurroundingWords() ? redactionServiceSettings.getNumberOfSurroundingWords() : wordsBefore.length;
+            int numberOfWordsBefore = wordsBefore.length > numberOfSurroundingWords ? numberOfSurroundingWords : wordsBefore.length;
             if (wordsBefore.length > 0) {
                 entity.setTextBefore(concatWordsBefore(wordsBefore, numberOfWordsBefore, textBefore.endsWith(" ")));
             }
         }
 
-        int endOffset = entityEndOffset + redactionServiceSettings.getSurroundingWordsOffsetWindow() > text.length() ? text.length() : entityEndOffset + redactionServiceSettings.getSurroundingWordsOffsetWindow();
+        int endOffset = entityEndOffset + surroundingWordsOffsetWindow > text.length() ? text.length() : entityEndOffset + surroundingWordsOffsetWindow;
         String textAfter = text.substring(entityEndOffset, endOffset);
         if (!textAfter.isBlank()) {
             String[] wordsAfter = textAfter.split(" ");
-            int numberOfWordsAfter = wordsAfter.length > redactionServiceSettings.getNumberOfSurroundingWords() ? redactionServiceSettings.getNumberOfSurroundingWords() : wordsAfter.length;
+            int numberOfWordsAfter = wordsAfter.length > numberOfSurroundingWords ? numberOfSurroundingWords : wordsAfter.length;
             if (wordsAfter.length > 0) {
                 entity.setTextAfter(concatWordsAfter(wordsAfter, numberOfWordsAfter, textAfter.startsWith(" ")));
             }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/EntitySearchUtils.java

@@ -1,10 +1,12 @@
 package com.iqser.red.service.redaction.v1.server.redaction.utils;
 
+import java.awt.geom.Rectangle2D;
 import java.util.ArrayList;
 import java.util.Comparator;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Iterator;
+import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
@@ -13,12 +15,15 @@ import java.util.stream.Collectors;
 
 import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus;
 import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
+import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
 import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
 import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryModel;
 import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
 import com.iqser.red.service.redaction.v1.server.redaction.model.EntityPositionSequence;
 import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType;
 import com.iqser.red.service.redaction.v1.server.redaction.model.Image;
+import com.iqser.red.service.redaction.v1.server.redaction.model.RedRectangle2D;
+import com.iqser.red.service.redaction.v1.server.redaction.model.SearchTextWithTextPositionModel;
 import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText;
 
 import lombok.experimental.UtilityClass;
@@ -29,6 +34,102 @@ import lombok.extern.slf4j.Slf4j;
 @SuppressWarnings("PMD")
 public final class EntitySearchUtils {
 
+    public static List<EntityPositionSequence> findEntityPositionSequences(String entityString, SearchTextWithTextPositionModel searchTextWithTextPositionModel) {
+
+        return findOccurrences(entityString, searchTextWithTextPositionModel.getSearchText()).stream()
+                .map(range -> mapToTextPositionCoordinates(range, searchTextWithTextPositionModel.getStringCoordsToPositionCoords()))
+                .map(range -> mapToTextPositions(range, searchTextWithTextPositionModel.getPositions()))
+                .map(EntitySearchUtils::mapToEntityPositionSequence)
+                .toList();
+    }
+
+
+    private static List<TextPositionSequence> mapToTextPositions(IndexRange range, List<RedRectangle2D> positions) {
+
+        var slicedPositions = positions.subList(range.startIdx(), range.endIdx());
+        List<TextPositionSequence> sequencesInRange = new LinkedList<>();
+        //sequencesInRange.add()
+        return sequencesInRange;
+    }
+
+
+    private static EntityPositionSequence mapToEntityPositionSequence(List<TextPositionSequence> textPositionSequences) {
+
+        return EntityPositionSequence.builder() //
+                .id(IdBuilder.buildId(textPositionSequences)) //
+                .pageNumber(textPositionSequences.get(0).getPage()) //
+                .sequences(textPositionSequences) //
+                .build();
+    }
+
+
+    private static TextPositionSequence sliceTextPositionSequence(TextPositionSequence sequence, int start, int end) {
+
+        int startIndex = Math.max(start, 0);
+        int endIndex = Math.min(end, sequence.length());
+        return TextPositionSequence.builder()
+                .dir(sequence.getDir())
+                .page(sequence.getPage())
+                .pageHeight(sequence.getPageHeight())
+                .pageWidth(sequence.getPageWidth())
+                .rotation(sequence.getRotation())
+                .textPositions(sequence.getTextPositions().subList(startIndex, endIndex))
+                .build();
+    }
+
+
+    private static boolean sequenceBeforeRange(int end, IndexRange range) {
+
+        return end <= range.startIdx;
+    }
+
+
+    private static boolean sequenceBehindRange(int start, IndexRange range) {
+
+        return start >= range.endIdx;
+    }
+
+
+    private static boolean partialSequenceInRange(int start, int length, IndexRange range) {
+
+        return start >= range.startIdx || (start + length) <= range.endIdx;
+    }
+
+
+    private static boolean fullSequenceInRange(int start, int length, IndexRange range) {
+
+        return start >= range.startIdx && (start + length) <= range.endIdx;
+    }
+
+
+    private static IndexRange mapToTextPositionCoordinates(IndexRange range, List<Integer> stringCoordsToTextPositionCoords) {
+
+        return new IndexRange(stringCoordsToTextPositionCoords.get(range.startIdx), stringCoordsToTextPositionCoords.get(range.endIdx));
+    }
+
+
+    private static List<IndexRange> findOccurrences(String subString, String searchString) {
+
+        List<IndexRange> found = new LinkedList<>();
+
+
+            String cleanValue = subString.trim();
+
+            int startIndex;
+            int stopIndex = 0;
+            do {
+                startIndex = searchString.indexOf(cleanValue, stopIndex);
+                stopIndex = startIndex + cleanValue.length();
+
+                if (startIndex > -1) {
+                    found.add(new IndexRange(startIndex, stopIndex));
+                }
+            } while (startIndex > -1);
+
+        return found;
+    }
+
+
     public boolean sectionContainsAny(String sectionText, SearchImplementation searchImplementation) {
 
         return searchImplementation.atLeastOneMatches(sectionText);
@@ -63,7 +164,7 @@ public final class EntitySearchUtils {
 
         Set<Entity> entities = new HashSet<>();
 
-        searchImplementation.getMatches(inputString).forEach(match -> validateAndAddEntity(entities, findEntityDetails, inputString, match.getStartIndex(), match.getEndIndex()));
+        searchImplementation.getMatches(inputString).forEach(match -> validateAndAddEntity(entities, findEntityDetails, inputString, match.startIndex(), match.endIndex()));
 
         return entities;
     }
@@ -79,6 +180,7 @@ public final class EntitySearchUtils {
                     stopIndex,
                     findEntityDetails.getHeadline(),
                     findEntityDetails.getSectionNumber(),
+                    -1,
                     findEntityDetails.isDictionaryEntry(),
                     findEntityDetails.isDossierDictionary(),
                     findEntityDetails.getEngine(),
@@ -99,7 +201,7 @@ public final class EntitySearchUtils {
 
             List<Entity> orderedEntities = entitiesByWord.get(word).stream().sorted(Comparator.comparing(Entity::getStart)).collect(Collectors.toList());
             Entity firstEntity = orderedEntities.get(0);
-            List<EntityPositionSequence> positionSequences = text.getSequences(firstEntity.getWord().trim(),
+            List<EntityPositionSequence> positionSequences = text.getSequences(word.trim(),
                     dictionary == null || dictionary.isCaseInsensitiveDictionary(firstEntity.getType()),
                     firstEntity.getTargetSequences());
 
@@ -358,7 +460,7 @@ public final class EntitySearchUtils {
 
         for (Entity existing : existingEntities) {
 
-            // skip if either start or end is equal
+            // skip if either startIdx or endIdx is equal
             if (existing.getStart().equals(found.getStart()) || existing.getEnd().equals(found.getEnd())) {
                 continue;
             }
@@ -374,4 +476,9 @@ public final class EntitySearchUtils {
         return false;
     }
 
+
+    record IndexRange(int startIdx, int endIdx) {
+
+    }
+
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/OffsetStringUtils.java

@@ -16,8 +16,8 @@ public final class OffsetStringUtils {
      * Same logic as in StringUtils.redactBetween, but returns a list of object with offsets insteadof on the Strings only.
      *
      * @param str – the String containing the substrings, null returns null, empty returns empty
-     * @param open – the String identifying the start of the substring, empty returns null
+     * @param open – the String identifying the startIdx of the substring, empty returns null
-     * @param close – the String identifying the end of the substring, empty returns null
+     * @param close – the String identifying the endIdx of the substring, empty returns null
      * @return a list of Strings with their offsets
      */
     public List<OffsetString> substringsBetween(final String str, final String open, final String close) {

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/SearchImplementation.java

@@ -1,10 +1,5 @@
 package com.iqser.red.service.redaction.v1.server.redaction.utils;
 
-import lombok.AllArgsConstructor;
-import lombok.Data;
-
-import org.ahocorasick.trie.Trie;
-
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.List;
@@ -12,6 +7,12 @@ import java.util.Locale;
 import java.util.regex.Pattern;
 import java.util.stream.Collectors;
 
+import org.ahocorasick.trie.Trie;
+
+import com.iqser.red.service.redaction.v1.server.document.graph.Boundary;
+
+import lombok.Data;
+
 @Data
 public class SearchImplementation {
 
@@ -74,7 +75,33 @@ public class SearchImplementation {
         } else {
             return this.trie.containsMatch(textToCheck);
         }
+    }
 
+
+    public List<Boundary> getMatches(CharSequence text) {
+
+        if (this.values.isEmpty()) {
+            return new ArrayList<>();
+        }
+        if (this.pattern != null) {
+            return this.pattern.matcher(text).results().map(r -> new Boundary(r.start(), r.end())).collect(Collectors.toList());
+        } else {
+            return this.trie.parseText(text).stream().map(r -> new Boundary(r.getStart(), r.getEnd() + 1)).collect(Collectors.toList());
+        }
+    }
+
+
+    public List<Boundary> getMatches(CharSequence text, int startOffset) {
+
+        if (this.values.isEmpty()) {
+            return new ArrayList<>();
+        }
+        CharSequence subSequence = text.subSequence(startOffset, startOffset + text.length());
+        if (this.pattern != null) {
+            return this.pattern.matcher(subSequence).results().map(r -> new Boundary(r.start() + startOffset, r.end() + startOffset)).collect(Collectors.toList());
+        } else {
+            return this.trie.parseText(subSequence).stream().map(r -> new Boundary(r.getStart() + startOffset, r.getEnd() + startOffset + 1)).collect(Collectors.toList());
+        }
     }
 
 
@@ -95,12 +122,7 @@ public class SearchImplementation {
     }
 
 
-    @Data
+    public record MatchPosition(int startIndex, int endIndex) {
-    @AllArgsConstructor
-    public static class MatchPosition {
-
-        private int startIndex;
-        private int endIndex;
 
     }
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/segmentation/SectionsBuilderService.java

@@ -15,7 +15,7 @@ import com.iqser.red.service.redaction.v1.server.classification.model.Document;
 import com.iqser.red.service.redaction.v1.server.classification.model.Footer;
 import com.iqser.red.service.redaction.v1.server.classification.model.Header;
 import com.iqser.red.service.redaction.v1.server.classification.model.Page;
-import com.iqser.red.service.redaction.v1.server.classification.model.Paragraph;
+import com.iqser.red.service.redaction.v1.server.classification.model.Section;
 import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
 import com.iqser.red.service.redaction.v1.server.classification.model.UnclassifiedText;
 import com.iqser.red.service.redaction.v1.server.redaction.model.PdfImage;
@@ -32,7 +32,7 @@ public class SectionsBuilderService {
     public void buildSections(Document document) {
 
         List<AbstractTextContainer> chunkWords = new ArrayList<>();
-        List<Paragraph> chunkBlockList = new ArrayList<>();
+        List<Section> chunkBlockList = new ArrayList<>();
         List<Header> headers = new ArrayList<>();
         List<Footer> footers = new ArrayList<>();
         List<UnclassifiedText> unclassifiedTexts = new ArrayList<>();
@@ -69,7 +69,7 @@ public class SectionsBuilderService {
                 }
 
                 if (prev != null && current.getClassification().startsWith("H ") && !prev.getClassification().startsWith("H ") || !document.isHeadlines()) {
-                    Paragraph chunkBlock = buildTextBlock(chunkWords, lastHeadline);
+                    Section chunkBlock = buildTextBlock(chunkWords, lastHeadline);
                     chunkBlock.setHeadline(lastHeadline);
                     if (document.isHeadlines()) {
                         lastHeadline = current.getText();
@@ -101,11 +101,11 @@ public class SectionsBuilderService {
             }
         }
 
-        Paragraph chunkBlock = buildTextBlock(chunkWords, lastHeadline);
+        Section chunkBlock = buildTextBlock(chunkWords, lastHeadline);
         chunkBlock.setHeadline(lastHeadline);
         chunkBlockList.add(chunkBlock);
 
-        document.setParagraphs(chunkBlockList);
+        document.setSections(chunkBlockList);
         document.setHeaders(headers);
         document.setFooters(footers);
         document.setUnclassifiedTexts(unclassifiedTexts);
@@ -114,8 +114,8 @@ public class SectionsBuilderService {
 
     public void addImagesToSections(Document document) {
 
-        Map<Integer, List<Paragraph>> paragraphMap = new HashMap<>();
+        Map<Integer, List<Section>> paragraphMap = new HashMap<>();
-        for (Paragraph paragraph : document.getParagraphs()) {
+        for (Section paragraph : document.getSections()) {
             for (AbstractTextContainer container : paragraph.getPageBlocks()) {
 
                 paragraphMap.computeIfAbsent(container.getPage(), c -> new ArrayList<>()).add(paragraph);
@@ -124,22 +124,22 @@ public class SectionsBuilderService {
         }
 
         if (paragraphMap.isEmpty()) {
-            Paragraph paragraph = new Paragraph();
+            Section paragraph = new Section();
-            document.getParagraphs().add(paragraph);
+            document.getSections().add(paragraph);
             paragraphMap.computeIfAbsent(1, x -> new ArrayList<>()).add(paragraph);
         }
 
         // first page is always a paragraph, else we can't process pages 1..N,
         // where N is the first found page with a paragraph
         if (paragraphMap.get(1) == null) {
-            Paragraph paragraph = new Paragraph();
+            Section paragraph = new Section();
-            document.getParagraphs().add(paragraph);
+            document.getSections().add(paragraph);
             paragraphMap.computeIfAbsent(1, x -> new ArrayList<>()).add(paragraph);
         }
 
         for (Page page : document.getPages()) {
             for (PdfImage image : page.getImages()) {
-                List<Paragraph> paragraphsOnPage = paragraphMap.get(page.getPageNumber());
+                List<Section> paragraphsOnPage = paragraphMap.get(page.getPageNumber());
                 if (paragraphsOnPage == null) {
                     int i = page.getPageNumber();
                     while (paragraphsOnPage == null) {
@@ -147,7 +147,7 @@ public class SectionsBuilderService {
                         i--;
                     }
                 }
-                for (Paragraph paragraph : paragraphsOnPage) {
+                for (Section paragraph : paragraphsOnPage) {
                     Float xMin = null;
                     Float yMin = null;
                     Float xMax = null;
@@ -239,9 +239,9 @@ public class SectionsBuilderService {
     }
 
 
-    private Paragraph buildTextBlock(List<AbstractTextContainer> wordBlockList, String lastHeadline) {
+    private Section buildTextBlock(List<AbstractTextContainer> wordBlockList, String lastHeadline) {
 
-        Paragraph paragraph = new Paragraph();
+        Section section = new Section();
         TextBlock textBlock = null;
 
         int pageBefore = -1;
@@ -268,40 +268,17 @@ public class SectionsBuilderService {
                 }
 
                 if (textBlock != null && !alreadyAdded) {
-                    paragraph.getPageBlocks().add(textBlock);
+                    section.getPageBlocks().add(textBlock);
                     alreadyAdded = true;
                 }
-                paragraph.getPageBlocks().add(table);
+                section.getPageBlocks().add(table);
                 continue;
             }
 
             TextBlock wordBlock = (TextBlock) container;
-
+            section.getPageBlocks().add(wordBlock);
-            if (textBlock == null) {
-                textBlock = new TextBlock(wordBlock.getMinX(), wordBlock.getMaxX(), wordBlock.getMinY(), wordBlock.getMaxY(), wordBlock.getSequences(), wordBlock.getRotation());
-                textBlock.setPage(wordBlock.getPage());
-            } else if (splitByTable) {
-                textBlock = new TextBlock(wordBlock.getMinX(), wordBlock.getMaxX(), wordBlock.getMinY(), wordBlock.getMaxY(), wordBlock.getSequences(), wordBlock.getRotation());
-                textBlock.setPage(wordBlock.getPage());
-                alreadyAdded = false;
-            } else if (pageBefore != -1 && wordBlock.getPage() != pageBefore) {
-                textBlock.setPage(pageBefore);
-                paragraph.getPageBlocks().add(textBlock);
-                textBlock = new TextBlock(wordBlock.getMinX(), wordBlock.getMaxX(), wordBlock.getMinY(), wordBlock.getMaxY(), wordBlock.getSequences(), wordBlock.getRotation());
-                textBlock.setPage(wordBlock.getPage());
-            } else {
-                TextBlock spatialEntity = textBlock.union(wordBlock);
-                textBlock.resize(spatialEntity.getMinX(), spatialEntity.getMinY(), spatialEntity.getWidth(), spatialEntity.getHeight());
-            }
-            pageBefore = wordBlock.getPage();
-            splitByTable = false;
-            previous = container;
         }
-
+        return section;
-        if (textBlock != null && !alreadyAdded) {
-            paragraph.getPageBlocks().add(textBlock);
-        }
-        return paragraph;
     }
 
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/visualization/service/PdfVisualisationService.java

@@ -12,7 +12,7 @@ import org.springframework.stereotype.Service;
 
 import com.iqser.red.service.redaction.v1.server.classification.model.Document;
 import com.iqser.red.service.redaction.v1.server.classification.model.Page;
-import com.iqser.red.service.redaction.v1.server.classification.model.Paragraph;
+import com.iqser.red.service.redaction.v1.server.classification.model.Section;
 import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.Cell;
@@ -37,7 +37,7 @@ public class PdfVisualisationService {
             PDPage pdPage = document.getPage(page - 1);
             PDPageContentStream contentStream = new PDPageContentStream(document, pdPage, PDPageContentStream.AppendMode.APPEND, true);
 
-            for (Paragraph paragraph : classifiedDoc.getParagraphs()) {
+            for (Section paragraph : classifiedDoc.getSections()) {
 
                 for (int i = 0; i <= paragraph.getPageBlocks().size() - 1; i++) {
 

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AbstractTestWithDictionaries.java

@@ -0,0 +1,533 @@
+package com.iqser.red.service.redaction.v1.server;
+
+import static org.mockito.Mockito.when;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.net.URL;
+import java.nio.charset.StandardCharsets;
+import java.time.OffsetDateTime;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.kie.api.KieServices;
+import org.kie.api.builder.KieBuilder;
+import org.kie.api.builder.KieFileSystem;
+import org.kie.api.builder.KieModule;
+import org.kie.api.builder.KieRepository;
+import org.kie.api.builder.ReleaseId;
+import org.kie.api.runtime.KieContainer;
+import org.kie.internal.io.ResourceFactory;
+import org.mockito.stubbing.Answer;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
+import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration;
+import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.boot.test.mock.mockito.MockBean;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.context.annotation.Import;
+import org.springframework.context.annotation.Primary;
+import org.springframework.core.io.ClassPathResource;
+import org.springframework.test.context.junit.jupiter.SpringExtension;
+
+import com.iqser.red.service.persistence.service.v1.api.model.common.JSONPrimitive;
+import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.configuration.Colors;
+import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
+import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntry;
+import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.Type;
+import com.iqser.red.service.redaction.v1.model.AnalyzeRequest;
+import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
+import com.iqser.red.service.redaction.v1.server.client.RulesClient;
+import com.iqser.red.service.redaction.v1.server.redaction.utils.ResourceLoader;
+import com.iqser.red.service.redaction.v1.server.redaction.utils.TextNormalizationUtilities;
+import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
+import com.iqser.red.storage.commons.StorageAutoConfiguration;
+import com.iqser.red.storage.commons.service.StorageService;
+
+import lombok.SneakyThrows;
+
+@ExtendWith(SpringExtension.class)
+@SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
+@Import(AbstractTestWithDictionaries.TestConfiguration.class)
+public class AbstractTestWithDictionaries {
+
+    protected static final String RULES = loadFromClassPath("drools/rules.drl");
+    protected static final String RULES_PATH = "drools/rules.drl";
+    protected static final String ENTITY_RULES_PATH = "drools/entity_rules.drl";
+    private static final String VERTEBRATE = "vertebrate";
+    private static final String ADDRESS = "CBI_address";
+    private static final String AUTHOR = "CBI_author";
+    private static final String SPONSOR = "CBI_sponsor";
+    private static final String NO_REDACTION_INDICATOR = "no_redaction_indicator";
+    private static final String REDACTION_INDICATOR = "redaction_indicator";
+    private static final String HINT_ONLY = "hint_only";
+    private static final String MUST_REDACT = "must_redact";
+    private static final String PUBLISHED_INFORMATION = "published_information";
+    private static final String TEST_METHOD = "test_method";
+    private static final String PURITY = "purity";
+    private static final String IMAGE = "image";
+    private static final String LOGO = "logo";
+    private static final String SIGNATURE = "signature";
+    private static final String FORMULA = "formula";
+    private static final String OCR = "ocr";
+    private static final String DOSSIER_REDACTIONS = "dossier_redactions";
+    private static final String IMPORTED_REDACTION = "imported_redaction";
+    private static final String PII = "PII";
+    private static final String ROTATE_SIMPLE = "RotateSimple";
+
+    protected final static String TEST_DOSSIER_TEMPLATE_ID = "123";
+    protected final static String TEST_DOSSIER_ID = "123";
+    protected final static String TEST_FILE_ID = "123";
+
+    @Autowired
+    protected StorageService storageService;
+
+    @MockBean
+    protected DictionaryClient dictionaryClient;
+
+    @MockBean
+    protected RulesClient rulesClient;
+
+    private final Map<String, List<String>> dictionary = new HashMap<>();
+    private final Map<String, List<String>> dossierDictionary = new HashMap<>();
+    private final Map<String, List<String>> falsePositive = new HashMap<>();
+    private final Map<String, List<String>> falseRecommendation = new HashMap<>();
+    private final Map<String, String> typeColorMap = new HashMap<>();
+    private final Map<String, Boolean> hintTypeMap = new HashMap<>();
+    private final Map<String, Boolean> caseInSensitiveMap = new HashMap<>();
+    private final Map<String, Boolean> recommendationTypeMap = new HashMap<>();
+    private final Map<String, Integer> rankTypeMap = new HashMap<>();
+    private final Colors colors = new Colors();
+    private final Map<String, Long> reanalysisVersions = new HashMap<>();
+    private final Set<String> deleted = new HashSet<>();
+
+
+    @BeforeEach
+    public void stubClients() {
+
+        when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L);
+        when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(JSONPrimitive.of(RULES));
+
+        loadDictionaryForTest();
+        loadTypeForTest();
+        loadNerForTest();
+        when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L);
+        when(dictionaryClient.getAllTypesForDossierTemplate(TEST_DOSSIER_TEMPLATE_ID, false)).thenReturn(getTypeResponse());
+
+        when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L);
+        when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, false)).thenReturn(List.of(Type.builder()
+                .id(DOSSIER_REDACTIONS + ":" + TEST_DOSSIER_TEMPLATE_ID)
+                .type(DOSSIER_REDACTIONS)
+                .dossierTemplateId(TEST_DOSSIER_ID)
+                .hexColor("#ffe187")
+                .isHint(hintTypeMap.get(DOSSIER_REDACTIONS))
+                .isCaseInsensitive(caseInSensitiveMap.get(DOSSIER_REDACTIONS))
+                .isRecommendation(recommendationTypeMap.get(DOSSIER_REDACTIONS))
+                .rank(rankTypeMap.get(DOSSIER_REDACTIONS))
+                .build()));
+
+        mockDictionaryCalls(null);
+        mockDictionaryCalls(0L);
+
+        when(dictionaryClient.getColors(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(colors);
+    }
+
+
+    private static String loadFromClassPath(String path) {
+
+        URL resource = ResourceLoader.class.getClassLoader().getResource(path);
+        if (resource == null) {
+            throw new IllegalArgumentException("could not load classpath resource: drools/rules.drl");
+        }
+        try (BufferedReader br = new BufferedReader(new InputStreamReader(resource.openStream(), StandardCharsets.UTF_8))) {
+            StringBuilder sb = new StringBuilder();
+            String str;
+            while ((str = br.readLine()) != null) {
+                sb.append(str).append("\n");
+            }
+            return sb.toString();
+        } catch (IOException e) {
+            throw new IllegalArgumentException("could not load classpath resource: " + path, e);
+        }
+    }
+
+
+    private void mockDictionaryCalls(Long version) {
+
+        when(dictionaryClient.getDictionaryForType(VERTEBRATE + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(VERTEBRATE,
+                false));
+        when(dictionaryClient.getDictionaryForType(ADDRESS + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(ADDRESS, false));
+        when(dictionaryClient.getDictionaryForType(AUTHOR + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(AUTHOR, false));
+        when(dictionaryClient.getDictionaryForType(SPONSOR + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(SPONSOR, false));
+        when(dictionaryClient.getDictionaryForType(NO_REDACTION_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(
+                NO_REDACTION_INDICATOR,
+                false));
+        when(dictionaryClient.getDictionaryForType(REDACTION_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(
+                REDACTION_INDICATOR,
+                false));
+        when(dictionaryClient.getDictionaryForType(HINT_ONLY + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(HINT_ONLY, false));
+        when(dictionaryClient.getDictionaryForType(MUST_REDACT + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(MUST_REDACT,
+                false));
+        when(dictionaryClient.getDictionaryForType(PUBLISHED_INFORMATION + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(
+                PUBLISHED_INFORMATION,
+                false));
+        when(dictionaryClient.getDictionaryForType(TEST_METHOD + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(TEST_METHOD,
+                false));
+        when(dictionaryClient.getDictionaryForType(PII + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(PII, false));
+        when(dictionaryClient.getDictionaryForType(PURITY + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(PURITY, false));
+        when(dictionaryClient.getDictionaryForType(IMAGE + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(IMAGE, false));
+        when(dictionaryClient.getDictionaryForType(OCR + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(OCR, false));
+        when(dictionaryClient.getDictionaryForType(LOGO + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(LOGO, false));
+        when(dictionaryClient.getDictionaryForType(SIGNATURE + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(SIGNATURE, false));
+        when(dictionaryClient.getDictionaryForType(FORMULA + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(FORMULA, false));
+        when(dictionaryClient.getDictionaryForType(ROTATE_SIMPLE + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(ROTATE_SIMPLE,
+                false));
+        when(dictionaryClient.getDictionaryForType(DOSSIER_REDACTIONS + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(
+                DOSSIER_REDACTIONS,
+                true));
+        when(dictionaryClient.getDictionaryForType(IMPORTED_REDACTION + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(
+                IMPORTED_REDACTION,
+                true));
+
+    }
+
+
+    private String cleanDictionaryEntry(String entry) {
+
+        return TextNormalizationUtilities.removeHyphenLineBreaks(entry).replaceAll("\\n", " ");
+    }
+
+
+    private List<DictionaryEntry> toDictionaryEntry(List<String> entries) {
+
+        if (entries == null) {
+            entries = Collections.emptyList();
+        }
+
+        List<DictionaryEntry> dictionaryEntries = new ArrayList<>();
+        entries.forEach(entry -> {
+            dictionaryEntries.add(DictionaryEntry.builder().value(entry).version(reanalysisVersions.getOrDefault(entry, 0L)).deleted(deleted.contains(entry)).build());
+        });
+        return dictionaryEntries;
+    }
+
+
+    private Type getDictionaryResponse(String type, boolean isDossierDictionary) {
+
+        return Type.builder()
+                .id(type + ":" + TEST_DOSSIER_TEMPLATE_ID)
+                .hexColor(typeColorMap.get(type))
+                .entries(isDossierDictionary ? toDictionaryEntry(dossierDictionary.get(type)) : toDictionaryEntry(dictionary.get(type)))
+                .falsePositiveEntries(falsePositive.containsKey(type) ? toDictionaryEntry(falsePositive.get(type)) : new ArrayList<>())
+                .falseRecommendationEntries(falseRecommendation.containsKey(type) ? toDictionaryEntry(falseRecommendation.get(type)) : new ArrayList<>())
+                .isHint(hintTypeMap.get(type))
+                .isCaseInsensitive(caseInSensitiveMap.get(type))
+                .isRecommendation(recommendationTypeMap.get(type))
+                .rank(rankTypeMap.get(type))
+                .build();
+    }
+
+
+    private void loadTypeForTest() {
+
+        typeColorMap.put(VERTEBRATE, "#ff85f7");
+        typeColorMap.put(ADDRESS, "#ffe187");
+        typeColorMap.put(AUTHOR, "#ffe187");
+        typeColorMap.put(SPONSOR, "#85ebff");
+        typeColorMap.put(NO_REDACTION_INDICATOR, "#be85ff");
+        typeColorMap.put(REDACTION_INDICATOR, "#caff85");
+        typeColorMap.put(HINT_ONLY, "#abc0c4");
+        typeColorMap.put(MUST_REDACT, "#fab4c0");
+        typeColorMap.put(PUBLISHED_INFORMATION, "#85ebff");
+        typeColorMap.put(TEST_METHOD, "#91fae8");
+        typeColorMap.put(PII, "#66ccff");
+        typeColorMap.put(PURITY, "#ffe187");
+        typeColorMap.put(IMAGE, "#fcc5fb");
+        typeColorMap.put(OCR, "#fcc5fb");
+        typeColorMap.put(LOGO, "#ffe187");
+        typeColorMap.put(FORMULA, "#ffe187");
+        typeColorMap.put(SIGNATURE, "#ffe187");
+        typeColorMap.put(IMPORTED_REDACTION, "#fcfbe6");
+        typeColorMap.put(ROTATE_SIMPLE, "#66ccff");
+
+        hintTypeMap.put(VERTEBRATE, true);
+        hintTypeMap.put(ADDRESS, false);
+        hintTypeMap.put(AUTHOR, false);
+        hintTypeMap.put(SPONSOR, false);
+        hintTypeMap.put(NO_REDACTION_INDICATOR, true);
+        hintTypeMap.put(REDACTION_INDICATOR, true);
+        hintTypeMap.put(HINT_ONLY, true);
+        hintTypeMap.put(MUST_REDACT, true);
+        hintTypeMap.put(PUBLISHED_INFORMATION, true);
+        hintTypeMap.put(TEST_METHOD, true);
+        hintTypeMap.put(PII, false);
+        hintTypeMap.put(PURITY, false);
+        hintTypeMap.put(IMAGE, true);
+        hintTypeMap.put(OCR, true);
+        hintTypeMap.put(FORMULA, false);
+        hintTypeMap.put(LOGO, false);
+        hintTypeMap.put(SIGNATURE, false);
+        hintTypeMap.put(DOSSIER_REDACTIONS, false);
+        hintTypeMap.put(IMPORTED_REDACTION, false);
+        hintTypeMap.put(ROTATE_SIMPLE, false);
+
+        caseInSensitiveMap.put(VERTEBRATE, true);
+        caseInSensitiveMap.put(ADDRESS, false);
+        caseInSensitiveMap.put(AUTHOR, false);
+        caseInSensitiveMap.put(SPONSOR, false);
+        caseInSensitiveMap.put(NO_REDACTION_INDICATOR, true);
+        caseInSensitiveMap.put(REDACTION_INDICATOR, true);
+        caseInSensitiveMap.put(HINT_ONLY, true);
+        caseInSensitiveMap.put(MUST_REDACT, true);
+        caseInSensitiveMap.put(PUBLISHED_INFORMATION, true);
+        caseInSensitiveMap.put(TEST_METHOD, false);
+        caseInSensitiveMap.put(PII, false);
+        caseInSensitiveMap.put(PURITY, false);
+        caseInSensitiveMap.put(IMAGE, true);
+        caseInSensitiveMap.put(OCR, true);
+        caseInSensitiveMap.put(SIGNATURE, true);
+        caseInSensitiveMap.put(LOGO, true);
+        caseInSensitiveMap.put(FORMULA, true);
+        caseInSensitiveMap.put(DOSSIER_REDACTIONS, false);
+        caseInSensitiveMap.put(IMPORTED_REDACTION, false);
+        caseInSensitiveMap.put(ROTATE_SIMPLE, true);
+
+        recommendationTypeMap.put(VERTEBRATE, false);
+        recommendationTypeMap.put(ADDRESS, false);
+        recommendationTypeMap.put(AUTHOR, false);
+        recommendationTypeMap.put(SPONSOR, false);
+        recommendationTypeMap.put(NO_REDACTION_INDICATOR, false);
+        recommendationTypeMap.put(REDACTION_INDICATOR, false);
+        recommendationTypeMap.put(HINT_ONLY, false);
+        recommendationTypeMap.put(MUST_REDACT, false);
+        recommendationTypeMap.put(PUBLISHED_INFORMATION, false);
+        recommendationTypeMap.put(TEST_METHOD, false);
+        recommendationTypeMap.put(PII, false);
+        recommendationTypeMap.put(PURITY, false);
+        recommendationTypeMap.put(IMAGE, false);
+        recommendationTypeMap.put(OCR, false);
+        recommendationTypeMap.put(FORMULA, false);
+        recommendationTypeMap.put(SIGNATURE, false);
+        recommendationTypeMap.put(LOGO, false);
+        recommendationTypeMap.put(DOSSIER_REDACTIONS, false);
+        recommendationTypeMap.put(IMPORTED_REDACTION, false);
+        recommendationTypeMap.put(ROTATE_SIMPLE, false);
+
+        rankTypeMap.put(PURITY, 155);
+        rankTypeMap.put(PII, 150);
+        rankTypeMap.put(ADDRESS, 140);
+        rankTypeMap.put(AUTHOR, 130);
+        rankTypeMap.put(SPONSOR, 120);
+        rankTypeMap.put(VERTEBRATE, 110);
+        rankTypeMap.put(MUST_REDACT, 100);
+        rankTypeMap.put(REDACTION_INDICATOR, 90);
+        rankTypeMap.put(NO_REDACTION_INDICATOR, 80);
+        rankTypeMap.put(PUBLISHED_INFORMATION, 70);
+        rankTypeMap.put(TEST_METHOD, 60);
+        rankTypeMap.put(HINT_ONLY, 50);
+        rankTypeMap.put(IMAGE, 30);
+        rankTypeMap.put(OCR, 29);
+        rankTypeMap.put(LOGO, 28);
+        rankTypeMap.put(SIGNATURE, 27);
+        rankTypeMap.put(FORMULA, 26);
+        rankTypeMap.put(DOSSIER_REDACTIONS, 200);
+        rankTypeMap.put(IMPORTED_REDACTION, 200);
+        rankTypeMap.put(ROTATE_SIMPLE, 150);
+
+        colors.setSkippedColor("#cccccc");
+        colors.setRequestAddColor("#04b093");
+        colors.setRequestRemoveColor("#04b093");
+    }
+
+
+    @SneakyThrows
+    private void loadNerForTest() {
+
+        ClassPathResource responseJson = new ClassPathResource("files/ner_response.json");
+        storageService.storeObject(RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.NER_ENTITIES), responseJson.getInputStream());
+    }
+
+
+    private List<Type> getTypeResponse() {
+
+        return typeColorMap.entrySet()
+                .stream()
+                .map(typeColor -> Type.builder()
+                        .id(typeColor.getKey() + ":" + TEST_DOSSIER_TEMPLATE_ID)
+                        .type(typeColor.getKey())
+                        .dossierTemplateId(TEST_DOSSIER_TEMPLATE_ID)
+                        .hexColor(typeColor.getValue())
+                        .isHint(hintTypeMap.get(typeColor.getKey()))
+                        .isCaseInsensitive(caseInSensitiveMap.get(typeColor.getKey()))
+                        .isRecommendation(recommendationTypeMap.get(typeColor.getKey()))
+                        .rank(rankTypeMap.get(typeColor.getKey()))
+                        .build())
+
+                .collect(Collectors.toList());
+    }
+
+    protected List<File> getPathsRecursively(File path) {
+
+        List<File> result = new ArrayList<>();
+        if (path == null || path.listFiles() == null) {
+            return result;
+        }
+        for (File f : path.listFiles()) {
+            if (f.isFile()) {
+                result.add(f);
+            } else {
+                result.addAll(getPathsRecursively(f));
+            }
+        }
+        return result;
+
+    }
+
+    protected void loadOnlyDictionaryForSimpleFile() {
+
+        dictionary.clear();
+        dictionary.computeIfAbsent(ROTATE_SIMPLE, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/RotateTestFileSimple.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
+    }
+
+
+    private void loadDictionaryForTest() {
+
+        dictionary.computeIfAbsent(AUTHOR, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/CBI_author.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
+        dictionary.computeIfAbsent(SPONSOR, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/CBI_sponsor.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
+        dictionary.computeIfAbsent(VERTEBRATE, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/vertebrate.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
+        dictionary.computeIfAbsent(ADDRESS, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/CBI_address.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
+        dictionary.computeIfAbsent(NO_REDACTION_INDICATOR, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/no_redaction_indicator.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
+        dictionary.computeIfAbsent(REDACTION_INDICATOR, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/redaction_indicator.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
+        dictionary.computeIfAbsent(HINT_ONLY, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/hint_only.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
+        dictionary.computeIfAbsent(MUST_REDACT, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/must_redact.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
+        dictionary.computeIfAbsent(PUBLISHED_INFORMATION, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/published_information.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
+        dictionary.computeIfAbsent(TEST_METHOD, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/test_method.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
+        dictionary.computeIfAbsent(PII, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/PII.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
+        dictionary.computeIfAbsent(PURITY, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/purity.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
+        dictionary.computeIfAbsent(IMAGE, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/empty.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
+        dictionary.computeIfAbsent(OCR, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/empty.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
+        dictionary.computeIfAbsent(LOGO, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/empty.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
+        dictionary.computeIfAbsent(SIGNATURE, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/empty.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
+        dictionary.computeIfAbsent(FORMULA, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/empty.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
+        dossierDictionary.computeIfAbsent(DOSSIER_REDACTIONS, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/dossier_redactions.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
+        dossierDictionary.put(IMPORTED_REDACTION, new ArrayList<>());
+
+        falsePositive.computeIfAbsent(PII, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/PII_false_positive.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
+
+    }
+
+
+    @SneakyThrows
+    protected AnalyzeRequest prepareStorage(String file) {
+
+        return prepareStorage(file, "files/cv_service_empty_response.json");
+    }
+
+
+    @SneakyThrows
+    protected AnalyzeRequest prepareStorage(String file, String cvServiceResponseFile) {
+
+        ClassPathResource pdfFileResource = new ClassPathResource(file);
+        ClassPathResource cvServiceResponseFileResource = new ClassPathResource(cvServiceResponseFile);
+
+        return prepareStorage(pdfFileResource.getInputStream(), cvServiceResponseFileResource.getInputStream());
+    }
+
+
+    @SneakyThrows
+    protected AnalyzeRequest prepareStorage(InputStream fileStream, InputStream cvServiceResponseFileStream) {
+
+        AnalyzeRequest request = AnalyzeRequest.builder()
+                .dossierTemplateId(TEST_DOSSIER_TEMPLATE_ID)
+                .dossierId(TEST_DOSSIER_ID)
+                .fileId(TEST_FILE_ID)
+                .lastProcessed(OffsetDateTime.now())
+                .build();
+
+        storageService.storeObject(RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.TABLES), cvServiceResponseFileStream);
+        storageService.storeObject(RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.ORIGIN), fileStream);
+
+        return request;
+    }
+
+
+    @AfterEach
+    public void cleanupStorage() {
+
+        if (this.storageService instanceof FileSystemBackedStorageService) {
+            ((FileSystemBackedStorageService) this.storageService).clearStorage();
+        }
+    }
+
+
+    @Configuration
+    @EnableAutoConfiguration(exclude = {RabbitAutoConfiguration.class, StorageAutoConfiguration.class})
+    public static class TestConfiguration {
+
+        @Bean
+        public KieContainer kieContainer() {
+
+            KieServices kieServices = KieServices.Factory.get();
+
+            KieFileSystem kieFileSystem = kieServices.newKieFileSystem();
+            //kieFileSystem.write(ResourceFactory.newClassPathResource(RULES_PATH, "UTF-8"));
+            kieFileSystem.write(ResourceFactory.newClassPathResource(ENTITY_RULES_PATH, "UTF-8"));
+
+            KieRepository kieRepository = kieServices.getRepository();
+
+            kieRepository.addKieModule(new KieModule() {
+                public ReleaseId getReleaseId() {
+                    return kieRepository.getDefaultReleaseId();
+                }
+            });
+
+            KieBuilder kieBuilder = kieServices
+                    .newKieBuilder(kieFileSystem)
+                    .buildAll();
+
+            return kieServices.newKieContainer(kieRepository.getDefaultReleaseId());
+        }
+
+        @Bean
+        @Primary
+        public StorageService inmemoryStorage() {
+
+            return new FileSystemBackedStorageService();
+        }
+
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DictionaryServiceTest.java

@@ -1,16 +1,16 @@
 package com.iqser.red.service.redaction.v1.server;
 
-import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.configuration.Colors;
+import static org.mockito.ArgumentMatchers.any;
-import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntry;
+import static org.mockito.ArgumentMatchers.anyBoolean;
-import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.Type;
+import static org.mockito.ArgumentMatchers.anyString;
-import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
+import static org.mockito.Mockito.when;
-import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryVersion;
-import com.iqser.red.service.redaction.v1.server.redaction.service.DictionaryService;
-import com.iqser.red.storage.commons.StorageAutoConfiguration;
-import com.iqser.red.storage.commons.service.StorageService;
 
-import org.junit.Test;
+import java.util.List;
-import org.junit.runner.RunWith;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
 import org.kie.api.runtime.KieContainer;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
@@ -21,16 +21,18 @@ import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;
 import org.springframework.context.annotation.Import;
 import org.springframework.context.annotation.Primary;
-import org.springframework.test.context.junit4.SpringRunner;
+import org.springframework.test.context.junit.jupiter.SpringExtension;
 
-import java.util.List;
+import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.configuration.Colors;
-import java.util.stream.Collectors;
+import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntry;
-import java.util.stream.Stream;
+import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.Type;
+import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
+import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryVersion;
+import com.iqser.red.service.redaction.v1.server.redaction.service.DictionaryService;
+import com.iqser.red.storage.commons.StorageAutoConfiguration;
+import com.iqser.red.storage.commons.service.StorageService;
 
-import static org.mockito.ArgumentMatchers.*;
+@ExtendWith(SpringExtension.class)
-import static org.mockito.Mockito.when;
-
-@RunWith(SpringRunner.class)
 @SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
 @Import(RedactionIntegrationTest.RedactionIntegrationTestConfiguration.class)
 public class DictionaryServiceTest {

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DocumentGraphIntegrationTest.java

@@ -0,0 +1,258 @@
+package com.iqser.red.service.redaction.v1.server;
+
+import static com.iqser.red.service.redaction.v1.server.utils.PdfDraw.drawRectangle2DList;
+
+import java.awt.Color;
+import java.awt.Shape;
+import java.awt.geom.AffineTransform;
+import java.awt.geom.Rectangle2D;
+import java.io.File;
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.pdfbox.pdmodel.PDDocument;
+import org.junit.jupiter.api.Test;
+import org.kie.api.runtime.KieContainer;
+import org.kie.api.runtime.KieSession;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.core.io.ClassPathResource;
+
+import com.iqser.red.service.redaction.v1.model.FileAttribute;
+import com.iqser.red.service.redaction.v1.server.document.data.DocumentData;
+import com.iqser.red.service.redaction.v1.server.document.graph.DocumentGraph;
+import com.iqser.red.service.redaction.v1.server.document.graph.nodes.EntityNode;
+import com.iqser.red.service.redaction.v1.server.document.graph.nodes.PageNode;
+import com.iqser.red.service.redaction.v1.server.document.graph.nodes.ParagraphNode;
+import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SectionNode;
+import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock;
+import com.iqser.red.service.redaction.v1.server.document.services.DocumentDataMapper;
+import com.iqser.red.service.redaction.v1.server.document.services.DocumentGraphFactory;
+import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
+import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType;
+import com.iqser.red.service.redaction.v1.server.redaction.service.DictionaryService;
+import com.iqser.red.service.redaction.v1.server.redaction.utils.SearchImplementation;
+import com.iqser.red.service.redaction.v1.server.segmentation.PdfSegmentationService;
+
+import lombok.SneakyThrows;
+
+public class DocumentGraphIntegrationTest extends AbstractTestWithDictionaries {
+
+    @Autowired
+    private DocumentGraphFactory documentGraphFactory;
+
+    @Autowired
+    private PdfSegmentationService segmentationService;
+
+    @Autowired
+    private DictionaryService dictionaryService;
+
+    @Autowired
+    private DocumentDataMapper documentDataMapper;
+
+    @Qualifier("kieContainer")
+    @Autowired
+    private KieContainer kieContainer;
+
+
+
+
+    @Test
+    @SneakyThrows
+    public void testDroolsOnDocumentGraph() {
+
+        String filename = "files/new/crafted document";
+
+        prepareStorage(filename + ".pdf");
+        ClassPathResource fileResource = new ClassPathResource(filename + ".pdf");
+
+        var classifiedDoc = segmentationService.parseDocument(TEST_DOSSIER_ID, TEST_FILE_ID, fileResource.getInputStream(), null);
+        DocumentGraph document = documentGraphFactory.buildDocumentGraph(classifiedDoc);
+
+        dictionaryService.updateDictionary(TEST_DOSSIER_TEMPLATE_ID, TEST_DOSSIER_ID);
+        Dictionary dictionary = dictionaryService.getDeepCopyDictionary(TEST_DOSSIER_TEMPLATE_ID, TEST_DOSSIER_ID);
+
+        List<EntityNode> foundEntities = new LinkedList<>();
+        for (var model : dictionary.getDictionaryModels()) {
+            findEntitiesWithSearchImplementationAndAddToGraph(document, model.getEntriesSearch(), EntityType.ENTITY, foundEntities, model.getType());
+            findEntitiesWithSearchImplementationAndAddToGraph(document, model.getFalsePositiveSearch(), EntityType.FALSE_POSITIVE, foundEntities, model.getType());
+            findEntitiesWithSearchImplementationAndAddToGraph(document, model.getFalseRecommendationsSearch(), EntityType.FALSE_RECOMMENDATION, foundEntities, model.getType());
+        }
+
+        KieSession kieSession = kieContainer.newKieSession();
+        kieSession.setGlobal("document", document);
+        document.getEntities().forEach(kieSession::insert);
+        document.getSections().forEach(kieSession::insert);
+        document.getPages().forEach(kieSession::insert);
+        document.getSections().forEach(sec -> sec.getParagraphs().forEach(kieSession::insert));
+        kieSession.insert(FileAttribute.builder().label("Vertebrate Study").value("Yes").build());
+        kieSession.fireAllRules();
+
+        drawAllEntities(filename, fileResource, document);
+    }
+
+
+    @Test
+    @SneakyThrows
+    public void testBuildTextBlockPerformance() {
+
+        int n = 10000;
+        String filename = "files/Metolachlor/S-Metolachlor_RAR_01_Volume_1_2018-09-06";
+
+        prepareStorage(filename + ".pdf");
+        ClassPathResource fileResource = new ClassPathResource(filename + ".pdf");
+
+        var classifiedDoc = segmentationService.parseDocument(TEST_DOSSIER_ID, TEST_FILE_ID, fileResource.getInputStream(), null);
+        DocumentGraph document = documentGraphFactory.buildDocumentGraph(classifiedDoc);
+        var start = System.currentTimeMillis();
+        for (int i = 0; i < n; i++) {
+            TextBlock textBlock = document.getText();
+        }
+        float durationMillis = ((float) (System.currentTimeMillis() - start));
+        System.out.printf("%d calls of buildTextBlock() on document took %f s, average is %f ms\n", n, durationMillis / 1000, durationMillis / n);
+
+        SectionNode section = document.getSections().get(8);
+        start = System.currentTimeMillis();
+        for (int i = 0; i < n; i++) {
+            section.buildTextBlock();
+        }
+        durationMillis = ((float) (System.currentTimeMillis() - start));
+        System.out.printf("%d calls of buildTextBlock() on section took %f s, average is %f ms\n", n, durationMillis / 1000, durationMillis / n);
+
+        ParagraphNode paragraph = document.getSections().get(8).getParagraphs().get(1);
+        start = System.currentTimeMillis();
+        for (int i = 0; i < n; i++) {
+            paragraph.buildTextBlock();
+        }
+        durationMillis = ((float) (System.currentTimeMillis() - start));
+        System.out.printf("%d calls of buildTextBlock() on paragraph took %f s, average is %f ms\n", n, durationMillis / 1000, durationMillis / n);
+    }
+
+
+    @Test
+    @SneakyThrows
+    public void testDictionarySearchOnDocumentGraph() {
+
+        String filename = "files/Metolachlor/S-Metolachlor_RAR_01_Volume_1_2018-09-06";
+
+        prepareStorage(filename + ".pdf");
+        ClassPathResource fileResource = new ClassPathResource(filename + ".pdf");
+
+        var classifiedDoc = segmentationService.parseDocument(TEST_DOSSIER_ID, TEST_FILE_ID, fileResource.getInputStream(), null);
+        DocumentGraph document = documentGraphFactory.buildDocumentGraph(classifiedDoc);
+
+        dictionaryService.updateDictionary(TEST_DOSSIER_TEMPLATE_ID, TEST_DOSSIER_ID);
+        Dictionary dictionary = dictionaryService.getDeepCopyDictionary(TEST_DOSSIER_TEMPLATE_ID, TEST_DOSSIER_ID);
+
+        int numberOfSearches = 25;
+        float totalTime = 0;
+        float totalGraphTime = 0;
+        List<EntityNode> foundEntities = new LinkedList<>();
+        for (int i = 0; i < numberOfSearches; i++) {
+
+            var graphStart = System.currentTimeMillis();
+            document = documentGraphFactory.buildDocumentGraph(classifiedDoc);
+            var graphTime = ((float) (System.currentTimeMillis() - graphStart)) / 1000;
+            totalGraphTime += graphTime;
+            var start = System.currentTimeMillis();
+            foundEntities = new LinkedList<>();
+
+            for (var model : dictionary.getDictionaryModels()) {
+                findEntitiesWithSearchImplementationAndAddToGraph(document, model.getEntriesSearch(), EntityType.ENTITY, foundEntities, model.getType());
+                findEntitiesWithSearchImplementationAndAddToGraph(document, model.getFalsePositiveSearch(), EntityType.FALSE_POSITIVE, foundEntities, model.getType());
+                findEntitiesWithSearchImplementationAndAddToGraph(document, model.getFalseRecommendationsSearch(), EntityType.FALSE_RECOMMENDATION, foundEntities, model.getType());
+            }
+            var time = ((float) (System.currentTimeMillis() - start)) / 1000;
+            totalTime += time;
+            System.out.printf("%d Search %fs; Graph construction %fs \n", i, time, graphTime);
+        }
+
+        System.out.printf("%d Searches took %f s, average %f s\n", numberOfSearches, totalTime, totalTime / numberOfSearches);
+        System.out.printf("%d Graph constructions took %f s, average %f s\n", numberOfSearches, totalGraphTime, totalGraphTime / numberOfSearches);
+        Set<EntityNode> distinctFoundEntities = new HashSet<>(foundEntities);
+        System.out.printf("Found %d entities and saved %d\n", distinctFoundEntities.size(), document.getEntities().size());
+        //assert document.getEntities().size() == distinctFoundEntities.size();
+
+        drawAllEntities(filename, fileResource, document);
+    }
+
+
+    private static void drawAllEntities(String filename, ClassPathResource fileResource, DocumentGraph document) throws IOException {
+
+        var tmpFileName = "/tmp/" + filename.split("/")[2] + "_ENTITY_BBOX.pdf";
+        try (var fileStream = fileResource.getInputStream()) {
+            PDDocument pdDocument = PDDocument.load(fileStream);
+
+            for (PageNode page : document.getPages()) {
+                AffineTransform mirrorY = new AffineTransform(1, 0, 0, -1, 0, page.getHeight() + 6);
+                List<Rectangle2D> entityPositionsOnPage = page.getEntities()
+                        .stream()
+                        .filter(EntityNode::isRedact)
+                        .map(EntityNode::getPositions)
+                        .flatMap(List::stream)
+                        .map(mirrorY::createTransformedShape)
+                        .map(Shape::getBounds2D)
+                        .toList();
+
+                drawRectangle2DList(pdDocument, page.getNumber(), entityPositionsOnPage, Color.BLACK);
+            }
+
+            for (PageNode page : document.getPages()) {
+                AffineTransform mirrorY = new AffineTransform(1, 0, 0, -1, 0, page.getHeight() + 6);
+                List<Rectangle2D> entityPositionsOnPage = page.getEntities()
+                        .stream()
+                        .filter(e -> !e.isRedact())
+                        .map(EntityNode::getPositions)
+                        .flatMap(List::stream)
+                        .map(mirrorY::createTransformedShape)
+                        .map(Shape::getBounds2D)
+                        .toList();
+
+                drawRectangle2DList(pdDocument, page.getNumber(), entityPositionsOnPage, Color.BLUE);
+            }
+            File outputFile = new File(tmpFileName);
+            pdDocument.save(outputFile);
+            pdDocument.close();
+        }
+    }
+
+
+    private void findEntitiesWithSearchImplementationAndAddToGraph(DocumentGraph documentGraph,
+                                                                   SearchImplementation searchImplementation,
+                                                                   EntityType entityType,
+                                                                   List<EntityNode> foundEntities,
+                                                                   String type) {
+
+        documentGraph.getSections().forEach( section -> {
+            TextBlock textBlock = section.buildTextBlock();
+            searchImplementation.getMatches(textBlock, textBlock.getBoundary().start())
+                    .stream()
+                    .map(bounds -> section.createAndAddEntity(bounds, type, entityType))
+                    .forEach(foundEntities::add);
+        });
+        documentGraph.getPages().forEach( page -> {
+            TextBlock textBlock = page.getHeader();
+            searchImplementation.getMatches(textBlock, textBlock.getBoundary().start())
+                    .stream()
+                    .map(bounds -> page.createAndAddEntity(bounds, type, entityType))
+                    .forEach(foundEntities::add);
+        });
+        documentGraph.getPages().forEach( page -> {
+            TextBlock textBlock = page.getFooter();
+            searchImplementation.getMatches(textBlock, textBlock.getBoundary().start())
+                    .stream()
+                    .map(bounds -> page.createAndAddEntity(bounds, type, entityType))
+                    .forEach(foundEntities::add);
+        });
+         /*
+        searchImplementation.getMatches(documentGraph.getText())
+                .stream()
+                .map(pos -> documentGraph.createAndAddEntity(new Boundary(pos.startIndex(), pos.endIndex()), type, entityType))
+                .forEach(foundEntities::add);
+
+         */
+    }
+}

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DocumentGraphMappingTest.java

@@ -0,0 +1,55 @@
+package com.iqser.red.service.redaction.v1.server;
+
+import org.junit.jupiter.api.Test;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.core.io.ClassPathResource;
+
+import com.iqser.red.service.redaction.v1.server.document.data.DocumentData;
+import com.iqser.red.service.redaction.v1.server.document.graph.DocumentGraph;
+import com.iqser.red.service.redaction.v1.server.document.services.DocumentDataMapper;
+import com.iqser.red.service.redaction.v1.server.document.services.DocumentGraphFactory;
+import com.iqser.red.service.redaction.v1.server.document.services.DocumentGraphMapper;
+import com.iqser.red.service.redaction.v1.server.segmentation.PdfSegmentationService;
+
+import lombok.SneakyThrows;
+
+public class DocumentGraphMappingTest extends AbstractTestWithDictionaries {
+
+    @Autowired
+    private DocumentGraphFactory documentGraphFactory;
+
+    @Autowired
+    private PdfSegmentationService segmentationService;
+
+    @Autowired
+    private DocumentDataMapper documentDataMapper;
+
+    @Autowired
+    private DocumentGraphMapper documentGraphMapper;
+
+
+    @Test
+    @SneakyThrows
+    public void testGraphMapping() {
+
+        String filename = "files/Metolachlor/S-Metolachlor_RAR_01_Volume_1_2018-09-06";
+
+        prepareStorage(filename + ".pdf");
+        ClassPathResource fileResource = new ClassPathResource(filename + ".pdf");
+
+        var classifiedDoc = segmentationService.parseDocument(TEST_DOSSIER_ID, TEST_FILE_ID, fileResource.getInputStream(), null);
+        DocumentGraph document = documentGraphFactory.buildDocumentGraph(classifiedDoc);
+        DocumentData documentData = documentDataMapper.toDocumentData(document);
+        storageService.storeJSONObject(filename + ".json", documentData);
+        DocumentGraph newDocumentGraph = documentGraphMapper.toDocumentGraph(documentData);
+
+        assert document.toString().equals(newDocumentGraph.toString());
+        assert document.getTableOfContents().toString().equals(newDocumentGraph.getTableOfContents().toString());
+        for (int pageIndex = 1; pageIndex < document.getNumberOfPages(); pageIndex++) {
+            var page = document.getPages().get(pageIndex);
+            var newPage = document.getPages().get(pageIndex);
+            assert page.toString().equals(newPage.toString());
+        }
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/HeadlinesGoldStandardIntegrationTest.java

@@ -20,10 +20,10 @@ import java.util.Set;
 import java.util.stream.Collectors;
 
 import org.assertj.core.api.Assertions;
-import org.junit.After;
+import org.junit.jupiter.api.AfterEach;
-import org.junit.Before;
+import org.junit.jupiter.api.BeforeEach;
-import org.junit.Test;
+import org.junit.jupiter.api.Test;
-import org.junit.runner.RunWith;
+import org.junit.jupiter.api.extension.ExtendWith;
 import org.kie.api.KieServices;
 import org.kie.api.builder.KieBuilder;
 import org.kie.api.builder.KieFileSystem;
@@ -41,7 +41,7 @@ import org.springframework.context.annotation.Configuration;
 import org.springframework.context.annotation.Import;
 import org.springframework.context.annotation.Primary;
 import org.springframework.core.io.ClassPathResource;
-import org.springframework.test.context.junit4.SpringRunner;
+import org.springframework.test.context.junit.jupiter.SpringExtension;
 
 import com.amazonaws.services.s3.AmazonS3;
 import com.fasterxml.jackson.databind.ObjectMapper;
@@ -72,7 +72,7 @@ import lombok.EqualsAndHashCode;
 import lombok.SneakyThrows;
 import lombok.ToString;
 
-@RunWith(SpringRunner.class)
+@ExtendWith(SpringExtension.class)
 @SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
 @Import(HeadlinesGoldStandardIntegrationTest.RedactionIntegrationTestConfiguration.class)
 public class HeadlinesGoldStandardIntegrationTest {
@@ -229,7 +229,7 @@ public class HeadlinesGoldStandardIntegrationTest {
     }
 
 
-    @After
+    @AfterEach
     public void cleanupStorage() {
 
         if (this.storageService instanceof FileSystemBackedStorageService) {
@@ -238,7 +238,7 @@ public class HeadlinesGoldStandardIntegrationTest {
     }
 
 
-    @Before
+    @BeforeEach
     public void stubClients() {
 
         when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L);

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java

@@ -24,11 +24,11 @@ import java.util.Set;
 import java.util.UUID;
 import java.util.stream.Collectors;
 
-import org.junit.After;
+import org.junit.jupiter.api.AfterEach;
-import org.junit.Before;
+import org.junit.jupiter.api.BeforeEach;
-import org.junit.Ignore;
+import org.junit.jupiter.api.Disabled;
-import org.junit.Test;
+import org.junit.jupiter.api.Test;
-import org.junit.runner.RunWith;
+import org.junit.jupiter.api.extension.ExtendWith;
 import org.kie.api.KieServices;
 import org.kie.api.builder.KieBuilder;
 import org.kie.api.builder.KieFileSystem;
@@ -46,7 +46,7 @@ import org.springframework.context.annotation.Configuration;
 import org.springframework.context.annotation.Import;
 import org.springframework.context.annotation.Primary;
 import org.springframework.core.io.ClassPathResource;
-import org.springframework.test.context.junit4.SpringRunner;
+import org.springframework.test.context.junit.jupiter.SpringExtension;
 
 import com.amazonaws.services.s3.AmazonS3;
 import com.fasterxml.jackson.core.type.TypeReference;
@@ -69,6 +69,7 @@ import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.ty
 import com.iqser.red.service.redaction.v1.model.AnalyzeRequest;
 import com.iqser.red.service.redaction.v1.model.AnalyzeResult;
 import com.iqser.red.service.redaction.v1.model.FileAttribute;
+import com.iqser.red.service.redaction.v1.model.RedactionLog;
 import com.iqser.red.service.redaction.v1.model.RedactionLogEntry;
 import com.iqser.red.service.redaction.v1.model.RedactionRequest;
 import com.iqser.red.service.redaction.v1.model.RedactionResult;
@@ -81,7 +82,9 @@ import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
 import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient;
 import com.iqser.red.service.redaction.v1.server.client.RulesClient;
 import com.iqser.red.service.redaction.v1.server.controller.RedactionController;
+import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
 import com.iqser.red.service.redaction.v1.server.redaction.service.AnalyzeService;
+import com.iqser.red.service.redaction.v1.server.redaction.service.DictionaryService;
 import com.iqser.red.service.redaction.v1.server.redaction.service.ManualRedactionSurroundingTextService;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.OsUtils;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.ResourceLoader;
@@ -92,7 +95,7 @@ import com.iqser.red.storage.commons.service.StorageService;
 
 import lombok.SneakyThrows;
 
-@RunWith(SpringRunner.class)
+@ExtendWith(SpringExtension.class)
 @SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
 @Import(RedactionIntegrationTest.RedactionIntegrationTestConfiguration.class)
 public class RedactionIntegrationTest {
@@ -155,6 +158,11 @@ public class RedactionIntegrationTest {
     @MockBean
     private LegalBasisClient legalBasisClient;
 
+    @Autowired
+    private DictionaryService dictionaryService;
+
+
+
     private final Map<String, List<String>> dictionary = new HashMap<>();
     private final Map<String, List<String>> dossierDictionary = new HashMap<>();
     private final Map<String, List<String>> falsePositive = new HashMap<>();
@@ -202,7 +210,7 @@ public class RedactionIntegrationTest {
     }
 
 
-    @After
+    @AfterEach
     public void cleanupStorage() {
 
         if (this.storageService instanceof FileSystemBackedStorageService) {
@@ -211,7 +219,7 @@ public class RedactionIntegrationTest {
     }
 
 
-    @Before
+    @BeforeEach
     public void stubClients() {
 
         when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L);
@@ -293,7 +301,7 @@ public class RedactionIntegrationTest {
 
 
     @Test
-    @Ignore
+    @Disabled
     public void testLargeScannedFileOOM() {
 
         AnalyzeRequest request = prepareStorage("scanned/VV-377031.pdf");
@@ -364,15 +372,14 @@ public class RedactionIntegrationTest {
     @Test
     public void titleExtraction() throws IOException {
 
-        AnalyzeRequest request = prepareStorage("files/new/APN3_Clean_6.1 (6.4.3.01-02)_Apple_211029.pdf");
+        AnalyzeRequest request = prepareStorage("files/new/crafted document.pdf");
 
         analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
         AnalyzeResult result = analyzeService.analyze(request);
 
-        var redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID);
+        RedactionLog redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID);
-
-        var text = redactionStorageService.getText(TEST_DOSSIER_ID, TEST_FILE_ID);
 
+        Dictionary dictionary = dictionaryService.getDeepCopyDictionary(TEST_DOSSIER_TEMPLATE_ID, TEST_DOSSIER_ID);
         AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder().dossierId(TEST_DOSSIER_ID).fileId(TEST_FILE_ID).build());
 
         String outputFileName = OsUtils.getTemporaryDirectory() + "/Annotated.pdf";
@@ -398,7 +405,7 @@ public class RedactionIntegrationTest {
 
 
     @Test
-    @Ignore
+    @Disabled
     @SneakyThrows
     public void testIgnoreHint() {
 
@@ -440,7 +447,7 @@ public class RedactionIntegrationTest {
 
 
     @Test
-    @Ignore
+    @Disabled
     public void noExceptionShouldBeThrownForAnyFiles() throws IOException {
 
         long start = System.currentTimeMillis();
@@ -1111,7 +1118,7 @@ public class RedactionIntegrationTest {
 
         System.out.println("classificationTest");
 
-        AnalyzeRequest request = prepareStorage("files/new/RotateTestFile.pdf");
+        AnalyzeRequest request = prepareStorage("files/Metolachlor/S-Metolachlor_RAR_01_Volume_1_2018-09-06.pdf");
 
         RedactionRequest redactionRequest = RedactionRequest.builder()
                 .dossierId(request.getDossierId())
@@ -1252,7 +1259,7 @@ public class RedactionIntegrationTest {
 
 
     @Test
-    @Ignore
+    @Disabled
     public void resizeRedactionTest() throws IOException {
 
         String pdfFile = "files/Minimal Examples/Single Table.pdf";
@@ -1369,7 +1376,7 @@ public class RedactionIntegrationTest {
 
 
     @Test
-    @Ignore
+    @Disabled
     public void testManualSurroundingText() throws IOException {
 
         String pdfFile = "files/new/S4.pdf";

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RulesTest.java

@@ -28,11 +28,11 @@ import java.util.Set;
 import java.util.stream.Collectors;
 
 import org.apache.commons.lang3.StringUtils;
-import org.junit.After;
+import org.junit.jupiter.api.AfterEach;
-import org.junit.Before;
+import org.junit.jupiter.api.BeforeEach;
-import org.junit.Ignore;
+import org.junit.jupiter.api.Disabled;
-import org.junit.Test;
+import org.junit.jupiter.api.Test;
-import org.junit.runner.RunWith;
+import org.junit.jupiter.api.extension.ExtendWith;
 import org.kie.api.KieServices;
 import org.kie.api.builder.KieBuilder;
 import org.kie.api.builder.KieFileSystem;
@@ -50,7 +50,7 @@ import org.springframework.context.annotation.Import;
 import org.springframework.context.annotation.Primary;
 import org.springframework.core.io.ClassPathResource;
 import org.springframework.test.annotation.IfProfileValue;
-import org.springframework.test.context.junit4.SpringRunner;
+import org.springframework.test.context.junit.jupiter.SpringExtension;
 
 import com.amazonaws.services.s3.AmazonS3;
 import com.fasterxml.jackson.databind.DeserializationFeature;
@@ -88,7 +88,7 @@ import lombok.SneakyThrows;
 import lombok.extern.slf4j.Slf4j;
 
 @Slf4j
-@RunWith(SpringRunner.class)
+@ExtendWith(SpringExtension.class)
 @SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
 @Import(RulesTest.RulesTestConfiguration.class)
 public class RulesTest {
@@ -247,7 +247,7 @@ public class RulesTest {
     private LegalBasisClient legalBasisClient;
 
 
-    @Before
+    @BeforeEach
     public void stubClients() {
 
         objectMapper.registerModule(new JavaTimeModule());
@@ -280,7 +280,7 @@ public class RulesTest {
     }
 
 
-    @After
+    @AfterEach
     public void cleanupStorage() {
 
         if (this.storageService instanceof FileSystemBackedStorageService) {
@@ -294,7 +294,7 @@ public class RulesTest {
      * If the RedactionLog already exists, it will be overwritten
      * Test is ignored, because it's for manual tests.
      */
-    @Ignore
+    @Disabled
     @Test
     public void generateRedactionLogForOneFile() {
 

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/BoundaryTest.java

@@ -0,0 +1,44 @@
+package com.iqser.red.service.redaction.v1.server.document.graph;
+
+import static org.junit.jupiter.api.Assertions.*;
+
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+class BoundaryTest {
+
+    Boundary startBoundary;
+    @BeforeEach
+    void setUp() {
+        startBoundary = new Boundary(10, 100);
+    }
+
+
+    @Test
+    void testContains() {
+        assertTrue(startBoundary.contains(11));
+        assertTrue(startBoundary.contains(50));
+        assertFalse(startBoundary.contains(9));
+        assertFalse(startBoundary.contains(100));
+        assertFalse(startBoundary.contains(150));
+        assertFalse(startBoundary.contains(-123));
+        assertTrue(startBoundary.contains(new Boundary(11, 99)));
+        assertTrue(startBoundary.contains(new Boundary(10, 100)));
+        assertTrue(startBoundary.contains(new Boundary(11, 11)));
+        assertFalse(startBoundary.contains(9, 100));
+        assertTrue(startBoundary.contains(100, 100));
+        assertFalse(startBoundary.contains(100, 101));
+        assertFalse(startBoundary.contains(150, 151));
+    }
+
+
+    @Test
+    void testIntersects() {
+        assertTrue(startBoundary.intersects(new Boundary(1, 11)));
+        assertTrue(startBoundary.intersects(new Boundary(11, 12)));
+        assertTrue(startBoundary.intersects(new Boundary(11, 100)));
+        assertFalse(startBoundary.intersects(new Boundary(100, 101)));
+        assertTrue(startBoundary.intersects(new Boundary(99, 101)));
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/realdata/AnalyseFileRealDataIntegrationTest.java

@@ -4,7 +4,7 @@ import java.time.OffsetDateTime;
 import java.util.List;
 import java.util.Set;
 
-import org.junit.Test;
+import org.junit.jupiter.api.Test;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.core.io.ClassPathResource;
 

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/realdata/LiveDataIntegrationTest.java

@@ -1,28 +1,21 @@
 package com.iqser.red.service.redaction.v1.server.realdata;
 
-import com.fasterxml.jackson.core.type.TypeReference;
+import static org.assertj.core.api.AssertionsForClassTypes.assertThat;
-import com.fasterxml.jackson.databind.ObjectMapper;
+import static org.mockito.ArgumentMatchers.any;
-import com.iqser.red.service.persistence.service.v1.api.model.common.JSONPrimitive;
+import static org.mockito.ArgumentMatchers.anyBoolean;
-import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.configuration.Colors;
+import static org.mockito.ArgumentMatchers.anyString;
-import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntry;
+import static org.mockito.ArgumentMatchers.nullable;
-import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.Type;
+import static org.mockito.Mockito.when;
-import com.iqser.red.service.redaction.v1.server.Application;
-import com.iqser.red.service.redaction.v1.server.FileSystemBackedStorageService;
-import com.iqser.red.service.redaction.v1.server.client.*;
-import com.iqser.red.service.redaction.v1.server.queue.RedactionMessageReceiver;
-import com.iqser.red.service.redaction.v1.server.redaction.service.DictionaryService;
-import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;
-import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
-import com.iqser.red.storage.commons.StorageAutoConfiguration;
-import com.iqser.red.storage.commons.service.StorageService;
 
-import io.micrometer.prometheus.PrometheusMeterRegistry;
+import java.util.ArrayList;
-import lombok.SneakyThrows;
+import java.util.List;
+import java.util.Objects;
+import java.util.stream.Collectors;
 
 import org.apache.commons.io.IOUtils;
-import org.junit.Before;
+import org.junit.jupiter.api.BeforeEach;
-import org.junit.Test;
+import org.junit.jupiter.api.Test;
-import org.junit.runner.RunWith;
+import org.junit.jupiter.api.extension.ExtendWith;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
 import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration;
@@ -35,18 +28,33 @@ import org.springframework.context.annotation.Primary;
 import org.springframework.core.io.ClassPathResource;
 import org.springframework.core.io.Resource;
 import org.springframework.core.io.support.ResourcePatternResolver;
+import org.springframework.test.context.junit.jupiter.SpringExtension;
 import org.springframework.test.context.junit4.SpringRunner;
 
-import java.util.ArrayList;
+import com.fasterxml.jackson.core.type.TypeReference;
-import java.util.List;
+import com.fasterxml.jackson.databind.ObjectMapper;
-import java.util.Objects;
+import com.iqser.red.service.persistence.service.v1.api.model.common.JSONPrimitive;
-import java.util.stream.Collectors;
+import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.configuration.Colors;
+import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntry;
+import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.Type;
+import com.iqser.red.service.redaction.v1.server.Application;
+import com.iqser.red.service.redaction.v1.server.FileSystemBackedStorageService;
+import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
+import com.iqser.red.service.redaction.v1.server.client.EntityRecognitionClient;
+import com.iqser.red.service.redaction.v1.server.client.FileStatusProcessingUpdateClient;
+import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient;
+import com.iqser.red.service.redaction.v1.server.client.RulesClient;
+import com.iqser.red.service.redaction.v1.server.queue.RedactionMessageReceiver;
+import com.iqser.red.service.redaction.v1.server.redaction.service.DictionaryService;
+import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;
+import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
+import com.iqser.red.storage.commons.StorageAutoConfiguration;
+import com.iqser.red.storage.commons.service.StorageService;
 
-import static org.assertj.core.api.AssertionsForClassTypes.assertThat;
+import io.micrometer.prometheus.PrometheusMeterRegistry;
-import static org.mockito.ArgumentMatchers.*;
+import lombok.SneakyThrows;
-import static org.mockito.Mockito.when;
 
-@RunWith(SpringRunner.class)
+@ExtendWith(SpringExtension.class)
 @SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
 @Import(LiveDataIntegrationTest.RedactionIntegrationTestConfiguration.class)
 public class LiveDataIntegrationTest {
@@ -108,7 +116,7 @@ public class LiveDataIntegrationTest {
 
 
     @SneakyThrows
-    @Before
+    @BeforeEach
     public void prepareTest() {
 
         when(dictionaryClient.getVersion(anyString())).thenReturn(1L);

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/rulebuilder/RuleBuilderModelServiceTest.java

@@ -2,10 +2,11 @@ package com.iqser.red.service.redaction.v1.server.redaction.rulebuilder;
 
 import com.iqser.red.service.redaction.v1.model.RuleBuilderModel;
 
-import org.junit.Test;
 
 import static org.assertj.core.api.AssertionsForClassTypes.assertThat;
 
+import org.junit.jupiter.api.Test;
+
 public class RuleBuilderModelServiceTest {
 
     @Test

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/utils/EntitySearchUtilsTest.java

@@ -5,7 +5,7 @@ import static org.assertj.core.api.Assertions.assertThat;
 import java.util.HashSet;
 import java.util.Set;
 
-import org.junit.Test;
+import org.junit.jupiter.api.Test;
 
 import com.iqser.red.service.redaction.v1.model.Engine;
 import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
@@ -17,8 +17,8 @@ public class EntitySearchUtilsTest {
     public void testNestedEntitiesRemoval() {
 
         Set<Entity> entities = new HashSet<>();
-        Entity nested = new Entity("nested", "fake type", 10, 16, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
+        Entity nested = new Entity("nested", "fake type", 10, 16, "fake headline", 0, 0, false, false, Engine.RULE, EntityType.ENTITY);
-        Entity nesting = new Entity("nesting nested", "fake type", 2, 16, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
+        Entity nesting = new Entity("nesting nested", "fake type", 2, 16, "fake headline", 0, 0, false, false, Engine.RULE, EntityType.ENTITY);
         entities.add(nested);
         entities.add(nesting);
         EntitySearchUtils.removeEntitiesContainedInLarger(entities);
@@ -40,14 +40,14 @@ public class EntitySearchUtilsTest {
 
         // Arrange
         Set<Entity> existingEntities = new HashSet<>();
-        Entity existingEntity1 = new Entity("Batman", "fake type", 0, 5, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
+        Entity existingEntity1 = new Entity("Batman", "fake type", 0, 5, "fake headline", 0, 0, false, false, Engine.RULE, EntityType.ENTITY);
-        Entity existingEntity2 = new Entity("Superman", "fake type", 10, 17, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
+        Entity existingEntity2 = new Entity("Superman", "fake type", 10, 17, "fake headline", 0, 0, false, false, Engine.RULE, EntityType.ENTITY);
         existingEntities.add(existingEntity1);
         existingEntities.add(existingEntity2);
 
         Set<Entity> foundEntities = new HashSet<>();
-        Entity foundEntities1 = new Entity("Batman X.", "fake type", 0, 8, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
+        Entity foundEntities1 = new Entity("Batman X.", "fake type", 0, 8, "fake headline", 0, 0, false, false, Engine.RULE, EntityType.ENTITY);
-        Entity foundEntities2 = new Entity("Superman Y.", "fake type", 10, 20, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
+        Entity foundEntities2 = new Entity("Superman Y.", "fake type", 10, 20, "fake headline", 0, 0, false, false, Engine.RULE, EntityType.ENTITY);
         foundEntities.add(foundEntities1);
         foundEntities.add(foundEntities2);
 
@@ -73,14 +73,14 @@ public class EntitySearchUtilsTest {
 
         // Arrange
         Set<Entity> existingEntities = new HashSet<>();
-        Entity existingEntity1 = new Entity("Batman X.", "fake type", 0, 8, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
+        Entity existingEntity1 = new Entity("Batman X.", "fake type", 0, 8, "fake headline", 0, 0, false, false, Engine.RULE, EntityType.ENTITY);
-        Entity existingEntity2 = new Entity("Superman", "fake type", 10, 17, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
+        Entity existingEntity2 = new Entity("Superman", "fake type", 10, 17, "fake headline", 0, 0, false, false, Engine.RULE, EntityType.ENTITY);
         existingEntities.add(existingEntity1);
         existingEntities.add(existingEntity2);
 
         Set<Entity> foundEntities = new HashSet<>();
-        Entity foundEntities1 = new Entity("Batman X.", "fake type", 0, 8, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
+        Entity foundEntities1 = new Entity("Batman X.", "fake type", 0, 8, "fake headline", 0, 0,false, false, Engine.RULE, EntityType.ENTITY);
-        Entity foundEntities2 = new Entity("X. Superman Y.", "fake type", 7, 20, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
+        Entity foundEntities2 = new Entity("X. Superman Y.", "fake type", 7, 20, "fake headline", 0, 0, false, false, Engine.RULE, EntityType.ENTITY);
         foundEntities.add(foundEntities1);
         foundEntities.add(foundEntities2);
 
@@ -105,14 +105,14 @@ public class EntitySearchUtilsTest {
 
         // Arrange
         Set<Entity> existingEntities = new HashSet<>();
-        Entity existingEntity1 = new Entity("Batman X.", "fake type", 0, 8, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
+        Entity existingEntity1 = new Entity("Batman X.", "fake type", 0, 8, "fake headline", 0, 0, false, false, Engine.RULE, EntityType.ENTITY);
-        Entity existingEntity2 = new Entity("Superman", "fake type", 10, 17, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
+        Entity existingEntity2 = new Entity("Superman", "fake type", 10, 17, "fake headline", 0, 0,false, false, Engine.RULE, EntityType.ENTITY);
         existingEntities.add(existingEntity1);
         existingEntities.add(existingEntity2);
 
         Set<Entity> foundEntities = new HashSet<>();
-        Entity foundEntities1 = new Entity("Batman X.", "fake type", 0, 8, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
+        Entity foundEntities1 = new Entity("Batman X.", "fake type", 0, 8, "fake headline", 0, 0,false, false, Engine.RULE, EntityType.ENTITY);
-        Entity foundEntities2 = new Entity("X. Superman", "fake type", 7, 17, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
+        Entity foundEntities2 = new Entity("X. Superman", "fake type", 7, 17, "fake headline", 0, 0,false, false, Engine.RULE, EntityType.ENTITY);
         foundEntities.add(foundEntities1);
         foundEntities.add(foundEntities2);
 
@@ -137,15 +137,15 @@ public class EntitySearchUtilsTest {
 
         // Arrange
         Set<Entity> existingEntities = new HashSet<>();
-        Entity existingEntity1 = new Entity("X. Superman", "fake type", 7, 17, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
+        Entity existingEntity1 = new Entity("X. Superman", "fake type", 7, 17, "fake headline", 0, 0,false, false, Engine.RULE, EntityType.ENTITY);
-        Entity existingEntity2 = new Entity("Batman", "fake type", 0, 5, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
+        Entity existingEntity2 = new Entity("Batman", "fake type", 0, 5, "fake headline", 0, 0,false, false, Engine.RULE, EntityType.ENTITY);
         existingEntities.add(existingEntity1);
         existingEntities.add(existingEntity2);
 
         Set<Entity> foundEntities = new HashSet<>();
-        Entity foundEntities1 = new Entity("Batman X.", "fake type", 0, 8, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
+        Entity foundEntities1 = new Entity("Batman X.", "fake type", 0, 8, "fake headline", 0, 0,false, false, Engine.RULE, EntityType.ENTITY);
-        Entity foundEntities2 = new Entity("Superman", "fake type", 10, 17, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
+        Entity foundEntities2 = new Entity("Superman", "fake type", 10, 17, "fake headline", 0, 0,false, false, Engine.RULE, EntityType.ENTITY);
-        Entity foundEntities3 = new Entity("Superman Y.", "fake type", 10, 20, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
+        Entity foundEntities3 = new Entity("Superman Y.", "fake type", 10, 20, "fake headline", 0, 0,false, false, Engine.RULE, EntityType.ENTITY);
         foundEntities.add(foundEntities1);
         foundEntities.add(foundEntities2);
         foundEntities.add(foundEntities3);
@@ -171,16 +171,16 @@ public class EntitySearchUtilsTest {
 
         // Arrange
         Set<Entity> existingEntities = new HashSet<>();
-        Entity existingEntity1 = new Entity("X. Superman", "fake type", 7, 17, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
+        Entity existingEntity1 = new Entity("X. Superman", "fake type", 7, 17, "fake headline", 0, 0,false, false, Engine.RULE, EntityType.ENTITY);
-        Entity existingEntity2 = new Entity("Batman", "fake type", 0, 5, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
+        Entity existingEntity2 = new Entity("Batman", "fake type", 0, 5, "fake headline", 0, 0,false, false, Engine.RULE, EntityType.ENTITY);
         existingEntities.add(existingEntity1);
         existingEntities.add(existingEntity2);
 
         Set<Entity> foundEntities = new HashSet<>();
-        Entity foundEntitiesOverlap1 = new Entity("Batman X. Superman Y.", "fake type", 0, 17, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
+        Entity foundEntitiesOverlap1 = new Entity("Batman X. Superman Y.", "fake type", 0, 17, "fake headline", 0, 0,false, false, Engine.RULE, EntityType.ENTITY);
-        Entity foundEntitiesOverlap2 = new Entity("Superman Y.", "fake type", 10, 20, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
+        Entity foundEntitiesOverlap2 = new Entity("Superman Y.", "fake type", 10, 20, "fake headline", 0, 0,false, false, Engine.RULE, EntityType.ENTITY);
-        Entity foundEntitiesSubset1 = new Entity("Batman X. Superman", "fake type", 0, 17, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
+        Entity foundEntitiesSubset1 = new Entity("Batman X. Superman", "fake type", 0, 17, "fake headline", 0, 0,false, false, Engine.RULE, EntityType.ENTITY);
-        Entity foundEntitiesSubset2 = new Entity("Superman", "fake type", 10, 17, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
+        Entity foundEntitiesSubset2 = new Entity("Superman", "fake type", 10, 17, "fake headline", 0, 0,false, false, Engine.RULE, EntityType.ENTITY);
         foundEntities.add(foundEntitiesOverlap1);
         foundEntities.add(foundEntitiesOverlap2);
         foundEntities.add(foundEntitiesSubset1);

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/utils/RegExPatternTest.java

@@ -6,7 +6,7 @@ import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
 import org.apache.commons.lang3.StringUtils;
-import org.junit.Test;
+import org.junit.jupiter.api.Test;
 
 public class RegExPatternTest {
 

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/utils/TextNormalizationUtilitiesTest.java

@@ -1,7 +1,7 @@
 package com.iqser.red.service.redaction.v1.server.redaction.utils;
 
 import org.assertj.core.api.Assertions;
-import org.junit.Test;
+import org.junit.jupiter.api.Test;
 
 public class TextNormalizationUtilitiesTest {
 

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/segmentation/PdfSegmentationServiceTest.java

@@ -11,8 +11,8 @@ import java.util.Locale;
 import java.util.Map;
 import java.util.stream.Collectors;
 
-import org.junit.Test;
+import org.junit.jupiter.api.Test;
-import org.junit.runner.RunWith;
+import org.junit.jupiter.api.extension.ExtendWith;
 import org.kie.api.runtime.KieContainer;
 import org.springframework.amqp.rabbit.core.RabbitTemplate;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -25,7 +25,7 @@ import org.springframework.context.annotation.Configuration;
 import org.springframework.context.annotation.Import;
 import org.springframework.context.annotation.Primary;
 import org.springframework.core.io.ClassPathResource;
-import org.springframework.test.context.junit4.SpringRunner;
+import org.springframework.test.context.junit.jupiter.SpringExtension;
 
 import com.amazonaws.services.s3.AmazonS3;
 import com.fasterxml.jackson.databind.ObjectMapper;
@@ -49,7 +49,7 @@ import com.iqser.red.storage.commons.service.StorageService;
 
 import lombok.SneakyThrows;
 
-@RunWith(SpringRunner.class)
+@ExtendWith(SpringExtension.class)
 @SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
 @Import(PdfSegmentationServiceTest.TestConfiguration.class)
 public class PdfSegmentationServiceTest {
@@ -131,8 +131,8 @@ public class PdfSegmentationServiceTest {
         ClassPathResource pdfFileResource = new ClassPathResource("files/Minimal Examples/Spanning Cells.pdf");
 
         Document document = pdfSegmentationService.parseDocument(TEST_DOSSIER_ID, TEST_FILE_ID, pdfFileResource.getInputStream(), null);
-        assertThat(document.getParagraphs().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList())).isNotEmpty();
+        assertThat(document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList())).isNotEmpty();
-        Table table = document.getParagraphs().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList()).get(0);
+        Table table = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList()).get(0);
         assertThat(table.getColCount()).isEqualTo(6);
         assertThat(table.getRowCount()).isEqualTo(13);
         assertThat(table.getRows().stream().mapToInt(List::size).sum()).isEqualTo(6 * 13);
@@ -146,11 +146,11 @@ public class PdfSegmentationServiceTest {
         ClassPathResource pdfFileResource = new ClassPathResource("files/Minimal Examples/Merge Table.pdf");
 
         Document document = pdfSegmentationService.parseDocument(TEST_DOSSIER_ID, TEST_FILE_ID, pdfFileResource.getInputStream(), null);
-        assertThat(document.getParagraphs().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList())).isNotEmpty();
+        assertThat(document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList())).isNotEmpty();
-        Table firstTable = document.getParagraphs().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList()).get(0);
+        Table firstTable = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList()).get(0);
         assertThat(firstTable.getColCount()).isEqualTo(8);
         assertThat(firstTable.getRowCount()).isEqualTo(1);
-        Table secondTable = document.getParagraphs().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList()).get(1);
+        Table secondTable = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList()).get(1);
         assertThat(secondTable.getColCount()).isEqualTo(8);
         assertThat(secondTable.getRowCount()).isEqualTo(2);
         List<List<Cell>> firstTableHeaderCells = firstTable.getRows().get(0).stream().map(Collections::singletonList).collect(Collectors.toList());
@@ -165,11 +165,11 @@ public class PdfSegmentationServiceTest {
         ClassPathResource pdfFileResource = new ClassPathResource("files/Minimal Examples/Merge Multi Page Table.pdf");
 
         Document document = pdfSegmentationService.parseDocument(TEST_DOSSIER_ID, TEST_FILE_ID, pdfFileResource.getInputStream(), null);
-        assertThat(document.getParagraphs().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList())).isNotEmpty();
+        assertThat(document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList())).isNotEmpty();
-        Table firstTable = document.getParagraphs().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList()).get(0);
+        Table firstTable = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList()).get(0);
         assertThat(firstTable.getColCount()).isEqualTo(9);
         assertThat(firstTable.getRowCount()).isEqualTo(5);
-        Table secondTable = document.getParagraphs().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList()).get(1);
+        Table secondTable = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList()).get(1);
         assertThat(secondTable.getColCount()).isEqualTo(9);
         assertThat(secondTable.getRowCount()).isEqualTo(6);
         List<List<Cell>> firstTableHeaderCells = firstTable.getRows().get(firstTable.getRowCount() - 1).stream().map(Cell::getHeaderCells).collect(Collectors.toList());
@@ -184,11 +184,11 @@ public class PdfSegmentationServiceTest {
         ClassPathResource pdfFileResource = new ClassPathResource("files/Minimal Examples/Rotated Table Headers.pdf");
 
         Document document = pdfSegmentationService.parseDocument(TEST_DOSSIER_ID, TEST_FILE_ID, pdfFileResource.getInputStream(), null);
-        assertThat(document.getParagraphs().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList())).isNotEmpty();
+        assertThat(document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList())).isNotEmpty();
-        Table firstTable = document.getParagraphs().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList()).get(0);
+        Table firstTable = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList()).get(0);
         assertThat(firstTable.getColCount()).isEqualTo(8);
         assertThat(firstTable.getRowCount()).isEqualTo(1);
-        Table secondTable = document.getParagraphs().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList()).get(1);
+        Table secondTable = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList()).get(1);
         assertThat(secondTable.getColCount()).isEqualTo(8);
         assertThat(secondTable.getRowCount()).isEqualTo(6);
         List<List<Cell>> firstTableHeaderCells = firstTable.getRows().get(0).stream().map(Collections::singletonList).collect(Collectors.toList());

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/stringmatching/StringMatchingPerformanceTest.java

@@ -1,24 +1,24 @@
 package com.iqser.red.service.redaction.v1.server.stringmatching;
 
-import lombok.AllArgsConstructor;
+import static org.assertj.core.api.AssertionsForClassTypes.assertThat;
-import lombok.EqualsAndHashCode;
-import lombok.SneakyThrows;
-
-import org.ahocorasick.trie.Trie;
-import org.apache.commons.io.IOUtils;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.springframework.core.io.ClassPathResource;
-import org.springframework.test.context.junit4.SpringRunner;
 
 import java.util.HashSet;
 import java.util.Set;
 import java.util.regex.Pattern;
 import java.util.stream.Collectors;
 
-import static org.assertj.core.api.AssertionsForClassTypes.assertThat;
+import org.ahocorasick.trie.Trie;
+import org.apache.commons.io.IOUtils;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.springframework.core.io.ClassPathResource;
+import org.springframework.test.context.junit.jupiter.SpringExtension;
 
-@RunWith(SpringRunner.class)
+import lombok.AllArgsConstructor;
+import lombok.EqualsAndHashCode;
+import lombok.SneakyThrows;
+
+@ExtendWith(SpringExtension.class)
 public class StringMatchingPerformanceTest {
 
     @Test

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/utils/PdfDraw.java

@@ -0,0 +1,27 @@
+package com.iqser.red.service.redaction.v1.server.utils;
+
+import java.awt.Color;
+import java.awt.geom.Rectangle2D;
+import java.util.List;
+
+import org.apache.pdfbox.pdmodel.PDDocument;
+import org.apache.pdfbox.pdmodel.PDPageContentStream;
+
+import lombok.SneakyThrows;
+import lombok.experimental.UtilityClass;
+
+@UtilityClass
+public class PdfDraw {
+
+    @SneakyThrows
+    public static void drawRectangle2DList(PDDocument document, int pageNumber, List<Rectangle2D> rectCollection, Color color) {
+        var pdPage = document.getPage(pageNumber - 1);
+        var contentStream = new PDPageContentStream(document, pdPage, PDPageContentStream.AppendMode.APPEND, true);
+        contentStream.setStrokingColor(color);
+        for (var r : rectCollection) {
+            contentStream.addRect((float) r.getMinX(), (float) r.getMinY(), (float) r.getWidth(), (float) r.getHeight());
+            contentStream.stroke();
+        }
+        contentStream.close();
+    }
+}

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/allAuthors.drl

@@ -19,8 +19,8 @@ rule "0: Expand CBI Authors with firstname initials"
     when
         Section(matchesType("CBI_author") || matchesType("recommendation_CBI_author"))
     then
-        section.expandByRegEx("CBI_author", "(,? [A-Z]\\.?( ?[A-Z]\\.?)?( ?[A-Z]\\.?)?\\b\\.?)", false, 1, "[^\\s]+");
+        section.expandByRegEx("CBI_author", "(,? [A-Z]\\.?( ?[A-Z]\\.?)?( ?[A-Z]\\.?)?\\b\\.?)", false, 1, "[^\\s]+",dictionary);
-        section.expandByRegEx("recommendation_CBI_author", "(,? [A-Z]\\.?( ?[A-Z]\\.?)?( ?[A-Z]\\.?)?\\b\\.?)", false, 1, "[^\\s]+");
+        section.expandByRegEx("recommendation_CBI_author", "(,? [A-Z]\\.?( ?[A-Z]\\.?)?( ?[A-Z]\\.?)?\\b\\.?)", false, 1, "[^\\s]+",dictionary);
     end
 
 
@@ -53,7 +53,7 @@ rule "4: Redact Author(s) cells in Tables with Author(s) header"
     when
         Section(hasTableHeader("Author(s)") && !hasTableHeader("Vertebrate study Y/N"))
     then
-        section.redactCell("Author(s)", 4, "CBI_author", false, "Author found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+        section.redactCell("Author(s)", 4, "CBI_author", false, "Author found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary);
     end
 
 
@@ -61,7 +61,7 @@ rule "5: Redact Author cells in Tables with Author header"
     when
         Section(hasTableHeader("Author") && !hasTableHeader("Vertebrate study Y/N"))
     then
-        section.redactCell("Author", 5, "CBI_author", false, "Author found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+        section.redactCell("Author", 5, "CBI_author", false, "Author found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary);
     end
 
 
@@ -69,7 +69,7 @@ rule "6: Redact and recommand Authors in Tables with Vertebrate study Y/N header
     when
         Section(rowEquals("Vertebrate study Y/N", "Y") || rowEquals("Vertebrate study Y/N", "Yes") || rowEquals("Vertebrate study Y/N", "N") || rowEquals("Vertebrate study Y/N", "No"))
     then
-        section.redactCell("Author(s)", 6, "CBI_author", true, "Author found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+        section.redactCell("Author(s)", 6, "CBI_author", true, "Author found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary);
     end
 
 
@@ -77,8 +77,8 @@ rule "7: Redact if CTL/* or BL/*  was found"
     when
         Section(searchText.contains("CTL/") || searchText.contains("BL/"))
     then
-        section.addRedaction("CTL", "must_redact", 7, "Laboratory for vertebrate studies found", "Article 39(1)(2) of Regulation (EC) No 178/2002" );
+        section.addRedaction("CTL", "must_redact", 7, "Laboratory for vertebrate studies found", "Article 39(1)(2) of Regulation (EC) No 178/2002", dictionary);
-        section.addRedaction("BL", "must_redact", 7, "Laboratory for vertebrate studies found", "Article 39(1)(2) of Regulation (EC) No 178/2002" );
+        section.addRedaction("BL", "must_redact", 7, "Laboratory for vertebrate studies found", "Article 39(1)(2) of Regulation (EC) No 178/2002", dictionary);
     end
 
 
@@ -86,7 +86,7 @@ rule "8: Redact and add recommendation for et al. author"
     when
         Section(searchText.contains("et al"))
     then
-		section.redactAndRecommendByRegEx("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", false, 1, "CBI_author", 8, "Author found", "Reg (EC) No 1107/2009 Art. 63 (2g)");
+		section.redactAndRecommendByRegEx("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", false, 1, "CBI_author", 8, "Author found", "Reg (EC) No 1107/2009 Art. 63 (2g)",dictionary);
     end
 
 
@@ -130,7 +130,7 @@ rule "13: Redact Emails by RegEx"
     when
         Section(searchText.contains("@"))
     then
-        section.redactByRegEx("\\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\\.[A-Z]{2,4}\\b", true, 0, "PII", 13, "PII (Personal Identification Information) found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+        section.redactByRegEx("\\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\\.[A-Z]{2,4}\\b", true, 0, "PII", 13, "PII (Personal Identification Information) found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary);
     end
 
 
@@ -153,25 +153,25 @@ rule "14: Redact contact information"
        || text.contains("Phone No.")
        || text.contains("European contact:"))
     then
-          section.redactLineAfter("Contact point:", "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+          section.redactLineAfter("Contact point:", "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary);
-              section.redactLineAfter("Phone:", "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+              section.redactLineAfter("Phone:", "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary);
-              section.redactLineAfter("Fax:", "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+              section.redactLineAfter("Fax:", "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary);
-              section.redactLineAfter("Tel.:", "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+              section.redactLineAfter("Tel.:", "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary);
-              section.redactLineAfter("Tel:", "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+              section.redactLineAfter("Tel:", "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary);
-              section.redactLineAfter("E-mail:", "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+              section.redactLineAfter("E-mail:", "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary);
-              section.redactLineAfter("Email:", "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+              section.redactLineAfter("Email:", "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary);
-              section.redactLineAfter("e-mail:", "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+              section.redactLineAfter("e-mail:", "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary);
-              section.redactLineAfter("E-mail address:", "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+              section.redactLineAfter("E-mail address:", "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary);
-              section.redactLineAfter("Contact:", "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+              section.redactLineAfter("Contact:", "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary);
-              section.redactLineAfter("Alternative contact:", "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+              section.redactLineAfter("Alternative contact:", "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary);
-              section.redactLineAfter("Telephone number:", "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+              section.redactLineAfter("Telephone number:", "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary);
-              section.redactLineAfter("Telephone No:", "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+              section.redactLineAfter("Telephone No:", "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary);
-              section.redactLineAfter("Fax number:", "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+              section.redactLineAfter("Fax number:", "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary);
-              section.redactLineAfter("Telephone:", "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+              section.redactLineAfter("Telephone:", "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary);
-              section.redactLineAfter("Phone No.", "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+              section.redactLineAfter("Phone No.", "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary);
-              section.redactBetween("No:", "Fax", "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+              section.redactBetween("No:", "Fax", "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary);
-              section.redactBetween("Contact:", "Tel.:", "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+              section.redactBetween("Contact:", "Tel.:", "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary);
-              section.redactLineAfter("European contact:",  "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+              section.redactLineAfter("European contact:",  "PII", 14, true, "Contact information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary);
      end
 
 
@@ -179,25 +179,25 @@ rule "15: Redact contact information if applicant is found"
     when
         Section(headlineContainsWord("applicant") || text.contains("Applicant") || headlineContainsWord("Primary contact") || headlineContainsWord("Alternative contact") || text.contains("Telephone number:"))
     then
-        section.redactLineAfter("Contact point:", "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Contact point:", "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary);
-        section.redactLineAfter("Phone:", "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Phone:", "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary);
-        section.redactLineAfter("Fax:", "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Fax:", "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary);
-        section.redactLineAfter("Tel.:", "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Tel.:", "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary);
-        section.redactLineAfter("Tel:", "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Tel:", "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary);
-        section.redactLineAfter("E-mail:", "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("E-mail:", "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary);
-        section.redactLineAfter("Email:", "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Email:", "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary);
-        section.redactLineAfter("e-mail:", "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("e-mail:", "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary);
-        section.redactLineAfter("E-mail address:", "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("E-mail address:", "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary);
-        section.redactLineAfter("Contact:", "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Contact:", "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary);
-        section.redactLineAfter("Alternative contact:", "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Alternative contact:", "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary);
-        section.redactLineAfter("Telephone number:", "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Telephone number:", "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary);
-        section.redactLineAfter("Telephone No:", "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Telephone No:", "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary);
-        section.redactLineAfter("Fax number:", "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Fax number:", "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary);
-        section.redactLineAfter("Telephone:", "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Telephone:", "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary);
-        section.redactLineAfter("Phone No.", "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Phone No.", "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary);
-        section.redactBetween("No:", "Fax", "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+        section.redactBetween("No:", "Fax", "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary);
-        section.redactBetween("Contact:", "Tel.:", "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+        section.redactBetween("Contact:", "Tel.:", "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary);
-        section.redactLineAfter("European contact:",  "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("European contact:",  "PII", 15, true, "Applicant information was found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary);
     end
 
 
@@ -205,17 +205,17 @@ rule "16: Redact contact information if Producer is found"
     when
         Section(text.toLowerCase().contains("producer of the plant protection") || text.toLowerCase().contains("producer of the active substance") || text.contains("Manufacturer of the active substance") || text.contains("Manufacturer:") || text.contains("Producer or producers of the active substance"))
     then
-        section.redactLineAfter("Contact:", "PII", 16, true, "Producer was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Contact:", "PII", 16, true, "Producer was found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary);
-        section.redactLineAfter("Telephone:", "PII", 16, true, "Producer was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Telephone:", "PII", 16, true, "Producer was found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary);
-        section.redactLineAfter("Phone:", "PII", 16, true, "Producer was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Phone:", "PII", 16, true, "Producer was found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary);
-        section.redactLineAfter("Fax:", "PII", 16, true, "Producer was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Fax:", "PII", 16, true, "Producer was found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary);
-        section.redactLineAfter("E-mail:", "PII", 16, true, "Producer was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("E-mail:", "PII", 16, true, "Producer was found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary);
-        section.redactLineAfter("Contact:", "PII", 16, true, "Producer was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Contact:", "PII", 16, true, "Producer was found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary);
-        section.redactLineAfter("Fax number:", "PII", 16, true, "Producer was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Fax number:", "PII", 16, true, "Producer was found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary);
-        section.redactLineAfter("Telephone number:", "PII", 16, true, "Producer was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Telephone number:", "PII", 16, true, "Producer was found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary);
-        section.redactLineAfter("Tel:", "PII", 16, true, "Producer was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Tel:", "PII", 16, true, "Producer was found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary);
-        section.redactLineAfter("Phone No.", "PII", 16, true, "Producer was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Phone No.", "PII", 16, true, "Producer was found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary);
-        section.redactBetween("No:", "Fax", "PII", 16, true, "Producer was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+        section.redactBetween("No:", "Fax", "PII", 16, true, "Producer was found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary);
     end
 
 
@@ -223,7 +223,7 @@ rule "17: Redact AUTHOR(S)"
     when
         Section(searchText.contains("AUTHOR(S):"))
     then
-        section.redactLinesBetween("AUTHOR(S):", "COMPLETION DATE:", "PII", 17, true, "AUTHOR(S) was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+        section.redactLinesBetween("AUTHOR(S):", "COMPLETION DATE:", "PII", 17, true, "AUTHOR(S) was found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary);
     end
 
 
@@ -231,7 +231,7 @@ rule "18: Redact PERFORMING LABORATORY"
     when
         Section(searchText.contains("PERFORMING LABORATORY:"))
     then
-        section.redactBetween("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "PII", 18, true, "PERFORMING LABORATORY was found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+        section.redactBetween("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "PII", 18, true, "PERFORMING LABORATORY was found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary);
     end
 
 
@@ -239,7 +239,7 @@ rule "19: Redact On behalf of Sequani Ltd.:"
     when
         Section(searchText.contains("On behalf of Sequani Ltd.: Name Title"))
     then
-        section.redactBetween("On behalf of Sequani Ltd.: Name Title", "On behalf of", "PII", 19, false , "PII (Personal Identification Information) found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+        section.redactBetween("On behalf of Sequani Ltd.: Name Title", "On behalf of", "PII", 19, false , "PII (Personal Identification Information) found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary);
     end
 
 
@@ -247,7 +247,7 @@ rule "20: Redact On behalf of Syngenta Ltd.:"
     when
         Section(searchText.contains("On behalf of Syngenta Ltd.: Name Title"))
     then
-        section.redactBetween("On behalf of Syngenta Ltd.: Name Title", "Study dates", "PII", 20, false , "PII (Personal Identification Information) found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
+        section.redactBetween("On behalf of Syngenta Ltd.: Name Title", "Study dates", "PII", 20, false , "PII (Personal Identification Information) found", "Article 39(1)(2) of Regulation (EC) No 178/2002",dictionary);
     end
 
 
@@ -257,7 +257,7 @@ rule "21: Purity Hint"
     when
         Section(searchText.toLowerCase().contains("purity"))
     then
-	    section.addHintAnnotationByRegEx("(purity ?( of|\\(.{1,20}\\))?( ?:)?) .{0,5}[\\d\\.]+( .{0,4}\\.)? ?%", true, 1, "hint_only");
+	    section.addHintAnnotationByRegEx("(purity ?( of|\\(.{1,20}\\))?( ?:)?) .{0,5}[\\d\\.]+( .{0,4}\\.)? ?%", true, 1, "hint_only",dictionary);
     end
 
 

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/entity_rules.drl

@@ -0,0 +1,43 @@
+package drools
+
+import static java.lang.String.format;
+import static com.iqser.red.service.redaction.v1.server.document.services.RegexMatcher.anyMatch;
+
+
+import java.util.List;
+import java.util.LinkedList;
+import java.util.HashSet;
+
+import com.iqser.red.service.redaction.v1.server.document.graph.*
+import com.iqser.red.service.redaction.v1.server.document.graph.nodes.*
+import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType;
+import com.iqser.red.service.redaction.v1.model.FileAttribute;
+import com.iqser.red.service.redaction.v1.model.Engine;
+import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntry;
+import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUtils;
+import java.util.Set;
+
+global DocumentGraph document
+
+
+rule "1: Redact CBI_author"
+
+    when
+        FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "yes")
+        entity: EntityNode(type == "CBI_author")
+    then
+        entity.setRedact(true);
+        update(entity)
+    end
+
+rule "2: do not redact genitive CBI_author"
+
+    when
+        entity: EntityNode(type == "CBI_author", anyMatch(textAfter, "['’’'ʼˈ´`‘′ʻ’']s"), redact == true)
+    then
+        entity.setRedact(false);
+        entity.setEntityType(EntityType.FALSE_POSITIVE);
+        update(entity)
+    end
+
+

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/headlines.drl

@@ -9,5 +9,5 @@ rule "1: Find headlines"
     when
         Section(text.length() > 1)
     then
-        section.redactHeadline("headline", 1, "Headline found", "n-a.");
+        section.redactHeadline("headline", 1, "Headline found", "n-a.",dictionary);
     end

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/prod_rules.drl

@@ -0,0 +1,383 @@
+package drools
+
+import java.util.Set;
+
+import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;
+import com.iqser.red.service.redaction.v1.server.redaction.model.*
+
+global RedactionServiceSettings redactionServiceSettings;
+global Dictionary dictionary;
+
+
+// --------------------------------------- AI rules -------------------------------------------------------------------
+rule "-1: find entities from dictionary"
+    salience 100
+    no-loop true
+    when
+        section: Section()
+    then
+        section.findDictionaryEntities(dictionary, redactionServiceSettings);
+        update(section);
+     end
+
+rule "0: Add CBI_author from ai"
+    when
+        section: Section(aiMatchesType("CBI_author"))
+    then
+        section.addAiEntities("CBI_author", "CBI_author", dictionary);
+    end
+
+rule "0: Combine address parts from ai to CBI_address (org is mandatory)"
+    when
+        section: Section(aiMatchesType("ORG"))
+    then
+        section.combineAiTypes("ORG", "STREET,POSTAL,COUNTRY,CARDINAL,CITY,STATE", 20, "CBI_address", 3, false, dictionary);
+    end
+
+rule "0: Combine address parts from ai to CBI_address (street is mandatory)"
+    when
+        section: Section(aiMatchesType("STREET"))
+    then
+        section.combineAiTypes("STREET", "ORG,POSTAL,COUNTRY,CARDINAL,CITY,STATE", 20, "CBI_address", 3, false, dictionary);
+    end
+
+rule "0: Combine address parts from ai to CBI_address (city is mandatory)"
+    when
+        section: Section(aiMatchesType("CITY"))
+    then
+        section.combineAiTypes("CITY", "ORG,STREET,POSTAL,COUNTRY,CARDINAL,STATE", 20, "CBI_address", 3, false, dictionary);
+    end
+
+
+// --------------------------------------- CBI rules -------------------------------------------------------------------
+
+rule "1: Redact CBI Authors (Non vertebrate study)"
+    when
+        section: Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesType("CBI_author"))
+    then
+        section.redact("CBI_author", 1, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+    end
+
+rule "2: Redact CBI Authors (Vertebrate study)"
+    when
+        section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesType("CBI_author"))
+    then
+        section.redact("CBI_author", 2, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+    end
+
+
+rule "3: Redact not CBI Address (Non vertebrate study)"
+    when
+        section: Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesType("CBI_address"))
+    then
+        section.redactNot("CBI_address", 3, "Address found for non vertebrate study");
+        section.ignoreRecommendations("CBI_address");
+    end
+
+rule "4: Redact CBI Address (Vertebrate study)"
+    when
+        section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesType("CBI_address"))
+    then
+        section.redact("CBI_address", 4, "Address found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+    end
+
+
+rule "5: Do not redact genitive CBI_author"
+    when
+        section: Section(matchesType("CBI_author"))
+    then
+        section.expandToFalsePositiveByRegEx("CBI_author", "['’’'ʼˈ´`‘′ʻ’']s", false, 0, dictionary);
+    end
+
+
+rule "6: Redact Author(s) cells in Tables with Author(s) header (Non vertebrate study)"
+    when
+        section: Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && hasTableHeader("Author(s)") && !hasTableHeader("Vertebrate study Y/N"))
+    then
+        section.redactCell("Author(s)", 6, "CBI_author", false, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
+    end
+
+rule "7: Redact Author(s) cells in Tables with Author(s) header (Vertebrate study)"
+    when
+        section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && hasTableHeader("Author(s)") && !hasTableHeader("Vertebrate study Y/N"))
+    then
+        section.redactCell("Author(s)", 7, "CBI_author", false, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
+    end
+
+
+rule "8: Redact Author cells in Tables with Author header (Non vertebrate study)"
+    when
+        section: Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && hasTableHeader("Author") && !hasTableHeader("Vertebrate study Y/N"))
+    then
+        section.redactCell("Author", 8, "CBI_author", false, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
+    end
+
+rule "9: Redact Author cells in Tables with Author header (Vertebrate study)"
+    when
+        section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && hasTableHeader("Author") && !hasTableHeader("Vertebrate study Y/N"))
+    then
+        section.redactCell("Author", 9, "CBI_author", false, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
+    end
+
+
+rule "10: Redact and recommand Authors in Tables with Vertebrate study Y/N header (Non vertebrate study)"
+    when
+        section: Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && (rowEquals("Vertebrate study Y/N", "Y") || rowEquals("Vertebrate study Y/N", "Yes") || rowEquals("Vertebrate study Y/N", "N") || rowEquals("Vertebrate study Y/N", "No")))
+    then
+        section.redactCell("Author(s)", 10, "CBI_author", true, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
+    end
+
+rule "11: Redact and recommand Authors in Tables with Vertebrate study Y/N header (Vertebrate study)"
+    when
+        section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && (rowEquals("Vertebrate study Y/N", "Y") || rowEquals("Vertebrate study Y/N", "Yes") || rowEquals("Vertebrate study Y/N", "N") || rowEquals("Vertebrate study Y/N", "No")))
+    then
+        section.redactCell("Author(s)", 11, "CBI_author", true, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
+    end
+
+rule "14: Redact and add recommendation for et al. author (Non vertebrate study)"
+    when
+        section: Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText contains "et al")
+    then
+        section.redactAndRecommendByRegEx("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", false, 1, "CBI_author", 14, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002", dictionary);
+    end
+
+rule "15: Redact and add recommendation for et al. author (Vertebrate study)"
+    when
+        section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText contains "et al")
+    then
+        section.redactAndRecommendByRegEx("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", false, 1, "CBI_author", 15, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002", dictionary);
+    end
+
+
+rule "16: Add recommendation for Addresses in Test Organism sections"
+    when
+        section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText (contains "Species:" &&  contains "Source:"))
+    then
+        section.recommendLineAfter("Source:", "CBI_address");
+    end
+
+rule "17: Add recommendation for Addresses in Test Animals sections"
+    when
+        section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText (contains "Species" &&  contains "Source") )
+    then
+        section.recommendLineAfter("Source", "CBI_address");
+    end
+
+/*
+
+rule "18: Do not redact Names and Addresses if Published Information found"
+    when
+        section: Section(matchesType("published_information"))
+    then
+        section.redactNotAndReference("CBI_author","published_information", 18, "Published Information found");
+        section.redactNotAndReference("CBI_address","published_information", 18, "Published Information found");
+    end
+
+*/
+
+// --------------------------------------- PII rules -------------------------------------------------------------------
+
+
+rule "19: Redacted PII Personal Identification Information (Non vertebrate study)"
+    when
+        section: Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesType("PII"))
+    then
+        section.redact("PII", 19, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+    end
+
+rule "20: Redacted PII Personal Identification Information (Vertebrate study)"
+    when
+        section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesType("PII"))
+    then
+        section.redact("PII", 20, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+    end
+
+
+rule "21: Redact Emails by RegEx (Non vertebrate study)"
+    when
+        section: Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText contains "@")
+    then
+        section.redactByRegEx("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", true, 1, "PII", 21, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002", dictionary);
+    end
+
+rule "22: Redact Emails by RegEx (Vertebrate study)"
+    when
+        section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText contains "@")
+    then
+        section.redactByRegEx("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", true, 1, "PII", 22, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
+    end
+
+
+rule "23: Redact contact information (Non vertebrate study)"
+    when
+       section: Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study", "Yes") &&
+       (text (contains "Contact point:" ||
+       contains "Contact:" ||
+       contains "Alternative contact:" ||
+       (contains "No:" && contains "Fax") ||
+       (contains "Contact:" && contains "Tel.:") ||
+       contains "European contact:")))
+    then
+        section.redactLineAfter("Contact point:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
+        section.redactLineAfter("Contact:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
+        section.redactLineAfter("Alternative contact:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
+        section.redactBetween("No:", "Fax", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
+        section.redactBetween("Contact:", "Tel.:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
+        section.redactLineAfter("European contact:",  "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
+     end
+
+rule "24: Redact contact information (Vertebrate study)"
+    when
+       section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") &&
+       (text (contains "Contact point:"
+            || contains "Contact:"
+            || contains"Alternative contact:"
+            || (contains "No:" &&  contains "Fax")
+            || (contains "Contact:" && contains "Tel.:")
+            || contains "European contact:"
+        )))
+    then
+        section.redactLineAfter("Contact point:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
+        section.redactLineAfter("Contact:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
+        section.redactLineAfter("Alternative contact:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
+        section.redactBetween("No:", "Fax", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
+        section.redactBetween("Contact:", "Tel.:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
+        section.redactLineAfter("European contact:",  "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
+     end
+
+rule "25: Redact Phone and Fax by RegEx (Non vertebrate study)"
+    when
+        section: Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && (
+               text contains "Contact"
+            || contains "Telephone"
+            || contains "Phone"
+            || contains "Fax"
+            || contains "Tel"
+            || contains "Ter"
+            || contains "Mobile"
+            || contains "Fel"
+            || contains "Fer"
+        ))
+    then
+        section.redactByRegEx("\\b(contact|telephone|phone|fax|tel|ter|mobile|fel|fer)[a-zA-Z\\s]{0,10}[:.\\s]{0,3}([\\+\\d\\(][\\s\\d\\(\\)\\-\\/\\.]{4,100}\\d)\\b", true, 2, "PII", 25, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002", dictionary);
+    end
+
+rule "26: Redact Phone and Fax by RegEx (Vertebrate study)"
+    when
+        section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && ( 
+            text contains "Contact"
+            || contains "Telephone"
+            || contains "Phone"
+            || contains "Fax"
+            || contains "Tel"
+            || contains "Ter"
+            || contains "Mobile"
+            || contains "Fel"
+            || contains "Fer"
+        ))
+    then
+        section.redactByRegEx("\\b(contact|telephone|phone|fax|tel|ter|mobile|fel|fer)[a-zA-Z\\s]{0,10}[:.\\s]{0,3}([\\+\\d\\(][\\s\\d\\(\\)\\-\\/\\.]{4,100}\\d)\\b", true, 2, "PII", 26, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002", dictionary);
+    end
+
+rule "27: Redact AUTHOR(S) (Non vertebrate study)"
+    when
+        section: Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes")
+            && (searchText contains "AUTHOR(S):"
+            && contains "COMPLETION DATE:"
+            && not contains "STUDY COMPLETION DATE:")
+        )
+    then
+        section.redactLinesBetween("AUTHOR(S):", "COMPLETION DATE:", "PII", 27, true, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002", dictionary);
+    end
+
+rule "28: Redact AUTHOR(S) (Vertebrate study)"
+    when
+        section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes")
+            && (searchText contains "AUTHOR(S):"
+            && contains "COMPLETION DATE:"
+            && not contains "STUDY COMPLETION DATE:")
+        )
+    then
+        section.redactLinesBetween("AUTHOR(S):", "COMPLETION DATE:", "PII", 28, true, "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002", dictionary);
+    end
+
+
+rule "29: Redact AUTHOR(S) (Non vertebrate study)"
+    when
+        section: Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes")
+            && (searchText contains "AUTHOR(S):"
+            && contains "STUDY COMPLETION DATE:")
+        )
+    then
+        section.redactLinesBetween("AUTHOR(S):", "STUDY COMPLETION DATE:", "PII", 29, true, "AUTHOR(S) was found", "Article 39(e)(3) of Regulation (EC) No 178/2002", dictionary);
+    end
+
+rule "30: Redact AUTHOR(S) (Vertebrate study)"
+    when
+        section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes")
+            && searchText contains "AUTHOR(S):"
+            && contains "STUDY COMPLETION DATE:"
+        )
+    then
+        section.redactLinesBetween("AUTHOR(S):", "STUDY COMPLETION DATE:", "PII", 30, true, "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002", dictionary);
+    end
+
+
+rule "31: Redact PERFORMING LABORATORY (Non vertebrate study)"
+    when
+        section: Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes")
+            && searchText contains "PERFORMING LABORATORY:"
+        )
+    then
+        section.redactBetween("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", 31, true, "PERFORMING LABORATORY was found", "Article 39(e)(3) of Regulation (EC) No 178/2002", dictionary);
+        section.redactNot("CBI_address", 31, "Performing laboratory found for non vertebrate study");
+    end
+
+rule "32: Redact PERFORMING LABORATORY (Vertebrate study)"
+    when
+        section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes")
+            && searchText contains "PERFORMING LABORATORY:")
+    then
+        section.redactBetween("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", 32, true, "PERFORMING LABORATORY was found", "Article 39(e)(2) of Regulation (EC) No 178/2002", dictionary);
+    end
+
+rule "33: Redact study director abbreviation"
+    when
+        section: Section(searchText contains "KATH" || contains "BECH" || contains "KML")
+    then
+        section.redactWordPartByRegEx("((KATH)|(BECH)|(KML)) ?(\\d{4})", true, 0, 1, "PII", 34, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002", dictionary);
+     end
+
+
+// --------------------------------------- other rules -------------------------------------------------------------------
+
+rule "34: Purity Hint"
+    when
+        section: Section(searchText.toLowerCase() contains "purity")
+    then
+	    section.addHintAnnotationByRegEx("(purity ?( of|\\(.{1,20}\\))?( ?:)?) .{0,5}[\\d\\.]+( .{0,4}\\.)? ?%", true, 1, "hint_only",dictionary);
+    end
+
+
+
+rule "35: Redact signatures (Non vertebrate study)"
+    when
+        section: Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesImageType("signature"))
+    then
+        section.redactImage("signature", 35, "Signature found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+    end
+
+rule "36: Redact signatures (Vertebrate study)"
+    when
+        section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesImageType("signature"))
+    then
+        section.redactImage("signature", 36, "Signature found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+    end
+
+
+rule "43: Redact Logos (Vertebrate study)"
+    when
+        section: Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && matchesImageType("logo"))
+    then
+        section.redactImage("logo", 43, "Logo found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+    end

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl

@@ -1,407 +1,40 @@
 package drools
 
-import com.iqser.red.service.redaction.v1.server.redaction.model.Section
+import static java.lang.String.format;
+import static com.iqser.red.service.redaction.v1.server.document.services.RegexMatcher.anyMatch;
 
-global Section section
+import java.util.List;
+import java.util.LinkedList;
+import java.util.HashSet;
+
+import com.iqser.red.service.redaction.v1.server.document.graph.*
+import com.iqser.red.service.redaction.v1.server.document.graph.nodes.*
+import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType;
+import com.iqser.red.service.redaction.v1.model.FileAttribute;
+import com.iqser.red.service.redaction.v1.model.Engine;
+import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntry;
+import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUtils;
+import java.util.Set;
+global DocumentGraph document
 
 
-// --------------------------------------- CBI rules -------------------------------------------------------------------
 
-//rule "0: Expand CBI Authors with firstname initials"
+rule "1: Redact CBI_author"
-//    when
-//        Section(matchesType("CBI_author"))
-//    then
-//        section.expandByRegEx("CBI_author", "(,? [A-Z]\\.?( ?[A-Z]\\.?)?( ?[A-Z]\\.?)?\\b\\.?)", false, 1);
-//    end
 
-
-rule "0: Add CBI_author from ai"
     when
-        Section(aiMatchesType("CBI_author"))
+        FileAttribute(label == "Vertebrate Study" && (value.toLowerCase() == "yes"))
+        entity: EntityNode(type == "CBI_author")
     then
-        section.addAiEntities("CBI_author", "CBI_author");
+        entity.setRedact(true);
+        update(entity)
     end
 
-rule "0: Combine ai types CBI_author from ai"
+rule "2: do not redact genitive CBI_author"
+
     when
-        Section(aiMatchesType("ORG"))
+        entity: EntityNode(type == "CBI_author", anyMatch("['’’'ʼˈ´`‘′ʻ’']s", textAfter))
     then
-        section.combineAiTypes("ORG", "STREET,POSTAL,COUNTRY,CARDINAL,CITY,STATE", 20, "CBI_address", 3, false);
+        entity.setRedact(false);
+        entity.setEntityType(EntityType.FALSE_POSITIVE);
+        update(entity)
     end
-
-rule "0: Expand CBI Authors with firstname initials"
-    when
-        Section(matchesType("CBI_author"))
-    then
-        section.expandByRegEx("CBI_author", "(,? [A-Z]\\.?( ?[A-Z]\\.?)?( ?[A-Z]\\.?)?\\b\\.?)", false, 1, "[^\\s]+");
-    end
-
-rule "0: Expand CBI_author and PII matches with salutation prefix"
-    when
-        Section((matchesType("CBI_author") || matchesType("PII")) && (
-            searchText.contains("Mr")
-            || searchText.contains("Mrs")
-            || searchText.contains("Ms")
-            || searchText.contains("Miss")
-            || searchText.contains("Sir")
-            || searchText.contains("Madam")
-            || searchText.contains("Madame")
-            || searchText.contains("Mme")
-        ))
-    then
-        section.expandByPrefixRegEx("CBI_author", "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*", false, 0);
-        section.expandByPrefixRegEx("PII", "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*", false, 0);
-    end
-
-
-rule "1: Redacted because Section contains Vertebrate"
-    when
-        Section(matchesType("vertebrate"))
-    then
-        section.redact("CBI_author", 1, "Vertebrate found", "Reg (EC) No 1107/2009 Art. 63 (2g)");
-        section.redact("CBI_address", 1, "Vertebrate found", "Reg (EC) No 1107/2009 Art. 63 (2g)");
-    end
-
-
-rule "2: Not Redacted because Section contains no Vertebrate"
-    when
-        Section(!matchesType("vertebrate"))
-    then
-        section.redactNot("CBI_author", 2, "No Vertebrate found");
-        section.redactNot("CBI_address", 2, "No Vertebrate found");
-    end
-
-
-rule "3: Do not redact Names and Addresses if no redaction Indicator is contained"
-    when
-        Section(matchesType("vertebrate"), matchesType("no_redaction_indicator"))
-    then
-        section.redactNot("CBI_author", 3, "Vertebrate and No Redaction Indicator found");
-        section.redactNot("CBI_address", 3, "Vertebrate and No Redaction Indicator found");
-    end
-
-
-rule "4: Redact Names and Addresses if no_redaction_indicator and redaction_indicator is contained"
-    when
-        Section(matchesType("vertebrate"), matchesType("no_redaction_indicator"), matchesType("redaction_indicator"))
-    then
-        section.redact("CBI_author", 4, "Vertebrate and Redaction Indicator found", "Reg (EC) No 1107/2009 Art. 63 (2g)");
-        section.redact("CBI_address", 4, "Vertebrate and Redaction Indicator found", "Reg (EC) No 1107/2009 Art. 63 (2g)");
-    end
-
-
-rule "5: Do not redact Names and Addresses if no redaction Indicator is contained"
-    when
-        Section(matchesType("vertebrate"), matchesType("published_information"))
-    then
-        section.redactNotAndReference("CBI_author","published_information", 5, "Vertebrate and Published Information found");
-        section.redactNotAndReference("CBI_address","published_information", 5, "Vertebrate and Published Information found");
-    end
-
-
-rule "6: Not redacted because Vertebrate Study = N"
-    when
-        Section(rowEquals("Vertebrate study Y/N", "N") || rowEquals("Vertebrate study Y/N", "No"))
-    then
-        section.redactNotCell("Author(s)", 6, "CBI_author", true, "Not redacted because row is not a vertebrate study");
-        section.redactNot("CBI_author", 6, "Not redacted because row is not a vertebrate study");
-        section.redactNot("CBI_address", 6, "Not redacted because row is not a vertebrate study");
-        section.highlightCell("Vertebrate study Y/N", 6, "hint_only");
-    end
-
-
-rule "7: Redact if must redact entry is found"
-    when
-        Section(matchesType("must_redact"))
-    then
-        section.redact("CBI_author", 7, "must_redact entry was found.", "Reg (EC) No 1107/2009 Art. 63 (2g)");
-        section.redact("CBI_address", 7, "must_redact entry was found.", "Reg (EC) No 1107/2009 Art. 63 (2g)");
-    end
-
-
-rule "8: Redact Authors and Addresses in Reference Table if it is a Vertebrate study"
-    when
-        Section(rowEquals("Vertebrate study Y/N", "Y") || rowEquals("Vertebrate study Y/N", "Yes"))
-    then
-        section.redactCell("Author(s)", 8, "CBI_author", true, "Redacted because row is a vertebrate study", "Reg (EC) No 1107/2009 Art. 63 (2g)");
-        section.redact("CBI_address", 8, "Redacted because row is a vertebrate study", "Reg (EC) No 1107/2009 Art. 63 (2g)");
-        section.highlightCell("Vertebrate study Y/N", 8, "must_redact");
-    end
-
-
-rule "9: Redact sponsor company"
-    when
-        Section(searchText.toLowerCase().contains("batches produced at"))
-    then
-        section.redactIfPrecededBy("batches produced at", "CBI_sponsor", 9, "Redacted because it represents a sponsor company", "Reg (EC) No 1107/2009 Art. 63 (2g)");
-        section.addHintAnnotation("batches produced at", "must_redact");
-    end
-
-
-rule "10: Redact determination of residues"
-    when
-        Section((
-            searchText.toLowerCase.contains("determination of residues") ||
-            searchText.toLowerCase.contains("determination of total residues")
-        ) && (
-            searchText.toLowerCase.contains("livestock") ||
-            searchText.toLowerCase.contains("live stock") ||
-            searchText.toLowerCase.contains("tissue") ||
-            searchText.toLowerCase.contains("tissues") ||
-            searchText.toLowerCase.contains("liver") ||
-            searchText.toLowerCase.contains("muscle") ||
-            searchText.toLowerCase.contains("bovine") ||
-            searchText.toLowerCase.contains("ruminant") ||
-            searchText.toLowerCase.contains("ruminants")
-        ))
-    then
-        section.redact("CBI_author", 10, "Determination of residues was found.", "Reg (EC) No 1107/2009 Art. 63 (2g)");
-        section.redact("CBI_address", 10, "Determination of residues was found.", "Reg (EC) No 1107/2009 Art. 63 (2g)");
-        section.addHintAnnotation("determination of residues", "must_redact");
-        section.addHintAnnotation("livestock", "must_redact");
-        section.addHintAnnotation("live stock", "must_redact");
-        section.addHintAnnotation("tissue", "must_redact");
-        section.addHintAnnotation("tissues", "must_redact");
-        section.addHintAnnotation("liver", "must_redact");
-        section.addHintAnnotation("muscle", "must_redact");
-        section.addHintAnnotation("bovine", "must_redact");
-        section.addHintAnnotation("ruminant", "must_redact");
-        section.addHintAnnotation("ruminants", "must_redact");
-    end
-
-
-rule "11: Redact if CTL/* or BL/*  was found"
-    when
-        Section(searchText.contains("CTL/") || searchText.contains("BL/"))
-    then
-        section.redact("CBI_author", 11, "Laboraty for vertebrate studies found", "Reg (EC) No 1107/2009 Art. 63 (2g)");
-        section.redact("CBI_address", 11, "Laboraty for vertebrate studies found", "Reg (EC) No 1107/2009 Art. 63 (2g)");
-        section.addHintAnnotation("CTL", "must_redact");
-        section.addHintAnnotation("BL", "must_redact");
-    end
-
-
-rule "12: Redact and add recommendation for et al. author"
-    when
-        Section(searchText.contains("et al"))
-    then
-		section.redactAndRecommendByRegEx("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", false, 1, "CBI_author", 12, "Author found", "Reg (EC) No 1107/2009 Art. 63 (2g)");
-    end
-
-
-rule "13: Add recommendation for Addresses in Test Organism sections"
-    when
-        Section(searchText.contains("Species:") && searchText.contains("Source:"))
-    then
-		section.recommendLineAfter("Source:", "CBI_address");
-    end
-
-
-rule "14: Add recommendation for Addresses in Test Animals sections"
-    when
-        Section(searchText.contains("Species") && searchText.contains("Source"))
-    then
-		section.recommendLineAfter("Source", "CBI_address");
-    end
-
-// --------------------------------------- PII rules -------------------------------------------------------------------
-
-
-rule "14: Redacted PII Personal Identification Information"
-    when
-        Section(matchesType("PII"))
-    then
-        section.redact("PII", 14, "PII (Personal Identification Information) found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-    end
-
-
-rule "15: Redact Emails by RegEx"
-    when
-        Section(searchText.contains("@"))
-    then
-        section.redactByRegEx("\\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\\.[A-Z]{2,4}\\b", true, 0, "PII", 15, "PII (Personal Identification Information) found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-    end
-
-
-rule "16: Redact contact information"
-    when
-       Section(text.contains("Contact point:")
-       || text.contains("Phone:")
-       || text.contains("Fax:")
-       || text.contains("Tel.:")
-       || text.contains("Tel:")
-       || text.contains("E-mail:")
-       || text.contains("Email:")
-       || text.contains("e-mail:")
-       || text.contains("E-mail address:")
-       || text.contains("Alternative contact:")
-       || text.contains("Telephone number:")
-       || text.contains("Telephone No:")
-       || text.contains("Fax number:")
-       || text.contains("Telephone:")
-       || text.contains("European contact:"))
-    then
-          section.redactLineAfter("Contact point:", "PII", 16, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-              section.redactLineAfter("Phone:", "PII", 16, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-              section.redactLineAfter("Fax:", "PII", 16, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-              section.redactLineAfter("Tel.:", "PII", 16, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-              section.redactLineAfter("Tel:", "PII", 16, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-              section.redactLineAfter("E-mail:", "PII", 16, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-              section.redactLineAfter("Email:", "PII", 16, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-              section.redactLineAfter("e-mail:", "PII", 16, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-              section.redactLineAfter("E-mail address:", "PII", 16, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-              section.redactLineAfter("Contact:", "PII", 16, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-              section.redactLineAfter("Alternative contact:", "PII", 16, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-              section.redactLineAfter("Telephone number:", "PII", 16, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-              section.redactLineAfter("Telephone No:", "PII", 16, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-              section.redactLineAfter("Fax number:", "PII", 16, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-              section.redactLineAfter("Telephone:", "PII", 16, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-              section.redactBetween("No:", "Fax", "PII", 16, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-              section.redactBetween("Contact:", "Tel.:", "PII", 16, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-              section.redactLineAfter("European contact:",  "PII", 16, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-     end
-
-
-rule "17: Redact contact information if applicant is found"
-    when
-        Section(headlineContainsWord("applicant") || text.contains("Applicant") || headlineContainsWord("Primary contact") || headlineContainsWord("Alternative contact") || text.contains("Telephone number:"))
-    then
-        section.redactLineAfter("Contact point:", "PII", 17, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-        section.redactLineAfter("Phone:", "PII", 17, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-        section.redactLineAfter("Fax:", "PII", 17, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-        section.redactLineAfter("Tel.:", "PII", 17, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-        section.redactLineAfter("Tel:", "PII", 17, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-        section.redactLineAfter("E-mail:", "PII", 17, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-        section.redactLineAfter("Email:", "PII", 17, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-        section.redactLineAfter("e-mail:", "PII", 17, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-        section.redactLineAfter("E-mail address:", "PII", 17, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-        section.redactLineAfter("Contact:", "PII", 17, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-        section.redactLineAfter("Alternative contact:", "PII", 17, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-        section.redactLineAfter("Telephone number:", "PII", 17, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-        section.redactLineAfter("Telephone No:", "PII", 17, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-        section.redactLineAfter("Fax number:", "PII", 17, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-        section.redactLineAfter("Telephone:", "PII", 17, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-        section.redactBetween("No:", "Fax", "PII", 17, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-        section.redactBetween("Contact:", "Tel.:", "PII", 17, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-        section.redactLineAfter("European contact:",  "PII", 17, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-    end
-
-
-rule "18: Redact contact information if Producer is found"
-    when
-        Section(text.toLowerCase().contains("producer of the plant protection") || text.toLowerCase().contains("producer of the active substance") || text.contains("Manufacturer of the active substance") || text.contains("Manufacturer:") || text.contains("Producer or producers of the active substance"))
-    then
-        section.redactLineAfter("Contact:", "PII", 18, true, "Producer was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-        section.redactLineAfter("Telephone:", "PII", 18, true, "Producer was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-        section.redactLineAfter("Phone:", "PII", 18, true, "Producer was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-        section.redactLineAfter("Fax:", "PII", 18, true, "Producer was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-        section.redactLineAfter("E-mail:", "PII", 18, true, "Producer was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-        section.redactLineAfter("Contact:", "PII", 18, true, "Producer was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-        section.redactLineAfter("Fax number:", "PII", 18, true, "Producer was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-        section.redactLineAfter("Telephone number:", "PII", 18, true, "Producer was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-        section.redactLineAfter("Tel:", "PII", 18, true, "Producer was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-        section.redactBetween("No:", "Fax", "PII", 18, true, "Producer was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-    end
-
-
-rule "19: Redact AUTHOR(S)"
-    when
-        Section(searchText.contains("AUTHOR(S):") && fileAttributeByPlaceholderEquals("{fileattributes.vertebrateStudy}", "true"))
-    then
-        section.redactLinesBetween("AUTHOR(S):", "COMPLETION DATE:", "PII", 19, true, "AUTHOR(S) was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-    end
-
-
-rule "20: Redact PERFORMING LABORATORY"
-    when
-        Section(searchText.contains("PERFORMING LABORATORY:"))
-    then
-        section.redactBetween("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "PII", 20, true, "PERFORMING LABORATORY was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-    end
-
-
-rule "21: Redact On behalf of Sequani Ltd.:"
-    when
-        Section(searchText.contains("On behalf of Sequani Ltd.: Name Title"))
-    then
-        section.redactBetween("On behalf of Sequani Ltd.: Name Title", "On behalf of", "PII", 21, false , "PII (Personal Identification Information) found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-    end
-
-
-rule "22: Redact On behalf of Syngenta Ltd.:"
-    when
-        Section(searchText.contains("On behalf of Syngenta Ltd.: Name Title"))
-    then
-        section.redactBetween("On behalf of Syngenta Ltd.: Name Title", "Study dates", "PII", 22, false , "PII (Personal Identification Information) found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-    end
-
-// --------------------------------------- other rules -------------------------------------------------------------------
-
-rule "25: Redact Purity"
-    when
-        Section(searchText.contains("purity"))
-    then
-	    section.redactByRegEx("purity ?:? (([\\d\\.]+)( .{0,4}\\.)? ?%)", true, 1, "purity", 17, "Purity found", "Reg (EC) No 1107/2009 Art. 63 (2a)");
-    end
-
-
-rule "26: Redact signatures"
-    when
-        Section(matchesImageType("signature"))
-    then
-        section.redactImage("signature", 26, "Signature found", "Reg (EC) No 1107/2009 Art. 63 (2g)");
-    end
-
-
-rule "27: Redact formula"
-    when
-        Section(matchesImageType("formula"))
-    then
-        section.redactImage("formula", 27, "Formula found", "Reg (EC) No 1107/2009 Art. 63 (2g)");
-    end
-
-
-rule "28: Redact Logos"
-    when
-        Section(matchesImageType("logo"))
-    then
-        section.redactImage("logo", 28, "Logo found", "Reg (EC) No 1107/2009 Art. 63 (2g)");
-    end
-
-
-rule "29: Redact Dossier Redactions"
-     when
-         Section(matchesType("dossier_redactions"))
-     then
-         section.redact("dossier_redactions", 29, "Dossier Redaction found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
-     end
-
-rule "30: Ignore dossier_redactions if confidential"
-    when
-        Section(!fileAttributeByLabelEqualsIgnoreCase("Confidentiality","confidential") && matchesType("dossier_redactions"));
-    then
-        section.ignore("dossier_redactions");
-    end
-
-// ex. "New Rules for PAD" - "Annex A" - page 21, page 35 (table without header), page 38 (in-text)
-// https://www.regexplanet.com/share/index.html?share=yyyypb71xkr
-rule "101: Redact CAS numbers"
-    when
-        Section(hasTableHeader("Sample #"))
-    then
-        section.redactCell("Sample #", 8, "PII", true, "Redacted because row is a vertebrate study", "Reg (EC) No 1107/2009 Art. 63 (2g)");
-    end
-
-rule "102: Guidelines FileAttributes"
-    when
-        Section((text.contains("DATA REQUIREMENT(S):") || text.contains("TEST GUIDELINE(S):")) && (text.contains("OECD") || text.contains("EPA") || text.contains("OPPTS")))
-    then
-        section.addFileAttribute("OECD Number", "OECD (No\\.? )?\\d{3}( \\(\\d{4}\\))?", false, 0);
-    end
-
-rule "8: Redact Author cells in Tables with Author header (Non vertebrate study)"
-    when
-        Section(hasTableHeader("h5.1"))
-    then
-        section.redactCell("h5.1", 8, "CBI_author", false, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
-    end

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules2.drl

@@ -0,0 +1 @@
+rules.drl

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/testRules.drl

@@ -11,28 +11,28 @@ rule "0: Add CBI_author from ai"
     when
         Section(aiMatchesType("CBI_author"))
     then
-        section.addAiEntities("CBI_author", "CBI_author");
+        section.addAiEntities("CBI_author", "CBI_author",dictionary);
     end
 
 rule "0: Combine address parts from ai to CBI_address (org is mandatory)"
     when
         Section(aiMatchesType("ORG"))
     then
-        section.combineAiTypes("ORG", "STREET,POSTAL,COUNTRY,CARDINAL,CITY,STATE", 20, "CBI_address", 3, false);
+        section.combineAiTypes("ORG", "STREET,POSTAL,COUNTRY,CARDINAL,CITY,STATE", 20, "CBI_address", 3, false, dictionary);
     end
 
 rule "0: Combine address parts from ai to CBI_address (street is mandatory)"
     when
         Section(aiMatchesType("STREET"))
     then
-        section.combineAiTypes("STREET", "ORG,POSTAL,COUNTRY,CARDINAL,CITY,STATE", 20, "CBI_address", 3, false);
+        section.combineAiTypes("STREET", "ORG,POSTAL,COUNTRY,CARDINAL,CITY,STATE", 20, "CBI_address", 3, false, dictionary);
     end
 
 rule "0: Combine address parts from ai to CBI_address (city is mandatory)"
     when
         Section(aiMatchesType("CITY"))
     then
-        section.combineAiTypes("CITY", "ORG,STREET,POSTAL,COUNTRY,CARDINAL,STATE", 20, "CBI_address", 3, false);
+        section.combineAiTypes("CITY", "ORG,STREET,POSTAL,COUNTRY,CARDINAL,STATE", 20, "CBI_address", 3, false, dictionary);
     end
 
 
@@ -73,7 +73,7 @@ rule "5: Do not redact genitive CBI_author"
     when
         Section(matchesType("CBI_author"))
     then
-        section.expandToFalsePositiveByRegEx("CBI_author", "['’’'ʼˈ´`‘′ʻ’']s", false, 0);
+        section.expandToFalsePositiveByRegEx("CBI_author", "['’’'ʼˈ´`‘′ʻ’']s", false, 0, dictionary);
     end
 
 
@@ -81,14 +81,14 @@ rule "6: Redact Author(s) cells in Tables with Author(s) header (Non vertebrate
     when
         Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && hasTableHeader("Author(s)") && !hasTableHeader("Vertebrate study Y/N"))
     then
-        section.redactCell("Author(s)", 6, "CBI_author", false, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        section.redactCell("Author(s)", 6, "CBI_author", false, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
     end
 
 rule "7: Redact Author(s) cells in Tables with Author(s) header (Vertebrate study)"
     when
         Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && hasTableHeader("Author(s)") && !hasTableHeader("Vertebrate study Y/N"))
     then
-        section.redactCell("Author(s)", 7, "CBI_author", false, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        section.redactCell("Author(s)", 7, "CBI_author", false, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
     end
 
 
@@ -96,14 +96,14 @@ rule "8: Redact Author cells in Tables with Author header (Non vertebrate study)
     when
         Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && hasTableHeader("Author") && !hasTableHeader("Vertebrate study Y/N"))
     then
-        section.redactCell("Author", 8, "CBI_author", false, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        section.redactCell("Author", 8, "CBI_author", false, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
     end
 
 rule "9: Redact Author cells in Tables with Author header (Vertebrate study)"
     when
         Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && hasTableHeader("Author") && !hasTableHeader("Vertebrate study Y/N"))
     then
-        section.redactCell("Author", 9, "CBI_author", false, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        section.redactCell("Author", 9, "CBI_author", false, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
     end
 
 
@@ -111,14 +111,14 @@ rule "10: Redact and recommand Authors in Tables with Vertebrate study Y/N heade
     when
         Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && (rowEquals("Vertebrate study Y/N", "Y") || rowEquals("Vertebrate study Y/N", "Yes") || rowEquals("Vertebrate study Y/N", "N") || rowEquals("Vertebrate study Y/N", "No")))
     then
-        section.redactCell("Author(s)", 10, "CBI_author", true, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        section.redactCell("Author(s)", 10, "CBI_author", true, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
     end
 
 rule "11: Redact and recommand Authors in Tables with Vertebrate study Y/N header (Vertebrate study)"
     when
         Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && (rowEquals("Vertebrate study Y/N", "Y") || rowEquals("Vertebrate study Y/N", "Yes") || rowEquals("Vertebrate study Y/N", "N") || rowEquals("Vertebrate study Y/N", "No")))
     then
-        section.redactCell("Author(s)", 11, "CBI_author", true, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        section.redactCell("Author(s)", 11, "CBI_author", true, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
     end
 
 /* Syngenta specific laboratory rule */
@@ -133,14 +133,14 @@ rule "14: Redact and add recommendation for et al. author (Non vertebrate study)
     when
         Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("et al"))
     then
-        section.redactAndRecommendByRegEx("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", false, 1, "CBI_author", 14, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        section.redactAndRecommendByRegEx("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", false, 1, "CBI_author", 14, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
     end
 
 rule "15: Redact and add recommendation for et al. author (Vertebrate study)"
     when
         Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("et al"))
     then
-        section.redactAndRecommendByRegEx("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", false, 1, "CBI_author", 15, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        section.redactAndRecommendByRegEx("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", false, 1, "CBI_author", 15, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
     end
 
 
@@ -190,14 +190,14 @@ rule "21: Redact Emails by RegEx (Non vertebrate study)"
     when
         Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("@"))
     then
-        section.redactByRegEx("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", true, 1, "PII", 21, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        section.redactByRegEx("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", true, 1, "PII", 21, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
     end
 
 rule "22: Redact Emails by RegEx (Vertebrate study)"
     when
         Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("@"))
     then
-        section.redactByRegEx("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", true, 1, "PII", 22, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        section.redactByRegEx("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", true, 1, "PII", 22, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
     end
 
 
@@ -224,25 +224,25 @@ rule "23: Redact contact information (Non vertebrate study)"
             || text.contains("European contact:")
         ))
     then
-        section.redactLineAfter("Contact point:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Contact point:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
-        section.redactLineAfter("Phone:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Phone:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
-        section.redactLineAfter("Fax:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Fax:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
-        section.redactLineAfter("Tel.:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Tel.:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
-        section.redactLineAfter("Tel:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Tel:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
-        section.redactLineAfter("E-mail:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("E-mail:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
-        section.redactLineAfter("Email:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Email:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
-        section.redactLineAfter("e-mail:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("e-mail:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
-        section.redactLineAfter("E-mail address:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("E-mail address:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
-        section.redactLineAfter("Contact:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Contact:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
-        section.redactLineAfter("Alternative contact:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Alternative contact:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
-        section.redactLineAfter("Telephone number:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Telephone number:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
-        section.redactLineAfter("Telephone No:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Telephone No:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
-        section.redactLineAfter("Fax number:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Fax number:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
-        section.redactLineAfter("Telephone:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Telephone:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
-        section.redactLineAfter("Phone No.", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Phone No.", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
-        section.redactBetween("No:", "Fax", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        section.redactBetween("No:", "Fax", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
-        section.redactBetween("Contact:", "Tel.:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        section.redactBetween("Contact:", "Tel.:", "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
-        section.redactLineAfter("European contact:",  "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("European contact:",  "PII", 23, true, "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
      end
 
 rule "24: Redact contact information (Vertebrate study)"
@@ -268,25 +268,25 @@ rule "24: Redact contact information (Vertebrate study)"
             || text.contains("European contact:")
         ))
     then
-        section.redactLineAfter("Contact point:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Contact point:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
-        section.redactLineAfter("Phone:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Phone:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
-        section.redactLineAfter("Fax:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Fax:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
-        section.redactLineAfter("Tel.:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Tel.:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
-        section.redactLineAfter("Tel:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Tel:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
-        section.redactLineAfter("E-mail:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("E-mail:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
-        section.redactLineAfter("Email:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Email:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
-        section.redactLineAfter("e-mail:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("e-mail:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
-        section.redactLineAfter("E-mail address:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("E-mail address:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
-        section.redactLineAfter("Contact:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Contact:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
-        section.redactLineAfter("Alternative contact:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Alternative contact:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
-        section.redactLineAfter("Telephone number:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Telephone number:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
-        section.redactLineAfter("Telephone No:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Telephone No:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
-        section.redactLineAfter("Fax number:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Fax number:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
-        section.redactLineAfter("Telephone:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Telephone:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
-        section.redactLineAfter("Phone No.", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Phone No.", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
-        section.redactBetween("No:", "Fax", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        section.redactBetween("No:", "Fax", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
-        section.redactBetween("Contact:", "Tel.:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        section.redactBetween("Contact:", "Tel.:", "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
-        section.redactLineAfter("European contact:",  "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("European contact:",  "PII", 24, true, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
      end
 
 
@@ -305,7 +305,7 @@ rule "25: Redact Phone and Fax by RegEx (Non vertebrate study)"
             || text.contains("Fer")
         ))
     then
-        section.redactByRegEx("\\b(telephone|phone|fax|tel|ter|cell|mobile|fel|fer)[:.\\s]{0,3}((\\(?\\+?[0-9])(\\(?[0-9\\/.\\-\\s]+\\)?)*([0-9]+\\)?))\\b", true, 2, "PII", 25, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        section.redactByRegEx("\\b(telephone|phone|fax|tel|ter|cell|mobile|fel|fer)[:.\\s]{0,3}((\\(?\\+?[0-9])(\\(?[0-9\\/.\\-\\s]+\\)?)*([0-9]+\\)?))\\b", true, 2, "PII", 25, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
     end
 
 rule "26: Redact Phone and Fax by RegEx (Vertebrate study)"
@@ -323,7 +323,7 @@ rule "26: Redact Phone and Fax by RegEx (Vertebrate study)"
             || text.contains("Fer")
         ))
     then
-        section.redactByRegEx("\\b(telephone|phone|fax|tel|ter|cell|mobile|fel|fer)[:.\\s]{0,3}((\\(?\\+?[0-9])(\\(?[0-9\\/.\\-\\s]+\\)?)*([0-9]+\\)?))\\b", true, 2, "PII", 26, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        section.redactByRegEx("\\b(telephone|phone|fax|tel|ter|cell|mobile|fel|fer)[:.\\s]{0,3}((\\(?\\+?[0-9])(\\(?[0-9\\/.\\-\\s]+\\)?)*([0-9]+\\)?))\\b", true, 2, "PII", 26, "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
     end
 
 
@@ -335,7 +335,7 @@ rule "27: Redact AUTHOR(S) (Non vertebrate study)"
             && !searchText.contains("STUDY COMPLETION DATE:")
         )
     then
-        section.redactLinesBetween("AUTHOR(S):", "COMPLETION DATE:", "PII", 27, true, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        section.redactLinesBetween("AUTHOR(S):", "COMPLETION DATE:", "PII", 27, true, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
     end
 
 rule "28: Redact AUTHOR(S) (Vertebrate study)"
@@ -346,7 +346,7 @@ rule "28: Redact AUTHOR(S) (Vertebrate study)"
             && !searchText.contains("STUDY COMPLETION DATE:")
         )
     then
-        section.redactLinesBetween("AUTHOR(S):", "COMPLETION DATE:", "PII", 28, true, "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        section.redactLinesBetween("AUTHOR(S):", "COMPLETION DATE:", "PII", 28, true, "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
     end
 
 
@@ -357,7 +357,7 @@ rule "29: Redact AUTHOR(S) (Non vertebrate study)"
             && searchText.contains("STUDY COMPLETION DATE:")
         )
     then
-        section.redactLinesBetween("AUTHOR(S):", "STUDY COMPLETION DATE:", "PII", 29, true, "AUTHOR(S) was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        section.redactLinesBetween("AUTHOR(S):", "STUDY COMPLETION DATE:", "PII", 29, true, "AUTHOR(S) was found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
     end
 
 rule "30: Redact AUTHOR(S) (Vertebrate study)"
@@ -367,7 +367,7 @@ rule "30: Redact AUTHOR(S) (Vertebrate study)"
             && searchText.contains("STUDY COMPLETION DATE:")
         )
     then
-        section.redactLinesBetween("AUTHOR(S):", "STUDY COMPLETION DATE:", "PII", 30, true, "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        section.redactLinesBetween("AUTHOR(S):", "STUDY COMPLETION DATE:", "PII", 30, true, "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
     end
 
 
@@ -377,7 +377,7 @@ rule "31: Redact PERFORMING LABORATORY (Non vertebrate study)"
             && searchText.contains("PERFORMING LABORATORY:")
         )
     then
-        section.redactBetween("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", 31, true, "PERFORMING LABORATORY was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        section.redactBetween("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", 31, true, "PERFORMING LABORATORY was found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
         section.redactNot("CBI_address", 31, "Performing laboratory found for non vertebrate study");
     end
 
@@ -386,7 +386,7 @@ rule "32: Redact PERFORMING LABORATORY (Vertebrate study)"
         Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes")
             && searchText.contains("PERFORMING LABORATORY:"))
     then
-        section.redactBetween("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", 32, true, "PERFORMING LABORATORY was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        section.redactBetween("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", 32, true, "PERFORMING LABORATORY was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
     end
 
 
@@ -396,7 +396,7 @@ rule "33: Purity Hint"
     when
         Section(searchText.toLowerCase().contains("purity"))
     then
-	    section.addHintAnnotationByRegEx("(purity ?( of|\\(.{1,20}\\))?( ?:)?) .{0,5}[\\d\\.]+( .{0,4}\\.)? ?%", true, 1, "hint_only");
+	    section.addHintAnnotationByRegEx("(purity ?( of|\\(.{1,20}\\))?( ?:)?) .{0,5}[\\d\\.]+( .{0,4}\\.)? ?%", true, 1, "hint_only",dictionary);
     end
 
 

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/test_rules.drl

@@ -0,0 +1,35 @@
+package drools
+
+import static java.lang.String.format;
+
+import java.util.List;
+import java.util.LinkedList;
+import java.util.HashSet;
+
+import com.iqser.red.service.redaction.v1.server.document.graph.*
+import com.iqser.red.service.redaction.v1.server.redaction.model.*;
+import com.iqser.red.service.redaction.v1.model.FileAttribute;
+import com.iqser.red.service.redaction.v1.model.Engine;
+import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntry;
+import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUtils;
+import java.util.Set;
+
+global DocumentGraph document
+
+rule "dummy"
+    when
+
+    then
+        System.out.println("This actually works");
+    end
+
+rule "1: Redact CBI_author"
+    when
+        //FileAttribute(label == "Vertebrate Study" && (value == "Yes"))
+        $entity: EntityNode(type == "CBI_author")
+    then
+        System.out.println("Entity found");
+        modify($entity){
+            $entity.setRedact(true)
+        }
+    end

redaction-service-v1/redaction-service-server-v1/src/test/resources/performance/dictionaries/EFSA_sanitisation_GFL_v1/rules.drl

@@ -10,7 +10,7 @@ rule "0: Add CBI_author from ai"
     when
         Section(aiMatchesType("CBI_author"))
     then
-        section.addAiEntities("CBI_author", "CBI_author");
+        section.addAiEntities("CBI_author", "CBI_author",dictionary);
     end
 
 
@@ -18,7 +18,7 @@ rule "0: Combine ai types CBI_author from ai"
     when
         Section(aiMatchesType("ORG"))
     then
-        section.combineAiTypes("ORG", "STREET,POSTAL,COUNTRY,CARDINAL,CITY,STATE", 20, "CBI_address", 3, false);
+        section.combineAiTypes("ORG", "STREET,POSTAL,COUNTRY,CARDINAL,CITY,STATE", 20, "CBI_address", 3, false, dictionary);
     end
 
 
@@ -59,7 +59,7 @@ rule "5: Do not redact genitive CBI_author"
     when
         Section(matchesType("CBI_author"))
     then
-        section.expandToFalsePositiveByRegEx("CBI_author", "['’’'ʼˈ´`‘′ʻ’']s", false, 0);
+        section.expandToFalsePositiveByRegEx("CBI_author", "['’’'ʼˈ´`‘′ʻ’']s", false, 0, dictionary);
     end
 
 
@@ -67,7 +67,7 @@ rule "6: Redact Author(s) cells in Tables with Author(s) header (Non vertebrate
     when
         Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && hasTableHeader("Author(s)") && !hasTableHeader("Vertebrate study Y/N"))
     then
-        section.redactCell("Author(s)", 6, "CBI_author", false, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        section.redactCell("Author(s)", 6, "CBI_author", false, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
     end
 
 
@@ -75,7 +75,7 @@ rule "7: Redact Author(s) cells in Tables with Author(s) header (Vertebrate stud
     when
         Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && hasTableHeader("Author(s)") && !hasTableHeader("Vertebrate study Y/N"))
     then
-        section.redactCell("Author(s)", 7, "CBI_author", false, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        section.redactCell("Author(s)", 7, "CBI_author", false, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
     end
 
 
@@ -83,7 +83,7 @@ rule "8: Redact Author cells in Tables with Author header (Non vertebrate study)
     when
         Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && hasTableHeader("Author") && !hasTableHeader("Vertebrate study Y/N"))
     then
-        section.redactCell("Author", 8, "CBI_author", false, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        section.redactCell("Author", 8, "CBI_author", false, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
     end
 
 
@@ -91,7 +91,7 @@ rule "9: Redact Author cells in Tables with Author header (Vertebrate study)"
     when
         Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && hasTableHeader("Author") && !hasTableHeader("Vertebrate study Y/N"))
     then
-        section.redactCell("Author", 9, "CBI_author", false, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        section.redactCell("Author", 9, "CBI_author", false, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
     end
 
 
@@ -99,7 +99,7 @@ rule "10: Redact and recommand Authors in Tables with Vertebrate study Y/N heade
     when
         Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && (rowEquals("Vertebrate study Y/N", "Y") || rowEquals("Vertebrate study Y/N", "Yes") || rowEquals("Vertebrate study Y/N", "N") || rowEquals("Vertebrate study Y/N", "No")))
     then
-        section.redactCell("Author(s)", 10, "CBI_author", true, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        section.redactCell("Author(s)", 10, "CBI_author", true, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
     end
 
 
@@ -107,7 +107,7 @@ rule "11: Redact and recommand Authors in Tables with Vertebrate study Y/N heade
     when
         Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && (rowEquals("Vertebrate study Y/N", "Y") || rowEquals("Vertebrate study Y/N", "Yes") || rowEquals("Vertebrate study Y/N", "N") || rowEquals("Vertebrate study Y/N", "No")))
     then
-        section.redactCell("Author(s)", 11, "CBI_author", true, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        section.redactCell("Author(s)", 11, "CBI_author", true, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
     end
 
 
@@ -115,7 +115,7 @@ rule "13: Redact addresses that start with BL or CTL"
     when
         Section(searchText.contains("BL") || searchText.contains("CT"))
     then
-        section.redactNotAndRecommendByRegEx("((\\b((([Cc]T(([1ILli\\/])| L|~P))|(BL))[\\. ]?([\\dA-Ziltphz~\\/.:!]| ?[\\(',][Ppi](\\(e)?|([\\(-?']\\/))+( ?[\\(\\/\\dA-Znasieg]+)?)\\b( ?\\/? ?\\d+)?)|(\\bCT[L1i]\\b))", true, 0, "CBI_address", 13, "Laboratory for vertebrate studies found");
+        section.redactNotAndRecommendByRegEx("((\\b((([Cc]T(([1ILli\\/])| L|~P))|(BL))[\\. ]?([\\dA-Ziltphz~\\/.:!]| ?[\\(',][Ppi](\\(e)?|([\\(-?']\\/))+( ?[\\(\\/\\dA-Znasieg]+)?)\\b( ?\\/? ?\\d+)?)|(\\bCT[L1i]\\b))", true, 0, "CBI_address", 13, "Laboratory for vertebrate studies found",dictionary);
     end
 
 
@@ -123,7 +123,7 @@ rule "14: Redact and add recommendation for et al. author (Non vertebrate study)
     when
         Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("et al"))
     then
-        section.redactAndRecommendByRegEx("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", false, 1, "CBI_author", 14, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        section.redactAndRecommendByRegEx("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", false, 1, "CBI_author", 14, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
     end
 
 
@@ -131,7 +131,7 @@ rule "15: Redact and add recommendation for et al. author (Vertebrate study)"
     when
         Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("et al"))
     then
-        section.redactAndRecommendByRegEx("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", false, 1, "CBI_author", 15, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        section.redactAndRecommendByRegEx("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", false, 1, "CBI_author", 15, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
     end
 
 
@@ -183,7 +183,7 @@ rule "21: Redact Emails by RegEx (Non vertebrate study)"
     when
         Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("@"))
     then
-        section.redactByRegEx("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", true, 1, "PII", 21, "PII (Personal Identification Information) found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        section.redactByRegEx("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", true, 1, "PII", 21, "PII (Personal Identification Information) found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
     end
 
 
@@ -191,7 +191,7 @@ rule "22: Redact Emails by RegEx (Vertebrate study)"
     when
         Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("@"))
     then
-        section.redactByRegEx("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", true, 1, "PII", 22, "PII (Personal Identification Information) found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        section.redactByRegEx("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", true, 1, "PII", 22, "PII (Personal Identification Information) found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
     end
 
 
@@ -199,7 +199,7 @@ rule "23: Redact telephone numbers by RegEx (Non vertebrate study)"
     when
         Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && containsRegEx("[+]\\d{2,}", true))
     then
-        section.redactByRegEx("((([+]\\d{2,3} (\\d{7,12})\\b)|([+]\\d{2,3}(\\d{3,12})\\b|[+]\\d{2,3}([ -]\\(?\\d{2,6}\\)?){2,4})|[+]\\d{2,3} ?((\\d{2,6}\\)?)([ -]\\d{2,6}){1,4}))(-\\d{1,3})?\\b)", true, 1, "PII", 23, "PII (Personal Identification Information) found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        section.redactByRegEx("((([+]\\d{2,3} (\\d{7,12})\\b)|([+]\\d{2,3}(\\d{3,12})\\b|[+]\\d{2,3}([ -]\\(?\\d{2,6}\\)?){2,4})|[+]\\d{2,3} ?((\\d{2,6}\\)?)([ -]\\d{2,6}){1,4}))(-\\d{1,3})?\\b)", true, 1, "PII", 23, "PII (Personal Identification Information) found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
     end
 
 
@@ -207,7 +207,7 @@ rule "24: Redact telephone numbers by RegEx (Vertebrate study)"
     when
         Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && containsRegEx("[+]\\d{2,}", true))
     then
-        section.redactByRegEx("((([+]\\d{2,3} (\\d{7,12})\\b)|([+]\\d{2,3}(\\d{3,12})\\b|[+]\\d{2,3}([ -]\\(?\\d{2,6}\\)?){2,4})|[+]\\d{2,3} ?((\\d{2,6}\\)?)([ -]\\d{2,6}){1,4}))(-\\d{1,3})?\\b)", true, 1, "PII", 24, "PII (Personal Identification Information) found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        section.redactByRegEx("((([+]\\d{2,3} (\\d{7,12})\\b)|([+]\\d{2,3}(\\d{3,12})\\b|[+]\\d{2,3}([ -]\\(?\\d{2,6}\\)?){2,4})|[+]\\d{2,3} ?((\\d{2,6}\\)?)([ -]\\d{2,6}){1,4}))(-\\d{1,3})?\\b)", true, 1, "PII", 24, "PII (Personal Identification Information) found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
     end
 
 
@@ -238,25 +238,25 @@ rule "26: Redact contact information (Non vertebrate study)"
        || text.contains("Phone No.")
        || text.contains("European contact:")))
     then
-          section.redactLineAfter("Contact point:", "PII", 26, true, "Contact information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+          section.redactLineAfter("Contact point:", "PII", 26, true, "Contact information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
-              section.redactLineAfter("Phone:", "PII", 26, true, "Contact information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+              section.redactLineAfter("Phone:", "PII", 26, true, "Contact information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
-              section.redactLineAfter("Fax:", "PII", 26, true, "Contact information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+              section.redactLineAfter("Fax:", "PII", 26, true, "Contact information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
-              section.redactLineAfter("Tel.:", "PII", 26, true, "Contact information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+              section.redactLineAfter("Tel.:", "PII", 26, true, "Contact information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
-              section.redactLineAfter("Tel:", "PII", 26, true, "Contact information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+              section.redactLineAfter("Tel:", "PII", 26, true, "Contact information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
-              section.redactLineAfter("E-mail:", "PII", 26, true, "Contact information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+              section.redactLineAfter("E-mail:", "PII", 26, true, "Contact information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
-              section.redactLineAfter("Email:", "PII", 26, true, "Contact information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+              section.redactLineAfter("Email:", "PII", 26, true, "Contact information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
-              section.redactLineAfter("e-mail:", "PII", 26, true, "Contact information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+              section.redactLineAfter("e-mail:", "PII", 26, true, "Contact information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
-              section.redactLineAfter("E-mail address:", "PII", 26, true, "Contact information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+              section.redactLineAfter("E-mail address:", "PII", 26, true, "Contact information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
-              section.redactLineAfter("Contact:", "PII", 26, true, "Contact information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+              section.redactLineAfter("Contact:", "PII", 26, true, "Contact information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
-              section.redactLineAfter("Alternative contact:", "PII", 26, true, "Contact information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+              section.redactLineAfter("Alternative contact:", "PII", 26, true, "Contact information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
-              section.redactLineAfter("Telephone number:", "PII", 26, true, "Contact information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+              section.redactLineAfter("Telephone number:", "PII", 26, true, "Contact information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
-              section.redactLineAfter("Telephone No:", "PII", 26, true, "Contact information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+              section.redactLineAfter("Telephone No:", "PII", 26, true, "Contact information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
-              section.redactLineAfter("Fax number:", "PII", 26, true, "Contact information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+              section.redactLineAfter("Fax number:", "PII", 26, true, "Contact information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
-              section.redactLineAfter("Telephone:", "PII", 26, true, "Contact information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+              section.redactLineAfter("Telephone:", "PII", 26, true, "Contact information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
-              section.redactLineAfter("Phone No.", "PII", 26, true, "Contact information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+              section.redactLineAfter("Phone No.", "PII", 26, true, "Contact information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
-              section.redactBetween("No:", "Fax", "PII", 26, true, "Contact information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+              section.redactBetween("No:", "Fax", "PII", 26, true, "Contact information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
-              section.redactBetween("Contact:", "Tel.:", "PII", 26, true, "Contact information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+              section.redactBetween("Contact:", "Tel.:", "PII", 26, true, "Contact information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
-              section.redactLineAfter("European contact:",  "PII", 26, true, "Contact information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+              section.redactLineAfter("European contact:",  "PII", 26, true, "Contact information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
      end
 
 
@@ -279,25 +279,25 @@ rule "27: Redact contact information (Vertebrate study)"
        || text.contains("Phone No.")
        || text.contains("European contact:")))
     then
-          section.redactLineAfter("Contact point:", "PII", 27, true, "Contact information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+          section.redactLineAfter("Contact point:", "PII", 27, true, "Contact information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
-              section.redactLineAfter("Phone:", "PII", 27, true, "Contact information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+              section.redactLineAfter("Phone:", "PII", 27, true, "Contact information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
-              section.redactLineAfter("Fax:", "PII", 27, true, "Contact information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+              section.redactLineAfter("Fax:", "PII", 27, true, "Contact information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
-              section.redactLineAfter("Tel.:", "PII", 27, true, "Contact information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+              section.redactLineAfter("Tel.:", "PII", 27, true, "Contact information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
-              section.redactLineAfter("Tel:", "PII", 27, true, "Contact information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+              section.redactLineAfter("Tel:", "PII", 27, true, "Contact information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
-              section.redactLineAfter("E-mail:", "PII", 27, true, "Contact information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+              section.redactLineAfter("E-mail:", "PII", 27, true, "Contact information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
-              section.redactLineAfter("Email:", "PII", 27, true, "Contact information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+              section.redactLineAfter("Email:", "PII", 27, true, "Contact information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
-              section.redactLineAfter("e-mail:", "PII", 27, true, "Contact information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+              section.redactLineAfter("e-mail:", "PII", 27, true, "Contact information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
-              section.redactLineAfter("E-mail address:", "PII", 27, true, "Contact information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+              section.redactLineAfter("E-mail address:", "PII", 27, true, "Contact information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
-              section.redactLineAfter("Contact:", "PII", 27, true, "Contact information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+              section.redactLineAfter("Contact:", "PII", 27, true, "Contact information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
-              section.redactLineAfter("Alternative contact:", "PII", 27, true, "Contact information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+              section.redactLineAfter("Alternative contact:", "PII", 27, true, "Contact information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
-              section.redactLineAfter("Telephone number:", "PII", 27, true, "Contact information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+              section.redactLineAfter("Telephone number:", "PII", 27, true, "Contact information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
-              section.redactLineAfter("Telephone No:", "PII", 27, true, "Contact information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+              section.redactLineAfter("Telephone No:", "PII", 27, true, "Contact information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
-              section.redactLineAfter("Fax number:", "PII", 27, true, "Contact information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+              section.redactLineAfter("Fax number:", "PII", 27, true, "Contact information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
-              section.redactLineAfter("Telephone:", "PII", 27, true, "Contact information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+              section.redactLineAfter("Telephone:", "PII", 27, true, "Contact information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
-              section.redactLineAfter("Phone No.", "PII", 27, true, "Contact information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+              section.redactLineAfter("Phone No.", "PII", 27, true, "Contact information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
-              section.redactBetween("No:", "Fax", "PII", 27, true, "Contact information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+              section.redactBetween("No:", "Fax", "PII", 27, true, "Contact information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
-              section.redactBetween("Contact:", "Tel.:", "PII", 27, true, "Contact information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+              section.redactBetween("Contact:", "Tel.:", "PII", 27, true, "Contact information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
-              section.redactLineAfter("European contact:",  "PII", 27, true, "Contact information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+              section.redactLineAfter("European contact:",  "PII", 27, true, "Contact information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
      end
 
 
@@ -305,25 +305,25 @@ rule "28: Redact contact information if applicant is found (Non vertebrate study
     when
         Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && (headlineContainsWord("applicant") || text.contains("Applicant") || headlineContainsWord("Primary contact") || headlineContainsWord("Alternative contact") || text.contains("Telephone number:")))
     then
-        section.redactLineAfter("Contact point:", "PII", 28, true, "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Contact point:", "PII", 28, true, "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
-        section.redactLineAfter("Phone:", "PII", 28, true, "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Phone:", "PII", 28, true, "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
-        section.redactLineAfter("Fax:", "PII", 28, true, "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Fax:", "PII", 28, true, "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
-        section.redactLineAfter("Tel.:", "PII", 28, true, "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Tel.:", "PII", 28, true, "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
-        section.redactLineAfter("Tel:", "PII", 28, true, "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Tel:", "PII", 28, true, "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
-        section.redactLineAfter("E-mail:", "PII", 28, true, "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("E-mail:", "PII", 28, true, "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
-        section.redactLineAfter("Email:", "PII", 28, true, "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Email:", "PII", 28, true, "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
-        section.redactLineAfter("e-mail:", "PII", 28, true, "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("e-mail:", "PII", 28, true, "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
-        section.redactLineAfter("E-mail address:", "PII", 28, true, "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("E-mail address:", "PII", 28, true, "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
-        section.redactLineAfter("Contact:", "PII", 28, true, "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Contact:", "PII", 28, true, "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
-        section.redactLineAfter("Alternative contact:", "PII", 28, true, "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Alternative contact:", "PII", 28, true, "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
-        section.redactLineAfter("Telephone number:", "PII", 28, true, "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Telephone number:", "PII", 28, true, "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
-        section.redactLineAfter("Telephone No:", "PII", 28, true, "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Telephone No:", "PII", 28, true, "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
-        section.redactLineAfter("Fax number:", "PII", 28, true, "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Fax number:", "PII", 28, true, "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
-        section.redactLineAfter("Telephone:", "PII", 28, true, "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Telephone:", "PII", 28, true, "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
-        section.redactLineAfter("Phone No.", "PII", 28, true, "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Phone No.", "PII", 28, true, "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
-        section.redactBetween("No:", "Fax", "PII", 28, true, "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        section.redactBetween("No:", "Fax", "PII", 28, true, "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
-        section.redactBetween("Contact:", "Tel.:", "PII", 28, true, "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        section.redactBetween("Contact:", "Tel.:", "PII", 28, true, "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
-        section.redactLineAfter("European contact:",  "PII", 28, true, "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("European contact:",  "PII", 28, true, "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
     end
 
 
@@ -331,25 +331,25 @@ rule "29: Redact contact information if applicant is found (Vertebrate study)"
     when
         Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && (headlineContainsWord("applicant") || text.contains("Applicant") || headlineContainsWord("Primary contact") || headlineContainsWord("Alternative contact") || text.contains("Telephone number:")))
     then
-        section.redactLineAfter("Contact point:", "PII", 29, true, "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Contact point:", "PII", 29, true, "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
-        section.redactLineAfter("Phone:", "PII", 29, true, "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Phone:", "PII", 29, true, "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
-        section.redactLineAfter("Fax:", "PII", 29, true, "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Fax:", "PII", 29, true, "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
-        section.redactLineAfter("Tel.:", "PII", 29, true, "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Tel.:", "PII", 29, true, "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
-        section.redactLineAfter("Tel:", "PII", 29, true, "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Tel:", "PII", 29, true, "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
-        section.redactLineAfter("E-mail:", "PII", 29, true, "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("E-mail:", "PII", 29, true, "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
-        section.redactLineAfter("Email:", "PII", 29, true, "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Email:", "PII", 29, true, "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
-        section.redactLineAfter("e-mail:", "PII", 29, true, "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("e-mail:", "PII", 29, true, "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
-        section.redactLineAfter("E-mail address:", "PII", 29, true, "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("E-mail address:", "PII", 29, true, "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
-        section.redactLineAfter("Contact:", "PII", 29, true, "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Contact:", "PII", 29, true, "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
-        section.redactLineAfter("Alternative contact:", "PII", 29, true, "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Alternative contact:", "PII", 29, true, "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
-        section.redactLineAfter("Telephone number:", "PII", 29, true, "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Telephone number:", "PII", 29, true, "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
-        section.redactLineAfter("Telephone No:", "PII", 29, true, "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Telephone No:", "PII", 29, true, "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
-        section.redactLineAfter("Fax number:", "PII", 29, true, "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Fax number:", "PII", 29, true, "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
-        section.redactLineAfter("Telephone:", "PII", 29, true, "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Telephone:", "PII", 29, true, "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
-        section.redactLineAfter("Phone No.", "PII", 29, true, "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Phone No.", "PII", 29, true, "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
-        section.redactBetween("No:", "Fax", "PII", 29, true, "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        section.redactBetween("No:", "Fax", "PII", 29, true, "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
-        section.redactBetween("Contact:", "Tel.:", "PII", 29, true, "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        section.redactBetween("Contact:", "Tel.:", "PII", 29, true, "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
-        section.redactLineAfter("European contact:",  "PII", 29, true, "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("European contact:",  "PII", 29, true, "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
     end
 
 
@@ -357,17 +357,17 @@ rule "30: Redact contact information if Producer is found (Non vertebrate study)
     when
         Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && (text.toLowerCase().contains("producer of the plant protection") || text.toLowerCase().contains("producer of the active substance") || text.contains("Manufacturer of the active substance") || text.contains("Manufacturer:") || text.contains("Producer or producers of the active substance")))
     then
-        section.redactLineAfter("Contact:", "PII", 30, true, "Producer was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Contact:", "PII", 30, true, "Producer was found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
-        section.redactLineAfter("Telephone:", "PII", 30, true, "Producer was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Telephone:", "PII", 30, true, "Producer was found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
-        section.redactLineAfter("Phone:", "PII", 30, true, "Producer was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Phone:", "PII", 30, true, "Producer was found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
-        section.redactLineAfter("Fax:", "PII", 30, true, "Producer was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Fax:", "PII", 30, true, "Producer was found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
-        section.redactLineAfter("E-mail:", "PII", 30, true, "Producer was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("E-mail:", "PII", 30, true, "Producer was found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
-        section.redactLineAfter("Contact:", "PII", 30, true, "Producer was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Contact:", "PII", 30, true, "Producer was found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
-        section.redactLineAfter("Fax number:", "PII", 30, true, "Producer was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Fax number:", "PII", 30, true, "Producer was found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
-        section.redactLineAfter("Telephone number:", "PII", 30, true, "Producer was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Telephone number:", "PII", 30, true, "Producer was found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
-        section.redactLineAfter("Tel:", "PII", 30, true, "Producer was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Tel:", "PII", 30, true, "Producer was found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
-        section.redactLineAfter("Phone No.", "PII", 30, true, "Producer was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Phone No.", "PII", 30, true, "Producer was found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
-        section.redactBetween("No:", "Fax", "PII", 30, true, "Producer was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        section.redactBetween("No:", "Fax", "PII", 30, true, "Producer was found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
     end
 
 
@@ -375,17 +375,17 @@ rule "31: Redact contact information if Producer is found (Vertebrate study)"
     when
         Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && (text.toLowerCase().contains("producer of the plant protection") || text.toLowerCase().contains("producer of the active substance") || text.contains("Manufacturer of the active substance") || text.contains("Manufacturer:") || text.contains("Producer or producers of the active substance")))
     then
-        section.redactLineAfter("Contact:", "PII", 31, true, "Producer was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Contact:", "PII", 31, true, "Producer was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
-        section.redactLineAfter("Telephone:", "PII", 31, true, "Producer was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Telephone:", "PII", 31, true, "Producer was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
-        section.redactLineAfter("Phone:", "PII", 31, true, "Producer was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Phone:", "PII", 31, true, "Producer was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
-        section.redactLineAfter("Fax:", "PII", 31, true, "Producer was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Fax:", "PII", 31, true, "Producer was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
-        section.redactLineAfter("E-mail:", "PII", 31, true, "Producer was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("E-mail:", "PII", 31, true, "Producer was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
-        section.redactLineAfter("Contact:", "PII", 31, true, "Producer was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Contact:", "PII", 31, true, "Producer was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
-        section.redactLineAfter("Fax number:", "PII", 31, true, "Producer was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Fax number:", "PII", 31, true, "Producer was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
-        section.redactLineAfter("Telephone number:", "PII", 31, true, "Producer was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Telephone number:", "PII", 31, true, "Producer was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
-        section.redactLineAfter("Tel:", "PII", 31, true, "Producer was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Tel:", "PII", 31, true, "Producer was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
-        section.redactLineAfter("Phone No.", "PII", 31, true, "Producer was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        section.redactLineAfter("Phone No.", "PII", 31, true, "Producer was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
-        section.redactBetween("No:", "Fax", "PII", 31, true, "Producer was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        section.redactBetween("No:", "Fax", "PII", 31, true, "Producer was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
     end
 
 
@@ -393,7 +393,7 @@ rule "32: Redact AUTHOR(S) (Non vertebrate study)"
     when
         Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("AUTHOR(S):") && searchText.contains("COMPLETION DATE:") && !searchText.contains("STUDY COMPLETION DATE:"))
     then
-        section.redactLinesBetween("AUTHOR(S):", "COMPLETION DATE:", "PII", 32, true, "AUTHOR(S) was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        section.redactLinesBetween("AUTHOR(S):", "COMPLETION DATE:", "PII", 32, true, "AUTHOR(S) was found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
     end
 
 
@@ -401,7 +401,7 @@ rule "33: Redact AUTHOR(S) (Vertebrate study)"
     when
         Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("AUTHOR(S):") && searchText.contains("COMPLETION DATE:") && !searchText.contains("STUDY COMPLETION DATE:"))
     then
-        section.redactLinesBetween("AUTHOR(S):", "COMPLETION DATE:", "PII", 33, true, "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        section.redactLinesBetween("AUTHOR(S):", "COMPLETION DATE:", "PII", 33, true, "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
     end
 
 
@@ -409,7 +409,7 @@ rule "34: Redact AUTHOR(S) (Non vertebrate study)"
     when
         Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("AUTHOR(S):") && searchText.contains("STUDY COMPLETION DATE:"))
     then
-        section.redactLinesBetween("AUTHOR(S):", "STUDY COMPLETION DATE:", "PII", 34, true, "AUTHOR(S) was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        section.redactLinesBetween("AUTHOR(S):", "STUDY COMPLETION DATE:", "PII", 34, true, "AUTHOR(S) was found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
     end
 
 
@@ -417,7 +417,7 @@ rule "35: Redact AUTHOR(S) (Vertebrate study)"
     when
         Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("AUTHOR(S):") && searchText.contains("STUDY COMPLETION DATE:"))
     then
-        section.redactLinesBetween("AUTHOR(S):", "STUDY COMPLETION DATE:", "PII", 35, true, "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        section.redactLinesBetween("AUTHOR(S):", "STUDY COMPLETION DATE:", "PII", 35, true, "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
     end
 
 
@@ -425,7 +425,7 @@ rule "36: Redact PERFORMING LABORATORY (Non vertebrate study)"
     when
         Section(!fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("PERFORMING LABORATORY:"))
     then
-        section.redactBetween("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", 36, true, "PERFORMING LABORATORY was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
+        section.redactBetween("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", 36, true, "PERFORMING LABORATORY was found", "Article 39(e)(3) of Regulation (EC) No 178/2002",dictionary);
         section.redactNot("CBI_address", 36, "Performing laboratory found for non vertebrate study");
     end
 
@@ -434,7 +434,7 @@ rule "37: Redact PERFORMING LABORATORY (Vertebrate study)"
     when
         Section(fileAttributeByLabelEqualsIgnoreCase("Vertebrate Study","Yes") && searchText.contains("PERFORMING LABORATORY:"))
     then
-        section.redactBetween("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", 37, true, "PERFORMING LABORATORY was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
+        section.redactBetween("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", 37, true, "PERFORMING LABORATORY was found", "Article 39(e)(2) of Regulation (EC) No 178/2002",dictionary);
     end
 
 
@@ -444,7 +444,7 @@ rule "50: Purity Hint"
     when
         Section(searchText.toLowerCase().contains("purity"))
     then
-	    section.addHintAnnotationByRegEx("(purity ?( of|\\(.{1,20}\\))?( ?:)?) .{0,5}[\\d\\.]+( .{0,4}\\.)? ?%", true, 1, "hint_only");
+	    section.addHintAnnotationByRegEx("(purity ?( of|\\(.{1,20}\\))?( ?:)?) .{0,5}[\\d\\.]+( .{0,4}\\.)? ?%", true, 1, "hint_only",dictionary);
     end