RED-6093: Prototype document structurewip dependency upgrade

wip

redaction-service-v1/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <artifactId>platform-dependency</artifactId>
         <groupId>com.iqser.red</groupId>
-        <version>1.13.0</version>
+        <version>1.17.0</version>
         <relativePath/>
     </parent>
     <modelVersion>4.0.0</modelVersion>
@@ -32,7 +32,7 @@
             <dependency>
                 <groupId>com.iqser.red</groupId>
                 <artifactId>platform-commons-dependency</artifactId>
-                <version>1.20.0</version>
+                <version>1.21.0</version>
                 <scope>import</scope>
                 <type>pom</type>
             </dependency>

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/model/Document.java

@@ -14,7 +14,7 @@ import lombok.NoArgsConstructor;
 public class Document {
 
     private List<Page> pages = new ArrayList<>();
-    private List<Paragraph> paragraphs = new ArrayList<>();
+    private List<Section> sections = new ArrayList<>();
     private List<Header> headers = new ArrayList<>();
     private List<Footer> footers = new ArrayList<>();
     private List<UnclassifiedText> unclassifiedTexts = new ArrayList<>();

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/model/Section.java

@@ -1,7 +1,13 @@
 package com.iqser.red.service.redaction.v1.server.classification.model;
 
+import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus;
+import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.IdRemoval;
+import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualImageRecategorization;
+import com.iqser.red.service.redaction.v1.server.redaction.model.Entities;
+import com.iqser.red.service.redaction.v1.server.redaction.model.Image;
 import com.iqser.red.service.redaction.v1.server.redaction.model.PdfImage;
 import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText;
+import com.iqser.red.service.redaction.v1.server.redaction.utils.IdBuilder;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;
 
@@ -10,10 +16,11 @@ import lombok.NoArgsConstructor;
 
 import java.util.ArrayList;
 import java.util.List;
+import java.util.stream.Collectors;
 
 @Data
 @NoArgsConstructor
-public class Paragraph implements Comparable {
+public class Section implements Comparable {
 
     private List<AbstractTextContainer> pageBlocks = new ArrayList<>();
     private List<PdfImage> images = new ArrayList<>();
@@ -62,4 +69,9 @@ public class Paragraph implements Comparable {
         return 0;
     }
 
+
+
+
+
+
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/model/SectionText.java

@@ -1,11 +1,19 @@
 package com.iqser.red.service.redaction.v1.server.classification.model;
 
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
 import com.dslplatform.json.CompiledJson;
 import com.dslplatform.json.JsonAttribute;
 import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.iqser.red.service.redaction.v1.model.SectionArea;
 import com.iqser.red.service.redaction.v1.server.redaction.model.CellValue;
 import com.iqser.red.service.redaction.v1.server.redaction.model.Image;
+import com.iqser.red.service.redaction.v1.server.redaction.model.Paragraph;
 import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText;
 
 import lombok.AllArgsConstructor;
@@ -13,8 +21,6 @@ import lombok.Builder;
 import lombok.Data;
 import lombok.NoArgsConstructor;
 
-import java.util.*;
-
 @Data
 @Builder
 @CompiledJson
@@ -27,6 +33,7 @@ public class SectionText {
 
     private boolean isTable;
     private String headline;
+    List<Paragraph> paragraphs;
 
     @Builder.Default
     private List<SectionArea> sectionAreas = new ArrayList<>();

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/model/TextBlock.java

@@ -29,6 +29,8 @@ public class TextBlock extends AbstractTextContainer {
     @JsonIgnore
     private int rotation;
 
+    private int indexOnPage;
+
     @JsonIgnore
     private String mostPopularWordFont;
 
@@ -184,8 +186,8 @@ public class TextBlock extends AbstractTextContainer {
     }
 
 
-    public TextBlock(float minX, float maxX, float minY, float maxY, List<TextPositionSequence> sequences, int rotation) {
+    public TextBlock(float minX, float maxX, float minY, float maxY, List<TextPositionSequence> sequences, int rotation, int indexOnPage) {
-
+        this.indexOnPage = indexOnPage;
         this.minX = minX;
         this.maxX = maxX;
         this.minY = minY;
@@ -248,7 +250,7 @@ public class TextBlock extends AbstractTextContainer {
 
     public TextBlock copy() {
 
-        return new TextBlock(minX, maxX, minY, maxY, sequences, rotation);
+        return new TextBlock(minX, maxX, minY, maxY, sequences, rotation, indexOnPage);
     }
 
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/service/BlockificationService.java

@@ -30,6 +30,7 @@ public class BlockificationService {
      * This method is building blocks by expanding the minX/maxX and minY/maxY value on each word that is not split by the conditions.
      * This method must use text direction adjusted postions (DirAdj). Where {0,0} is on the upper left. Never try to change this!
      * Rulings (Table lines) must be adjusted to the text directions as well, when checking if a block is split by a ruling.
+     *
      * @param textPositions         The words of a page.
      * @param horizontalRulingLines Horizontal table lines.
      * @param verticalRulingLines   Vertical table lines.
@@ -37,6 +38,7 @@ public class BlockificationService {
      */
     public Page blockify(List<TextPositionSequence> textPositions, List<Ruling> horizontalRulingLines, List<Ruling> verticalRulingLines) {
 
+        int indexOnPage = 0;
         List<TextPositionSequence> chunkWords = new ArrayList<>();
         List<AbstractTextContainer> chunkBlockList1 = new ArrayList<>();
 
@@ -62,7 +64,9 @@ public class BlockificationService {
                     prevOrientation = chunkBlockList1.get(chunkBlockList1.size() - 1).getOrientation();
                 }
 
-                TextBlock cb1 = buildTextBlock(chunkWords);
+                TextBlock cb1 = buildTextBlock(chunkWords, indexOnPage);
+                indexOnPage++;
+
                 chunkBlockList1.add(cb1);
                 chunkWords = new ArrayList<>();
 
@@ -102,7 +106,7 @@ public class BlockificationService {
             }
         }
 
-        TextBlock cb1 = buildTextBlock(chunkWords);
+        TextBlock cb1 = buildTextBlock(chunkWords, indexOnPage);
         if (cb1 != null) {
             chunkBlockList1.add(cb1);
         }
@@ -163,7 +167,7 @@ public class BlockificationService {
     }
 
 
-    private TextBlock buildTextBlock(List<TextPositionSequence> wordBlockList) {
+    private TextBlock buildTextBlock(List<TextPositionSequence> wordBlockList, int indexOnPage) {
 
         TextBlock textBlock = null;
 
@@ -182,7 +186,13 @@ public class BlockificationService {
             styleFrequencyCounter.add(wordBlock.getFontStyle());
 
             if (textBlock == null) {
-                textBlock = new TextBlock(wordBlock.getMinXDirAdj(), wordBlock.getMaxXDirAdj(), wordBlock.getMinYDirAdj(), wordBlock.getMaxYDirAdj(), wordBlockList, wordBlock.getRotation());
+                textBlock = new TextBlock(wordBlock.getMinXDirAdj(),
+                        wordBlock.getMaxXDirAdj(),
+                        wordBlock.getMinYDirAdj(),
+                        wordBlock.getMaxYDirAdj(),
+                        wordBlockList,
+                        wordBlock.getRotation(),
+                        indexOnPage);
             } else {
                 TextBlock spatialEntity = textBlock.union(wordBlock);
                 textBlock.resize(spatialEntity.getMinX(), spatialEntity.getMinY(), spatialEntity.getWidth(), spatialEntity.getHeight());
@@ -213,10 +223,38 @@ public class BlockificationService {
                                     List<Ruling> horizontalRulingLines,
                                     List<Ruling> verticalRulingLines) {
 
-        return isSplitByRuling(maxX, minY, word.getMinXDirAdj(), word.getMinYDirAdj(), verticalRulingLines, word.getDir().getDegrees(), word.getPageWidth(), word.getPageHeight()) //
+        return isSplitByRuling(maxX,
-                || isSplitByRuling(minX, minY, word.getMinXDirAdj(), word.getMaxYDirAdj(), horizontalRulingLines, word.getDir().getDegrees(), word.getPageWidth(), word.getPageHeight()) //
+                minY,
-                || isSplitByRuling(maxX, minY, word.getMinXDirAdj(), word.getMinYDirAdj(), horizontalRulingLines, word.getDir().getDegrees(), word.getPageWidth(), word.getPageHeight()) //
+                word.getMinXDirAdj(),
-                || isSplitByRuling(minX, minY, word.getMinXDirAdj(), word.getMaxYDirAdj(), verticalRulingLines, word.getDir().getDegrees(), word.getPageWidth(), word.getPageHeight()); //
+                word.getMinYDirAdj(),
+                verticalRulingLines,
+                word.getDir().getDegrees(),
+                word.getPageWidth(),
+                word.getPageHeight()) //
+                || isSplitByRuling(minX,
+                minY,
+                word.getMinXDirAdj(),
+                word.getMaxYDirAdj(),
+                horizontalRulingLines,
+                word.getDir().getDegrees(),
+                word.getPageWidth(),
+                word.getPageHeight()) //
+                || isSplitByRuling(maxX,
+                minY,
+                word.getMinXDirAdj(),
+                word.getMinYDirAdj(),
+                horizontalRulingLines,
+                word.getDir().getDegrees(),
+                word.getPageWidth(),
+                word.getPageHeight()) //
+                || isSplitByRuling(minX,
+                minY,
+                word.getMinXDirAdj(),
+                word.getMaxYDirAdj(),
+                verticalRulingLines,
+                word.getDir().getDegrees(),
+                word.getPageWidth(),
+                word.getPageHeight()); //
     }
 
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/AtomicTextBlockEntity.java

@@ -0,0 +1,97 @@
+package com.iqser.red.service.redaction.v1.server.document.graph;
+
+import java.util.List;
+
+import com.iqser.red.service.redaction.v1.server.redaction.model.RedRectangle2D;
+
+import lombok.AccessLevel;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.experimental.FieldDefaults;
+
+@Data
+@Builder
+@AllArgsConstructor
+@FieldDefaults(level = AccessLevel.PRIVATE)
+public class AtomicTextBlockEntity implements CharSequence {
+
+    Integer id;
+    //string coords
+    Integer offset;
+    String searchText;
+    List<Integer> lineBreaks;
+
+    //position coords
+    List<Integer> stringIdxToPositionIdx;
+    List<RedRectangle2D> positions;
+
+    Node parent;
+
+
+    public int indexOf(String searchTerm) {
+
+        int pos = searchText.indexOf(searchTerm);
+        return pos == -1 ? -1 : pos + offset;
+    }
+
+
+    public int numberOfLines() {
+
+        return lineBreaks.size();
+    }
+
+
+    public int getNextLinebreak(int startIndex) {
+
+        return lineBreaks.stream()//
+                .filter(linebreak -> linebreak > startIndex) //
+                .findFirst() //
+                .orElse(searchText.length()) + offset;
+    }
+
+
+    public RedRectangle2D getPosition(int stringIdx) {
+
+        return positions.get(stringIdxToPositionIdx.get(stringIdx - offset));
+    }
+
+
+    public List<RedRectangle2D> getPositions(int startStringIdx, int endStringIdx) {
+
+        return positions.subList(startStringIdx - offset, endStringIdx - offset);
+    }
+
+
+    @Override
+    public int length() {
+
+        return searchText.length();
+    }
+
+
+    @Override
+    public char charAt(int index) {
+
+        return searchText.charAt(index - offset);
+    }
+
+
+    @Override
+    public CharSequence subSequence(int start, int end) {
+
+        throw new UnsupportedOperationException("AtomicTextBlockEntity can not be sliced!");
+    }
+
+
+    public String getFirstLine() {
+
+        return searchText.substring(0, getNextLinebreak(0) - offset);
+    }
+
+    @Override
+    public String toString() {
+        return searchText;
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentEntity.java

@@ -0,0 +1,49 @@
+package com.iqser.red.service.redaction.v1.server.document.graph;
+
+import java.util.List;
+import java.util.stream.Stream;
+
+import lombok.AccessLevel;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.experimental.FieldDefaults;
+
+@Data
+@Builder
+@AllArgsConstructor
+@FieldDefaults(level = AccessLevel.PRIVATE)
+public class DocumentEntity implements Node {
+
+    List<SectionEntity> sections;
+    List<PageEntity> pages;
+    TableOfContents tableOfContents;
+    Integer numberOfPages;
+
+
+    @Override
+    public TextBlockEntity buildTextBlock() {
+
+        return streamAllNodes()
+                .filter(Node::hasTextBlock)
+                .map(Node::getTextBlock)
+                .collect(TextBlockCollector.collect());
+    }
+
+
+    private Stream<Node> streamAllNodes() {
+
+        return Stream.concat(Stream.concat(//
+                tableOfContents.streamEntriesInOrder().map(TableOfContents.Entry::node), //
+                pages.stream().map(PageEntity::getHeader)), //
+                pages.stream().map(PageEntity::getFooter)); //
+    }
+
+
+    @Override
+    public String toString() {
+
+        return tableOfContents.toString();
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/Entity2.java

@@ -0,0 +1,14 @@
+package com.iqser.red.service.redaction.v1.server.document.graph;
+
+import java.awt.geom.Rectangle2D;
+import java.util.List;
+
+public class Entity2 {
+
+    String uid;
+    String value;
+    List<Rectangle2D> positions;
+    Boolean redact;
+
+    List<Node> neighbours;
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/FooterEntity.java

@@ -0,0 +1,47 @@
+package com.iqser.red.service.redaction.v1.server.document.graph;
+
+import java.util.List;
+
+import lombok.AccessLevel;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.experimental.FieldDefaults;
+
+@Data
+@Builder
+@AllArgsConstructor
+@FieldDefaults(level = AccessLevel.PRIVATE)
+public class FooterEntity implements Node {
+
+    PageEntity page;
+    TextBlockEntity textBlock;
+    List<Entity2> entities;
+
+
+    @Override
+    public TextBlockEntity buildTextBlock() {
+
+        return getTextBlock();
+    }
+
+
+    @Override
+    public boolean hasTextBlock() {
+
+        return true;
+    }
+
+
+    @Override
+    public TextBlockEntity getTextBlock() {
+
+        return textBlock;
+    }
+
+    @Override
+    public String toString() {
+        return textBlock.toString();
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/HeaderEntity.java

@@ -0,0 +1,47 @@
+package com.iqser.red.service.redaction.v1.server.document.graph;
+
+import java.util.List;
+
+import lombok.AccessLevel;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.experimental.FieldDefaults;
+
+@Data
+@Builder
+@AllArgsConstructor
+@FieldDefaults(level = AccessLevel.PRIVATE)
+public class HeaderEntity implements Node {
+
+    PageEntity page;
+    TextBlockEntity textBlock;
+    List<Entity2> entities;
+
+
+    @Override
+    public TextBlockEntity buildTextBlock() {
+
+        return getTextBlock();
+    }
+
+
+    @Override
+    public boolean hasTextBlock() {
+
+        return true;
+    }
+
+
+    @Override
+    public TextBlockEntity getTextBlock() {
+
+        return textBlock;
+    }
+
+    @Override
+    public String toString() {
+        return textBlock.toString();
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/Node.java

@@ -0,0 +1,18 @@
+package com.iqser.red.service.redaction.v1.server.document.graph;
+
+public interface Node {
+    TextBlockEntity buildTextBlock();
+
+
+    default boolean hasTextBlock() {
+
+        return false;
+    }
+
+
+    default TextBlockEntity getTextBlock() {
+
+        throw new UnsupportedOperationException("Only terminal Nodes have direct access to TextBlocks");
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/PageEntity.java

@@ -0,0 +1,37 @@
+package com.iqser.red.service.redaction.v1.server.document.graph;
+
+import java.util.List;
+
+import lombok.AccessLevel;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.experimental.FieldDefaults;
+
+@Data
+@Builder
+@AllArgsConstructor
+@FieldDefaults(level = AccessLevel.PRIVATE)
+public class PageEntity implements Node {
+
+    Integer number;
+    Integer height;
+    Integer width;
+    List<SectionEntity> sections;
+    List<ParagraphEntity> paragraphs;
+    HeaderEntity header;
+    FooterEntity footer;
+
+
+    @Override
+    public TextBlockEntity buildTextBlock() {
+
+        return null;
+    }
+
+    @Override
+    public String toString() {
+        return number.toString();
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/ParagraphEntity.java

@@ -0,0 +1,54 @@
+package com.iqser.red.service.redaction.v1.server.document.graph;
+
+import java.util.List;
+
+import lombok.AccessLevel;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.experimental.FieldDefaults;
+
+@Data
+@Builder
+@AllArgsConstructor
+@FieldDefaults(level = AccessLevel.PRIVATE)
+public class ParagraphEntity implements Node {
+
+    String tocId;
+    Integer id;
+    Integer numberOnPage;
+    Integer numberInSection;
+
+    //foreign key
+    SectionEntity parentSection;
+    PageEntity page;
+    TextBlockEntity textBlock;
+    List<Entity2> entities;
+
+
+    @Override
+    public TextBlockEntity buildTextBlock() {
+
+        return getTextBlock();
+    }
+
+
+    @Override
+    public boolean hasTextBlock() {
+
+        return true;
+    }
+
+
+    @Override
+    public TextBlockEntity getTextBlock() {
+
+        return textBlock;
+    }
+
+
+    @Override
+    public String toString() {
+        return textBlock.toString();
+    }
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/SectionEntity.java

@@ -0,0 +1,56 @@
+package com.iqser.red.service.redaction.v1.server.document.graph;
+
+import java.util.List;
+
+import lombok.AccessLevel;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.experimental.FieldDefaults;
+
+@Data
+@Builder
+@AllArgsConstructor
+@FieldDefaults(level = AccessLevel.PRIVATE)
+public class SectionEntity implements Node {
+
+    Integer id;
+    TableOfContents tableOfContents;
+    String tocId;
+    Node parent;
+    //foreign key
+    TextBlockEntity headline;
+    List<SectionEntity> subSections;
+    List<ParagraphEntity> paragraphs;
+    List<Integer> pages;
+    List<Entity2> entities;
+
+
+    @Override
+    public String toString() {
+
+        return tocId + ": " + headline.toString();
+    }
+
+
+    @Override
+    public TextBlockEntity buildTextBlock() {
+
+        return tableOfContents.streamSubEntriesInOrder(tocId).map(TableOfContents.Entry::node).map(Node::getTextBlock).collect(TextBlockCollector.collect());
+    }
+
+
+    @Override
+    public boolean hasTextBlock() {
+
+        return true;
+    }
+
+
+    @Override
+    public TextBlockEntity getTextBlock() {
+
+        return headline;
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/TableCellEntity.java

@@ -0,0 +1,44 @@
+package com.iqser.red.service.redaction.v1.server.document.graph;
+
+import java.util.List;
+
+import lombok.AccessLevel;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.experimental.FieldDefaults;
+
+@Data
+@Builder
+@AllArgsConstructor
+@FieldDefaults(level = AccessLevel.PRIVATE)
+public class TableCellEntity implements Node {
+
+    //foreign key
+    Integer table;
+    TextBlockEntity textBlock;
+    List<Entity2> entities;
+
+
+    @Override
+    public TextBlockEntity buildTextBlock() {
+
+        return textBlock;
+    }
+
+
+    @Override
+    public boolean hasTextBlock() {
+
+        return true;
+    }
+
+
+    @Override
+    public TextBlockEntity getTextBlock() {
+
+        return textBlock;
+    }
+
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/TableEntity.java

@@ -0,0 +1,53 @@
+package com.iqser.red.service.redaction.v1.server.document.graph;
+
+import java.util.List;
+import java.util.stream.Stream;
+
+import lombok.AccessLevel;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.experimental.FieldDefaults;
+
+@Data
+@Builder
+@AllArgsConstructor
+@FieldDefaults(level = AccessLevel.PRIVATE)
+public class TableEntity implements Node {
+
+    Integer id;
+    Integer numberOfRows;
+    Integer numberOfCols;
+    List<List<TableCellEntity>> tableCells;
+
+    // graph connection
+    SectionEntity section;
+    PageEntity page;
+    List<Entity2> entities;
+
+
+    private Stream<TableCellEntity> streamTableCells() {
+
+        return tableCells.stream().flatMap(List::stream);
+    }
+
+
+    private Stream<TableCellEntity> streamTableRow(int row) {
+
+        return tableCells.get(row).stream();
+    }
+
+
+    private Stream<TableCellEntity> streamTableCol(int col) {
+
+        return tableCells.stream().map(row -> row.get(col));
+    }
+
+
+    @Override
+    public TextBlockEntity buildTextBlock() {
+
+        return streamTableCells().map(TableCellEntity::buildTextBlock).collect(TextBlockCollector.collect());
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/TableOfContents.java

@@ -0,0 +1,109 @@
+package com.iqser.red.service.redaction.v1.server.document.graph;
+
+import static java.lang.String.format;
+
+import java.util.Arrays;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.stream.Stream;
+
+import javax.management.openmbean.InvalidKeyException;
+
+import lombok.Data;
+
+@Data
+public class TableOfContents {
+
+    public static final String SECTION = "Sec";
+    public static final String HEADLINE = "H";
+    public static final String PARAGRAPH = "Par";
+    public static final String TABLE = "Tab";
+
+    List<Entry> entries;
+
+
+    public TableOfContents() {
+
+        entries = new LinkedList<>();
+    }
+
+
+    public String createNewEntryAndReturnId(String keyword, String summary, Node node) {
+
+        String id = String.format("%d", entries.size());
+        entries.add(new Entry(keyword, id, summary, new LinkedList<>(), node));
+        return id;
+    }
+
+
+    public String createNewChildEntryAndReturnId(String parentId, String keyword, String summary, Node node) {
+
+        Entry parent = getEntryById(parentId);
+        String childId = parentId + String.format(".%d", parent.children().size());
+        parent.children().add(new Entry(keyword, childId, summary, new LinkedList<>(), node));
+        return childId;
+    }
+
+
+    public Entry getEntryById(String parentId) {
+
+        List<Integer> ids = getIds(parentId);
+        if (ids.size() < 1) {
+            throw new InvalidKeyException(format("Section Identifier: \"%s\" is not valid.", parentId));
+        }
+        Entry entry = entries.get(ids.get(0));
+        for (int id : ids.subList(1, ids.size())) {
+            entry = entry.children().get(id);
+        }
+        return entry;
+    }
+
+
+    @Override
+    public String toString() {
+
+        return String.join("\n", streamEntriesInOrder().map(Entry::toString).toList());
+    }
+
+
+    public String toString(String id) {
+
+        return String.join("\n", streamSubEntriesInOrder(id).map(Entry::toString).toList());
+    }
+
+
+    public Stream<Entry> streamEntriesInOrder() {
+
+        return entries.stream().flatMap(TableOfContents::flatten);
+    }
+
+
+    public Stream<Entry> streamSubEntriesInOrder(String parentId) {
+
+        return Stream.of(getEntryById(parentId)).flatMap(TableOfContents::flatten);
+    }
+
+
+    private static List<Integer> getIds(String idsAsString) {
+
+        return Arrays.stream(idsAsString.split("\\.")).map(Integer::valueOf).toList();
+    }
+
+
+    private static Stream<Entry> flatten(Entry entry) {
+
+        return Stream.concat(Stream.of(entry), entry.children().stream().flatMap(TableOfContents::flatten));
+    }
+
+
+    public record Entry(String type, String id, String summary, List<Entry> children, Node node) {
+
+        @Override
+        public String toString() {
+
+            return id + ": " + type + ".: " + summary;
+        }
+
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/TextBlockCollector.java

@@ -0,0 +1,53 @@
+package com.iqser.red.service.redaction.v1.server.document.graph;
+
+import java.util.Collections;
+import java.util.Set;
+import java.util.function.BiConsumer;
+import java.util.function.BinaryOperator;
+import java.util.function.Function;
+import java.util.function.Supplier;
+import java.util.stream.Collector;
+
+public class TextBlockCollector implements Collector<TextBlockEntity, TextBlockEntity, TextBlockEntity> {
+
+    public static Collector<TextBlockEntity, TextBlockEntity, TextBlockEntity> collect() {
+
+        return new TextBlockCollector();
+    }
+
+
+    @Override
+    public Supplier<TextBlockEntity> supplier() {
+
+        return new TextBlockEntity(Collections.emptyList());
+    }
+
+
+    @Override
+    public BiConsumer<TextBlockEntity, TextBlockEntity> accumulator() {
+
+        return TextBlockEntity::concat;
+    }
+
+
+    @Override
+    public BinaryOperator<TextBlockEntity> combiner() {
+
+        return TextBlockEntity::concat;
+    }
+
+
+    @Override
+    public Function<TextBlockEntity, TextBlockEntity> finisher() {
+
+        return Function.identity();
+    }
+
+
+    @Override
+    public Set<Characteristics> characteristics() {
+
+        return Set.of(Characteristics.IDENTITY_FINISH, Characteristics.CONCURRENT);
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/TextBlockEntity.java

@@ -0,0 +1,166 @@
+package com.iqser.red.service.redaction.v1.server.document.graph;
+
+import java.awt.geom.Rectangle2D;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.function.Supplier;
+
+import com.iqser.red.service.redaction.v1.server.redaction.model.RedRectangle2D;
+
+import lombok.AccessLevel;
+import lombok.Data;
+import lombok.experimental.FieldDefaults;
+
+@Data
+@FieldDefaults(level = AccessLevel.PRIVATE)
+public class TextBlockEntity implements CharSequence, Supplier<TextBlockEntity> {
+
+    List<AtomicTextBlockEntity> atomicTextBlocks;
+    StringBuilder searchTextBuilder;
+
+    //string coords
+    Integer offset;
+
+
+    public TextBlockEntity(List<AtomicTextBlockEntity> atomicTextBlocks) {
+
+        this.atomicTextBlocks = new LinkedList<>();
+        this.searchTextBuilder = new StringBuilder();
+        if (atomicTextBlocks.isEmpty()) {
+            offset = -1;
+            return;
+        }
+        var firstTextBlock = atomicTextBlocks.get(0);
+        this.atomicTextBlocks.add(firstTextBlock);
+        this.offset = firstTextBlock.getOffset();
+        this.searchTextBuilder.append(firstTextBlock.getSearchText());
+
+        atomicTextBlocks.subList(1, atomicTextBlocks.size()).forEach(this::concat);
+    }
+
+
+    public TextBlockEntity concat(TextBlockEntity textBlock) {
+
+        if (this.atomicTextBlocks.isEmpty() && this.offset == -1) {
+            this.offset = textBlock.getOffset();
+        } else if (textBlock.getOffset() != offset + length()) {
+            throw new UnsupportedOperationException("Can only concat consecutive TextBlocks");
+        }
+
+        this.searchTextBuilder.append(textBlock.getSearchTextBuilder());
+        this.atomicTextBlocks.addAll(textBlock.getAtomicTextBlocks());
+        return this;
+    }
+
+
+    public void concat(AtomicTextBlockEntity atomicTextBlock) {
+
+        if (this.atomicTextBlocks.isEmpty() && this.offset == -1) {
+            this.offset = atomicTextBlock.getOffset();
+        } else if (atomicTextBlock.getOffset() != offset + length()) {
+            throw new UnsupportedOperationException("Can only concat consecutive TextBlocks");
+        }
+
+        this.searchTextBuilder.append(atomicTextBlock.getSearchText());
+        this.atomicTextBlocks.add(atomicTextBlock);
+    }
+
+
+    public int indexOf(String searchTerm) {
+
+        int pos = this.searchTextBuilder.indexOf(searchTerm);
+        return pos == -1 ? -1 : pos + offset;
+    }
+
+
+    public int numberOfLines() {
+
+        return atomicTextBlocks.stream().map(AtomicTextBlockEntity::getLineBreaks).mapToInt(List::size).sum();
+    }
+
+
+    public int getNextLinebreak(int startIndex) {
+
+        return getAtomicTextBlockByStringIndex(startIndex).getNextLinebreak(startIndex);
+    }
+
+
+    public RedRectangle2D getPositions(int stringIdx) {
+
+        return getAtomicTextBlockByStringIndex(stringIdx).getPosition(stringIdx);
+    }
+
+
+    public List<RedRectangle2D> getPositions(int startStringIdx, int endStringIdx) {
+
+        List<AtomicTextBlockEntity> textBlocks = getAllAtomicTextBlocksPartiallyInStringIdxRange(startStringIdx, endStringIdx);
+
+        if (textBlocks.size() == 1) {
+            return textBlocks.get(0).getPositions(startStringIdx, endStringIdx);
+        }
+
+        var firstTextBlock = textBlocks.get(0);
+        List<RedRectangle2D> positions = new LinkedList<>(firstTextBlock.getPositions(startStringIdx, firstTextBlock.getOffset() + firstTextBlock.length()));
+
+        for (var textBlock : textBlocks.subList(1, textBlocks.size() - 1)) {
+            positions.addAll(textBlock.getPositions());
+        }
+
+        var lastTextBlock = textBlocks.get(textBlocks.size() - 1);
+        positions.addAll(lastTextBlock.getPositions(lastTextBlock.getOffset(), endStringIdx));
+
+        return positions;
+    }
+
+
+    private AtomicTextBlockEntity getAtomicTextBlockByStringIndex(int stringIdx) {
+
+        return atomicTextBlocks.stream().filter(textBlock -> (textBlock.getOffset() + textBlock.length()) > stringIdx).findFirst().orElseThrow(IndexOutOfBoundsException::new);
+    }
+
+
+    private List<AtomicTextBlockEntity> getAllAtomicTextBlocksPartiallyInStringIdxRange(int startStringIdx, int endStringIdx) {
+
+        if (offset > endStringIdx || offset + length() <= startStringIdx) {
+            throw new IndexOutOfBoundsException();
+        }
+
+        return atomicTextBlocks.stream().filter(tb -> tb.getOffset() <= endStringIdx && tb.getOffset() + tb.length() > startStringIdx).toList();
+    }
+
+
+    public String getSearchText() {
+
+        return searchTextBuilder.toString();
+    }
+
+
+    @Override
+    public int length() {
+
+        return this.searchTextBuilder.length();
+    }
+
+
+    @Override
+    public char charAt(int index) {
+
+        return searchTextBuilder.charAt(index);
+    }
+
+
+    @Override
+    public CharSequence subSequence(int start, int end) {
+
+        throw new UnsupportedOperationException("AtomicTextBlockEntity can not be sliced!");
+    }
+
+
+    @Override
+    public TextBlockEntity get() {
+
+        return this;
+    }
+
+}
+

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/DocumentGraphFactory.java

@@ -0,0 +1,209 @@
+package com.iqser.red.service.redaction.v1.server.document.services;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.springframework.stereotype.Service;
+
+import com.iqser.red.service.redaction.v1.server.classification.model.Document;
+import com.iqser.red.service.redaction.v1.server.classification.model.Footer;
+import com.iqser.red.service.redaction.v1.server.classification.model.Header;
+import com.iqser.red.service.redaction.v1.server.classification.model.Page;
+import com.iqser.red.service.redaction.v1.server.classification.model.Section;
+import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
+import com.iqser.red.service.redaction.v1.server.document.graph.AtomicTextBlockEntity;
+import com.iqser.red.service.redaction.v1.server.document.graph.DocumentEntity;
+import com.iqser.red.service.redaction.v1.server.document.graph.FooterEntity;
+import com.iqser.red.service.redaction.v1.server.document.graph.HeaderEntity;
+import com.iqser.red.service.redaction.v1.server.document.graph.Node;
+import com.iqser.red.service.redaction.v1.server.document.graph.PageEntity;
+import com.iqser.red.service.redaction.v1.server.document.graph.ParagraphEntity;
+import com.iqser.red.service.redaction.v1.server.document.graph.SectionEntity;
+import com.iqser.red.service.redaction.v1.server.document.graph.TableOfContents;
+import com.iqser.red.service.redaction.v1.server.document.graph.TextBlockEntity;
+import com.iqser.red.service.redaction.v1.server.redaction.model.SearchTextWithTextPositionModel;
+import com.iqser.red.service.redaction.v1.server.redaction.service.SearchTextWithTextPositionFactory;
+import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
+
+import lombok.RequiredArgsConstructor;
+
+@Service
+@RequiredArgsConstructor
+public class DocumentGraphFactory {
+
+    private final SearchTextWithTextPositionFactory searchTextWithTextPositionFactory;
+
+    private Context context;
+
+
+    public DocumentEntity buildEntityFromDocument(Document document) {
+
+        context = new Context(new TableOfContents(), new LinkedList<>(), new LinkedList<>(), new LinkedList<>(), new LinkedList<>(), new AtomicInteger(0), new AtomicInteger(0));
+
+        context.pages.addAll(document.getPages().stream().map(this::buildPage).toList());
+
+        // is tracked by Table of Contents
+        for (int sectionIdx = 0; sectionIdx < document.getSections().size(); sectionIdx++) {
+            addSection(document.getSections().get(sectionIdx), sectionIdx);
+        }
+
+        // not tracked by Table of Contents
+        document.getHeaders().stream().map(Header::getTextBlocks).flatMap(List::stream).forEach(this::addHeader);
+        document.getFooters().stream().map(Footer::getTextBlocks).flatMap(List::stream).forEach(this::addFooter);
+
+        return DocumentEntity.builder().numberOfPages(context.pages.size()).pages(context.pages).sections(context.sections).tableOfContents(context.tableOfContents).build();
+    }
+
+
+    private void addSection(Section section, int sectionIdx) {
+
+        SectionEntity sectionEntity = SectionEntity.builder()
+                .id(sectionIdx)
+                .entities(new LinkedList<>())
+                .pages(new LinkedList<>())
+                .paragraphs(new LinkedList<>())
+                .tableOfContents(context.tableOfContents())
+                .subSections(new LinkedList<>())
+                .build();
+        context.sections().add(sectionEntity);
+        if (section.getPageBlocks().get(0) instanceof TextBlock) {
+            sectionEntity.setHeadline(new TextBlockEntity(List.of(buildAtomicTextBlock((TextBlock) section.getPageBlocks().get(0), sectionEntity))));
+            section.getPageBlocks().remove(0);
+        } else {
+            sectionEntity.setHeadline(emptyTextBlock(sectionEntity));
+        }
+
+        String sectionId = context.tableOfContents.createNewEntryAndReturnId(TableOfContents.SECTION,
+                buildSummary(sectionEntity.getHeadline().getAtomicTextBlocks().get(0)),
+                sectionEntity);
+        sectionEntity.setTocId(sectionId);
+
+        int paragraphIdx = 0;
+        for (AbstractTextContainer abstractTextContainer : section.getPageBlocks()) {
+            if (abstractTextContainer instanceof TextBlock) {
+                addParagraph(sectionEntity, (TextBlock) abstractTextContainer, paragraphIdx);
+                paragraphIdx++;
+            }
+        }
+    }
+
+
+    private TextBlockEntity emptyTextBlock(Node parent) {
+
+        return new TextBlockEntity(List.of(AtomicTextBlockEntity.builder()
+                .id(context.textBlockIdx.getAndIncrement())
+                .offset(context.stringOffset.get())
+                .searchText("")
+                .lineBreaks(Collections.emptyList())
+                .stringIdxToPositionIdx(Collections.emptyList())
+                .positions(Collections.emptyList())
+                .parent(parent)
+                .build()));
+    }
+
+
+    private void addParagraph(SectionEntity sectionEntity, TextBlock originalTextBlock, int idx) {
+
+        PageEntity page = getPage(originalTextBlock.getPage());
+        ParagraphEntity paragraph = ParagraphEntity.builder().id(idx).numberOnPage(originalTextBlock.getIndexOnPage()).page(page).parentSection(sectionEntity).build();
+
+        sectionEntity.getParagraphs().add(paragraph);
+        page.getParagraphs().add(paragraph);
+
+        if (!page.getSections().contains(sectionEntity)) {
+            page.getSections().add(sectionEntity);
+        }
+
+        var textBlock = buildAtomicTextBlock(originalTextBlock, paragraph);
+        paragraph.setTextBlock(new TextBlockEntity(List.of(textBlock)));
+
+        String tocId = context.tableOfContents.createNewChildEntryAndReturnId(sectionEntity.getTocId(), TableOfContents.PARAGRAPH, buildSummary(textBlock), paragraph);
+        paragraph.setTocId(tocId);
+    }
+
+
+    private static String buildSummary(AtomicTextBlockEntity textBlock) {
+
+        if (textBlock == null) {
+            return " probably a table";
+        }
+
+        String[] words = textBlock.getFirstLine().split(" ");
+        int bound = Math.min(words.length, 4);
+        List<String> list = new ArrayList<>(Arrays.asList(words).subList(0, bound));
+
+        return String.join(" ", list);
+    }
+
+
+    private PageEntity buildPage(Page p) {
+
+        return PageEntity.builder()
+                .height((int) p.getPageHeight())
+                .width((int) p.getPageWidth())
+                .number(p.getPageNumber())
+                .paragraphs(new LinkedList<>())
+                .sections(new LinkedList<>())
+                .build();
+    }
+
+
+    private void addFooter(TextBlock textBlock) {
+
+        PageEntity page = getPage(textBlock.getPage());
+        FooterEntity footer = FooterEntity.builder().page(page).build();
+        AtomicTextBlockEntity textBlockEntity = buildAtomicTextBlock(textBlock, footer);
+        footer.setTextBlock(new TextBlockEntity(List.of(textBlockEntity)));
+        page.setFooter(footer);
+    }
+
+
+    public void addHeader(TextBlock textBlock) {
+
+        PageEntity page = getPage(textBlock.getPage());
+        HeaderEntity header = HeaderEntity.builder().page(page).build();
+        AtomicTextBlockEntity textBlockEntity = buildAtomicTextBlock(textBlock, header);
+        header.setTextBlock(new TextBlockEntity(List.of(textBlockEntity)));
+        page.setHeader(header);
+    }
+
+
+    public AtomicTextBlockEntity buildAtomicTextBlock(TextBlock textBlock, Node parent) {
+
+        SearchTextWithTextPositionModel searchTextWithTextPositionModel = searchTextWithTextPositionFactory.buildSearchTextToTextPositionModel(textBlock.getSequences());
+        int offset = context.stringOffset().getAndAdd(searchTextWithTextPositionModel.getSearchText().length());
+
+        return AtomicTextBlockEntity.builder()
+                .id(context.textBlockIdx.getAndIncrement())
+                .parent(parent)
+                .searchText(searchTextWithTextPositionModel.getSearchText())
+                .lineBreaks(searchTextWithTextPositionModel.getLineBreaks())
+                .positions(searchTextWithTextPositionModel.getPositions())
+                .stringIdxToPositionIdx(searchTextWithTextPositionModel.getStringCoordsToPositionCoords())
+                .offset(offset)
+                .build();
+    }
+
+
+    public PageEntity getPage(int i) {
+
+        return context.pages.stream().filter(page -> page.getNumber() == i).findFirst().orElseThrow(NoSuchFieldError::new);
+    }
+
+
+    record Context(
+            TableOfContents tableOfContents,
+            List<PageEntity> pages,
+            List<SectionEntity> sections,
+            List<HeaderEntity> headers,
+            List<FooterEntity> footers,
+            AtomicInteger stringOffset,
+            AtomicInteger textBlockIdx) {
+
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Entity.java

@@ -1,22 +1,25 @@
 package com.iqser.red.service.redaction.v1.server.redaction.model;
 
-import com.iqser.red.service.redaction.v1.model.Engine;
-import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
-
-import lombok.*;
-
 import java.util.ArrayList;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
 
+import com.iqser.red.service.redaction.v1.model.Engine;
+import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.EqualsAndHashCode;
+import lombok.NoArgsConstructor;
+
 @Data
 @Builder
 @NoArgsConstructor
 @AllArgsConstructor
 @EqualsAndHashCode(onlyExplicitlyIncluded = true)
 public class Entity implements ReasonHolder {
-
     private String word;
     private String type;
     private boolean redaction;
@@ -26,6 +29,7 @@ public class Entity implements ReasonHolder {
     private List<EntityPositionSequence> positionSequences = new ArrayList<>();
     private List<TextPositionSequence> targetSequences;
 
+
     @EqualsAndHashCode.Include
     private Integer start;
     @EqualsAndHashCode.Include
@@ -38,6 +42,9 @@ public class Entity implements ReasonHolder {
     @EqualsAndHashCode.Include
     private int sectionNumber;
 
+    @EqualsAndHashCode.Include
+    private int paragraphNumber;
+
     private boolean isDictionaryEntry;
 
     private String textBefore;
@@ -66,6 +73,7 @@ public class Entity implements ReasonHolder {
                   String headline,
                   int matchedRule,
                   int sectionNumber,
+                  int paragraphNumber,
                   String legalBasis,
                   boolean isDictionaryEntry,
                   String textBefore,
@@ -85,6 +93,7 @@ public class Entity implements ReasonHolder {
         this.headline = headline;
         this.matchedRule = matchedRule;
         this.sectionNumber = sectionNumber;
+        this.paragraphNumber = paragraphNumber;
         this.legalBasis = legalBasis;
         this.isDictionaryEntry = isDictionaryEntry;
         this.textBefore = textBefore;
@@ -104,6 +113,7 @@ public class Entity implements ReasonHolder {
                   Integer end,
                   String headline,
                   int sectionNumber,
+                  int paragraphNumber,
                   boolean isDictionaryEntry,
                   boolean isDossierDictionaryEntry,
                   Engine engine,
@@ -115,6 +125,7 @@ public class Entity implements ReasonHolder {
         this.end = end;
         this.headline = headline;
         this.sectionNumber = sectionNumber;
+        this.paragraphNumber = paragraphNumber;
         this.isDictionaryEntry = isDictionaryEntry;
         this.isDossierDictionaryEntry = isDossierDictionaryEntry;
         this.engines.add(engine);

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Paragraph.java

@@ -0,0 +1,16 @@
+package com.iqser.red.service.redaction.v1.server.redaction.model;
+
+import lombok.AccessLevel;
+import lombok.Builder;
+import lombok.Getter;
+import lombok.experimental.FieldDefaults;
+
+@Getter
+@Builder
+@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
+public class Paragraph {
+
+    SearchTextWithTextPositionModel searchTextToTextPosition;
+    int sectionNumber;
+    int paragraphNumber;
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/SearchTextWithTextPositionModel.java

@@ -0,0 +1,19 @@
+package com.iqser.red.service.redaction.v1.server.redaction.model;
+
+import java.util.List;
+
+import lombok.AccessLevel;
+import lombok.Builder;
+import lombok.Getter;
+import lombok.experimental.FieldDefaults;
+
+@Builder
+@Getter
+@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
+public class SearchTextWithTextPositionModel {
+
+    String searchText;
+    List<Integer> lineBreaks;
+    List<Integer> stringCoordsToPositionCoords;
+    List<RedRectangle2D> positions;
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/SearchableText.java

@@ -1,5 +1,11 @@
 package com.iqser.red.service.redaction.v1.server.redaction.model;
 
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.List;
+import java.util.stream.Collectors;
+
 import com.dslplatform.json.JsonAttribute;
 import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
@@ -7,12 +13,6 @@ import com.iqser.red.service.redaction.v1.server.redaction.utils.IdBuilder;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.SeparatorUtils;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.TextNormalizationUtilities;
 
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.List;
-import java.util.stream.Collectors;
-
 import lombok.Getter;
 
 public class SearchableText {
@@ -186,25 +186,13 @@ public class SearchableText {
         return stringRepresentation;
     }
 
-
     public static String buildString(List<TextPositionSequence> sequences) {
 
         StringBuilder sb = new StringBuilder();
-
-        TextPositionSequence previous = null;
         for (TextPositionSequence word : sequences) {
-
+            sb.append(word.toString());
-            if (previous != null) {
-                if (Math.abs(previous.getMaxYDirAdj() - word.getMaxYDirAdj()) > word.getTextHeight()) {
-                    sb.append('\n');
-                } else {
             sb.append(' ');
         }
-            }
-            sb.append(word.toString());
-            previous = word;
-        }
-
         return TextNormalizationUtilities.removeHyphenLineBreaks(sb.toString()).replaceAll("\n", " ").replaceAll(" {2}", " ");
     }
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java

@@ -18,7 +18,10 @@ import java.util.stream.Collectors;
 
 import org.apache.commons.lang3.StringUtils;
 
+import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus;
 import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
+import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.IdRemoval;
+import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualImageRecategorization;
 import com.iqser.red.service.redaction.v1.model.ArgumentType;
 import com.iqser.red.service.redaction.v1.model.Engine;
 import com.iqser.red.service.redaction.v1.model.FileAttribute;
@@ -26,12 +29,14 @@ import com.iqser.red.service.redaction.v1.model.SectionArea;
 import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
 import com.iqser.red.service.redaction.v1.server.parsing.model.RedTextPosition;
 import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
+import com.iqser.red.service.redaction.v1.server.redaction.service.SurroundingWordsService;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUtils;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.FindEntityDetails;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.IdBuilder;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.OffsetStringUtils;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.Patterns;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.SearchImplementation;
+import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;
 
 import lombok.Builder;
 import lombok.Data;
@@ -49,7 +54,8 @@ public class Section {
     @Builder.Default
     private Map<String, Set<String>> localDictionaryAdds = new HashMap<>();
 
-    private Set<Entity> entities;
+    @Builder.Default
+    private Set<Entity> entities = new HashSet<>();
 
     private Set<Entity> nerEntities;
 
@@ -85,12 +91,15 @@ public class Section {
 
     private boolean isInTable;
 
+    @Builder.Default
+    private List<Integer> cellStarts = new ArrayList<>();
+
+    private RedactionServiceSettings redactionServiceSettings;
 
     @Deprecated
     @SuppressWarnings("unused")
     @ThenAction
     public void addAiEntities(@Argument(ArgumentType.TYPE) String type, @Argument(ArgumentType.TYPE) String asType) {
-
         redactOrRecommendAiEntities(type, asType, false, 0, null, null);
     }
 
@@ -1292,7 +1301,7 @@ public class Section {
     }
 
 
-    private Set<Entity> findEntities(String value,
+    public Set<Entity> findEntities(String value,
                                      String asType,
                                      boolean caseInsensitive,
                                      boolean redacted,
@@ -1363,6 +1372,7 @@ public class Section {
                     value.getRowSpanStart() + word.length(),
                     headline,
                     sectionNumber,
+                    -1,
                     false,
                     false,
                     Engine.RULE,
@@ -1675,10 +1685,92 @@ public class Section {
 
     }
 
+
+
+    public boolean findDictionaryEntities(){
+
+        findEntities();
+
+        if (cellStarts != null && !cellStarts.isEmpty()) {
+            SurroundingWordsService.addSurroundingText(entities, searchableText, dictionary, cellStarts, redactionServiceSettings.getSurroundingWordsOffsetWindow(), redactionServiceSettings.getNumberOfSurroundingWords());
+        } else {
+            SurroundingWordsService.addSurroundingText(entities, searchableText, dictionary, redactionServiceSettings.getSurroundingWordsOffsetWindow(), redactionServiceSettings.getNumberOfSurroundingWords());
+        }
+
+        if (!isLocal && manualRedactions != null) {
+
+            var approvedForceRedactions = manualRedactions
+                    .getForceRedactions()
+                    .stream()
+                    .filter(fr -> fr.getStatus() == AnnotationStatus.APPROVED)
+                    .filter(fr -> fr.getRequestDate() != null)
+                    .collect(Collectors.toList());
+            // only approved id removals, that haven't been forced back afterwards
+            var idsToRemove = manualRedactions
+                    .getIdsToRemove()
+                    .stream()
+                    .filter(idr -> idr.getStatus() == AnnotationStatus.APPROVED && !idr.isRemoveFromDictionary())
+                    .filter(idr -> idr.getRequestDate() != null)
+                    .filter(idr -> approvedForceRedactions.stream()
+                            .noneMatch(forceRedact -> forceRedact.getAnnotationId().equals(idr.getAnnotationId()) && forceRedact.getRequestDate()
+                                    .isAfter(idr.getRequestDate())))
+                    .map(IdRemoval::getAnnotationId)
+                    .collect(Collectors.toSet());
+
+            if (images != null && !images.isEmpty() && manualRedactions.getImageRecategorization() != null) {
+                for (Image image : images) {
+                    String imageId = IdBuilder.buildId(image.getPosition(), image.getPage());
+                    for (ManualImageRecategorization imageRecategorization : manualRedactions.getImageRecategorization()) {
+                        if (imageRecategorization.getStatus().equals(AnnotationStatus.APPROVED) && imageRecategorization.getAnnotationId().equals(imageId)) {
+                            image.setType(imageRecategorization.getType());
+                        }
+                    }
+                    if (idsToRemove.contains(imageId)) {
+                        image.setIgnored(true);
+                    }
+                }
+            }
+
+            entities.forEach(entity -> entity.getPositionSequences().forEach(ps -> {
+                if (idsToRemove.contains(ps.getId())) {
+                    entity.setIgnored(true);
+                }
+            }));
+        }
+
+        return false;
     }
 
 
+    private void findEntities() {
 
+        Set<Entity> found = new HashSet<>();
+        String searchableString = searchableText.asString();
 
+        if (StringUtils.isEmpty(searchableString)) {
+            entities = new HashSet<>();
+            return;
+        }
 
+        String lowercaseInputString = searchableString.toLowerCase();
+        for (DictionaryModel model : dictionary.getDictionaryModels()) {
 
+            var searchImplementation = isLocal ? model.getLocalSearch() : model.getEntriesSearch();
+            var entities = EntitySearchUtils.findEntities(model.isCaseInsensitive() ? lowercaseInputString : searchableString,
+                    searchImplementation,
+                    model,
+                    new FindEntityDetails(model.getType(),
+                            headline,
+                            sectionNumber,
+                            !isLocal,
+                            model.isDossierDictionary(),
+                            isLocal ? Engine.RULE : Engine.DICTIONARY,
+                            isLocal ? EntityType.RECOMMENDATION : EntityType.ENTITY));
+
+            EntitySearchUtils.addOrAddEngine(found, entities);
+        }
+
+        entities = EntitySearchUtils.clearAndFindPositions(found, searchableText, dictionary, manualRedactions);
+        nerEntities = EntitySearchUtils.clearAndFindPositions(nerEntities, searchableText, dictionary, manualRedactions);
+    }
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/SectionSearchableTextPair.java

@@ -1,16 +0,0 @@
-package com.iqser.red.service.redaction.v1.server.redaction.model;
-
-import java.util.List;
-
-import lombok.AllArgsConstructor;
-import lombok.Data;
-
-@Data
-@AllArgsConstructor
-public class SectionSearchableTextPair {
-
-    private Section section;
-    private SearchableText searchableText;
-    private List<Integer> cellStarts;
-
-}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/AnalyzeService.java

@@ -41,6 +41,7 @@ import com.iqser.red.service.redaction.v1.server.classification.model.Simplified
 import com.iqser.red.service.redaction.v1.server.classification.model.Text;
 import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient;
 import com.iqser.red.service.redaction.v1.server.client.model.NerEntities;
+import com.iqser.red.service.redaction.v1.server.document.services.DocumentGraphFactory;
 import com.iqser.red.service.redaction.v1.server.exception.RedactionException;
 import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
 import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryIncrement;
@@ -80,6 +81,7 @@ public class AnalyzeService {
     private final SectionGridCreatorService sectionGridCreatorService;
     private final ImageService imageService;
     private final ImportedRedactionService importedRedactionService;
+    private final DocumentGraphFactory documentGraphFactory;
 
 
     @Timed("redactmanager_analyzeDocumentStructure")
@@ -107,7 +109,6 @@ public class AnalyzeService {
         }
 
         List<SectionText> sectionTexts = sectionTextBuilderService.buildSectionText(classifiedDoc);
-
         sectionGridCreatorService.createSectionGrid(classifiedDoc, pageCount);
 
         Text text = new Text(pageCount, sectionTexts);

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DroolsExecutionService.java

@@ -1,11 +1,12 @@
 package com.iqser.red.service.redaction.v1.server.redaction.service;
 
-import com.iqser.red.service.redaction.v1.server.client.RulesClient;
+import java.io.ByteArrayInputStream;
-import com.iqser.red.service.redaction.v1.server.exception.RulesValidationException;
+import java.io.InputStream;
-import com.iqser.red.service.redaction.v1.server.redaction.model.Section;
+import java.nio.charset.StandardCharsets;
-
+import java.util.HashMap;
-import io.micrometer.core.annotation.Timed;
+import java.util.LinkedList;
-import lombok.RequiredArgsConstructor;
+import java.util.List;
+import java.util.Map;
 
 import org.apache.commons.lang3.StringUtils;
 import org.kie.api.KieServices;
@@ -14,13 +15,19 @@ import org.kie.api.builder.KieFileSystem;
 import org.kie.api.builder.KieModule;
 import org.kie.api.runtime.KieContainer;
 import org.kie.api.runtime.KieSession;
+import org.kie.api.runtime.rule.QueryResults;
+import org.kie.api.runtime.rule.QueryResultsRow;
 import org.springframework.stereotype.Service;
 
-import java.io.ByteArrayInputStream;
+import com.iqser.red.service.redaction.v1.server.client.RulesClient;
-import java.io.InputStream;
+import com.iqser.red.service.redaction.v1.server.exception.RulesValidationException;
-import java.nio.charset.StandardCharsets;
+import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
-import java.util.HashMap;
+import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
-import java.util.Map;
+import com.iqser.red.service.redaction.v1.server.redaction.model.Paragraph;
+import com.iqser.red.service.redaction.v1.server.redaction.model.Section;
+
+import io.micrometer.core.annotation.Timed;
+import lombok.RequiredArgsConstructor;
 
 @Service
 @RequiredArgsConstructor
@@ -45,18 +52,29 @@ public class DroolsExecutionService {
 
 
     @Timed("redactmanager_executeRules")
-    public Section executeRules(KieContainer kieContainer, Section section) {
+    public List<Entity> executeRules(KieContainer kieContainer, List<Section> sections, List<Paragraph> paragraphs, Dictionary dictionary) {
 
         KieSession kieSession = kieContainer.newKieSession();
-        kieSession.setGlobal("section", section);
+        kieSession.setGlobal("dictionary", dictionary);
-        kieSession.insert(section);
+        sections.forEach(kieSession::insert);
+        paragraphs.forEach(kieSession::insert);
         kieSession.fireAllRules();
+        List<Entity> entities = getEntities(kieSession);
         kieSession.dispose();
 
-        return section;
+        return entities;
 
     }
 
+    public List<Entity> getEntities(KieSession ks) {
+        List<Entity> entities = new LinkedList<>();
+        QueryResults entitiesResult = ks.getQueryResults("getEntities");
+        for (QueryResultsRow resultsRow : entitiesResult) {
+            entities.add((Entity) resultsRow.get("$result"));
+        }
+        return entities;
+    }
+
 
     public KieContainer updateRules(String dossierTemplateId) {
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionService.java

@@ -1,33 +1,43 @@
 package com.iqser.red.service.redaction.v1.server.redaction.service;
 
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import org.apache.commons.lang3.StringUtils;
+import org.kie.api.runtime.KieContainer;
+import org.springframework.stereotype.Service;
+
 import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.IdRemoval;
-import com.iqser.red.service.persistence.service.v1.api.model.annotations.entitymapped.ManualImageRecategorization;
 import com.iqser.red.service.redaction.v1.model.AnalyzeRequest;
 import com.iqser.red.service.redaction.v1.model.Engine;
 import com.iqser.red.service.redaction.v1.model.FileAttribute;
 import com.iqser.red.service.redaction.v1.server.classification.model.SectionText;
 import com.iqser.red.service.redaction.v1.server.client.model.NerEntities;
 import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
-import com.iqser.red.service.redaction.v1.server.redaction.model.*;
+import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryModel;
+import com.iqser.red.service.redaction.v1.server.redaction.model.Entities;
+import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
+import com.iqser.red.service.redaction.v1.server.redaction.model.EntityPositionSequence;
+import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType;
+import com.iqser.red.service.redaction.v1.server.redaction.model.FindEntitiesResult;
+import com.iqser.red.service.redaction.v1.server.redaction.model.Image;
+import com.iqser.red.service.redaction.v1.server.redaction.model.PageEntities;
+import com.iqser.red.service.redaction.v1.server.redaction.model.Paragraph;
+import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText;
+import com.iqser.red.service.redaction.v1.server.redaction.model.Section;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUtils;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.FindEntityDetails;
-import com.iqser.red.service.redaction.v1.server.redaction.utils.IdBuilder;
 import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;
 
 import io.micrometer.core.annotation.Timed;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
 
-import org.apache.commons.lang3.StringUtils;
-import org.kie.api.runtime.KieContainer;
-import org.springframework.stereotype.Service;
-
-import java.util.*;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
-
 @Slf4j
 @Service
 @RequiredArgsConstructor
@@ -35,7 +45,7 @@ public class EntityRedactionService {
 
     private final RedactionServiceSettings redactionServiceSettings;
     private final DroolsExecutionService droolsExecutionService;
-    private final SurroundingWordsService surroundingWordsService;
+    private final SearchTextWithTextPositionFactory searchTextWithTextPositionFactory;
 
 
     public PageEntities findEntities(Dictionary dictionary, List<SectionText> sectionTexts, KieContainer kieContainer, AnalyzeRequest analyzeRequest, NerEntities nerEntities) {
@@ -54,7 +64,14 @@ public class EntityRedactionService {
             }
 
             Map<Integer, Set<Entity>> hintsPerSectionNumber = getHintsPerSection(findEntitiesResult.getEntities(), dictionary);
-            FindEntitiesResult foundByLocalEntitiesResult = findEntities(sectionTexts, dictionary, kieContainer, analyzeRequest, true, hintsPerSectionNumber, imagesPerPage, nerEntities);
+            FindEntitiesResult foundByLocalEntitiesResult = findEntities(sectionTexts,
+                    dictionary,
+                    kieContainer,
+                    analyzeRequest,
+                    true,
+                    hintsPerSectionNumber,
+                    imagesPerPage,
+                    nerEntities);
             EntitySearchUtils.addEntitiesWithHigherRank(findEntitiesResult.getEntities(), foundByLocalEntitiesResult.getEntities(), dictionary);
             EntitySearchUtils.removeEntitiesContainedInLarger(findEntitiesResult.getEntities());
         }
@@ -75,73 +92,20 @@ public class EntityRedactionService {
                                            Map<Integer, Set<Image>> imagesPerPage,
                                            NerEntities nerEntities) {
 
-        List<SectionSearchableTextPair> sectionSearchableTextPairs = new ArrayList<>();
+        List<Section> sections = new ArrayList<>(reanalysisSections.size());
+
         for (SectionText reanalysisSection : reanalysisSections) {
 
-            Entities entities = findEntities(reanalysisSection.getSearchableText(),
+            Set<Entity> nerFound = new HashSet<>();
-                    reanalysisSection.getHeadline(),
+            if (!local) {
-                    reanalysisSection.getSectionNumber(),
+                nerFound.addAll(getNerValues(reanalysisSection.getSectionNumber(), nerEntities, reanalysisSection.getCellStarts(), reanalysisSection.getHeadline()));
-                    dictionary,
-                    local,
-                    nerEntities,
-                    reanalysisSection.getCellStarts(),
-                    analyzeRequest.getManualRedactions());
-
-            if (reanalysisSection.getCellStarts() != null && !reanalysisSection.getCellStarts().isEmpty()) {
-                surroundingWordsService.addSurroundingText(entities.getEntities(), reanalysisSection.getSearchableText(), dictionary, reanalysisSection.getCellStarts());
-            } else {
-                surroundingWordsService.addSurroundingText(entities.getEntities(), reanalysisSection.getSearchableText(), dictionary);
-            }
-
-            if (!local && analyzeRequest.getManualRedactions() != null) {
-
-                var approvedForceRedactions = analyzeRequest.getManualRedactions()
-                        .getForceRedactions()
-                        .stream()
-                        .filter(fr -> fr.getStatus() == AnnotationStatus.APPROVED)
-                        .filter(fr -> fr.getRequestDate() != null)
-                        .collect(Collectors.toList());
-                // only approved id removals, that haven't been forced back afterwards
-                var idsToRemove = analyzeRequest.getManualRedactions()
-                        .getIdsToRemove()
-                        .stream()
-                        .filter(idr -> idr.getStatus() == AnnotationStatus.APPROVED && !idr.isRemoveFromDictionary())
-                        .filter(idr -> idr.getRequestDate() != null)
-                        .filter(idr -> approvedForceRedactions.stream()
-                                .noneMatch(forceRedact -> forceRedact.getAnnotationId().equals(idr.getAnnotationId()) && forceRedact.getRequestDate()
-                                        .isAfter(idr.getRequestDate())))
-                        .map(IdRemoval::getAnnotationId)
-                        .collect(Collectors.toSet());
-
-                if (reanalysisSection.getImages() != null && !reanalysisSection.getImages().isEmpty() && analyzeRequest.getManualRedactions().getImageRecategorization() != null) {
-                    for (Image image : reanalysisSection.getImages()) {
-                        String imageId = IdBuilder.buildId(image.getPosition(), image.getPage());
-                        for (ManualImageRecategorization imageRecategorization : analyzeRequest.getManualRedactions().getImageRecategorization()) {
-                            if (imageRecategorization.getStatus().equals(AnnotationStatus.APPROVED) && imageRecategorization.getAnnotationId().equals(imageId)) {
-                                image.setType(imageRecategorization.getType());
-                            }
-                        }
-                        if (idsToRemove.contains(imageId)) {
-                            image.setIgnored(true);
-                        }
-                    }
-                }
-
-                entities.getEntities().forEach(entity -> entity.getPositionSequences().forEach(ps -> {
-                    if (idsToRemove.contains(ps.getId())) {
-                        entity.setIgnored(true);
-                    }
-                }));
             }
 
             log.debug("Section {}, Images: {}", reanalysisSection.getSectionNumber(), reanalysisSection.getImages());
-
+            sections.add(Section.builder()
-            sectionSearchableTextPairs.add(new SectionSearchableTextPair(Section.builder()
                     .isLocal(false)
                     .dictionaryTypes(dictionary.getTypes())
-                    .entities(hintsPerSectionNumber != null && hintsPerSectionNumber.containsKey(reanalysisSection.getSectionNumber()) ? Stream.concat(entities.getEntities()
+                    .nerEntities(nerFound)
-                            .stream(), hintsPerSectionNumber.get(reanalysisSection.getSectionNumber()).stream()).collect(Collectors.toSet()) : entities.getEntities())
-                    .nerEntities(entities.getNerEntities())
                     .text(reanalysisSection.getSearchableText().getAsStringWithLinebreaks())
                     .searchText(reanalysisSection.getSearchableText().toString())
                     .headline(reanalysisSection.getHeadline())
@@ -154,45 +118,63 @@ public class EntityRedactionService {
                     .fileAttributes(analyzeRequest.getFileAttributes())
                     .manualRedactions(analyzeRequest.getManualRedactions())
                     .isInTable(reanalysisSection.isTable())
-                    .build(), reanalysisSection.getSearchableText(), reanalysisSection.getCellStarts()));
+                    .redactionServiceSettings(redactionServiceSettings)
+                    .cellStarts(reanalysisSection.getCellStarts())
+                    .build());
 
         }
 
-
         Set<FileAttribute> addedFileAttributes = new HashSet<>();
-        Set<Entity> entities = new HashSet<>();
+        sections.forEach(section -> {
-        sectionSearchableTextPairs.forEach(sectionSearchableTextPair -> {
 
             if (!addedFileAttributes.isEmpty()) {
                 //Section.Builder provides immutable list.
                 List<FileAttribute> mergedFileAttributes = new ArrayList<>();
-                mergedFileAttributes.addAll(sectionSearchableTextPair.getSection().getAddedFileAttributes());
+                mergedFileAttributes.addAll(section.getAddedFileAttributes());
                 mergedFileAttributes.addAll(addedFileAttributes);
-                sectionSearchableTextPair.getSection().setFileAttributes(mergedFileAttributes);
+                section.setFileAttributes(mergedFileAttributes);
+            }
+        });
+
+        List<Paragraph> paragraphs = new ArrayList<>();
+        for (var reanalysisSection : reanalysisSections) {
+            for (int i = 0; i < reanalysisSection.getTextBlocks().size(); ++i) {
+                paragraphs.add(Paragraph.builder()
+                        .paragraphNumber(i)
+                        .sectionNumber(reanalysisSection.getSectionNumber())
+                        .searchTextToTextPosition(searchTextWithTextPositionFactory.buildSearchTextToTextPositionModel(reanalysisSection.getTextBlocks().get(i).getSequences()))
+                        .build());
+            }
         }
 
-            Section analysedSection = droolsExecutionService.executeRules(kieContainer, sectionSearchableTextPair.getSection());
+        List<Entity> entitiesList = droolsExecutionService.executeRules(kieContainer, sections, paragraphs, dictionary);
+        Set<Entity> entities = new HashSet<>(entitiesList);
 
+        sections.forEach(analysedSection -> {
             addedFileAttributes.addAll(analysedSection.getAddedFileAttributes());
 
-            EntitySearchUtils.removeEntitiesContainedInLarger(analysedSection.getEntities());
+            EntitySearchUtils.removeEntitiesContainedInLarger(entities);
 
-            var entriesWithoutSurroundingText = analysedSection.getEntities()
+            var entriesWithoutSurroundingText = entities.stream()
-                    .stream()
+                    .filter(e -> e.getSectionNumber() == analysedSection.getSectionNumber())
                     .filter(e -> e.getTextAfter() == null && e.getTextBefore() == null)
                     .collect(Collectors.toSet());
 
-            if (sectionSearchableTextPair.getCellStarts() != null && !sectionSearchableTextPair.getCellStarts().isEmpty()) {
+            if (analysedSection.getCellStarts() != null && !analysedSection.getCellStarts().isEmpty()) {
-                surroundingWordsService.addSurroundingText(entriesWithoutSurroundingText,
+                SurroundingWordsService.addSurroundingText(entriesWithoutSurroundingText,
-                        sectionSearchableTextPair.getSearchableText(),
+                        analysedSection.getSearchableText(),
                         dictionary,
-                        sectionSearchableTextPair.getCellStarts());
+                        analysedSection.getCellStarts(),
+                        redactionServiceSettings.getSurroundingWordsOffsetWindow(),
+                        redactionServiceSettings.getNumberOfSurroundingWords());
             } else {
-                surroundingWordsService.addSurroundingText(entriesWithoutSurroundingText, sectionSearchableTextPair.getSearchableText(), dictionary);
+                SurroundingWordsService.addSurroundingText(entriesWithoutSurroundingText,
+                        analysedSection.getSearchableText(),
+                        dictionary,
+                        redactionServiceSettings.getSurroundingWordsOffsetWindow(),
+                        redactionServiceSettings.getNumberOfSurroundingWords());
             }
 
-            entities.addAll(analysedSection.getEntities());
-
             if (!local) {
                 for (Image image : analysedSection.getImages()) {
                     imagesPerPage.computeIfAbsent(image.getPage(), (a) -> new HashSet<>()).add(image);
@@ -225,6 +207,7 @@ public class EntityRedactionService {
                                 entity.getHeadline(),
                                 entity.getMatchedRule(),
                                 entity.getSectionNumber(),
+                                -1,
                                 entity.getLegalBasis(),
                                 entity.isDictionaryEntry(),
                                 entity.getTextBefore(),
@@ -328,6 +311,7 @@ public class EntityRedactionService {
                             res.getEndOffset(),
                             headline,
                             sectionNumber,
+                            -1,
                             false,
                             false,
                             Engine.NER,
@@ -347,6 +331,7 @@ public class EntityRedactionService {
                                 res.getEndOffset(),
                                 headline,
                                 sectionNumber,
+                                -1,
                                 false,
                                 false,
                                 Engine.NER,

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ManualRedactionSurroundingTextService.java

@@ -22,6 +22,7 @@ import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUtils;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.FindEntityDetails;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.SearchImplementation;
+import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;
 import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
 
 import io.micrometer.core.annotation.Timed;
@@ -34,8 +35,7 @@ import lombok.extern.slf4j.Slf4j;
 public class ManualRedactionSurroundingTextService {
 
     private final RedactionStorageService redactionStorageService;
-    private final SurroundingWordsService surroundingWordsService;
+    private final RedactionServiceSettings redactionServiceSettings;
-
 
     @Timed("redactmanager_surroundingTextAnalysis")
     public AnalyzeResult addSurroundingText(String dossierId, String fileId, ManualRedactions manualRedactions) {
@@ -87,9 +87,9 @@ public class ManualRedactionSurroundingTextService {
         }
 
         if (sectionText.getCellStarts() != null && !sectionText.getCellStarts().isEmpty()) {
-            surroundingWordsService.addSurroundingText(Set.of(correctEntity), sectionText.getSearchableText(), null, sectionText.getCellStarts());
+            SurroundingWordsService.addSurroundingText(Set.of(correctEntity), sectionText.getSearchableText(), null, sectionText.getCellStarts(), redactionServiceSettings.getSurroundingWordsOffsetWindow(), redactionServiceSettings.getNumberOfSurroundingWords());
         } else {
-            surroundingWordsService.addSurroundingText(Set.of(correctEntity), sectionText.getSearchableText(), null);
+            SurroundingWordsService.addSurroundingText(Set.of(correctEntity), sectionText.getSearchableText(), null, redactionServiceSettings.getSurroundingWordsOffsetWindow(), redactionServiceSettings.getNumberOfSurroundingWords());
         }
 
         return Pair.of(correctEntity.getTextBefore(), correctEntity.getTextAfter());

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/SearchTextWithTextPositionFactory.java

@@ -0,0 +1,91 @@
+package com.iqser.red.service.redaction.v1.server.redaction.service;
+
+import java.awt.geom.Rectangle2D;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Objects;
+
+import org.springframework.stereotype.Service;
+
+import com.iqser.red.service.redaction.v1.server.parsing.model.RedTextPosition;
+import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
+import com.iqser.red.service.redaction.v1.server.redaction.model.RedRectangle2D;
+import com.iqser.red.service.redaction.v1.server.redaction.model.SearchTextWithTextPositionModel;
+
+@Service
+public class SearchTextWithTextPositionFactory {
+
+    public SearchTextWithTextPositionModel buildSearchTextToTextPositionModel(List<TextPositionSequence> sequences) {
+
+        List<Integer> stringIdxToPositionIdx = new LinkedList<>();
+        List<Integer> lineBreaksStringIdx = new LinkedList<>();
+        StringBuilder sb = new StringBuilder();
+
+        int stringIdx = 0;
+        int positionIdx = 0;
+
+        String currentUnicode;
+        String previousUnicode = " ";
+
+        for (TextPositionSequence word : sequences) {
+            for (int i = 0; i < word.getTextPositions().size(); ++i) {
+
+                currentUnicode = word.getTextPositions().get(i).getUnicode();
+
+                if (isLineBreak(currentUnicode)) {
+                    lineBreaksStringIdx.add(stringIdx);
+                } else if (!isRepeatedWhitespace(currentUnicode, previousUnicode) && //
+                        !isHyphenLinebreak(currentUnicode)) {
+
+                    sb.append(currentUnicode);
+                    stringIdxToPositionIdx.add(positionIdx);
+                    ++stringIdx;
+                }
+
+                previousUnicode = currentUnicode;
+                ++positionIdx;
+            }
+
+            previousUnicode = " ";
+            sb.append(previousUnicode);
+            stringIdxToPositionIdx.add(positionIdx);
+            ++stringIdx;
+        }
+
+        assert sb.length() == stringIdxToPositionIdx.size();
+
+        List<RedRectangle2D> positions = sequences.stream()//
+                .map(TextPositionSequence::getTextPositions)//
+                .flatMap(List::stream)//
+                .map(RedTextPosition::getPosition)//
+                .map(a -> (RedRectangle2D) new RedRectangle2D(a[0], a[1], a[2], a[3]))
+                .toList();
+
+        return SearchTextWithTextPositionModel.builder()
+                .searchText(sb.toString())
+                .lineBreaks(lineBreaksStringIdx)
+                .stringCoordsToPositionCoords(stringIdxToPositionIdx)
+                .positions(positions)
+                .build();
+    }
+
+
+    private boolean isLineBreak(String currentUnicode) {
+
+        return Objects.equals(currentUnicode, "\n");
+    }
+
+
+    private boolean isRepeatedWhitespace(String currentUnicode, String previousUnicode) {
+
+        return Objects.equals(previousUnicode, " ") && Objects.equals(currentUnicode, " ");
+    }
+
+
+    private boolean isHyphenLinebreak(String unicodeCharacter) {
+
+        return unicodeCharacter.matches("([^\\s\\d\\-]{2,500})[\\-\\u00AD]\\R");
+
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/SectionGridCreatorService.java

@@ -4,7 +4,7 @@ import com.iqser.red.service.redaction.v1.model.CellRectangle;
 import com.iqser.red.service.redaction.v1.model.Point;
 import com.iqser.red.service.redaction.v1.model.SectionRectangle;
 import com.iqser.red.service.redaction.v1.server.classification.model.Document;
-import com.iqser.red.service.redaction.v1.server.classification.model.Paragraph;
+import com.iqser.red.service.redaction.v1.server.classification.model.Section;
 import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.Cell;
@@ -31,7 +31,7 @@ public class SectionGridCreatorService {
 
     private void addSectionGrid(Document classifiedDoc, int page) {
 
-        for (Paragraph paragraph : classifiedDoc.getParagraphs()) {
+        for (Section paragraph : classifiedDoc.getSections()) {
 
             for (int i = 0; i <= paragraph.getPageBlocks().size() - 1; i++) {
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/SectionTextBuilderService.java

@@ -16,7 +16,7 @@ import com.iqser.red.service.redaction.v1.model.SectionArea;
 import com.iqser.red.service.redaction.v1.server.classification.model.Document;
 import com.iqser.red.service.redaction.v1.server.classification.model.Footer;
 import com.iqser.red.service.redaction.v1.server.classification.model.Header;
-import com.iqser.red.service.redaction.v1.server.classification.model.Paragraph;
+import com.iqser.red.service.redaction.v1.server.classification.model.Section;
 import com.iqser.red.service.redaction.v1.server.classification.model.SectionText;
 import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
 import com.iqser.red.service.redaction.v1.server.classification.model.UnclassifiedText;
@@ -40,14 +40,14 @@ public class SectionTextBuilderService {
 
         List<SectionText> sectionTexts = new ArrayList<>();
         AtomicInteger sectionNumber = new AtomicInteger(1);
-        for (Paragraph paragraph : classifiedDoc.getParagraphs()) {
+        for (Section section : classifiedDoc.getSections()) {
 
-            List<Table> tables = paragraph.getTables();
+            List<Table> tables = section.getTables();
             for (Table table : tables) {
                 sectionTexts.addAll(processTablePerRow(table, sectionNumber));
                 sectionNumber.incrementAndGet();
             }
-            sectionTexts.add(processText(paragraph.getSearchableText(), paragraph.getTextBlocks(), paragraph.getHeadline(), sectionNumber, paragraph.getImages()));
+            sectionTexts.add(processText(section.getSearchableText(), section.getTextBlocks(), section.getHeadline(), sectionNumber, section.getImages()));
             sectionNumber.incrementAndGet();
         }
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/SurroundingWordsService.java

@@ -7,6 +7,7 @@ import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettin
 
 import io.micrometer.core.annotation.Timed;
 import lombok.RequiredArgsConstructor;
+import lombok.experimental.UtilityClass;
 import lombok.extern.slf4j.Slf4j;
 
 import org.springframework.stereotype.Service;
@@ -15,15 +16,12 @@ import java.util.List;
 import java.util.Set;
 
 @Slf4j
-@Service
+@UtilityClass
-@RequiredArgsConstructor
 public class SurroundingWordsService {
 
-    private final RedactionServiceSettings redactionServiceSettings;
-
 
     @Timed("redactmanager_addSurroundingText")
-    public void addSurroundingText(Set<Entity> entities, SearchableText searchableText, Dictionary dictionary) {
+    public void addSurroundingText(Set<Entity> entities, SearchableText searchableText, Dictionary dictionary, int surroundingWordsOffsetWindow, int numberOfSurroundingWords) {
 
         if (entities.isEmpty()) {
             return;
@@ -35,7 +33,7 @@ public class SurroundingWordsService {
                 if (dictionary != null && dictionary.isHint(entity.getType())) {
                     continue;
                 }
-                findSurroundingWords(entity, searchableText.asString(), entity.getStart(), entity.getEnd());
+                findSurroundingWords(entity, searchableText.asString(), entity.getStart(), entity.getEnd(), surroundingWordsOffsetWindow, numberOfSurroundingWords);
             }
         } catch (Exception e) {
             log.warn("Could not get surrounding text!");
@@ -44,7 +42,7 @@ public class SurroundingWordsService {
 
 
     @Timed("redactmanager_addSurroundingTextTables")
-    public void addSurroundingText(Set<Entity> entities, SearchableText searchableText, Dictionary dictionary, List<Integer> cellstarts) {
+    public void addSurroundingText(Set<Entity> entities, SearchableText searchableText, Dictionary dictionary, List<Integer> cellstarts, int surroundingWordsOffsetWindow, int numberOfSurroundingWords) {
 
         if (entities.isEmpty()) {
             return;
@@ -75,7 +73,7 @@ public class SurroundingWordsService {
                         if (entity.getStart() >= startOffset && entity.getEnd() <= endOffset) {
                             int entityStartOffset = entity.getStart() - startOffset;
                             int entityEndOffset = entity.getEnd() - startOffset;
-                            findSurroundingWords(entity, text, entityStartOffset, entityEndOffset);
+                            findSurroundingWords(entity, text, entityStartOffset, entityEndOffset, surroundingWordsOffsetWindow, numberOfSurroundingWords);
                         }
                     }
                 }
@@ -86,23 +84,23 @@ public class SurroundingWordsService {
     }
 
 
-    private void findSurroundingWords(Entity entity, String text, int entityStartOffset, int entityEndOffset) {
+    private void findSurroundingWords(Entity entity, String text, int entityStartOffset, int entityEndOffset, int surroundingWordsOffsetWindow, int numberOfSurroundingWords) {
 
-        int offsetBefore = entityStartOffset - redactionServiceSettings.getSurroundingWordsOffsetWindow() < 0 ? 0 : entityStartOffset - redactionServiceSettings.getSurroundingWordsOffsetWindow();
+        int offsetBefore = entityStartOffset - surroundingWordsOffsetWindow < 0 ? 0 : entityStartOffset - surroundingWordsOffsetWindow;
         String textBefore = text.substring(offsetBefore, entityStartOffset);
         if (!textBefore.isBlank()) {
             String[] wordsBefore = textBefore.split(" ");
-            int numberOfWordsBefore = wordsBefore.length > redactionServiceSettings.getNumberOfSurroundingWords() ? redactionServiceSettings.getNumberOfSurroundingWords() : wordsBefore.length;
+            int numberOfWordsBefore = wordsBefore.length > numberOfSurroundingWords ? numberOfSurroundingWords : wordsBefore.length;
             if (wordsBefore.length > 0) {
                 entity.setTextBefore(concatWordsBefore(wordsBefore, numberOfWordsBefore, textBefore.endsWith(" ")));
             }
         }
 
-        int endOffset = entityEndOffset + redactionServiceSettings.getSurroundingWordsOffsetWindow() > text.length() ? text.length() : entityEndOffset + redactionServiceSettings.getSurroundingWordsOffsetWindow();
+        int endOffset = entityEndOffset + surroundingWordsOffsetWindow > text.length() ? text.length() : entityEndOffset + surroundingWordsOffsetWindow;
         String textAfter = text.substring(entityEndOffset, endOffset);
         if (!textAfter.isBlank()) {
             String[] wordsAfter = textAfter.split(" ");
-            int numberOfWordsAfter = wordsAfter.length > redactionServiceSettings.getNumberOfSurroundingWords() ? redactionServiceSettings.getNumberOfSurroundingWords() : wordsAfter.length;
+            int numberOfWordsAfter = wordsAfter.length > numberOfSurroundingWords ? numberOfSurroundingWords : wordsAfter.length;
             if (wordsAfter.length > 0) {
                 entity.setTextAfter(concatWordsAfter(wordsAfter, numberOfWordsAfter, textAfter.startsWith(" ")));
             }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/EntitySearchUtils.java

@@ -1,10 +1,12 @@
 package com.iqser.red.service.redaction.v1.server.redaction.utils;
 
+import java.awt.geom.Rectangle2D;
 import java.util.ArrayList;
 import java.util.Comparator;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Iterator;
+import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
@@ -13,12 +15,15 @@ import java.util.stream.Collectors;
 
 import com.iqser.red.service.persistence.service.v1.api.model.annotations.AnnotationStatus;
 import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
+import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
 import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
 import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryModel;
 import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
 import com.iqser.red.service.redaction.v1.server.redaction.model.EntityPositionSequence;
 import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType;
 import com.iqser.red.service.redaction.v1.server.redaction.model.Image;
+import com.iqser.red.service.redaction.v1.server.redaction.model.RedRectangle2D;
+import com.iqser.red.service.redaction.v1.server.redaction.model.SearchTextWithTextPositionModel;
 import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText;
 
 import lombok.experimental.UtilityClass;
@@ -29,6 +34,102 @@ import lombok.extern.slf4j.Slf4j;
 @SuppressWarnings("PMD")
 public final class EntitySearchUtils {
 
+    public static List<EntityPositionSequence> findEntityPositionSequences(String entityString, SearchTextWithTextPositionModel searchTextWithTextPositionModel) {
+
+        return findOccurrences(entityString, searchTextWithTextPositionModel.getSearchText()).stream()
+                .map(range -> mapToTextPositionCoordinates(range, searchTextWithTextPositionModel.getStringCoordsToPositionCoords()))
+                .map(range -> mapToTextPositions(range, searchTextWithTextPositionModel.getPositions()))
+                .map(EntitySearchUtils::mapToEntityPositionSequence)
+                .toList();
+    }
+
+
+    private static List<TextPositionSequence> mapToTextPositions(IndexRange range, List<RedRectangle2D> positions) {
+
+        var slicedPositions = positions.subList(range.startIdx(), range.endIdx());
+        List<TextPositionSequence> sequencesInRange = new LinkedList<>();
+        //sequencesInRange.add()
+        return sequencesInRange;
+    }
+
+
+    private static EntityPositionSequence mapToEntityPositionSequence(List<TextPositionSequence> textPositionSequences) {
+
+        return EntityPositionSequence.builder() //
+                .id(IdBuilder.buildId(textPositionSequences)) //
+                .pageNumber(textPositionSequences.get(0).getPage()) //
+                .sequences(textPositionSequences) //
+                .build();
+    }
+
+
+    private static TextPositionSequence sliceTextPositionSequence(TextPositionSequence sequence, int start, int end) {
+
+        int startIndex = Math.max(start, 0);
+        int endIndex = Math.min(end, sequence.length());
+        return TextPositionSequence.builder()
+                .dir(sequence.getDir())
+                .page(sequence.getPage())
+                .pageHeight(sequence.getPageHeight())
+                .pageWidth(sequence.getPageWidth())
+                .rotation(sequence.getRotation())
+                .textPositions(sequence.getTextPositions().subList(startIndex, endIndex))
+                .build();
+    }
+
+
+    private static boolean sequenceBeforeRange(int end, IndexRange range) {
+
+        return end <= range.startIdx;
+    }
+
+
+    private static boolean sequenceBehindRange(int start, IndexRange range) {
+
+        return start >= range.endIdx;
+    }
+
+
+    private static boolean partialSequenceInRange(int start, int length, IndexRange range) {
+
+        return start >= range.startIdx || (start + length) <= range.endIdx;
+    }
+
+
+    private static boolean fullSequenceInRange(int start, int length, IndexRange range) {
+
+        return start >= range.startIdx && (start + length) <= range.endIdx;
+    }
+
+
+    private static IndexRange mapToTextPositionCoordinates(IndexRange range, List<Integer> stringCoordsToTextPositionCoords) {
+
+        return new IndexRange(stringCoordsToTextPositionCoords.get(range.startIdx), stringCoordsToTextPositionCoords.get(range.endIdx));
+    }
+
+
+    private static List<IndexRange> findOccurrences(String subString, String searchString) {
+
+        List<IndexRange> found = new LinkedList<>();
+
+
+            String cleanValue = subString.trim();
+
+            int startIndex;
+            int stopIndex = 0;
+            do {
+                startIndex = searchString.indexOf(cleanValue, stopIndex);
+                stopIndex = startIndex + cleanValue.length();
+
+                if (startIndex > -1) {
+                    found.add(new IndexRange(startIndex, stopIndex));
+                }
+            } while (startIndex > -1);
+
+        return found;
+    }
+
+
     public boolean sectionContainsAny(String sectionText, SearchImplementation searchImplementation) {
 
         return searchImplementation.atLeastOneMatches(sectionText);
@@ -63,7 +164,7 @@ public final class EntitySearchUtils {
 
         Set<Entity> entities = new HashSet<>();
 
-        searchImplementation.getMatches(inputString).forEach(match -> validateAndAddEntity(entities, findEntityDetails, inputString, match.getStartIndex(), match.getEndIndex()));
+        searchImplementation.getMatches(inputString).forEach(match -> validateAndAddEntity(entities, findEntityDetails, inputString, match.startIndex(), match.endIndex()));
 
         return entities;
     }
@@ -79,6 +180,7 @@ public final class EntitySearchUtils {
                     stopIndex,
                     findEntityDetails.getHeadline(),
                     findEntityDetails.getSectionNumber(),
+                    -1,
                     findEntityDetails.isDictionaryEntry(),
                     findEntityDetails.isDossierDictionary(),
                     findEntityDetails.getEngine(),
@@ -99,7 +201,7 @@ public final class EntitySearchUtils {
 
             List<Entity> orderedEntities = entitiesByWord.get(word).stream().sorted(Comparator.comparing(Entity::getStart)).collect(Collectors.toList());
             Entity firstEntity = orderedEntities.get(0);
-            List<EntityPositionSequence> positionSequences = text.getSequences(firstEntity.getWord().trim(),
+            List<EntityPositionSequence> positionSequences = text.getSequences(word.trim(),
                     dictionary == null || dictionary.isCaseInsensitiveDictionary(firstEntity.getType()),
                     firstEntity.getTargetSequences());
 
@@ -358,7 +460,7 @@ public final class EntitySearchUtils {
 
         for (Entity existing : existingEntities) {
 
-            // skip if either start or end is equal
+            // skip if either startIdx or endIdx is equal
             if (existing.getStart().equals(found.getStart()) || existing.getEnd().equals(found.getEnd())) {
                 continue;
             }
@@ -374,4 +476,9 @@ public final class EntitySearchUtils {
         return false;
     }
 
+
+    record IndexRange(int startIdx, int endIdx) {
+
+    }
+
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/OffsetStringUtils.java

@@ -16,8 +16,8 @@ public final class OffsetStringUtils {
      * Same logic as in StringUtils.redactBetween, but returns a list of object with offsets insteadof on the Strings only.
      *
      * @param str – the String containing the substrings, null returns null, empty returns empty
-     * @param open – the String identifying the start of the substring, empty returns null
+     * @param open – the String identifying the startIdx of the substring, empty returns null
-     * @param close – the String identifying the end of the substring, empty returns null
+     * @param close – the String identifying the endIdx of the substring, empty returns null
      * @return a list of Strings with their offsets
      */
     public List<OffsetString> substringsBetween(final String str, final String open, final String close) {

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/SearchImplementation.java

@@ -1,10 +1,5 @@
 package com.iqser.red.service.redaction.v1.server.redaction.utils;
 
-import lombok.AllArgsConstructor;
-import lombok.Data;
-
-import org.ahocorasick.trie.Trie;
-
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.List;
@@ -12,6 +7,10 @@ import java.util.Locale;
 import java.util.regex.Pattern;
 import java.util.stream.Collectors;
 
+import org.ahocorasick.trie.Trie;
+
+import lombok.Data;
+
 @Data
 public class SearchImplementation {
 
@@ -74,7 +73,18 @@ public class SearchImplementation {
         } else {
             return this.trie.containsMatch(textToCheck);
         }
+    }
 
+    public List<MatchPosition> getMatches(CharSequence text) {
+        if (this.values.isEmpty()) {
+            return new ArrayList<>();
+        }
+        if (this.pattern != null) {
+
+            return this.pattern.matcher(text).results().map(r -> new MatchPosition(r.start(), r.end())).collect(Collectors.toList());
+        } else {
+            return this.trie.parseText(text).stream().map(r -> new MatchPosition(r.getStart(), r.getEnd() + 1)).collect(Collectors.toList());
+        }
     }
 
 
@@ -94,13 +104,7 @@ public class SearchImplementation {
         }
     }
 
-
+    public record MatchPosition(int startIndex, int endIndex) {
-    @Data
-    @AllArgsConstructor
-    public static class MatchPosition {
-
-        private int startIndex;
-        private int endIndex;
 
     }
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/segmentation/SectionsBuilderService.java

@@ -15,7 +15,7 @@ import com.iqser.red.service.redaction.v1.server.classification.model.Document;
 import com.iqser.red.service.redaction.v1.server.classification.model.Footer;
 import com.iqser.red.service.redaction.v1.server.classification.model.Header;
 import com.iqser.red.service.redaction.v1.server.classification.model.Page;
-import com.iqser.red.service.redaction.v1.server.classification.model.Paragraph;
+import com.iqser.red.service.redaction.v1.server.classification.model.Section;
 import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
 import com.iqser.red.service.redaction.v1.server.classification.model.UnclassifiedText;
 import com.iqser.red.service.redaction.v1.server.redaction.model.PdfImage;
@@ -32,7 +32,7 @@ public class SectionsBuilderService {
     public void buildSections(Document document) {
 
         List<AbstractTextContainer> chunkWords = new ArrayList<>();
-        List<Paragraph> chunkBlockList = new ArrayList<>();
+        List<Section> chunkBlockList = new ArrayList<>();
         List<Header> headers = new ArrayList<>();
         List<Footer> footers = new ArrayList<>();
         List<UnclassifiedText> unclassifiedTexts = new ArrayList<>();
@@ -69,7 +69,7 @@ public class SectionsBuilderService {
                 }
 
                 if (prev != null && current.getClassification().startsWith("H ") && !prev.getClassification().startsWith("H ") || !document.isHeadlines()) {
-                    Paragraph chunkBlock = buildTextBlock(chunkWords, lastHeadline);
+                    Section chunkBlock = buildTextBlock(chunkWords, lastHeadline);
                     chunkBlock.setHeadline(lastHeadline);
                     if (document.isHeadlines()) {
                         lastHeadline = current.getText();
@@ -101,11 +101,11 @@ public class SectionsBuilderService {
             }
         }
 
-        Paragraph chunkBlock = buildTextBlock(chunkWords, lastHeadline);
+        Section chunkBlock = buildTextBlock(chunkWords, lastHeadline);
         chunkBlock.setHeadline(lastHeadline);
         chunkBlockList.add(chunkBlock);
 
-        document.setParagraphs(chunkBlockList);
+        document.setSections(chunkBlockList);
         document.setHeaders(headers);
         document.setFooters(footers);
         document.setUnclassifiedTexts(unclassifiedTexts);
@@ -114,8 +114,8 @@ public class SectionsBuilderService {
 
     public void addImagesToSections(Document document) {
 
-        Map<Integer, List<Paragraph>> paragraphMap = new HashMap<>();
+        Map<Integer, List<Section>> paragraphMap = new HashMap<>();
-        for (Paragraph paragraph : document.getParagraphs()) {
+        for (Section paragraph : document.getSections()) {
             for (AbstractTextContainer container : paragraph.getPageBlocks()) {
 
                 paragraphMap.computeIfAbsent(container.getPage(), c -> new ArrayList<>()).add(paragraph);
@@ -124,22 +124,22 @@ public class SectionsBuilderService {
         }
 
         if (paragraphMap.isEmpty()) {
-            Paragraph paragraph = new Paragraph();
+            Section paragraph = new Section();
-            document.getParagraphs().add(paragraph);
+            document.getSections().add(paragraph);
             paragraphMap.computeIfAbsent(1, x -> new ArrayList<>()).add(paragraph);
         }
 
         // first page is always a paragraph, else we can't process pages 1..N,
         // where N is the first found page with a paragraph
         if (paragraphMap.get(1) == null) {
-            Paragraph paragraph = new Paragraph();
+            Section paragraph = new Section();
-            document.getParagraphs().add(paragraph);
+            document.getSections().add(paragraph);
             paragraphMap.computeIfAbsent(1, x -> new ArrayList<>()).add(paragraph);
         }
 
         for (Page page : document.getPages()) {
             for (PdfImage image : page.getImages()) {
-                List<Paragraph> paragraphsOnPage = paragraphMap.get(page.getPageNumber());
+                List<Section> paragraphsOnPage = paragraphMap.get(page.getPageNumber());
                 if (paragraphsOnPage == null) {
                     int i = page.getPageNumber();
                     while (paragraphsOnPage == null) {
@@ -147,7 +147,7 @@ public class SectionsBuilderService {
                         i--;
                     }
                 }
-                for (Paragraph paragraph : paragraphsOnPage) {
+                for (Section paragraph : paragraphsOnPage) {
                     Float xMin = null;
                     Float yMin = null;
                     Float xMax = null;
@@ -239,9 +239,9 @@ public class SectionsBuilderService {
     }
 
 
-    private Paragraph buildTextBlock(List<AbstractTextContainer> wordBlockList, String lastHeadline) {
+    private Section buildTextBlock(List<AbstractTextContainer> wordBlockList, String lastHeadline) {
 
-        Paragraph paragraph = new Paragraph();
+        Section section = new Section();
         TextBlock textBlock = null;
 
         int pageBefore = -1;
@@ -268,40 +268,17 @@ public class SectionsBuilderService {
                 }
 
                 if (textBlock != null && !alreadyAdded) {
-                    paragraph.getPageBlocks().add(textBlock);
+                    section.getPageBlocks().add(textBlock);
                     alreadyAdded = true;
                 }
-                paragraph.getPageBlocks().add(table);
+                section.getPageBlocks().add(table);
                 continue;
             }
 
             TextBlock wordBlock = (TextBlock) container;
-
+            section.getPageBlocks().add(wordBlock);
-            if (textBlock == null) {
-                textBlock = new TextBlock(wordBlock.getMinX(), wordBlock.getMaxX(), wordBlock.getMinY(), wordBlock.getMaxY(), wordBlock.getSequences(), wordBlock.getRotation());
-                textBlock.setPage(wordBlock.getPage());
-            } else if (splitByTable) {
-                textBlock = new TextBlock(wordBlock.getMinX(), wordBlock.getMaxX(), wordBlock.getMinY(), wordBlock.getMaxY(), wordBlock.getSequences(), wordBlock.getRotation());
-                textBlock.setPage(wordBlock.getPage());
-                alreadyAdded = false;
-            } else if (pageBefore != -1 && wordBlock.getPage() != pageBefore) {
-                textBlock.setPage(pageBefore);
-                paragraph.getPageBlocks().add(textBlock);
-                textBlock = new TextBlock(wordBlock.getMinX(), wordBlock.getMaxX(), wordBlock.getMinY(), wordBlock.getMaxY(), wordBlock.getSequences(), wordBlock.getRotation());
-                textBlock.setPage(wordBlock.getPage());
-            } else {
-                TextBlock spatialEntity = textBlock.union(wordBlock);
-                textBlock.resize(spatialEntity.getMinX(), spatialEntity.getMinY(), spatialEntity.getWidth(), spatialEntity.getHeight());
         }
-            pageBefore = wordBlock.getPage();
+        return section;
-            splitByTable = false;
-            previous = container;
-        }
-
-        if (textBlock != null && !alreadyAdded) {
-            paragraph.getPageBlocks().add(textBlock);
-        }
-        return paragraph;
     }
 
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/visualization/service/PdfVisualisationService.java

@@ -12,7 +12,7 @@ import org.springframework.stereotype.Service;
 
 import com.iqser.red.service.redaction.v1.server.classification.model.Document;
 import com.iqser.red.service.redaction.v1.server.classification.model.Page;
-import com.iqser.red.service.redaction.v1.server.classification.model.Paragraph;
+import com.iqser.red.service.redaction.v1.server.classification.model.Section;
 import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.Cell;
@@ -37,7 +37,7 @@ public class PdfVisualisationService {
             PDPage pdPage = document.getPage(page - 1);
             PDPageContentStream contentStream = new PDPageContentStream(document, pdPage, PDPageContentStream.AppendMode.APPEND, true);
 
-            for (Paragraph paragraph : classifiedDoc.getParagraphs()) {
+            for (Section paragraph : classifiedDoc.getSections()) {
 
                 for (int i = 0; i <= paragraph.getPageBlocks().size() - 1; i++) {
 

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AbstractTestWithDictionaries.java

@@ -0,0 +1,520 @@
+package com.iqser.red.service.redaction.v1.server;
+
+import static org.mockito.Mockito.when;
+
+import java.io.BufferedReader;
+import java.io.ByteArrayInputStream;
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.net.URL;
+import java.nio.charset.StandardCharsets;
+import java.time.OffsetDateTime;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.kie.api.KieServices;
+import org.kie.api.builder.KieBuilder;
+import org.kie.api.builder.KieFileSystem;
+import org.kie.api.builder.KieModule;
+import org.kie.api.runtime.KieContainer;
+import org.mockito.stubbing.Answer;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
+import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration;
+import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.boot.test.mock.mockito.MockBean;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.context.annotation.Import;
+import org.springframework.context.annotation.Primary;
+import org.springframework.core.io.ClassPathResource;
+import org.springframework.test.context.junit.jupiter.SpringExtension;
+
+import com.iqser.red.service.persistence.service.v1.api.model.common.JSONPrimitive;
+import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.configuration.Colors;
+import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
+import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntry;
+import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.Type;
+import com.iqser.red.service.redaction.v1.model.AnalyzeRequest;
+import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
+import com.iqser.red.service.redaction.v1.server.client.RulesClient;
+import com.iqser.red.service.redaction.v1.server.redaction.utils.ResourceLoader;
+import com.iqser.red.service.redaction.v1.server.redaction.utils.TextNormalizationUtilities;
+import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
+import com.iqser.red.storage.commons.StorageAutoConfiguration;
+import com.iqser.red.storage.commons.service.StorageService;
+
+import lombok.SneakyThrows;
+
+@ExtendWith(SpringExtension.class)
+@SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
+@Import(AbstractTestWithDictionaries.TestConfiguration.class)
+public class AbstractTestWithDictionaries {
+
+    private static final String RULES = loadFromClassPath("drools/rules.drl");
+    private static final String VERTEBRATE = "vertebrate";
+    private static final String ADDRESS = "CBI_address";
+    private static final String AUTHOR = "CBI_author";
+    private static final String SPONSOR = "CBI_sponsor";
+    private static final String NO_REDACTION_INDICATOR = "no_redaction_indicator";
+    private static final String REDACTION_INDICATOR = "redaction_indicator";
+    private static final String HINT_ONLY = "hint_only";
+    private static final String MUST_REDACT = "must_redact";
+    private static final String PUBLISHED_INFORMATION = "published_information";
+    private static final String TEST_METHOD = "test_method";
+    private static final String PURITY = "purity";
+    private static final String IMAGE = "image";
+    private static final String LOGO = "logo";
+    private static final String SIGNATURE = "signature";
+    private static final String FORMULA = "formula";
+    private static final String OCR = "ocr";
+    private static final String DOSSIER_REDACTIONS = "dossier_redactions";
+    private static final String IMPORTED_REDACTION = "imported_redaction";
+    private static final String PII = "PII";
+    private static final String ROTATE_SIMPLE = "RotateSimple";
+
+    protected final static String TEST_DOSSIER_TEMPLATE_ID = "123";
+    protected final static String TEST_DOSSIER_ID = "123";
+    protected final static String TEST_FILE_ID = "123";
+
+    @Autowired
+    private StorageService storageService;
+
+    @MockBean
+    private DictionaryClient dictionaryClient;
+
+    @MockBean
+    private RulesClient rulesClient;
+
+    private final Map<String, List<String>> dictionary = new HashMap<>();
+    private final Map<String, List<String>> dossierDictionary = new HashMap<>();
+    private final Map<String, List<String>> falsePositive = new HashMap<>();
+    private final Map<String, List<String>> falseRecommendation = new HashMap<>();
+    private final Map<String, String> typeColorMap = new HashMap<>();
+    private final Map<String, Boolean> hintTypeMap = new HashMap<>();
+    private final Map<String, Boolean> caseInSensitiveMap = new HashMap<>();
+    private final Map<String, Boolean> recommendationTypeMap = new HashMap<>();
+    private final Map<String, Integer> rankTypeMap = new HashMap<>();
+    private final Colors colors = new Colors();
+    private final Map<String, Long> reanalysisVersions = new HashMap<>();
+    private final Set<String> deleted = new HashSet<>();
+
+
+    @BeforeEach
+    public void stubClients() {
+
+        when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L);
+        when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(JSONPrimitive.of(RULES));
+
+        loadDictionaryForTest();
+        loadTypeForTest();
+        loadNerForTest();
+        when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L);
+        when(dictionaryClient.getAllTypesForDossierTemplate(TEST_DOSSIER_TEMPLATE_ID, false)).thenReturn(getTypeResponse());
+
+        when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L);
+        when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, false)).thenReturn(List.of(Type.builder()
+                .id(DOSSIER_REDACTIONS + ":" + TEST_DOSSIER_TEMPLATE_ID)
+                .type(DOSSIER_REDACTIONS)
+                .dossierTemplateId(TEST_DOSSIER_ID)
+                .hexColor("#ffe187")
+                .isHint(hintTypeMap.get(DOSSIER_REDACTIONS))
+                .isCaseInsensitive(caseInSensitiveMap.get(DOSSIER_REDACTIONS))
+                .isRecommendation(recommendationTypeMap.get(DOSSIER_REDACTIONS))
+                .rank(rankTypeMap.get(DOSSIER_REDACTIONS))
+                .build()));
+
+        mockDictionaryCalls(null);
+        mockDictionaryCalls(0L);
+
+        when(dictionaryClient.getColors(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(colors);
+    }
+
+
+    private static String loadFromClassPath(String path) {
+
+        URL resource = ResourceLoader.class.getClassLoader().getResource(path);
+        if (resource == null) {
+            throw new IllegalArgumentException("could not load classpath resource: drools/rules.drl");
+        }
+        try (BufferedReader br = new BufferedReader(new InputStreamReader(resource.openStream(), StandardCharsets.UTF_8))) {
+            StringBuilder sb = new StringBuilder();
+            String str;
+            while ((str = br.readLine()) != null) {
+                sb.append(str).append("\n");
+            }
+            return sb.toString();
+        } catch (IOException e) {
+            throw new IllegalArgumentException("could not load classpath resource: " + path, e);
+        }
+    }
+
+
+    private void mockDictionaryCalls(Long version) {
+
+        when(dictionaryClient.getDictionaryForType(VERTEBRATE + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(VERTEBRATE,
+                false));
+        when(dictionaryClient.getDictionaryForType(ADDRESS + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(ADDRESS, false));
+        when(dictionaryClient.getDictionaryForType(AUTHOR + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(AUTHOR, false));
+        when(dictionaryClient.getDictionaryForType(SPONSOR + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(SPONSOR, false));
+        when(dictionaryClient.getDictionaryForType(NO_REDACTION_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(
+                NO_REDACTION_INDICATOR,
+                false));
+        when(dictionaryClient.getDictionaryForType(REDACTION_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(
+                REDACTION_INDICATOR,
+                false));
+        when(dictionaryClient.getDictionaryForType(HINT_ONLY + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(HINT_ONLY, false));
+        when(dictionaryClient.getDictionaryForType(MUST_REDACT + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(MUST_REDACT,
+                false));
+        when(dictionaryClient.getDictionaryForType(PUBLISHED_INFORMATION + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(
+                PUBLISHED_INFORMATION,
+                false));
+        when(dictionaryClient.getDictionaryForType(TEST_METHOD + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(TEST_METHOD,
+                false));
+        when(dictionaryClient.getDictionaryForType(PII + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(PII, false));
+        when(dictionaryClient.getDictionaryForType(PURITY + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(PURITY, false));
+        when(dictionaryClient.getDictionaryForType(IMAGE + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(IMAGE, false));
+        when(dictionaryClient.getDictionaryForType(OCR + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(OCR, false));
+        when(dictionaryClient.getDictionaryForType(LOGO + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(LOGO, false));
+        when(dictionaryClient.getDictionaryForType(SIGNATURE + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(SIGNATURE, false));
+        when(dictionaryClient.getDictionaryForType(FORMULA + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(FORMULA, false));
+        when(dictionaryClient.getDictionaryForType(ROTATE_SIMPLE + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(ROTATE_SIMPLE,
+                false));
+        when(dictionaryClient.getDictionaryForType(DOSSIER_REDACTIONS + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(
+                DOSSIER_REDACTIONS,
+                true));
+        when(dictionaryClient.getDictionaryForType(IMPORTED_REDACTION + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(
+                IMPORTED_REDACTION,
+                true));
+
+    }
+
+
+    private String cleanDictionaryEntry(String entry) {
+
+        return TextNormalizationUtilities.removeHyphenLineBreaks(entry).replaceAll("\\n", " ");
+    }
+
+
+    private List<DictionaryEntry> toDictionaryEntry(List<String> entries) {
+
+        if (entries == null) {
+            entries = Collections.emptyList();
+        }
+
+        List<DictionaryEntry> dictionaryEntries = new ArrayList<>();
+        entries.forEach(entry -> {
+            dictionaryEntries.add(DictionaryEntry.builder().value(entry).version(reanalysisVersions.getOrDefault(entry, 0L)).deleted(deleted.contains(entry)).build());
+        });
+        return dictionaryEntries;
+    }
+
+
+    private Type getDictionaryResponse(String type, boolean isDossierDictionary) {
+
+        return Type.builder()
+                .id(type + ":" + TEST_DOSSIER_TEMPLATE_ID)
+                .hexColor(typeColorMap.get(type))
+                .entries(isDossierDictionary ? toDictionaryEntry(dossierDictionary.get(type)) : toDictionaryEntry(dictionary.get(type)))
+                .falsePositiveEntries(falsePositive.containsKey(type) ? toDictionaryEntry(falsePositive.get(type)) : new ArrayList<>())
+                .falseRecommendationEntries(falseRecommendation.containsKey(type) ? toDictionaryEntry(falseRecommendation.get(type)) : new ArrayList<>())
+                .isHint(hintTypeMap.get(type))
+                .isCaseInsensitive(caseInSensitiveMap.get(type))
+                .isRecommendation(recommendationTypeMap.get(type))
+                .rank(rankTypeMap.get(type))
+                .build();
+    }
+
+
+    private void loadTypeForTest() {
+
+        typeColorMap.put(VERTEBRATE, "#ff85f7");
+        typeColorMap.put(ADDRESS, "#ffe187");
+        typeColorMap.put(AUTHOR, "#ffe187");
+        typeColorMap.put(SPONSOR, "#85ebff");
+        typeColorMap.put(NO_REDACTION_INDICATOR, "#be85ff");
+        typeColorMap.put(REDACTION_INDICATOR, "#caff85");
+        typeColorMap.put(HINT_ONLY, "#abc0c4");
+        typeColorMap.put(MUST_REDACT, "#fab4c0");
+        typeColorMap.put(PUBLISHED_INFORMATION, "#85ebff");
+        typeColorMap.put(TEST_METHOD, "#91fae8");
+        typeColorMap.put(PII, "#66ccff");
+        typeColorMap.put(PURITY, "#ffe187");
+        typeColorMap.put(IMAGE, "#fcc5fb");
+        typeColorMap.put(OCR, "#fcc5fb");
+        typeColorMap.put(LOGO, "#ffe187");
+        typeColorMap.put(FORMULA, "#ffe187");
+        typeColorMap.put(SIGNATURE, "#ffe187");
+        typeColorMap.put(IMPORTED_REDACTION, "#fcfbe6");
+        typeColorMap.put(ROTATE_SIMPLE, "#66ccff");
+
+        hintTypeMap.put(VERTEBRATE, true);
+        hintTypeMap.put(ADDRESS, false);
+        hintTypeMap.put(AUTHOR, false);
+        hintTypeMap.put(SPONSOR, false);
+        hintTypeMap.put(NO_REDACTION_INDICATOR, true);
+        hintTypeMap.put(REDACTION_INDICATOR, true);
+        hintTypeMap.put(HINT_ONLY, true);
+        hintTypeMap.put(MUST_REDACT, true);
+        hintTypeMap.put(PUBLISHED_INFORMATION, true);
+        hintTypeMap.put(TEST_METHOD, true);
+        hintTypeMap.put(PII, false);
+        hintTypeMap.put(PURITY, false);
+        hintTypeMap.put(IMAGE, true);
+        hintTypeMap.put(OCR, true);
+        hintTypeMap.put(FORMULA, false);
+        hintTypeMap.put(LOGO, false);
+        hintTypeMap.put(SIGNATURE, false);
+        hintTypeMap.put(DOSSIER_REDACTIONS, false);
+        hintTypeMap.put(IMPORTED_REDACTION, false);
+        hintTypeMap.put(ROTATE_SIMPLE, false);
+
+        caseInSensitiveMap.put(VERTEBRATE, true);
+        caseInSensitiveMap.put(ADDRESS, false);
+        caseInSensitiveMap.put(AUTHOR, false);
+        caseInSensitiveMap.put(SPONSOR, false);
+        caseInSensitiveMap.put(NO_REDACTION_INDICATOR, true);
+        caseInSensitiveMap.put(REDACTION_INDICATOR, true);
+        caseInSensitiveMap.put(HINT_ONLY, true);
+        caseInSensitiveMap.put(MUST_REDACT, true);
+        caseInSensitiveMap.put(PUBLISHED_INFORMATION, true);
+        caseInSensitiveMap.put(TEST_METHOD, false);
+        caseInSensitiveMap.put(PII, false);
+        caseInSensitiveMap.put(PURITY, false);
+        caseInSensitiveMap.put(IMAGE, true);
+        caseInSensitiveMap.put(OCR, true);
+        caseInSensitiveMap.put(SIGNATURE, true);
+        caseInSensitiveMap.put(LOGO, true);
+        caseInSensitiveMap.put(FORMULA, true);
+        caseInSensitiveMap.put(DOSSIER_REDACTIONS, false);
+        caseInSensitiveMap.put(IMPORTED_REDACTION, false);
+        caseInSensitiveMap.put(ROTATE_SIMPLE, true);
+
+        recommendationTypeMap.put(VERTEBRATE, false);
+        recommendationTypeMap.put(ADDRESS, false);
+        recommendationTypeMap.put(AUTHOR, false);
+        recommendationTypeMap.put(SPONSOR, false);
+        recommendationTypeMap.put(NO_REDACTION_INDICATOR, false);
+        recommendationTypeMap.put(REDACTION_INDICATOR, false);
+        recommendationTypeMap.put(HINT_ONLY, false);
+        recommendationTypeMap.put(MUST_REDACT, false);
+        recommendationTypeMap.put(PUBLISHED_INFORMATION, false);
+        recommendationTypeMap.put(TEST_METHOD, false);
+        recommendationTypeMap.put(PII, false);
+        recommendationTypeMap.put(PURITY, false);
+        recommendationTypeMap.put(IMAGE, false);
+        recommendationTypeMap.put(OCR, false);
+        recommendationTypeMap.put(FORMULA, false);
+        recommendationTypeMap.put(SIGNATURE, false);
+        recommendationTypeMap.put(LOGO, false);
+        recommendationTypeMap.put(DOSSIER_REDACTIONS, false);
+        recommendationTypeMap.put(IMPORTED_REDACTION, false);
+        recommendationTypeMap.put(ROTATE_SIMPLE, false);
+
+        rankTypeMap.put(PURITY, 155);
+        rankTypeMap.put(PII, 150);
+        rankTypeMap.put(ADDRESS, 140);
+        rankTypeMap.put(AUTHOR, 130);
+        rankTypeMap.put(SPONSOR, 120);
+        rankTypeMap.put(VERTEBRATE, 110);
+        rankTypeMap.put(MUST_REDACT, 100);
+        rankTypeMap.put(REDACTION_INDICATOR, 90);
+        rankTypeMap.put(NO_REDACTION_INDICATOR, 80);
+        rankTypeMap.put(PUBLISHED_INFORMATION, 70);
+        rankTypeMap.put(TEST_METHOD, 60);
+        rankTypeMap.put(HINT_ONLY, 50);
+        rankTypeMap.put(IMAGE, 30);
+        rankTypeMap.put(OCR, 29);
+        rankTypeMap.put(LOGO, 28);
+        rankTypeMap.put(SIGNATURE, 27);
+        rankTypeMap.put(FORMULA, 26);
+        rankTypeMap.put(DOSSIER_REDACTIONS, 200);
+        rankTypeMap.put(IMPORTED_REDACTION, 200);
+        rankTypeMap.put(ROTATE_SIMPLE, 150);
+
+        colors.setSkippedColor("#cccccc");
+        colors.setRequestAddColor("#04b093");
+        colors.setRequestRemoveColor("#04b093");
+    }
+
+
+    @SneakyThrows
+    private void loadNerForTest() {
+
+        ClassPathResource responseJson = new ClassPathResource("files/ner_response.json");
+        storageService.storeObject(RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.NER_ENTITIES), responseJson.getInputStream());
+    }
+
+
+    private List<Type> getTypeResponse() {
+
+        return typeColorMap.entrySet()
+                .stream()
+                .map(typeColor -> Type.builder()
+                        .id(typeColor.getKey() + ":" + TEST_DOSSIER_TEMPLATE_ID)
+                        .type(typeColor.getKey())
+                        .dossierTemplateId(TEST_DOSSIER_TEMPLATE_ID)
+                        .hexColor(typeColor.getValue())
+                        .isHint(hintTypeMap.get(typeColor.getKey()))
+                        .isCaseInsensitive(caseInSensitiveMap.get(typeColor.getKey()))
+                        .isRecommendation(recommendationTypeMap.get(typeColor.getKey()))
+                        .rank(rankTypeMap.get(typeColor.getKey()))
+                        .build())
+
+                .collect(Collectors.toList());
+    }
+
+    protected List<File> getPathsRecursively(File path) {
+
+        List<File> result = new ArrayList<>();
+        if (path == null || path.listFiles() == null) {
+            return result;
+        }
+        for (File f : path.listFiles()) {
+            if (f.isFile()) {
+                result.add(f);
+            } else {
+                result.addAll(getPathsRecursively(f));
+            }
+        }
+        return result;
+
+    }
+
+    protected void loadOnlyDictionaryForSimpleFile() {
+
+        dictionary.clear();
+        dictionary.computeIfAbsent(ROTATE_SIMPLE, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/RotateTestFileSimple.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
+    }
+
+
+    private void loadDictionaryForTest() {
+
+        dictionary.computeIfAbsent(AUTHOR, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/CBI_author.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
+        dictionary.computeIfAbsent(SPONSOR, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/CBI_sponsor.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
+        dictionary.computeIfAbsent(VERTEBRATE, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/vertebrate.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
+        dictionary.computeIfAbsent(ADDRESS, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/CBI_address.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
+        dictionary.computeIfAbsent(NO_REDACTION_INDICATOR, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/no_redaction_indicator.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
+        dictionary.computeIfAbsent(REDACTION_INDICATOR, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/redaction_indicator.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
+        dictionary.computeIfAbsent(HINT_ONLY, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/hint_only.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
+        dictionary.computeIfAbsent(MUST_REDACT, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/must_redact.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
+        dictionary.computeIfAbsent(PUBLISHED_INFORMATION, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/published_information.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
+        dictionary.computeIfAbsent(TEST_METHOD, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/test_method.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
+        dictionary.computeIfAbsent(PII, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/PII.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
+        dictionary.computeIfAbsent(PURITY, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/purity.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
+        dictionary.computeIfAbsent(IMAGE, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/empty.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
+        dictionary.computeIfAbsent(OCR, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/empty.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
+        dictionary.computeIfAbsent(LOGO, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/empty.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
+        dictionary.computeIfAbsent(SIGNATURE, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/empty.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
+        dictionary.computeIfAbsent(FORMULA, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/empty.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
+        dossierDictionary.computeIfAbsent(DOSSIER_REDACTIONS, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/dossier_redactions.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
+        dossierDictionary.put(IMPORTED_REDACTION, new ArrayList<>());
+
+        falsePositive.computeIfAbsent(PII, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/PII_false_positive.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
+
+    }
+
+
+    @SneakyThrows
+    protected AnalyzeRequest prepareStorage(String file) {
+
+        return prepareStorage(file, "files/cv_service_empty_response.json");
+    }
+
+
+    @SneakyThrows
+    protected AnalyzeRequest prepareStorage(String file, String cvServiceResponseFile) {
+
+        ClassPathResource pdfFileResource = new ClassPathResource(file);
+        ClassPathResource cvServiceResponseFileResource = new ClassPathResource(cvServiceResponseFile);
+
+        return prepareStorage(pdfFileResource.getInputStream(), cvServiceResponseFileResource.getInputStream());
+    }
+
+
+    @SneakyThrows
+    protected AnalyzeRequest prepareStorage(InputStream fileStream, InputStream cvServiceResponseFileStream) {
+
+        AnalyzeRequest request = AnalyzeRequest.builder()
+                .dossierTemplateId(TEST_DOSSIER_TEMPLATE_ID)
+                .dossierId(TEST_DOSSIER_ID)
+                .fileId(TEST_FILE_ID)
+                .lastProcessed(OffsetDateTime.now())
+                .build();
+
+        storageService.storeObject(RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.TABLES), cvServiceResponseFileStream);
+        storageService.storeObject(RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.ORIGIN), fileStream);
+
+        return request;
+    }
+
+
+    @AfterEach
+    public void cleanupStorage() {
+
+        if (this.storageService instanceof FileSystemBackedStorageService) {
+            ((FileSystemBackedStorageService) this.storageService).clearStorage();
+        }
+    }
+
+
+    @Configuration
+    @EnableAutoConfiguration(exclude = {RabbitAutoConfiguration.class, StorageAutoConfiguration.class})
+    public static class TestConfiguration {
+
+        @Bean
+        public KieContainer kieContainer() {
+
+            KieServices kieServices = KieServices.Factory.get();
+
+            KieFileSystem kieFileSystem = kieServices.newKieFileSystem();
+            InputStream input = new ByteArrayInputStream(RULES.getBytes(StandardCharsets.UTF_8));
+            kieFileSystem.write("src/test/resources/drools/rules.drl", kieServices.getResources().newInputStreamResource(input));
+            KieBuilder kieBuilder = kieServices.newKieBuilder(kieFileSystem);
+            kieBuilder.buildAll();
+            KieModule kieModule = kieBuilder.getKieModule();
+
+            return kieServices.newKieContainer(kieModule.getReleaseId());
+        }
+
+        @Bean
+        @Primary
+        public StorageService inmemoryStorage() {
+
+            return new FileSystemBackedStorageService();
+        }
+
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DictionaryServiceTest.java

@@ -1,16 +1,16 @@
 package com.iqser.red.service.redaction.v1.server;
 
-import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.configuration.Colors;
+import static org.mockito.ArgumentMatchers.any;
-import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntry;
+import static org.mockito.ArgumentMatchers.anyBoolean;
-import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.Type;
+import static org.mockito.ArgumentMatchers.anyString;
-import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
+import static org.mockito.Mockito.when;
-import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryVersion;
-import com.iqser.red.service.redaction.v1.server.redaction.service.DictionaryService;
-import com.iqser.red.storage.commons.StorageAutoConfiguration;
-import com.iqser.red.storage.commons.service.StorageService;
 
-import org.junit.Test;
+import java.util.List;
-import org.junit.runner.RunWith;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
 import org.kie.api.runtime.KieContainer;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
@@ -21,16 +21,18 @@ import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;
 import org.springframework.context.annotation.Import;
 import org.springframework.context.annotation.Primary;
-import org.springframework.test.context.junit4.SpringRunner;
+import org.springframework.test.context.junit.jupiter.SpringExtension;
 
-import java.util.List;
+import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.configuration.Colors;
-import java.util.stream.Collectors;
+import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntry;
-import java.util.stream.Stream;
+import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.Type;
+import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
+import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryVersion;
+import com.iqser.red.service.redaction.v1.server.redaction.service.DictionaryService;
+import com.iqser.red.storage.commons.StorageAutoConfiguration;
+import com.iqser.red.storage.commons.service.StorageService;
 
-import static org.mockito.ArgumentMatchers.*;
+@ExtendWith(SpringExtension.class)
-import static org.mockito.Mockito.when;
-
-@RunWith(SpringRunner.class)
 @SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
 @Import(RedactionIntegrationTest.RedactionIntegrationTestConfiguration.class)
 public class DictionaryServiceTest {

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DocumentGraphIntegrationTest.java

@@ -0,0 +1,65 @@
+package com.iqser.red.service.redaction.v1.server;
+
+import java.util.List;
+
+import org.junit.jupiter.api.Test;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.core.io.ClassPathResource;
+
+import com.iqser.red.service.redaction.v1.server.document.graph.DocumentEntity;
+import com.iqser.red.service.redaction.v1.server.document.graph.TextBlockEntity;
+import com.iqser.red.service.redaction.v1.server.document.services.DocumentGraphFactory;
+import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
+import com.iqser.red.service.redaction.v1.server.redaction.service.DictionaryService;
+import com.iqser.red.service.redaction.v1.server.redaction.utils.SearchImplementation;
+import com.iqser.red.service.redaction.v1.server.segmentation.PdfSegmentationService;
+
+import lombok.SneakyThrows;
+
+public class DocumentGraphIntegrationTest extends AbstractTestWithDictionaries {
+
+    @Autowired
+    private DocumentGraphFactory documentGraphFactory;
+
+    @Autowired
+    private PdfSegmentationService segmentationService;
+
+    @Autowired
+    private DictionaryService dictionaryService;
+
+
+    @Test
+    @SneakyThrows
+    public void testBuildDocumentEntity() {
+
+        String filename = "files/new/crafted document";
+
+        prepareStorage(filename + ".pdf");
+        ClassPathResource fileResource = new ClassPathResource(filename + ".pdf");
+
+        var classifiedDoc = segmentationService.parseDocument(TEST_DOSSIER_ID, TEST_FILE_ID, fileResource.getInputStream(), null);
+        DocumentEntity document = documentGraphFactory.buildEntityFromDocument(classifiedDoc);
+
+        var start = System.currentTimeMillis();
+
+        TextBlockEntity textBlock = document.buildTextBlock();
+        dictionaryService.updateDictionary(TEST_DOSSIER_TEMPLATE_ID, TEST_DOSSIER_ID);
+        Dictionary dictionary = dictionaryService.getDeepCopyDictionary(TEST_DOSSIER_TEMPLATE_ID, TEST_DOSSIER_ID);
+
+        for (var model : dictionary.getDictionaryModels()) {
+            List<SearchImplementation.MatchPosition> entries = model.getEntriesSearch().getMatches(textBlock);
+            List<SearchImplementation.MatchPosition> falsePositives = model.getFalsePositiveSearch().getMatches(textBlock);
+            List<SearchImplementation.MatchPosition> falseRecommendations = model.getFalseRecommendationsSearch().getMatches(textBlock);
+
+            if (!entries.isEmpty()) {
+                var pos = entries.get(0);
+                System.out.println(textBlock.getPositions(pos.startIndex(), pos.endIndex()));
+            }
+
+        }
+
+        System.out.printf("Search took %fs", ((float) (System.currentTimeMillis() - start)) / 1000);
+
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/HeadlinesGoldStandardIntegrationTest.java

@@ -20,10 +20,10 @@ import java.util.Set;
 import java.util.stream.Collectors;
 
 import org.assertj.core.api.Assertions;
-import org.junit.After;
+import org.junit.jupiter.api.AfterEach;
-import org.junit.Before;
+import org.junit.jupiter.api.BeforeEach;
-import org.junit.Test;
+import org.junit.jupiter.api.Test;
-import org.junit.runner.RunWith;
+import org.junit.jupiter.api.extension.ExtendWith;
 import org.kie.api.KieServices;
 import org.kie.api.builder.KieBuilder;
 import org.kie.api.builder.KieFileSystem;
@@ -41,7 +41,7 @@ import org.springframework.context.annotation.Configuration;
 import org.springframework.context.annotation.Import;
 import org.springframework.context.annotation.Primary;
 import org.springframework.core.io.ClassPathResource;
-import org.springframework.test.context.junit4.SpringRunner;
+import org.springframework.test.context.junit.jupiter.SpringExtension;
 
 import com.amazonaws.services.s3.AmazonS3;
 import com.fasterxml.jackson.databind.ObjectMapper;
@@ -72,7 +72,7 @@ import lombok.EqualsAndHashCode;
 import lombok.SneakyThrows;
 import lombok.ToString;
 
-@RunWith(SpringRunner.class)
+@ExtendWith(SpringExtension.class)
 @SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
 @Import(HeadlinesGoldStandardIntegrationTest.RedactionIntegrationTestConfiguration.class)
 public class HeadlinesGoldStandardIntegrationTest {
@@ -229,7 +229,7 @@ public class HeadlinesGoldStandardIntegrationTest {
     }
 
 
-    @After
+    @AfterEach
     public void cleanupStorage() {
 
         if (this.storageService instanceof FileSystemBackedStorageService) {
@@ -238,7 +238,7 @@ public class HeadlinesGoldStandardIntegrationTest {
     }
 
 
-    @Before
+    @BeforeEach
     public void stubClients() {
 
         when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L);

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java

@@ -24,11 +24,11 @@ import java.util.Set;
 import java.util.UUID;
 import java.util.stream.Collectors;
 
-import org.junit.After;
+import org.junit.jupiter.api.AfterEach;
-import org.junit.Before;
+import org.junit.jupiter.api.BeforeEach;
-import org.junit.Ignore;
+import org.junit.jupiter.api.Disabled;
-import org.junit.Test;
+import org.junit.jupiter.api.Test;
-import org.junit.runner.RunWith;
+import org.junit.jupiter.api.extension.ExtendWith;
 import org.kie.api.KieServices;
 import org.kie.api.builder.KieBuilder;
 import org.kie.api.builder.KieFileSystem;
@@ -46,7 +46,7 @@ import org.springframework.context.annotation.Configuration;
 import org.springframework.context.annotation.Import;
 import org.springframework.context.annotation.Primary;
 import org.springframework.core.io.ClassPathResource;
-import org.springframework.test.context.junit4.SpringRunner;
+import org.springframework.test.context.junit.jupiter.SpringExtension;
 
 import com.amazonaws.services.s3.AmazonS3;
 import com.fasterxml.jackson.core.type.TypeReference;
@@ -81,7 +81,9 @@ import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
 import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient;
 import com.iqser.red.service.redaction.v1.server.client.RulesClient;
 import com.iqser.red.service.redaction.v1.server.controller.RedactionController;
+import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
 import com.iqser.red.service.redaction.v1.server.redaction.service.AnalyzeService;
+import com.iqser.red.service.redaction.v1.server.redaction.service.DictionaryService;
 import com.iqser.red.service.redaction.v1.server.redaction.service.ManualRedactionSurroundingTextService;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.OsUtils;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.ResourceLoader;
@@ -92,7 +94,7 @@ import com.iqser.red.storage.commons.service.StorageService;
 
 import lombok.SneakyThrows;
 
-@RunWith(SpringRunner.class)
+@ExtendWith(SpringExtension.class)
 @SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
 @Import(RedactionIntegrationTest.RedactionIntegrationTestConfiguration.class)
 public class RedactionIntegrationTest {
@@ -155,6 +157,11 @@ public class RedactionIntegrationTest {
     @MockBean
     private LegalBasisClient legalBasisClient;
 
+    @Autowired
+    private DictionaryService dictionaryService;
+
+
+
     private final Map<String, List<String>> dictionary = new HashMap<>();
     private final Map<String, List<String>> dossierDictionary = new HashMap<>();
     private final Map<String, List<String>> falsePositive = new HashMap<>();
@@ -202,7 +209,7 @@ public class RedactionIntegrationTest {
     }
 
 
-    @After
+    @AfterEach
     public void cleanupStorage() {
 
         if (this.storageService instanceof FileSystemBackedStorageService) {
@@ -211,7 +218,7 @@ public class RedactionIntegrationTest {
     }
 
 
-    @Before
+    @BeforeEach
     public void stubClients() {
 
         when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L);
@@ -293,7 +300,7 @@ public class RedactionIntegrationTest {
 
 
     @Test
-    @Ignore
+    @Disabled
     public void testLargeScannedFileOOM() {
 
         AnalyzeRequest request = prepareStorage("scanned/VV-377031.pdf");
@@ -364,15 +371,14 @@ public class RedactionIntegrationTest {
     @Test
     public void titleExtraction() throws IOException {
 
-        AnalyzeRequest request = prepareStorage("files/new/APN3_Clean_6.1 (6.4.3.01-02)_Apple_211029.pdf");
+        AnalyzeRequest request = prepareStorage("files/new/crafted document.pdf");
 
         analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
         AnalyzeResult result = analyzeService.analyze(request);
 
         var redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID);
 
-        var text = redactionStorageService.getText(TEST_DOSSIER_ID, TEST_FILE_ID);
+        Dictionary dictionary = dictionaryService.getDeepCopyDictionary(TEST_DOSSIER_TEMPLATE_ID, TEST_DOSSIER_ID);
-
         AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder().dossierId(TEST_DOSSIER_ID).fileId(TEST_FILE_ID).build());
 
         String outputFileName = OsUtils.getTemporaryDirectory() + "/Annotated.pdf";
@@ -398,7 +404,7 @@ public class RedactionIntegrationTest {
 
 
     @Test
-    @Ignore
+    @Disabled
     @SneakyThrows
     public void testIgnoreHint() {
 
@@ -440,7 +446,7 @@ public class RedactionIntegrationTest {
 
 
     @Test
-    @Ignore
+    @Disabled
     public void noExceptionShouldBeThrownForAnyFiles() throws IOException {
 
         long start = System.currentTimeMillis();
@@ -1111,7 +1117,7 @@ public class RedactionIntegrationTest {
 
         System.out.println("classificationTest");
 
-        AnalyzeRequest request = prepareStorage("files/new/RotateTestFile.pdf");
+        AnalyzeRequest request = prepareStorage("files/RSS/03 - Acute Oral Toxicity Up and Down Procedur.pdf");
 
         RedactionRequest redactionRequest = RedactionRequest.builder()
                 .dossierId(request.getDossierId())
@@ -1252,7 +1258,7 @@ public class RedactionIntegrationTest {
 
 
     @Test
-    @Ignore
+    @Disabled
     public void resizeRedactionTest() throws IOException {
 
         String pdfFile = "files/Minimal Examples/Single Table.pdf";
@@ -1369,7 +1375,7 @@ public class RedactionIntegrationTest {
 
 
     @Test
-    @Ignore
+    @Disabled
     public void testManualSurroundingText() throws IOException {
 
         String pdfFile = "files/new/S4.pdf";

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RulesTest.java

@@ -28,11 +28,11 @@ import java.util.Set;
 import java.util.stream.Collectors;
 
 import org.apache.commons.lang3.StringUtils;
-import org.junit.After;
+import org.junit.jupiter.api.AfterEach;
-import org.junit.Before;
+import org.junit.jupiter.api.BeforeEach;
-import org.junit.Ignore;
+import org.junit.jupiter.api.Disabled;
-import org.junit.Test;
+import org.junit.jupiter.api.Test;
-import org.junit.runner.RunWith;
+import org.junit.jupiter.api.extension.ExtendWith;
 import org.kie.api.KieServices;
 import org.kie.api.builder.KieBuilder;
 import org.kie.api.builder.KieFileSystem;
@@ -50,7 +50,7 @@ import org.springframework.context.annotation.Import;
 import org.springframework.context.annotation.Primary;
 import org.springframework.core.io.ClassPathResource;
 import org.springframework.test.annotation.IfProfileValue;
-import org.springframework.test.context.junit4.SpringRunner;
+import org.springframework.test.context.junit.jupiter.SpringExtension;
 
 import com.amazonaws.services.s3.AmazonS3;
 import com.fasterxml.jackson.databind.DeserializationFeature;
@@ -88,7 +88,7 @@ import lombok.SneakyThrows;
 import lombok.extern.slf4j.Slf4j;
 
 @Slf4j
-@RunWith(SpringRunner.class)
+@ExtendWith(SpringExtension.class)
 @SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
 @Import(RulesTest.RulesTestConfiguration.class)
 public class RulesTest {
@@ -247,7 +247,7 @@ public class RulesTest {
     private LegalBasisClient legalBasisClient;
 
 
-    @Before
+    @BeforeEach
     public void stubClients() {
 
         objectMapper.registerModule(new JavaTimeModule());
@@ -280,7 +280,7 @@ public class RulesTest {
     }
 
 
-    @After
+    @AfterEach
     public void cleanupStorage() {
 
         if (this.storageService instanceof FileSystemBackedStorageService) {
@@ -294,7 +294,7 @@ public class RulesTest {
      * If the RedactionLog already exists, it will be overwritten
      * Test is ignored, because it's for manual tests.
      */
-    @Ignore
+    @Disabled
     @Test
     public void generateRedactionLogForOneFile() {
 

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/realdata/AnalyseFileRealDataIntegrationTest.java

@@ -4,7 +4,7 @@ import java.time.OffsetDateTime;
 import java.util.List;
 import java.util.Set;
 
-import org.junit.Test;
+import org.junit.jupiter.api.Test;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.core.io.ClassPathResource;
 

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/realdata/LiveDataIntegrationTest.java

@@ -1,28 +1,21 @@
 package com.iqser.red.service.redaction.v1.server.realdata;
 
-import com.fasterxml.jackson.core.type.TypeReference;
+import static org.assertj.core.api.AssertionsForClassTypes.assertThat;
-import com.fasterxml.jackson.databind.ObjectMapper;
+import static org.mockito.ArgumentMatchers.any;
-import com.iqser.red.service.persistence.service.v1.api.model.common.JSONPrimitive;
+import static org.mockito.ArgumentMatchers.anyBoolean;
-import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.configuration.Colors;
+import static org.mockito.ArgumentMatchers.anyString;
-import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntry;
+import static org.mockito.ArgumentMatchers.nullable;
-import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.Type;
+import static org.mockito.Mockito.when;
-import com.iqser.red.service.redaction.v1.server.Application;
-import com.iqser.red.service.redaction.v1.server.FileSystemBackedStorageService;
-import com.iqser.red.service.redaction.v1.server.client.*;
-import com.iqser.red.service.redaction.v1.server.queue.RedactionMessageReceiver;
-import com.iqser.red.service.redaction.v1.server.redaction.service.DictionaryService;
-import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;
-import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
-import com.iqser.red.storage.commons.StorageAutoConfiguration;
-import com.iqser.red.storage.commons.service.StorageService;
 
-import io.micrometer.prometheus.PrometheusMeterRegistry;
+import java.util.ArrayList;
-import lombok.SneakyThrows;
+import java.util.List;
+import java.util.Objects;
+import java.util.stream.Collectors;
 
 import org.apache.commons.io.IOUtils;
-import org.junit.Before;
+import org.junit.jupiter.api.BeforeEach;
-import org.junit.Test;
+import org.junit.jupiter.api.Test;
-import org.junit.runner.RunWith;
+import org.junit.jupiter.api.extension.ExtendWith;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
 import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration;
@@ -35,18 +28,33 @@ import org.springframework.context.annotation.Primary;
 import org.springframework.core.io.ClassPathResource;
 import org.springframework.core.io.Resource;
 import org.springframework.core.io.support.ResourcePatternResolver;
+import org.springframework.test.context.junit.jupiter.SpringExtension;
 import org.springframework.test.context.junit4.SpringRunner;
 
-import java.util.ArrayList;
+import com.fasterxml.jackson.core.type.TypeReference;
-import java.util.List;
+import com.fasterxml.jackson.databind.ObjectMapper;
-import java.util.Objects;
+import com.iqser.red.service.persistence.service.v1.api.model.common.JSONPrimitive;
-import java.util.stream.Collectors;
+import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.configuration.Colors;
+import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntry;
+import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.Type;
+import com.iqser.red.service.redaction.v1.server.Application;
+import com.iqser.red.service.redaction.v1.server.FileSystemBackedStorageService;
+import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
+import com.iqser.red.service.redaction.v1.server.client.EntityRecognitionClient;
+import com.iqser.red.service.redaction.v1.server.client.FileStatusProcessingUpdateClient;
+import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient;
+import com.iqser.red.service.redaction.v1.server.client.RulesClient;
+import com.iqser.red.service.redaction.v1.server.queue.RedactionMessageReceiver;
+import com.iqser.red.service.redaction.v1.server.redaction.service.DictionaryService;
+import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;
+import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
+import com.iqser.red.storage.commons.StorageAutoConfiguration;
+import com.iqser.red.storage.commons.service.StorageService;
 
-import static org.assertj.core.api.AssertionsForClassTypes.assertThat;
+import io.micrometer.prometheus.PrometheusMeterRegistry;
-import static org.mockito.ArgumentMatchers.*;
+import lombok.SneakyThrows;
-import static org.mockito.Mockito.when;
 
-@RunWith(SpringRunner.class)
+@ExtendWith(SpringExtension.class)
 @SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
 @Import(LiveDataIntegrationTest.RedactionIntegrationTestConfiguration.class)
 public class LiveDataIntegrationTest {
@@ -108,7 +116,7 @@ public class LiveDataIntegrationTest {
 
 
     @SneakyThrows
-    @Before
+    @BeforeEach
     public void prepareTest() {
 
         when(dictionaryClient.getVersion(anyString())).thenReturn(1L);

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/rulebuilder/RuleBuilderModelServiceTest.java

@@ -2,10 +2,11 @@ package com.iqser.red.service.redaction.v1.server.redaction.rulebuilder;
 
 import com.iqser.red.service.redaction.v1.model.RuleBuilderModel;
 
-import org.junit.Test;
 
 import static org.assertj.core.api.AssertionsForClassTypes.assertThat;
 
+import org.junit.jupiter.api.Test;
+
 public class RuleBuilderModelServiceTest {
 
     @Test

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/utils/EntitySearchUtilsTest.java

@@ -5,7 +5,7 @@ import static org.assertj.core.api.Assertions.assertThat;
 import java.util.HashSet;
 import java.util.Set;
 
-import org.junit.Test;
+import org.junit.jupiter.api.Test;
 
 import com.iqser.red.service.redaction.v1.model.Engine;
 import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
@@ -17,8 +17,8 @@ public class EntitySearchUtilsTest {
     public void testNestedEntitiesRemoval() {
 
         Set<Entity> entities = new HashSet<>();
-        Entity nested = new Entity("nested", "fake type", 10, 16, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
+        Entity nested = new Entity("nested", "fake type", 10, 16, "fake headline", 0, 0, false, false, Engine.RULE, EntityType.ENTITY);
-        Entity nesting = new Entity("nesting nested", "fake type", 2, 16, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
+        Entity nesting = new Entity("nesting nested", "fake type", 2, 16, "fake headline", 0, 0, false, false, Engine.RULE, EntityType.ENTITY);
         entities.add(nested);
         entities.add(nesting);
         EntitySearchUtils.removeEntitiesContainedInLarger(entities);
@@ -40,14 +40,14 @@ public class EntitySearchUtilsTest {
 
         // Arrange
         Set<Entity> existingEntities = new HashSet<>();
-        Entity existingEntity1 = new Entity("Batman", "fake type", 0, 5, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
+        Entity existingEntity1 = new Entity("Batman", "fake type", 0, 5, "fake headline", 0, 0, false, false, Engine.RULE, EntityType.ENTITY);
-        Entity existingEntity2 = new Entity("Superman", "fake type", 10, 17, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
+        Entity existingEntity2 = new Entity("Superman", "fake type", 10, 17, "fake headline", 0, 0, false, false, Engine.RULE, EntityType.ENTITY);
         existingEntities.add(existingEntity1);
         existingEntities.add(existingEntity2);
 
         Set<Entity> foundEntities = new HashSet<>();
-        Entity foundEntities1 = new Entity("Batman X.", "fake type", 0, 8, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
+        Entity foundEntities1 = new Entity("Batman X.", "fake type", 0, 8, "fake headline", 0, 0, false, false, Engine.RULE, EntityType.ENTITY);
-        Entity foundEntities2 = new Entity("Superman Y.", "fake type", 10, 20, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
+        Entity foundEntities2 = new Entity("Superman Y.", "fake type", 10, 20, "fake headline", 0, 0, false, false, Engine.RULE, EntityType.ENTITY);
         foundEntities.add(foundEntities1);
         foundEntities.add(foundEntities2);
 
@@ -73,14 +73,14 @@ public class EntitySearchUtilsTest {
 
         // Arrange
         Set<Entity> existingEntities = new HashSet<>();
-        Entity existingEntity1 = new Entity("Batman X.", "fake type", 0, 8, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
+        Entity existingEntity1 = new Entity("Batman X.", "fake type", 0, 8, "fake headline", 0, 0, false, false, Engine.RULE, EntityType.ENTITY);
-        Entity existingEntity2 = new Entity("Superman", "fake type", 10, 17, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
+        Entity existingEntity2 = new Entity("Superman", "fake type", 10, 17, "fake headline", 0, 0, false, false, Engine.RULE, EntityType.ENTITY);
         existingEntities.add(existingEntity1);
         existingEntities.add(existingEntity2);
 
         Set<Entity> foundEntities = new HashSet<>();
-        Entity foundEntities1 = new Entity("Batman X.", "fake type", 0, 8, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
+        Entity foundEntities1 = new Entity("Batman X.", "fake type", 0, 8, "fake headline", 0, 0,false, false, Engine.RULE, EntityType.ENTITY);
-        Entity foundEntities2 = new Entity("X. Superman Y.", "fake type", 7, 20, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
+        Entity foundEntities2 = new Entity("X. Superman Y.", "fake type", 7, 20, "fake headline", 0, 0, false, false, Engine.RULE, EntityType.ENTITY);
         foundEntities.add(foundEntities1);
         foundEntities.add(foundEntities2);
 
@@ -105,14 +105,14 @@ public class EntitySearchUtilsTest {
 
         // Arrange
         Set<Entity> existingEntities = new HashSet<>();
-        Entity existingEntity1 = new Entity("Batman X.", "fake type", 0, 8, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
+        Entity existingEntity1 = new Entity("Batman X.", "fake type", 0, 8, "fake headline", 0, 0, false, false, Engine.RULE, EntityType.ENTITY);
-        Entity existingEntity2 = new Entity("Superman", "fake type", 10, 17, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
+        Entity existingEntity2 = new Entity("Superman", "fake type", 10, 17, "fake headline", 0, 0,false, false, Engine.RULE, EntityType.ENTITY);
         existingEntities.add(existingEntity1);
         existingEntities.add(existingEntity2);
 
         Set<Entity> foundEntities = new HashSet<>();
-        Entity foundEntities1 = new Entity("Batman X.", "fake type", 0, 8, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
+        Entity foundEntities1 = new Entity("Batman X.", "fake type", 0, 8, "fake headline", 0, 0,false, false, Engine.RULE, EntityType.ENTITY);
-        Entity foundEntities2 = new Entity("X. Superman", "fake type", 7, 17, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
+        Entity foundEntities2 = new Entity("X. Superman", "fake type", 7, 17, "fake headline", 0, 0,false, false, Engine.RULE, EntityType.ENTITY);
         foundEntities.add(foundEntities1);
         foundEntities.add(foundEntities2);
 
@@ -137,15 +137,15 @@ public class EntitySearchUtilsTest {
 
         // Arrange
         Set<Entity> existingEntities = new HashSet<>();
-        Entity existingEntity1 = new Entity("X. Superman", "fake type", 7, 17, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
+        Entity existingEntity1 = new Entity("X. Superman", "fake type", 7, 17, "fake headline", 0, 0,false, false, Engine.RULE, EntityType.ENTITY);
-        Entity existingEntity2 = new Entity("Batman", "fake type", 0, 5, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
+        Entity existingEntity2 = new Entity("Batman", "fake type", 0, 5, "fake headline", 0, 0,false, false, Engine.RULE, EntityType.ENTITY);
         existingEntities.add(existingEntity1);
         existingEntities.add(existingEntity2);
 
         Set<Entity> foundEntities = new HashSet<>();
-        Entity foundEntities1 = new Entity("Batman X.", "fake type", 0, 8, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
+        Entity foundEntities1 = new Entity("Batman X.", "fake type", 0, 8, "fake headline", 0, 0,false, false, Engine.RULE, EntityType.ENTITY);
-        Entity foundEntities2 = new Entity("Superman", "fake type", 10, 17, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
+        Entity foundEntities2 = new Entity("Superman", "fake type", 10, 17, "fake headline", 0, 0,false, false, Engine.RULE, EntityType.ENTITY);
-        Entity foundEntities3 = new Entity("Superman Y.", "fake type", 10, 20, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
+        Entity foundEntities3 = new Entity("Superman Y.", "fake type", 10, 20, "fake headline", 0, 0,false, false, Engine.RULE, EntityType.ENTITY);
         foundEntities.add(foundEntities1);
         foundEntities.add(foundEntities2);
         foundEntities.add(foundEntities3);
@@ -171,16 +171,16 @@ public class EntitySearchUtilsTest {
 
         // Arrange
         Set<Entity> existingEntities = new HashSet<>();
-        Entity existingEntity1 = new Entity("X. Superman", "fake type", 7, 17, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
+        Entity existingEntity1 = new Entity("X. Superman", "fake type", 7, 17, "fake headline", 0, 0,false, false, Engine.RULE, EntityType.ENTITY);
-        Entity existingEntity2 = new Entity("Batman", "fake type", 0, 5, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
+        Entity existingEntity2 = new Entity("Batman", "fake type", 0, 5, "fake headline", 0, 0,false, false, Engine.RULE, EntityType.ENTITY);
         existingEntities.add(existingEntity1);
         existingEntities.add(existingEntity2);
 
         Set<Entity> foundEntities = new HashSet<>();
-        Entity foundEntitiesOverlap1 = new Entity("Batman X. Superman Y.", "fake type", 0, 17, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
+        Entity foundEntitiesOverlap1 = new Entity("Batman X. Superman Y.", "fake type", 0, 17, "fake headline", 0, 0,false, false, Engine.RULE, EntityType.ENTITY);
-        Entity foundEntitiesOverlap2 = new Entity("Superman Y.", "fake type", 10, 20, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
+        Entity foundEntitiesOverlap2 = new Entity("Superman Y.", "fake type", 10, 20, "fake headline", 0, 0,false, false, Engine.RULE, EntityType.ENTITY);
-        Entity foundEntitiesSubset1 = new Entity("Batman X. Superman", "fake type", 0, 17, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
+        Entity foundEntitiesSubset1 = new Entity("Batman X. Superman", "fake type", 0, 17, "fake headline", 0, 0,false, false, Engine.RULE, EntityType.ENTITY);
-        Entity foundEntitiesSubset2 = new Entity("Superman", "fake type", 10, 17, "fake headline", 0, false, false, Engine.RULE, EntityType.ENTITY);
+        Entity foundEntitiesSubset2 = new Entity("Superman", "fake type", 10, 17, "fake headline", 0, 0,false, false, Engine.RULE, EntityType.ENTITY);
         foundEntities.add(foundEntitiesOverlap1);
         foundEntities.add(foundEntitiesOverlap2);
         foundEntities.add(foundEntitiesSubset1);

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/utils/RegExPatternTest.java

@@ -6,7 +6,7 @@ import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
 import org.apache.commons.lang3.StringUtils;
-import org.junit.Test;
+import org.junit.jupiter.api.Test;
 
 public class RegExPatternTest {
 

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/utils/TextNormalizationUtilitiesTest.java

@@ -1,7 +1,7 @@
 package com.iqser.red.service.redaction.v1.server.redaction.utils;
 
 import org.assertj.core.api.Assertions;
-import org.junit.Test;
+import org.junit.jupiter.api.Test;
 
 public class TextNormalizationUtilitiesTest {
 

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/segmentation/PdfSegmentationServiceTest.java

@@ -11,8 +11,8 @@ import java.util.Locale;
 import java.util.Map;
 import java.util.stream.Collectors;
 
-import org.junit.Test;
+import org.junit.jupiter.api.Test;
-import org.junit.runner.RunWith;
+import org.junit.jupiter.api.extension.ExtendWith;
 import org.kie.api.runtime.KieContainer;
 import org.springframework.amqp.rabbit.core.RabbitTemplate;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -25,7 +25,7 @@ import org.springframework.context.annotation.Configuration;
 import org.springframework.context.annotation.Import;
 import org.springframework.context.annotation.Primary;
 import org.springframework.core.io.ClassPathResource;
-import org.springframework.test.context.junit4.SpringRunner;
+import org.springframework.test.context.junit.jupiter.SpringExtension;
 
 import com.amazonaws.services.s3.AmazonS3;
 import com.fasterxml.jackson.databind.ObjectMapper;
@@ -49,7 +49,7 @@ import com.iqser.red.storage.commons.service.StorageService;
 
 import lombok.SneakyThrows;
 
-@RunWith(SpringRunner.class)
+@ExtendWith(SpringExtension.class)
 @SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
 @Import(PdfSegmentationServiceTest.TestConfiguration.class)
 public class PdfSegmentationServiceTest {
@@ -131,8 +131,8 @@ public class PdfSegmentationServiceTest {
         ClassPathResource pdfFileResource = new ClassPathResource("files/Minimal Examples/Spanning Cells.pdf");
 
         Document document = pdfSegmentationService.parseDocument(TEST_DOSSIER_ID, TEST_FILE_ID, pdfFileResource.getInputStream(), null);
-        assertThat(document.getParagraphs().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList())).isNotEmpty();
+        assertThat(document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList())).isNotEmpty();
-        Table table = document.getParagraphs().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList()).get(0);
+        Table table = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList()).get(0);
         assertThat(table.getColCount()).isEqualTo(6);
         assertThat(table.getRowCount()).isEqualTo(13);
         assertThat(table.getRows().stream().mapToInt(List::size).sum()).isEqualTo(6 * 13);
@@ -146,11 +146,11 @@ public class PdfSegmentationServiceTest {
         ClassPathResource pdfFileResource = new ClassPathResource("files/Minimal Examples/Merge Table.pdf");
 
         Document document = pdfSegmentationService.parseDocument(TEST_DOSSIER_ID, TEST_FILE_ID, pdfFileResource.getInputStream(), null);
-        assertThat(document.getParagraphs().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList())).isNotEmpty();
+        assertThat(document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList())).isNotEmpty();
-        Table firstTable = document.getParagraphs().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList()).get(0);
+        Table firstTable = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList()).get(0);
         assertThat(firstTable.getColCount()).isEqualTo(8);
         assertThat(firstTable.getRowCount()).isEqualTo(1);
-        Table secondTable = document.getParagraphs().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList()).get(1);
+        Table secondTable = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList()).get(1);
         assertThat(secondTable.getColCount()).isEqualTo(8);
         assertThat(secondTable.getRowCount()).isEqualTo(2);
         List<List<Cell>> firstTableHeaderCells = firstTable.getRows().get(0).stream().map(Collections::singletonList).collect(Collectors.toList());
@@ -165,11 +165,11 @@ public class PdfSegmentationServiceTest {
         ClassPathResource pdfFileResource = new ClassPathResource("files/Minimal Examples/Merge Multi Page Table.pdf");
 
         Document document = pdfSegmentationService.parseDocument(TEST_DOSSIER_ID, TEST_FILE_ID, pdfFileResource.getInputStream(), null);
-        assertThat(document.getParagraphs().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList())).isNotEmpty();
+        assertThat(document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList())).isNotEmpty();
-        Table firstTable = document.getParagraphs().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList()).get(0);
+        Table firstTable = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList()).get(0);
         assertThat(firstTable.getColCount()).isEqualTo(9);
         assertThat(firstTable.getRowCount()).isEqualTo(5);
-        Table secondTable = document.getParagraphs().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList()).get(1);
+        Table secondTable = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList()).get(1);
         assertThat(secondTable.getColCount()).isEqualTo(9);
         assertThat(secondTable.getRowCount()).isEqualTo(6);
         List<List<Cell>> firstTableHeaderCells = firstTable.getRows().get(firstTable.getRowCount() - 1).stream().map(Cell::getHeaderCells).collect(Collectors.toList());
@@ -184,11 +184,11 @@ public class PdfSegmentationServiceTest {
         ClassPathResource pdfFileResource = new ClassPathResource("files/Minimal Examples/Rotated Table Headers.pdf");
 
         Document document = pdfSegmentationService.parseDocument(TEST_DOSSIER_ID, TEST_FILE_ID, pdfFileResource.getInputStream(), null);
-        assertThat(document.getParagraphs().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList())).isNotEmpty();
+        assertThat(document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList())).isNotEmpty();
-        Table firstTable = document.getParagraphs().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList()).get(0);
+        Table firstTable = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList()).get(0);
         assertThat(firstTable.getColCount()).isEqualTo(8);
         assertThat(firstTable.getRowCount()).isEqualTo(1);
-        Table secondTable = document.getParagraphs().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList()).get(1);
+        Table secondTable = document.getSections().stream().flatMap(paragraph -> paragraph.getTables().stream()).collect(Collectors.toList()).get(1);
         assertThat(secondTable.getColCount()).isEqualTo(8);
         assertThat(secondTable.getRowCount()).isEqualTo(6);
         List<List<Cell>> firstTableHeaderCells = firstTable.getRows().get(0).stream().map(Collections::singletonList).collect(Collectors.toList());

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/stringmatching/StringMatchingPerformanceTest.java

@@ -1,24 +1,24 @@
 package com.iqser.red.service.redaction.v1.server.stringmatching;
 
-import lombok.AllArgsConstructor;
+import static org.assertj.core.api.AssertionsForClassTypes.assertThat;
-import lombok.EqualsAndHashCode;
-import lombok.SneakyThrows;
-
-import org.ahocorasick.trie.Trie;
-import org.apache.commons.io.IOUtils;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.springframework.core.io.ClassPathResource;
-import org.springframework.test.context.junit4.SpringRunner;
 
 import java.util.HashSet;
 import java.util.Set;
 import java.util.regex.Pattern;
 import java.util.stream.Collectors;
 
-import static org.assertj.core.api.AssertionsForClassTypes.assertThat;
+import org.ahocorasick.trie.Trie;
+import org.apache.commons.io.IOUtils;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.springframework.core.io.ClassPathResource;
+import org.springframework.test.context.junit.jupiter.SpringExtension;
 
-@RunWith(SpringRunner.class)
+import lombok.AllArgsConstructor;
+import lombok.EqualsAndHashCode;
+import lombok.SneakyThrows;
+
+@ExtendWith(SpringExtension.class)
 public class StringMatchingPerformanceTest {
 
     @Test

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl

@@ -1,407 +1,67 @@
 package drools
 
-import com.iqser.red.service.redaction.v1.server.redaction.model.Section
+import static java.lang.String.format;
 
-global Section section
+import java.util.List;
+import java.util.LinkedList;
+import java.util.HashSet;
 
+import com.iqser.red.service.redaction.v1.server.redaction.model.*;
+import com.iqser.red.service.redaction.v1.model.Engine;
+import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntry;
+import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUtils;
+import java.util.Set;
 
-// --------------------------------------- CBI rules -------------------------------------------------------------------
+global Dictionary dictionary
 
-//rule "0: Expand CBI Authors with firstname initials"
+query "getEntities"
-//    when
+    $result: Entity()
-//        Section(matchesType("CBI_author"))
-//    then
-//        section.expandByRegEx("CBI_author", "(,? [A-Z]\\.?( ?[A-Z]\\.?)?( ?[A-Z]\\.?)?\\b\\.?)", false, 1);
-//    end
-
-
-rule "0: Add CBI_author from ai"
-    when
-        Section(aiMatchesType("CBI_author"))
-    then
-        section.addAiEntities("CBI_author", "CBI_author");
 end
 
-rule "0: Combine ai types CBI_author from ai"
+// --------------------------------------- AI rules -------------------------------------------------------------------
+rule "0: find entities from entry dictionary"
+    salience 100
     when
-        Section(aiMatchesType("ORG"))
+        $section: Section()
+        $paragraph: Paragraph(sectionNumber == $section.sectionNumber)
+        $entityPositionSequence: EntityPositionSequence() from EntitySearchUtils.findEntityPositionSequences("M. Must", $paragraph.getSearchTextToTextPosition())
     then
-        section.combineAiTypes("ORG", "STREET,POSTAL,COUNTRY,CARDINAL,CITY,STATE", 20, "CBI_address", 3, false);
+        List<EntityPositionSequence> eps = new LinkedList<>();
+        eps.add($entityPositionSequence);
+        Set<Engine> engineSet = new HashSet<>();
+        engineSet.add(Engine.DICTIONARY);
+        Set<Entity> empty = new HashSet<>();
+        insert(new Entity("CBI_author",
+               "custom",
+               true,
+               format("Found in Dictionary: %s", "custom"),
+               eps,
+               $section.getHeadline(),
+               0,
+               $section.getSectionNumber(),
+               $paragraph.getParagraphNumber(),
+               "",
+               true,
+               "",
+               "",
+               -1,
+               -1,
+               false,
+               engineSet,
+               empty,
+               EntityType.ENTITY
+               ));
     end
 
-rule "0: Expand CBI Authors with firstname initials"
+/*
+rule "44: Don't redact Author if contained in paragraph with published information"
     when
-        Section(matchesType("CBI_author"))
+        $section: Section()
+        $paragraph: Paragraph($section.sectionNumber == sectionNumber)
+
+        $authorEntity: Entity(type == "CBI_author", paragraphNumber == $paragraph.getParagraphNumber())
+        $publishedEntity: Entity(type == "published_information", paragraphNumber == $paragraph.getParagraphNumber())
     then
-        section.expandByRegEx("CBI_author", "(,? [A-Z]\\.?( ?[A-Z]\\.?)?( ?[A-Z]\\.?)?\\b\\.?)", false, 1, "[^\\s]+");
+        $authorEntity.setRedaction(false);
-    end
-
-rule "0: Expand CBI_author and PII matches with salutation prefix"
-    when
-        Section((matchesType("CBI_author") || matchesType("PII")) && (
-            searchText.contains("Mr")
-            || searchText.contains("Mrs")
-            || searchText.contains("Ms")
-            || searchText.contains("Miss")
-            || searchText.contains("Sir")
-            || searchText.contains("Madam")
-            || searchText.contains("Madame")
-            || searchText.contains("Mme")
-        ))
-    then
-        section.expandByPrefixRegEx("CBI_author", "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*", false, 0);
-        section.expandByPrefixRegEx("PII", "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*", false, 0);
-    end
-
-
-rule "1: Redacted because Section contains Vertebrate"
-    when
-        Section(matchesType("vertebrate"))
-    then
-        section.redact("CBI_author", 1, "Vertebrate found", "Reg (EC) No 1107/2009 Art. 63 (2g)");
-        section.redact("CBI_address", 1, "Vertebrate found", "Reg (EC) No 1107/2009 Art. 63 (2g)");
-    end
-
-
-rule "2: Not Redacted because Section contains no Vertebrate"
-    when
-        Section(!matchesType("vertebrate"))
-    then
-        section.redactNot("CBI_author", 2, "No Vertebrate found");
-        section.redactNot("CBI_address", 2, "No Vertebrate found");
-    end
-
-
-rule "3: Do not redact Names and Addresses if no redaction Indicator is contained"
-    when
-        Section(matchesType("vertebrate"), matchesType("no_redaction_indicator"))
-    then
-        section.redactNot("CBI_author", 3, "Vertebrate and No Redaction Indicator found");
-        section.redactNot("CBI_address", 3, "Vertebrate and No Redaction Indicator found");
-    end
-
-
-rule "4: Redact Names and Addresses if no_redaction_indicator and redaction_indicator is contained"
-    when
-        Section(matchesType("vertebrate"), matchesType("no_redaction_indicator"), matchesType("redaction_indicator"))
-    then
-        section.redact("CBI_author", 4, "Vertebrate and Redaction Indicator found", "Reg (EC) No 1107/2009 Art. 63 (2g)");
-        section.redact("CBI_address", 4, "Vertebrate and Redaction Indicator found", "Reg (EC) No 1107/2009 Art. 63 (2g)");
-    end
-
-
-rule "5: Do not redact Names and Addresses if no redaction Indicator is contained"
-    when
-        Section(matchesType("vertebrate"), matchesType("published_information"))
-    then
-        section.redactNotAndReference("CBI_author","published_information", 5, "Vertebrate and Published Information found");
-        section.redactNotAndReference("CBI_address","published_information", 5, "Vertebrate and Published Information found");
-    end
-
-
-rule "6: Not redacted because Vertebrate Study = N"
-    when
-        Section(rowEquals("Vertebrate study Y/N", "N") || rowEquals("Vertebrate study Y/N", "No"))
-    then
-        section.redactNotCell("Author(s)", 6, "CBI_author", true, "Not redacted because row is not a vertebrate study");
-        section.redactNot("CBI_author", 6, "Not redacted because row is not a vertebrate study");
-        section.redactNot("CBI_address", 6, "Not redacted because row is not a vertebrate study");
-        section.highlightCell("Vertebrate study Y/N", 6, "hint_only");
-    end
-
-
-rule "7: Redact if must redact entry is found"
-    when
-        Section(matchesType("must_redact"))
-    then
-        section.redact("CBI_author", 7, "must_redact entry was found.", "Reg (EC) No 1107/2009 Art. 63 (2g)");
-        section.redact("CBI_address", 7, "must_redact entry was found.", "Reg (EC) No 1107/2009 Art. 63 (2g)");
-    end
-
-
-rule "8: Redact Authors and Addresses in Reference Table if it is a Vertebrate study"
-    when
-        Section(rowEquals("Vertebrate study Y/N", "Y") || rowEquals("Vertebrate study Y/N", "Yes"))
-    then
-        section.redactCell("Author(s)", 8, "CBI_author", true, "Redacted because row is a vertebrate study", "Reg (EC) No 1107/2009 Art. 63 (2g)");
-        section.redact("CBI_address", 8, "Redacted because row is a vertebrate study", "Reg (EC) No 1107/2009 Art. 63 (2g)");
-        section.highlightCell("Vertebrate study Y/N", 8, "must_redact");
-    end
-
-
-rule "9: Redact sponsor company"
-    when
-        Section(searchText.toLowerCase().contains("batches produced at"))
-    then
-        section.redactIfPrecededBy("batches produced at", "CBI_sponsor", 9, "Redacted because it represents a sponsor company", "Reg (EC) No 1107/2009 Art. 63 (2g)");
-        section.addHintAnnotation("batches produced at", "must_redact");
-    end
-
-
-rule "10: Redact determination of residues"
-    when
-        Section((
-            searchText.toLowerCase.contains("determination of residues") ||
-            searchText.toLowerCase.contains("determination of total residues")
-        ) && (
-            searchText.toLowerCase.contains("livestock") ||
-            searchText.toLowerCase.contains("live stock") ||
-            searchText.toLowerCase.contains("tissue") ||
-            searchText.toLowerCase.contains("tissues") ||
-            searchText.toLowerCase.contains("liver") ||
-            searchText.toLowerCase.contains("muscle") ||
-            searchText.toLowerCase.contains("bovine") ||
-            searchText.toLowerCase.contains("ruminant") ||
-            searchText.toLowerCase.contains("ruminants")
-        ))
-    then
-        section.redact("CBI_author", 10, "Determination of residues was found.", "Reg (EC) No 1107/2009 Art. 63 (2g)");
-        section.redact("CBI_address", 10, "Determination of residues was found.", "Reg (EC) No 1107/2009 Art. 63 (2g)");
-        section.addHintAnnotation("determination of residues", "must_redact");
-        section.addHintAnnotation("livestock", "must_redact");
-        section.addHintAnnotation("live stock", "must_redact");
-        section.addHintAnnotation("tissue", "must_redact");
-        section.addHintAnnotation("tissues", "must_redact");
-        section.addHintAnnotation("liver", "must_redact");
-        section.addHintAnnotation("muscle", "must_redact");
-        section.addHintAnnotation("bovine", "must_redact");
-        section.addHintAnnotation("ruminant", "must_redact");
-        section.addHintAnnotation("ruminants", "must_redact");
-    end
-
-
-rule "11: Redact if CTL/* or BL/*  was found"
-    when
-        Section(searchText.contains("CTL/") || searchText.contains("BL/"))
-    then
-        section.redact("CBI_author", 11, "Laboraty for vertebrate studies found", "Reg (EC) No 1107/2009 Art. 63 (2g)");
-        section.redact("CBI_address", 11, "Laboraty for vertebrate studies found", "Reg (EC) No 1107/2009 Art. 63 (2g)");
-        section.addHintAnnotation("CTL", "must_redact");
-        section.addHintAnnotation("BL", "must_redact");
-    end
-
-
-rule "12: Redact and add recommendation for et al. author"
-    when
-        Section(searchText.contains("et al"))
-    then
-		section.redactAndRecommendByRegEx("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", false, 1, "CBI_author", 12, "Author found", "Reg (EC) No 1107/2009 Art. 63 (2g)");
-    end
-
-
-rule "13: Add recommendation for Addresses in Test Organism sections"
-    when
-        Section(searchText.contains("Species:") && searchText.contains("Source:"))
-    then
-		section.recommendLineAfter("Source:", "CBI_address");
-    end
-
-
-rule "14: Add recommendation for Addresses in Test Animals sections"
-    when
-        Section(searchText.contains("Species") && searchText.contains("Source"))
-    then
-		section.recommendLineAfter("Source", "CBI_address");
-    end
-
-// --------------------------------------- PII rules -------------------------------------------------------------------
-
-
-rule "14: Redacted PII Personal Identification Information"
-    when
-        Section(matchesType("PII"))
-    then
-        section.redact("PII", 14, "PII (Personal Identification Information) found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-    end
-
-
-rule "15: Redact Emails by RegEx"
-    when
-        Section(searchText.contains("@"))
-    then
-        section.redactByRegEx("\\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\\.[A-Z]{2,4}\\b", true, 0, "PII", 15, "PII (Personal Identification Information) found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-    end
-
-
-rule "16: Redact contact information"
-    when
-       Section(text.contains("Contact point:")
-       || text.contains("Phone:")
-       || text.contains("Fax:")
-       || text.contains("Tel.:")
-       || text.contains("Tel:")
-       || text.contains("E-mail:")
-       || text.contains("Email:")
-       || text.contains("e-mail:")
-       || text.contains("E-mail address:")
-       || text.contains("Alternative contact:")
-       || text.contains("Telephone number:")
-       || text.contains("Telephone No:")
-       || text.contains("Fax number:")
-       || text.contains("Telephone:")
-       || text.contains("European contact:"))
-    then
-          section.redactLineAfter("Contact point:", "PII", 16, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-              section.redactLineAfter("Phone:", "PII", 16, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-              section.redactLineAfter("Fax:", "PII", 16, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-              section.redactLineAfter("Tel.:", "PII", 16, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-              section.redactLineAfter("Tel:", "PII", 16, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-              section.redactLineAfter("E-mail:", "PII", 16, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-              section.redactLineAfter("Email:", "PII", 16, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-              section.redactLineAfter("e-mail:", "PII", 16, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-              section.redactLineAfter("E-mail address:", "PII", 16, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-              section.redactLineAfter("Contact:", "PII", 16, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-              section.redactLineAfter("Alternative contact:", "PII", 16, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-              section.redactLineAfter("Telephone number:", "PII", 16, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-              section.redactLineAfter("Telephone No:", "PII", 16, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-              section.redactLineAfter("Fax number:", "PII", 16, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-              section.redactLineAfter("Telephone:", "PII", 16, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-              section.redactBetween("No:", "Fax", "PII", 16, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-              section.redactBetween("Contact:", "Tel.:", "PII", 16, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-              section.redactLineAfter("European contact:",  "PII", 16, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-     end
-
-
-rule "17: Redact contact information if applicant is found"
-    when
-        Section(headlineContainsWord("applicant") || text.contains("Applicant") || headlineContainsWord("Primary contact") || headlineContainsWord("Alternative contact") || text.contains("Telephone number:"))
-    then
-        section.redactLineAfter("Contact point:", "PII", 17, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-        section.redactLineAfter("Phone:", "PII", 17, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-        section.redactLineAfter("Fax:", "PII", 17, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-        section.redactLineAfter("Tel.:", "PII", 17, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-        section.redactLineAfter("Tel:", "PII", 17, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-        section.redactLineAfter("E-mail:", "PII", 17, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-        section.redactLineAfter("Email:", "PII", 17, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-        section.redactLineAfter("e-mail:", "PII", 17, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-        section.redactLineAfter("E-mail address:", "PII", 17, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-        section.redactLineAfter("Contact:", "PII", 17, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-        section.redactLineAfter("Alternative contact:", "PII", 17, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-        section.redactLineAfter("Telephone number:", "PII", 17, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-        section.redactLineAfter("Telephone No:", "PII", 17, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-        section.redactLineAfter("Fax number:", "PII", 17, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-        section.redactLineAfter("Telephone:", "PII", 17, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-        section.redactBetween("No:", "Fax", "PII", 17, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-        section.redactBetween("Contact:", "Tel.:", "PII", 17, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-        section.redactLineAfter("European contact:",  "PII", 17, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-    end
-
-
-rule "18: Redact contact information if Producer is found"
-    when
-        Section(text.toLowerCase().contains("producer of the plant protection") || text.toLowerCase().contains("producer of the active substance") || text.contains("Manufacturer of the active substance") || text.contains("Manufacturer:") || text.contains("Producer or producers of the active substance"))
-    then
-        section.redactLineAfter("Contact:", "PII", 18, true, "Producer was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-        section.redactLineAfter("Telephone:", "PII", 18, true, "Producer was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-        section.redactLineAfter("Phone:", "PII", 18, true, "Producer was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-        section.redactLineAfter("Fax:", "PII", 18, true, "Producer was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-        section.redactLineAfter("E-mail:", "PII", 18, true, "Producer was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-        section.redactLineAfter("Contact:", "PII", 18, true, "Producer was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-        section.redactLineAfter("Fax number:", "PII", 18, true, "Producer was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-        section.redactLineAfter("Telephone number:", "PII", 18, true, "Producer was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-        section.redactLineAfter("Tel:", "PII", 18, true, "Producer was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-        section.redactBetween("No:", "Fax", "PII", 18, true, "Producer was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-    end
-
-
-rule "19: Redact AUTHOR(S)"
-    when
-        Section(searchText.contains("AUTHOR(S):") && fileAttributeByPlaceholderEquals("{fileattributes.vertebrateStudy}", "true"))
-    then
-        section.redactLinesBetween("AUTHOR(S):", "COMPLETION DATE:", "PII", 19, true, "AUTHOR(S) was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-    end
-
-
-rule "20: Redact PERFORMING LABORATORY"
-    when
-        Section(searchText.contains("PERFORMING LABORATORY:"))
-    then
-        section.redactBetween("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "PII", 20, true, "PERFORMING LABORATORY was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-    end
-
-
-rule "21: Redact On behalf of Sequani Ltd.:"
-    when
-        Section(searchText.contains("On behalf of Sequani Ltd.: Name Title"))
-    then
-        section.redactBetween("On behalf of Sequani Ltd.: Name Title", "On behalf of", "PII", 21, false , "PII (Personal Identification Information) found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-    end
-
-
-rule "22: Redact On behalf of Syngenta Ltd.:"
-    when
-        Section(searchText.contains("On behalf of Syngenta Ltd.: Name Title"))
-    then
-        section.redactBetween("On behalf of Syngenta Ltd.: Name Title", "Study dates", "PII", 22, false , "PII (Personal Identification Information) found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
-    end
-
-// --------------------------------------- other rules -------------------------------------------------------------------
-
-rule "25: Redact Purity"
-    when
-        Section(searchText.contains("purity"))
-    then
-	    section.redactByRegEx("purity ?:? (([\\d\\.]+)( .{0,4}\\.)? ?%)", true, 1, "purity", 17, "Purity found", "Reg (EC) No 1107/2009 Art. 63 (2a)");
-    end
-
-
-rule "26: Redact signatures"
-    when
-        Section(matchesImageType("signature"))
-    then
-        section.redactImage("signature", 26, "Signature found", "Reg (EC) No 1107/2009 Art. 63 (2g)");
-    end
-
-
-rule "27: Redact formula"
-    when
-        Section(matchesImageType("formula"))
-    then
-        section.redactImage("formula", 27, "Formula found", "Reg (EC) No 1107/2009 Art. 63 (2g)");
-    end
-
-
-rule "28: Redact Logos"
-    when
-        Section(matchesImageType("logo"))
-    then
-        section.redactImage("logo", 28, "Logo found", "Reg (EC) No 1107/2009 Art. 63 (2g)");
-    end
-
-
-rule "29: Redact Dossier Redactions"
-     when
-         Section(matchesType("dossier_redactions"))
-     then
-         section.redact("dossier_redactions", 29, "Dossier Redaction found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
-     end
-
-rule "30: Ignore dossier_redactions if confidential"
-    when
-        Section(!fileAttributeByLabelEqualsIgnoreCase("Confidentiality","confidential") && matchesType("dossier_redactions"));
-    then
-        section.ignore("dossier_redactions");
-    end
-
-// ex. "New Rules for PAD" - "Annex A" - page 21, page 35 (table without header), page 38 (in-text)
-// https://www.regexplanet.com/share/index.html?share=yyyypb71xkr
-rule "101: Redact CAS numbers"
-    when
-        Section(hasTableHeader("Sample #"))
-    then
-        section.redactCell("Sample #", 8, "PII", true, "Redacted because row is a vertebrate study", "Reg (EC) No 1107/2009 Art. 63 (2g)");
-    end
-
-rule "102: Guidelines FileAttributes"
-    when
-        Section((text.contains("DATA REQUIREMENT(S):") || text.contains("TEST GUIDELINE(S):")) && (text.contains("OECD") || text.contains("EPA") || text.contains("OPPTS")))
-    then
-        section.addFileAttribute("OECD Number", "OECD (No\\.? )?\\d{3}( \\(\\d{4}\\))?", false, 0);
-    end
-
-rule "8: Redact Author cells in Tables with Author header (Non vertebrate study)"
-    when
-        Section(hasTableHeader("h5.1"))
-    then
-        section.redactCell("h5.1", 8, "CBI_author", false, "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
     end
+*/