RED-6093: Prototype document structure

wip

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/model/SectionText.java

@@ -7,10 +7,20 @@ import java.util.List;
 import java.util.Map;
 import java.util.Set;
 
+import com.dslplatform.json.CompiledJson;
+import com.dslplatform.json.JsonAttribute;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
 import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.SectionArea;
 import com.iqser.red.service.redaction.v1.server.redaction.model.CellValue;
 import com.iqser.red.service.redaction.v1.server.redaction.model.Image;
+import com.iqser.red.service.redaction.v1.server.redaction.model.Paragraph;
 import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText;
 
 import lombok.AllArgsConstructor;
@@ -29,6 +39,7 @@ public class SectionText {
 
     private boolean isTable;
     private String headline;
+    List<Paragraph> paragraphs;
 
     @Builder.Default
     private List<SectionArea> sectionAreas = new ArrayList<>();

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/model/TextBlock.java

@@ -26,6 +26,8 @@ public class TextBlock extends AbstractTextContainer {
     @JsonIgnore
     private int rotation;
 
+    private int indexOnPage;
+
     @JsonIgnore
     private String mostPopularWordFont;
 
@@ -174,8 +176,8 @@ public class TextBlock extends AbstractTextContainer {
     }
 
 
-    public TextBlock(float minX, float maxX, float minY, float maxY, List<TextPositionSequence> sequences, int rotation) {
+    public TextBlock(float minX, float maxX, float minY, float maxY, List<TextPositionSequence> sequences, int rotation, int indexOnPage) {
-
+        this.indexOnPage = indexOnPage;
         this.minX = minX;
         this.maxX = maxX;
         this.minY = minY;
@@ -238,7 +240,7 @@ public class TextBlock extends AbstractTextContainer {
 
     public TextBlock copy() {
 
-        return new TextBlock(minX, maxX, minY, maxY, sequences, rotation);
+        return new TextBlock(minX, maxX, minY, maxY, sequences, rotation, indexOnPage);
     }
 
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/service/BlockificationService.java

@@ -30,13 +30,15 @@ public class BlockificationService {
      * This method is building blocks by expanding the minX/maxX and minY/maxY value on each word that is not split by the conditions.
      * This method must use text direction adjusted postions (DirAdj). Where {0,0} is on the upper left. Never try to change this!
      * Rulings (Table lines) must be adjusted to the text directions as well, when checking if a block is split by a ruling.
-     * @param textPositions The words of a page.
+     *
+     * @param textPositions         The words of a page.
      * @param horizontalRulingLines Horizontal table lines.
-     * @param verticalRulingLines Vertical table lines.
+     * @param verticalRulingLines   Vertical table lines.
      * @return Page object that contains the Textblock and text statistics.
      */
     public Page blockify(List<TextPositionSequence> textPositions, List<Ruling> horizontalRulingLines, List<Ruling> verticalRulingLines) {
 
+        int indexOnPage = 0;
         List<TextPositionSequence> chunkWords = new ArrayList<>();
         List<AbstractTextContainer> chunkBlockList1 = new ArrayList<>();
 
@@ -62,7 +64,9 @@ public class BlockificationService {
                     prevOrientation = chunkBlockList1.get(chunkBlockList1.size() - 1).getOrientation();
                 }
 
-                TextBlock cb1 = buildTextBlock(chunkWords);
+                TextBlock cb1 = buildTextBlock(chunkWords, indexOnPage);
+                indexOnPage++;
+
                 chunkBlockList1.add(cb1);
                 chunkWords = new ArrayList<>();
 
@@ -102,7 +106,7 @@ public class BlockificationService {
             }
         }
 
-        TextBlock cb1 = buildTextBlock(chunkWords);
+        TextBlock cb1 = buildTextBlock(chunkWords, indexOnPage);
         if (cb1 != null) {
             chunkBlockList1.add(cb1);
         }
@@ -163,7 +167,7 @@ public class BlockificationService {
     }
 
 
-    private TextBlock buildTextBlock(List<TextPositionSequence> wordBlockList) {
+    private TextBlock buildTextBlock(List<TextPositionSequence> wordBlockList, int indexOnPage) {
 
         TextBlock textBlock = null;
 
@@ -182,7 +186,13 @@ public class BlockificationService {
             styleFrequencyCounter.add(wordBlock.getFontStyle());
 
             if (textBlock == null) {
-                textBlock = new TextBlock(wordBlock.getMinXDirAdj(), wordBlock.getMaxXDirAdj(), wordBlock.getMinYDirAdj(), wordBlock.getMaxYDirAdj(), wordBlockList, wordBlock.getRotation());
+                textBlock = new TextBlock(wordBlock.getMinXDirAdj(),
+                        wordBlock.getMaxXDirAdj(),
+                        wordBlock.getMinYDirAdj(),
+                        wordBlock.getMaxYDirAdj(),
+                        wordBlockList,
+                        wordBlock.getRotation(),
+                        indexOnPage);
             } else {
                 TextBlock spatialEntity = textBlock.union(wordBlock);
                 textBlock.resize(spatialEntity.getMinX(), spatialEntity.getMinY(), spatialEntity.getWidth(), spatialEntity.getHeight());
@@ -213,10 +223,38 @@ public class BlockificationService {
                                     List<Ruling> horizontalRulingLines,
                                     List<Ruling> verticalRulingLines) {
 
-        return isSplitByRuling(maxX, minY, word.getMinXDirAdj(), word.getMinYDirAdj(), verticalRulingLines, word.getDir().getDegrees(), word.getPageWidth(), word.getPageHeight()) //
+        return isSplitByRuling(maxX,
-                || isSplitByRuling(minX, minY, word.getMinXDirAdj(), word.getMaxYDirAdj(), horizontalRulingLines, word.getDir().getDegrees(), word.getPageWidth(), word.getPageHeight()) //
+                minY,
-                || isSplitByRuling(maxX, minY, word.getMinXDirAdj(), word.getMinYDirAdj(), horizontalRulingLines, word.getDir().getDegrees(), word.getPageWidth(), word.getPageHeight()) //
+                word.getMinXDirAdj(),
-                || isSplitByRuling(minX, minY, word.getMinXDirAdj(), word.getMaxYDirAdj(), verticalRulingLines, word.getDir().getDegrees(), word.getPageWidth(), word.getPageHeight()); //
+                word.getMinYDirAdj(),
+                verticalRulingLines,
+                word.getDir().getDegrees(),
+                word.getPageWidth(),
+                word.getPageHeight()) //
+                || isSplitByRuling(minX,
+                minY,
+                word.getMinXDirAdj(),
+                word.getMaxYDirAdj(),
+                horizontalRulingLines,
+                word.getDir().getDegrees(),
+                word.getPageWidth(),
+                word.getPageHeight()) //
+                || isSplitByRuling(maxX,
+                minY,
+                word.getMinXDirAdj(),
+                word.getMinYDirAdj(),
+                horizontalRulingLines,
+                word.getDir().getDegrees(),
+                word.getPageWidth(),
+                word.getPageHeight()) //
+                || isSplitByRuling(minX,
+                minY,
+                word.getMinXDirAdj(),
+                word.getMaxYDirAdj(),
+                verticalRulingLines,
+                word.getDir().getDegrees(),
+                word.getPageWidth(),
+                word.getPageHeight()); //
     }
 
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/AtomicTextBlockEntity.java

@@ -0,0 +1,96 @@
+package com.iqser.red.service.redaction.v1.server.document.graph;
+
+import java.awt.geom.Rectangle2D;
+import java.util.List;
+
+import lombok.AccessLevel;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.experimental.FieldDefaults;
+
+@Data
+@Builder
+@AllArgsConstructor
+@FieldDefaults(level = AccessLevel.PRIVATE)
+public class AtomicTextBlockEntity implements CharSequence {
+
+    Integer id;
+    //string coords
+    Integer offset;
+    String searchText;
+    List<Integer> lineBreaks;
+
+    //position coords
+    List<Integer> stringIdxToPositionIdx;
+    List<Rectangle2D> positions;
+
+    Node parent;
+
+
+    public int indexOf(String searchTerm) {
+
+        int pos = searchText.indexOf(searchTerm);
+        return pos == -1 ? -1 : pos + offset;
+    }
+
+
+    public int numberOfLines() {
+
+        return lineBreaks.size();
+    }
+
+
+    public int getNextLinebreak(int startIndex) {
+
+        return lineBreaks.stream()//
+                .filter(linebreak -> linebreak > startIndex) //
+                .findFirst() //
+                .orElse(searchText.length()) + offset;
+    }
+
+
+    public Rectangle2D getPosition(int stringIdx) {
+
+        return positions.get(stringIdxToPositionIdx.get(stringIdx - offset));
+    }
+
+
+    public List<Rectangle2D> getPositions(int startStringIdx, int endStringIdx) {
+
+        return positions.subList(startStringIdx - offset, endStringIdx - offset);
+    }
+
+
+    @Override
+    public int length() {
+
+        return searchText.length();
+    }
+
+
+    @Override
+    public char charAt(int index) {
+
+        return searchText.charAt(index - offset);
+    }
+
+
+    @Override
+    public CharSequence subSequence(int start, int end) {
+
+        throw new UnsupportedOperationException("AtomicTextBlockEntity can not be sliced!");
+    }
+
+
+    public String getFirstLine() {
+
+        return searchText.substring(0, getNextLinebreak(0) - offset);
+    }
+
+    @Override
+    public String toString() {
+        return searchText;
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentEntity.java

@@ -0,0 +1,49 @@
+package com.iqser.red.service.redaction.v1.server.document.graph;
+
+import java.util.List;
+import java.util.stream.Stream;
+
+import lombok.AccessLevel;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.experimental.FieldDefaults;
+
+@Data
+@Builder
+@AllArgsConstructor
+@FieldDefaults(level = AccessLevel.PRIVATE)
+public class DocumentEntity implements Node {
+
+    List<SectionEntity> sections;
+    List<PageEntity> pages;
+    TableOfContents tableOfContents;
+    Integer numberOfPages;
+
+
+    @Override
+    public TextBlockEntity buildTextBlock() {
+
+        return streamAllNodes()
+                .filter(Node::hasTextBlock)
+                .map(Node::getTextBlock)
+                .collect(TextBlockCollector.collect());
+    }
+
+
+    private Stream<Node> streamAllNodes() {
+
+        return Stream.concat(Stream.concat(//
+                tableOfContents.streamEntriesInOrder().map(TableOfContents.Entry::node), //
+                pages.stream().map(PageEntity::getHeader)), //
+                pages.stream().map(PageEntity::getFooter)); //
+    }
+
+
+    @Override
+    public String toString() {
+
+        return tableOfContents.toString();
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/Entity2.java

@@ -0,0 +1,14 @@
+package com.iqser.red.service.redaction.v1.server.document.graph;
+
+import java.awt.geom.Rectangle2D;
+import java.util.List;
+
+public class Entity2 {
+
+    String uid;
+    String value;
+    List<Rectangle2D> positions;
+    Boolean redact;
+
+    List<Node> neighbours;
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/FooterEntity.java

@@ -0,0 +1,47 @@
+package com.iqser.red.service.redaction.v1.server.document.graph;
+
+import java.util.List;
+
+import lombok.AccessLevel;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.experimental.FieldDefaults;
+
+@Data
+@Builder
+@AllArgsConstructor
+@FieldDefaults(level = AccessLevel.PRIVATE)
+public class FooterEntity implements Node {
+
+    PageEntity page;
+    TextBlockEntity textBlock;
+    List<Entity2> entities;
+
+
+    @Override
+    public TextBlockEntity buildTextBlock() {
+
+        return getTextBlock();
+    }
+
+
+    @Override
+    public boolean hasTextBlock() {
+
+        return true;
+    }
+
+
+    @Override
+    public TextBlockEntity getTextBlock() {
+
+        return textBlock;
+    }
+
+    @Override
+    public String toString() {
+        return textBlock.toString();
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/HeaderEntity.java

@@ -0,0 +1,47 @@
+package com.iqser.red.service.redaction.v1.server.document.graph;
+
+import java.util.List;
+
+import lombok.AccessLevel;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.experimental.FieldDefaults;
+
+@Data
+@Builder
+@AllArgsConstructor
+@FieldDefaults(level = AccessLevel.PRIVATE)
+public class HeaderEntity implements Node {
+
+    PageEntity page;
+    TextBlockEntity textBlock;
+    List<Entity2> entities;
+
+
+    @Override
+    public TextBlockEntity buildTextBlock() {
+
+        return getTextBlock();
+    }
+
+
+    @Override
+    public boolean hasTextBlock() {
+
+        return true;
+    }
+
+
+    @Override
+    public TextBlockEntity getTextBlock() {
+
+        return textBlock;
+    }
+
+    @Override
+    public String toString() {
+        return textBlock.toString();
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/Node.java

@@ -0,0 +1,18 @@
+package com.iqser.red.service.redaction.v1.server.document.graph;
+
+public interface Node {
+    TextBlockEntity buildTextBlock();
+
+
+    default boolean hasTextBlock() {
+
+        return false;
+    }
+
+
+    default TextBlockEntity getTextBlock() {
+
+        throw new UnsupportedOperationException("Only terminal Nodes have direct access to TextBlocks");
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/PageEntity.java

@@ -0,0 +1,37 @@
+package com.iqser.red.service.redaction.v1.server.document.graph;
+
+import java.util.List;
+
+import lombok.AccessLevel;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.experimental.FieldDefaults;
+
+@Data
+@Builder
+@AllArgsConstructor
+@FieldDefaults(level = AccessLevel.PRIVATE)
+public class PageEntity implements Node {
+
+    Integer number;
+    Integer height;
+    Integer width;
+    List<SectionEntity> sections;
+    List<ParagraphEntity> paragraphs;
+    HeaderEntity header;
+    FooterEntity footer;
+
+
+    @Override
+    public TextBlockEntity buildTextBlock() {
+
+        return null;
+    }
+
+    @Override
+    public String toString() {
+        return number.toString();
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/ParagraphEntity.java

@@ -0,0 +1,54 @@
+package com.iqser.red.service.redaction.v1.server.document.graph;
+
+import java.util.List;
+
+import lombok.AccessLevel;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.experimental.FieldDefaults;
+
+@Data
+@Builder
+@AllArgsConstructor
+@FieldDefaults(level = AccessLevel.PRIVATE)
+public class ParagraphEntity implements Node {
+
+    String tocId;
+    Integer id;
+    Integer numberOnPage;
+    Integer numberInSection;
+
+    //foreign key
+    SectionEntity parentSection;
+    PageEntity page;
+    TextBlockEntity textBlock;
+    List<Entity2> entities;
+
+
+    @Override
+    public TextBlockEntity buildTextBlock() {
+
+        return getTextBlock();
+    }
+
+
+    @Override
+    public boolean hasTextBlock() {
+
+        return true;
+    }
+
+
+    @Override
+    public TextBlockEntity getTextBlock() {
+
+        return textBlock;
+    }
+
+
+    @Override
+    public String toString() {
+        return textBlock.toString();
+    }
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/SectionEntity.java

@@ -0,0 +1,56 @@
+package com.iqser.red.service.redaction.v1.server.document.graph;
+
+import java.util.List;
+
+import lombok.AccessLevel;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.experimental.FieldDefaults;
+
+@Data
+@Builder
+@AllArgsConstructor
+@FieldDefaults(level = AccessLevel.PRIVATE)
+public class SectionEntity implements Node {
+
+    Integer id;
+    TableOfContents tableOfContents;
+    String tocId;
+    Node parent;
+    //foreign key
+    TextBlockEntity headline;
+    List<SectionEntity> subSections;
+    List<ParagraphEntity> paragraphs;
+    List<Integer> pages;
+    List<Entity2> entities;
+
+
+    @Override
+    public String toString() {
+
+        return tocId + ": " + headline.toString();
+    }
+
+
+    @Override
+    public TextBlockEntity buildTextBlock() {
+
+        return tableOfContents.streamSubEntriesInOrder(tocId).map(TableOfContents.Entry::node).map(Node::getTextBlock).collect(TextBlockCollector.collect());
+    }
+
+
+    @Override
+    public boolean hasTextBlock() {
+
+        return true;
+    }
+
+
+    @Override
+    public TextBlockEntity getTextBlock() {
+
+        return headline;
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/TableCellEntity.java

@@ -0,0 +1,44 @@
+package com.iqser.red.service.redaction.v1.server.document.graph;
+
+import java.util.List;
+
+import lombok.AccessLevel;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.experimental.FieldDefaults;
+
+@Data
+@Builder
+@AllArgsConstructor
+@FieldDefaults(level = AccessLevel.PRIVATE)
+public class TableCellEntity implements Node {
+
+    //foreign key
+    Integer table;
+    TextBlockEntity textBlock;
+    List<Entity2> entities;
+
+
+    @Override
+    public TextBlockEntity buildTextBlock() {
+
+        return textBlock;
+    }
+
+
+    @Override
+    public boolean hasTextBlock() {
+
+        return true;
+    }
+
+
+    @Override
+    public TextBlockEntity getTextBlock() {
+
+        return textBlock;
+    }
+
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/TableEntity.java

@@ -0,0 +1,53 @@
+package com.iqser.red.service.redaction.v1.server.document.graph;
+
+import java.util.List;
+import java.util.stream.Stream;
+
+import lombok.AccessLevel;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.experimental.FieldDefaults;
+
+@Data
+@Builder
+@AllArgsConstructor
+@FieldDefaults(level = AccessLevel.PRIVATE)
+public class TableEntity implements Node {
+
+    Integer id;
+    Integer numberOfRows;
+    Integer numberOfCols;
+    List<List<TableCellEntity>> tableCells;
+
+    // graph connection
+    SectionEntity section;
+    PageEntity page;
+    List<Entity2> entities;
+
+
+    private Stream<TableCellEntity> streamTableCells() {
+
+        return tableCells.stream().flatMap(List::stream);
+    }
+
+
+    private Stream<TableCellEntity> streamTableRow(int row) {
+
+        return tableCells.get(row).stream();
+    }
+
+
+    private Stream<TableCellEntity> streamTableCol(int col) {
+
+        return tableCells.stream().map(row -> row.get(col));
+    }
+
+
+    @Override
+    public TextBlockEntity buildTextBlock() {
+
+        return streamTableCells().map(TableCellEntity::buildTextBlock).collect(TextBlockCollector.collect());
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/TableOfContents.java

@@ -0,0 +1,109 @@
+package com.iqser.red.service.redaction.v1.server.document.graph;
+
+import static java.lang.String.format;
+
+import java.util.Arrays;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.stream.Stream;
+
+import javax.management.openmbean.InvalidKeyException;
+
+import lombok.Data;
+
+@Data
+public class TableOfContents {
+
+    public static final String SECTION = "Sec";
+    public static final String HEADLINE = "H";
+    public static final String PARAGRAPH = "Par";
+    public static final String TABLE = "Tab";
+
+    List<Entry> entries;
+
+
+    public TableOfContents() {
+
+        entries = new LinkedList<>();
+    }
+
+
+    public String createNewEntryAndReturnId(String keyword, String summary, Node node) {
+
+        String id = String.format("%d", entries.size());
+        entries.add(new Entry(keyword, id, summary, new LinkedList<>(), node));
+        return id;
+    }
+
+
+    public String createNewChildEntryAndReturnId(String parentId, String keyword, String summary, Node node) {
+
+        Entry parent = getEntryById(parentId);
+        String childId = parentId + String.format(".%d", parent.children().size());
+        parent.children().add(new Entry(keyword, childId, summary, new LinkedList<>(), node));
+        return childId;
+    }
+
+
+    public Entry getEntryById(String parentId) {
+
+        List<Integer> ids = getIds(parentId);
+        if (ids.size() < 1) {
+            throw new InvalidKeyException(format("Section Identifier: \"%s\" is not valid.", parentId));
+        }
+        Entry entry = entries.get(ids.get(0));
+        for (int id : ids.subList(1, ids.size())) {
+            entry = entry.children().get(id);
+        }
+        return entry;
+    }
+
+
+    @Override
+    public String toString() {
+
+        return String.join("\n", streamEntriesInOrder().map(Entry::toString).toList());
+    }
+
+
+    public String toString(String id) {
+
+        return String.join("\n", streamSubEntriesInOrder(id).map(Entry::toString).toList());
+    }
+
+
+    public Stream<Entry> streamEntriesInOrder() {
+
+        return entries.stream().flatMap(TableOfContents::flatten);
+    }
+
+
+    public Stream<Entry> streamSubEntriesInOrder(String parentId) {
+
+        return Stream.of(getEntryById(parentId)).flatMap(TableOfContents::flatten);
+    }
+
+
+    private static List<Integer> getIds(String idsAsString) {
+
+        return Arrays.stream(idsAsString.split("\\.")).map(Integer::valueOf).toList();
+    }
+
+
+    private static Stream<Entry> flatten(Entry entry) {
+
+        return Stream.concat(Stream.of(entry), entry.children().stream().flatMap(TableOfContents::flatten));
+    }
+
+
+    public record Entry(String type, String id, String summary, List<Entry> children, Node node) {
+
+        @Override
+        public String toString() {
+
+            return id + ": " + type + ".: " + summary;
+        }
+
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/TextBlockCollector.java

@@ -0,0 +1,53 @@
+package com.iqser.red.service.redaction.v1.server.document.graph;
+
+import java.util.Collections;
+import java.util.Set;
+import java.util.function.BiConsumer;
+import java.util.function.BinaryOperator;
+import java.util.function.Function;
+import java.util.function.Supplier;
+import java.util.stream.Collector;
+
+public class TextBlockCollector implements Collector<TextBlockEntity, TextBlockEntity, TextBlockEntity> {
+
+    public static Collector<TextBlockEntity, TextBlockEntity, TextBlockEntity> collect() {
+
+        return new TextBlockCollector();
+    }
+
+
+    @Override
+    public Supplier<TextBlockEntity> supplier() {
+
+        return new TextBlockEntity(Collections.emptyList());
+    }
+
+
+    @Override
+    public BiConsumer<TextBlockEntity, TextBlockEntity> accumulator() {
+
+        return TextBlockEntity::concat;
+    }
+
+
+    @Override
+    public BinaryOperator<TextBlockEntity> combiner() {
+
+        return TextBlockEntity::concat;
+    }
+
+
+    @Override
+    public Function<TextBlockEntity, TextBlockEntity> finisher() {
+
+        return Function.identity();
+    }
+
+
+    @Override
+    public Set<Characteristics> characteristics() {
+
+        return Set.of(Characteristics.IDENTITY_FINISH, Characteristics.CONCURRENT);
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/TextBlockEntity.java

@@ -0,0 +1,164 @@
+package com.iqser.red.service.redaction.v1.server.document.graph;
+
+import java.awt.geom.Rectangle2D;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.function.Supplier;
+
+import lombok.AccessLevel;
+import lombok.Data;
+import lombok.experimental.FieldDefaults;
+
+@Data
+@FieldDefaults(level = AccessLevel.PRIVATE)
+public class TextBlockEntity implements CharSequence, Supplier<TextBlockEntity> {
+
+    List<AtomicTextBlockEntity> atomicTextBlocks;
+    StringBuilder searchTextBuilder;
+
+    //string coords
+    Integer offset;
+
+
+    public TextBlockEntity(List<AtomicTextBlockEntity> atomicTextBlocks) {
+
+        this.atomicTextBlocks = new LinkedList<>();
+        this.searchTextBuilder = new StringBuilder();
+        if (atomicTextBlocks.isEmpty()) {
+            offset = -1;
+            return;
+        }
+        var firstTextBlock = atomicTextBlocks.get(0);
+        this.atomicTextBlocks.add(firstTextBlock);
+        this.offset = firstTextBlock.getOffset();
+        this.searchTextBuilder.append(firstTextBlock.getSearchText());
+
+        atomicTextBlocks.subList(1, atomicTextBlocks.size()).forEach(this::concat);
+    }
+
+
+    public TextBlockEntity concat(TextBlockEntity textBlock) {
+
+        if (this.atomicTextBlocks.isEmpty() && this.offset == -1) {
+            this.offset = textBlock.getOffset();
+        } else if (textBlock.getOffset() != offset + length()) {
+            throw new UnsupportedOperationException("Can only concat consecutive TextBlocks");
+        }
+
+        this.searchTextBuilder.append(textBlock.getSearchTextBuilder());
+        this.atomicTextBlocks.addAll(textBlock.getAtomicTextBlocks());
+        return this;
+    }
+
+
+    public void concat(AtomicTextBlockEntity atomicTextBlock) {
+
+        if (this.atomicTextBlocks.isEmpty() && this.offset == -1) {
+            this.offset = atomicTextBlock.getOffset();
+        } else if (atomicTextBlock.getOffset() != offset + length()) {
+            throw new UnsupportedOperationException("Can only concat consecutive TextBlocks");
+        }
+
+        this.searchTextBuilder.append(atomicTextBlock.getSearchText());
+        this.atomicTextBlocks.add(atomicTextBlock);
+    }
+
+
+    public int indexOf(String searchTerm) {
+
+        int pos = this.searchTextBuilder.indexOf(searchTerm);
+        return pos == -1 ? -1 : pos + offset;
+    }
+
+
+    public int numberOfLines() {
+
+        return atomicTextBlocks.stream().map(AtomicTextBlockEntity::getLineBreaks).mapToInt(List::size).sum();
+    }
+
+
+    public int getNextLinebreak(int startIndex) {
+
+        return getAtomicTextBlockByStringIndex(startIndex).getNextLinebreak(startIndex);
+    }
+
+
+    public Rectangle2D getPositions(int stringIdx) {
+
+        return getAtomicTextBlockByStringIndex(stringIdx).getPosition(stringIdx);
+    }
+
+
+    public List<Rectangle2D> getPositions(int startStringIdx, int endStringIdx) {
+
+        List<AtomicTextBlockEntity> textBlocks = getAllAtomicTextBlocksPartiallyInStringIdxRange(startStringIdx, endStringIdx);
+
+        if (textBlocks.size() == 1) {
+            return textBlocks.get(0).getPositions(startStringIdx, endStringIdx);
+        }
+
+        var firstTextBlock = textBlocks.get(0);
+        List<Rectangle2D> positions = new LinkedList<>(firstTextBlock.getPositions(startStringIdx, firstTextBlock.getOffset() + firstTextBlock.length()));
+
+        for (var textBlock : textBlocks.subList(1, textBlocks.size() - 1)) {
+            positions.addAll(textBlock.getPositions());
+        }
+
+        var lastTextBlock = textBlocks.get(textBlocks.size() - 1);
+        positions.addAll(lastTextBlock.getPositions(lastTextBlock.getOffset(), endStringIdx));
+
+        return positions;
+    }
+
+
+    private AtomicTextBlockEntity getAtomicTextBlockByStringIndex(int stringIdx) {
+
+        return atomicTextBlocks.stream().filter(textBlock -> (textBlock.getOffset() + textBlock.length()) > stringIdx).findFirst().orElseThrow(IndexOutOfBoundsException::new);
+    }
+
+
+    private List<AtomicTextBlockEntity> getAllAtomicTextBlocksPartiallyInStringIdxRange(int startStringIdx, int endStringIdx) {
+
+        if (offset > endStringIdx || offset + length() <= startStringIdx) {
+            throw new IndexOutOfBoundsException();
+        }
+
+        return atomicTextBlocks.stream().filter(tb -> tb.getOffset() <= endStringIdx && tb.getOffset() + tb.length() > startStringIdx).toList();
+    }
+
+
+    public String getSearchText() {
+
+        return searchTextBuilder.toString();
+    }
+
+
+    @Override
+    public int length() {
+
+        return this.searchTextBuilder.length();
+    }
+
+
+    @Override
+    public char charAt(int index) {
+
+        return searchTextBuilder.charAt(index);
+    }
+
+
+    @Override
+    public CharSequence subSequence(int start, int end) {
+
+        throw new UnsupportedOperationException("AtomicTextBlockEntity can not be sliced!");
+    }
+
+
+    @Override
+    public TextBlockEntity get() {
+
+        return this;
+    }
+
+}
+

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/DocumentGraphFactory.java

@@ -0,0 +1,209 @@
+package com.iqser.red.service.redaction.v1.server.document.services;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.springframework.stereotype.Service;
+
+import com.iqser.red.service.redaction.v1.server.classification.model.Document;
+import com.iqser.red.service.redaction.v1.server.classification.model.Footer;
+import com.iqser.red.service.redaction.v1.server.classification.model.Header;
+import com.iqser.red.service.redaction.v1.server.classification.model.Page;
+import com.iqser.red.service.redaction.v1.server.classification.model.Section;
+import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
+import com.iqser.red.service.redaction.v1.server.document.graph.AtomicTextBlockEntity;
+import com.iqser.red.service.redaction.v1.server.document.graph.DocumentEntity;
+import com.iqser.red.service.redaction.v1.server.document.graph.FooterEntity;
+import com.iqser.red.service.redaction.v1.server.document.graph.HeaderEntity;
+import com.iqser.red.service.redaction.v1.server.document.graph.Node;
+import com.iqser.red.service.redaction.v1.server.document.graph.PageEntity;
+import com.iqser.red.service.redaction.v1.server.document.graph.ParagraphEntity;
+import com.iqser.red.service.redaction.v1.server.document.graph.SectionEntity;
+import com.iqser.red.service.redaction.v1.server.document.graph.TableOfContents;
+import com.iqser.red.service.redaction.v1.server.document.graph.TextBlockEntity;
+import com.iqser.red.service.redaction.v1.server.redaction.model.SearchTextWithTextPositionModel;
+import com.iqser.red.service.redaction.v1.server.redaction.service.SearchTextWithTextPositionFactory;
+import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
+
+import lombok.RequiredArgsConstructor;
+
+@Service
+@RequiredArgsConstructor
+public class DocumentGraphFactory {
+
+    private final SearchTextWithTextPositionFactory searchTextWithTextPositionFactory;
+
+    private Context context;
+
+
+    public DocumentEntity buildEntityFromDocument(Document document) {
+
+        context = new Context(new TableOfContents(), new LinkedList<>(), new LinkedList<>(), new LinkedList<>(), new LinkedList<>(), new AtomicInteger(0), new AtomicInteger(0));
+
+        context.pages.addAll(document.getPages().stream().map(this::buildPage).toList());
+
+        // is tracked by Table of Contents
+        for (int sectionIdx = 0; sectionIdx < document.getSections().size(); sectionIdx++) {
+            addSection(document.getSections().get(sectionIdx), sectionIdx);
+        }
+
+        // not tracked by Table of Contents
+        document.getHeaders().stream().map(Header::getTextBlocks).flatMap(List::stream).forEach(this::addHeader);
+        document.getFooters().stream().map(Footer::getTextBlocks).flatMap(List::stream).forEach(this::addFooter);
+
+        return DocumentEntity.builder().numberOfPages(context.pages.size()).pages(context.pages).sections(context.sections).tableOfContents(context.tableOfContents).build();
+    }
+
+
+    private void addSection(Section section, int sectionIdx) {
+
+        SectionEntity sectionEntity = SectionEntity.builder()
+                .id(sectionIdx)
+                .entities(new LinkedList<>())
+                .pages(new LinkedList<>())
+                .paragraphs(new LinkedList<>())
+                .tableOfContents(context.tableOfContents())
+                .subSections(new LinkedList<>())
+                .build();
+        context.sections().add(sectionEntity);
+        if (section.getPageBlocks().get(0) instanceof TextBlock) {
+            sectionEntity.setHeadline(new TextBlockEntity(List.of(buildAtomicTextBlock((TextBlock) section.getPageBlocks().get(0), sectionEntity))));
+            section.getPageBlocks().remove(0);
+        } else {
+            sectionEntity.setHeadline(emptyTextBlock(sectionEntity));
+        }
+
+        String sectionId = context.tableOfContents.createNewEntryAndReturnId(TableOfContents.SECTION,
+                buildSummary(sectionEntity.getHeadline().getAtomicTextBlocks().get(0)),
+                sectionEntity);
+        sectionEntity.setTocId(sectionId);
+
+        int paragraphIdx = 0;
+        for (AbstractTextContainer abstractTextContainer : section.getPageBlocks()) {
+            if (abstractTextContainer instanceof TextBlock) {
+                addParagraph(sectionEntity, (TextBlock) abstractTextContainer, paragraphIdx);
+                paragraphIdx++;
+            }
+        }
+    }
+
+
+    private TextBlockEntity emptyTextBlock(Node parent) {
+
+        return new TextBlockEntity(List.of(AtomicTextBlockEntity.builder()
+                .id(context.textBlockIdx.getAndIncrement())
+                .offset(context.stringOffset.get())
+                .searchText("")
+                .lineBreaks(Collections.emptyList())
+                .stringIdxToPositionIdx(Collections.emptyList())
+                .positions(Collections.emptyList())
+                .parent(parent)
+                .build()));
+    }
+
+
+    private void addParagraph(SectionEntity sectionEntity, TextBlock originalTextBlock, int idx) {
+
+        PageEntity page = getPage(originalTextBlock.getPage());
+        ParagraphEntity paragraph = ParagraphEntity.builder().id(idx).numberOnPage(originalTextBlock.getIndexOnPage()).page(page).parentSection(sectionEntity).build();
+
+        sectionEntity.getParagraphs().add(paragraph);
+        page.getParagraphs().add(paragraph);
+
+        if (!page.getSections().contains(sectionEntity)) {
+            page.getSections().add(sectionEntity);
+        }
+
+        var textBlock = buildAtomicTextBlock(originalTextBlock, paragraph);
+        paragraph.setTextBlock(new TextBlockEntity(List.of(textBlock)));
+
+        String tocId = context.tableOfContents.createNewChildEntryAndReturnId(sectionEntity.getTocId(), TableOfContents.PARAGRAPH, buildSummary(textBlock), paragraph);
+        paragraph.setTocId(tocId);
+    }
+
+
+    private static String buildSummary(AtomicTextBlockEntity textBlock) {
+
+        if (textBlock == null) {
+            return " probably a table";
+        }
+
+        String[] words = textBlock.getFirstLine().split(" ");
+        int bound = Math.min(words.length, 4);
+        List<String> list = new ArrayList<>(Arrays.asList(words).subList(0, bound));
+
+        return String.join(" ", list);
+    }
+
+
+    private PageEntity buildPage(Page p) {
+
+        return PageEntity.builder()
+                .height((int) p.getPageHeight())
+                .width((int) p.getPageWidth())
+                .number(p.getPageNumber())
+                .paragraphs(new LinkedList<>())
+                .sections(new LinkedList<>())
+                .build();
+    }
+
+
+    private void addFooter(TextBlock textBlock) {
+
+        PageEntity page = getPage(textBlock.getPage());
+        FooterEntity footer = FooterEntity.builder().page(page).build();
+        AtomicTextBlockEntity textBlockEntity = buildAtomicTextBlock(textBlock, footer);
+        footer.setTextBlock(new TextBlockEntity(List.of(textBlockEntity)));
+        page.setFooter(footer);
+    }
+
+
+    public void addHeader(TextBlock textBlock) {
+
+        PageEntity page = getPage(textBlock.getPage());
+        HeaderEntity header = HeaderEntity.builder().page(page).build();
+        AtomicTextBlockEntity textBlockEntity = buildAtomicTextBlock(textBlock, header);
+        header.setTextBlock(new TextBlockEntity(List.of(textBlockEntity)));
+        page.setHeader(header);
+    }
+
+
+    public AtomicTextBlockEntity buildAtomicTextBlock(TextBlock textBlock, Node parent) {
+
+        SearchTextWithTextPositionModel searchTextWithTextPositionModel = searchTextWithTextPositionFactory.buildSearchTextToTextPositionModel(textBlock.getSequences());
+        int offset = context.stringOffset().getAndAdd(searchTextWithTextPositionModel.getSearchText().length());
+
+        return AtomicTextBlockEntity.builder()
+                .id(context.textBlockIdx.getAndIncrement())
+                .parent(parent)
+                .searchText(searchTextWithTextPositionModel.getSearchText())
+                .lineBreaks(searchTextWithTextPositionModel.getLineBreaks())
+                .positions(searchTextWithTextPositionModel.getPositions())
+                .stringIdxToPositionIdx(searchTextWithTextPositionModel.getStringCoordsToPositionCoords())
+                .offset(offset)
+                .build();
+    }
+
+
+    public PageEntity getPage(int i) {
+
+        return context.pages.stream().filter(page -> page.getNumber() == i).findFirst().orElseThrow(NoSuchFieldError::new);
+    }
+
+
+    record Context(
+            TableOfContents tableOfContents,
+            List<PageEntity> pages,
+            List<SectionEntity> sections,
+            List<HeaderEntity> headers,
+            List<FooterEntity> footers,
+            AtomicInteger stringOffset,
+            AtomicInteger textBlockIdx) {
+
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/SearchTextWithTextPositionModel.java

@@ -1,9 +1,7 @@
 package com.iqser.red.service.redaction.v1.server.redaction.model;
 
+import java.awt.geom.Rectangle2D;
 import java.util.List;
-import java.util.Map;
-
-import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
 
 import lombok.AccessLevel;
 import lombok.Builder;
@@ -16,7 +14,7 @@ import lombok.experimental.FieldDefaults;
 public class SearchTextWithTextPositionModel {
 
     String searchText;
-    Map<Integer, Integer> stringCoordsToTextPositionCoords;
+    List<Integer> lineBreaks;
-    List<Integer> lineBreaksStringCoords;
+    List<Integer> stringCoordsToPositionCoords;
-    List<TextPositionSequence> textPositionSequences;
+    List<Rectangle2D> positions;
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java

@@ -1292,7 +1292,7 @@ public class Section {
     }
 
 
-    private Set<Entity> findEntities(String value,
+    public Set<Entity> findEntities(String value,
                                      String asType,
                                      boolean caseInsensitive,
                                      boolean redacted,

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/SearchTextWithTextPositionFactory.java

@@ -1,13 +1,13 @@
 package com.iqser.red.service.redaction.v1.server.redaction.service;
 
-import java.util.HashMap;
+import java.awt.geom.Rectangle2D;
 import java.util.LinkedList;
 import java.util.List;
-import java.util.Map;
 import java.util.Objects;
 
 import org.springframework.stereotype.Service;
 
+import com.iqser.red.service.redaction.v1.server.parsing.model.RedTextPosition;
 import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
 import com.iqser.red.service.redaction.v1.server.redaction.model.SearchTextWithTextPositionModel;
 
@@ -16,7 +16,7 @@ public class SearchTextWithTextPositionFactory {
 
     public SearchTextWithTextPositionModel buildSearchTextToTextPositionModel(List<TextPositionSequence> sequences) {
 
-        Map<Integer, Integer> stringIdxToPositionIdx = new HashMap<>();
+        List<Integer> stringIdxToPositionIdx = new LinkedList<>();
         List<Integer> lineBreaksStringIdx = new LinkedList<>();
         StringBuilder sb = new StringBuilder();
 
@@ -37,7 +37,7 @@ public class SearchTextWithTextPositionFactory {
                         !isHyphenLinebreak(currentUnicode)) {
 
                     sb.append(currentUnicode);
-                    stringIdxToPositionIdx.put(stringIdx, positionIdx);
+                    stringIdxToPositionIdx.add(positionIdx);
                     ++stringIdx;
                 }
 
@@ -47,15 +47,24 @@ public class SearchTextWithTextPositionFactory {
 
             previousUnicode = " ";
             sb.append(previousUnicode);
-            stringIdxToPositionIdx.put(stringIdx, positionIdx);
+            stringIdxToPositionIdx.add(positionIdx);
             ++stringIdx;
         }
 
+        assert sb.length() == stringIdxToPositionIdx.size();
+
+        List<Rectangle2D> positions = sequences.stream()//
+                .map(TextPositionSequence::getTextPositions)//
+                .flatMap(List::stream)//
+                .map(RedTextPosition::getPosition)//
+                .map(a -> (Rectangle2D) new Rectangle2D.Float(a[0], a[1], a[2], a[3]))
+                .toList();
+
         return SearchTextWithTextPositionModel.builder()
-                .searchText(sb.toString().stripTrailing())
+                .searchText(sb.toString())
-                .lineBreaksStringCoords(lineBreaksStringIdx)
+                .lineBreaks(lineBreaksStringIdx)
-                .stringCoordsToTextPositionCoords(stringIdxToPositionIdx)
+                .stringCoordsToPositionCoords(stringIdxToPositionIdx)
-                .textPositionSequences(sequences)
+                .positions(positions)
                 .build();
     }
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/OffsetStringUtils.java

@@ -16,8 +16,8 @@ public final class OffsetStringUtils {
      * Same logic as in StringUtils.redactBetween, but returns a list of object with offsets insteadof on the Strings only.
      *
      * @param str – the String containing the substrings, null returns null, empty returns empty
-     * @param open – the String identifying the start of the substring, empty returns null
+     * @param open – the String identifying the startIdx of the substring, empty returns null
-     * @param close – the String identifying the end of the substring, empty returns null
+     * @param close – the String identifying the endIdx of the substring, empty returns null
      * @return a list of Strings with their offsets
      */
     public List<OffsetString> substringsBetween(final String str, final String open, final String close) {

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/SearchImplementation.java

@@ -73,7 +73,18 @@ public class SearchImplementation {
         } else {
             return this.trie.containsMatch(textToCheck);
         }
+    }
 
+    public List<MatchPosition> getMatches(CharSequence text) {
+        if (this.values.isEmpty()) {
+            return new ArrayList<>();
+        }
+        if (this.pattern != null) {
+
+            return this.pattern.matcher(text).results().map(r -> new MatchPosition(r.start(), r.end())).collect(Collectors.toList());
+        } else {
+            return this.trie.parseText(text).stream().map(r -> new MatchPosition(r.getStart(), r.getEnd() + 1)).collect(Collectors.toList());
+        }
     }
 
 

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AbstractTestWithDictionaries.java

@@ -0,0 +1,520 @@
+package com.iqser.red.service.redaction.v1.server;
+
+import static org.mockito.Mockito.when;
+
+import java.io.BufferedReader;
+import java.io.ByteArrayInputStream;
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.net.URL;
+import java.nio.charset.StandardCharsets;
+import java.time.OffsetDateTime;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.runner.RunWith;
+import org.kie.api.KieServices;
+import org.kie.api.builder.KieBuilder;
+import org.kie.api.builder.KieFileSystem;
+import org.kie.api.builder.KieModule;
+import org.kie.api.runtime.KieContainer;
+import org.mockito.stubbing.Answer;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
+import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration;
+import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.boot.test.mock.mockito.MockBean;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.context.annotation.Import;
+import org.springframework.context.annotation.Primary;
+import org.springframework.core.io.ClassPathResource;
+import org.springframework.test.context.junit4.SpringRunner;
+
+import com.iqser.red.service.persistence.service.v1.api.model.common.JSONPrimitive;
+import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.configuration.Colors;
+import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
+import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntry;
+import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.Type;
+import com.iqser.red.service.redaction.v1.model.AnalyzeRequest;
+import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
+import com.iqser.red.service.redaction.v1.server.client.RulesClient;
+import com.iqser.red.service.redaction.v1.server.redaction.utils.ResourceLoader;
+import com.iqser.red.service.redaction.v1.server.redaction.utils.TextNormalizationUtilities;
+import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
+import com.iqser.red.storage.commons.StorageAutoConfiguration;
+import com.iqser.red.storage.commons.service.StorageService;
+
+import lombok.SneakyThrows;
+
+@RunWith(SpringRunner.class)
+@SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
+@Import(AbstractTestWithDictionaries.TestConfiguration.class)
+public class AbstractTestWithDictionaries {
+
+    private static final String RULES = loadFromClassPath("drools/rules.drl");
+    private static final String VERTEBRATE = "vertebrate";
+    private static final String ADDRESS = "CBI_address";
+    private static final String AUTHOR = "CBI_author";
+    private static final String SPONSOR = "CBI_sponsor";
+    private static final String NO_REDACTION_INDICATOR = "no_redaction_indicator";
+    private static final String REDACTION_INDICATOR = "redaction_indicator";
+    private static final String HINT_ONLY = "hint_only";
+    private static final String MUST_REDACT = "must_redact";
+    private static final String PUBLISHED_INFORMATION = "published_information";
+    private static final String TEST_METHOD = "test_method";
+    private static final String PURITY = "purity";
+    private static final String IMAGE = "image";
+    private static final String LOGO = "logo";
+    private static final String SIGNATURE = "signature";
+    private static final String FORMULA = "formula";
+    private static final String OCR = "ocr";
+    private static final String DOSSIER_REDACTIONS = "dossier_redactions";
+    private static final String IMPORTED_REDACTION = "imported_redaction";
+    private static final String PII = "PII";
+    private static final String ROTATE_SIMPLE = "RotateSimple";
+
+    protected final static String TEST_DOSSIER_TEMPLATE_ID = "123";
+    protected final static String TEST_DOSSIER_ID = "123";
+    protected final static String TEST_FILE_ID = "123";
+
+    @Autowired
+    private StorageService storageService;
+
+    @MockBean
+    private DictionaryClient dictionaryClient;
+
+    @MockBean
+    private RulesClient rulesClient;
+
+    private final Map<String, List<String>> dictionary = new HashMap<>();
+    private final Map<String, List<String>> dossierDictionary = new HashMap<>();
+    private final Map<String, List<String>> falsePositive = new HashMap<>();
+    private final Map<String, List<String>> falseRecommendation = new HashMap<>();
+    private final Map<String, String> typeColorMap = new HashMap<>();
+    private final Map<String, Boolean> hintTypeMap = new HashMap<>();
+    private final Map<String, Boolean> caseInSensitiveMap = new HashMap<>();
+    private final Map<String, Boolean> recommendationTypeMap = new HashMap<>();
+    private final Map<String, Integer> rankTypeMap = new HashMap<>();
+    private final Colors colors = new Colors();
+    private final Map<String, Long> reanalysisVersions = new HashMap<>();
+    private final Set<String> deleted = new HashSet<>();
+
+
+    @Before
+    public void stubClients() {
+
+        when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L);
+        when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(JSONPrimitive.of(RULES));
+
+        loadDictionaryForTest();
+        loadTypeForTest();
+        loadNerForTest();
+        when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L);
+        when(dictionaryClient.getAllTypesForDossierTemplate(TEST_DOSSIER_TEMPLATE_ID, false)).thenReturn(getTypeResponse());
+
+        when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L);
+        when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, false)).thenReturn(List.of(Type.builder()
+                .id(DOSSIER_REDACTIONS + ":" + TEST_DOSSIER_TEMPLATE_ID)
+                .type(DOSSIER_REDACTIONS)
+                .dossierTemplateId(TEST_DOSSIER_ID)
+                .hexColor("#ffe187")
+                .isHint(hintTypeMap.get(DOSSIER_REDACTIONS))
+                .isCaseInsensitive(caseInSensitiveMap.get(DOSSIER_REDACTIONS))
+                .isRecommendation(recommendationTypeMap.get(DOSSIER_REDACTIONS))
+                .rank(rankTypeMap.get(DOSSIER_REDACTIONS))
+                .build()));
+
+        mockDictionaryCalls(null);
+        mockDictionaryCalls(0L);
+
+        when(dictionaryClient.getColors(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(colors);
+    }
+
+
+    private static String loadFromClassPath(String path) {
+
+        URL resource = ResourceLoader.class.getClassLoader().getResource(path);
+        if (resource == null) {
+            throw new IllegalArgumentException("could not load classpath resource: drools/rules.drl");
+        }
+        try (BufferedReader br = new BufferedReader(new InputStreamReader(resource.openStream(), StandardCharsets.UTF_8))) {
+            StringBuilder sb = new StringBuilder();
+            String str;
+            while ((str = br.readLine()) != null) {
+                sb.append(str).append("\n");
+            }
+            return sb.toString();
+        } catch (IOException e) {
+            throw new IllegalArgumentException("could not load classpath resource: " + path, e);
+        }
+    }
+
+
+    private void mockDictionaryCalls(Long version) {
+
+        when(dictionaryClient.getDictionaryForType(VERTEBRATE + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(VERTEBRATE,
+                false));
+        when(dictionaryClient.getDictionaryForType(ADDRESS + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(ADDRESS, false));
+        when(dictionaryClient.getDictionaryForType(AUTHOR + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(AUTHOR, false));
+        when(dictionaryClient.getDictionaryForType(SPONSOR + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(SPONSOR, false));
+        when(dictionaryClient.getDictionaryForType(NO_REDACTION_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(
+                NO_REDACTION_INDICATOR,
+                false));
+        when(dictionaryClient.getDictionaryForType(REDACTION_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(
+                REDACTION_INDICATOR,
+                false));
+        when(dictionaryClient.getDictionaryForType(HINT_ONLY + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(HINT_ONLY, false));
+        when(dictionaryClient.getDictionaryForType(MUST_REDACT + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(MUST_REDACT,
+                false));
+        when(dictionaryClient.getDictionaryForType(PUBLISHED_INFORMATION + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(
+                PUBLISHED_INFORMATION,
+                false));
+        when(dictionaryClient.getDictionaryForType(TEST_METHOD + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(TEST_METHOD,
+                false));
+        when(dictionaryClient.getDictionaryForType(PII + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(PII, false));
+        when(dictionaryClient.getDictionaryForType(PURITY + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(PURITY, false));
+        when(dictionaryClient.getDictionaryForType(IMAGE + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(IMAGE, false));
+        when(dictionaryClient.getDictionaryForType(OCR + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(OCR, false));
+        when(dictionaryClient.getDictionaryForType(LOGO + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(LOGO, false));
+        when(dictionaryClient.getDictionaryForType(SIGNATURE + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(SIGNATURE, false));
+        when(dictionaryClient.getDictionaryForType(FORMULA + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(FORMULA, false));
+        when(dictionaryClient.getDictionaryForType(ROTATE_SIMPLE + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(ROTATE_SIMPLE,
+                false));
+        when(dictionaryClient.getDictionaryForType(DOSSIER_REDACTIONS + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(
+                DOSSIER_REDACTIONS,
+                true));
+        when(dictionaryClient.getDictionaryForType(IMPORTED_REDACTION + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(
+                IMPORTED_REDACTION,
+                true));
+
+    }
+
+
+    private String cleanDictionaryEntry(String entry) {
+
+        return TextNormalizationUtilities.removeHyphenLineBreaks(entry).replaceAll("\\n", " ");
+    }
+
+
+    private List<DictionaryEntry> toDictionaryEntry(List<String> entries) {
+
+        if (entries == null) {
+            entries = Collections.emptyList();
+        }
+
+        List<DictionaryEntry> dictionaryEntries = new ArrayList<>();
+        entries.forEach(entry -> {
+            dictionaryEntries.add(DictionaryEntry.builder().value(entry).version(reanalysisVersions.getOrDefault(entry, 0L)).deleted(deleted.contains(entry)).build());
+        });
+        return dictionaryEntries;
+    }
+
+
+    private Type getDictionaryResponse(String type, boolean isDossierDictionary) {
+
+        return Type.builder()
+                .id(type + ":" + TEST_DOSSIER_TEMPLATE_ID)
+                .hexColor(typeColorMap.get(type))
+                .entries(isDossierDictionary ? toDictionaryEntry(dossierDictionary.get(type)) : toDictionaryEntry(dictionary.get(type)))
+                .falsePositiveEntries(falsePositive.containsKey(type) ? toDictionaryEntry(falsePositive.get(type)) : new ArrayList<>())
+                .falseRecommendationEntries(falseRecommendation.containsKey(type) ? toDictionaryEntry(falseRecommendation.get(type)) : new ArrayList<>())
+                .isHint(hintTypeMap.get(type))
+                .isCaseInsensitive(caseInSensitiveMap.get(type))
+                .isRecommendation(recommendationTypeMap.get(type))
+                .rank(rankTypeMap.get(type))
+                .build();
+    }
+
+
+    private void loadTypeForTest() {
+
+        typeColorMap.put(VERTEBRATE, "#ff85f7");
+        typeColorMap.put(ADDRESS, "#ffe187");
+        typeColorMap.put(AUTHOR, "#ffe187");
+        typeColorMap.put(SPONSOR, "#85ebff");
+        typeColorMap.put(NO_REDACTION_INDICATOR, "#be85ff");
+        typeColorMap.put(REDACTION_INDICATOR, "#caff85");
+        typeColorMap.put(HINT_ONLY, "#abc0c4");
+        typeColorMap.put(MUST_REDACT, "#fab4c0");
+        typeColorMap.put(PUBLISHED_INFORMATION, "#85ebff");
+        typeColorMap.put(TEST_METHOD, "#91fae8");
+        typeColorMap.put(PII, "#66ccff");
+        typeColorMap.put(PURITY, "#ffe187");
+        typeColorMap.put(IMAGE, "#fcc5fb");
+        typeColorMap.put(OCR, "#fcc5fb");
+        typeColorMap.put(LOGO, "#ffe187");
+        typeColorMap.put(FORMULA, "#ffe187");
+        typeColorMap.put(SIGNATURE, "#ffe187");
+        typeColorMap.put(IMPORTED_REDACTION, "#fcfbe6");
+        typeColorMap.put(ROTATE_SIMPLE, "#66ccff");
+
+        hintTypeMap.put(VERTEBRATE, true);
+        hintTypeMap.put(ADDRESS, false);
+        hintTypeMap.put(AUTHOR, false);
+        hintTypeMap.put(SPONSOR, false);
+        hintTypeMap.put(NO_REDACTION_INDICATOR, true);
+        hintTypeMap.put(REDACTION_INDICATOR, true);
+        hintTypeMap.put(HINT_ONLY, true);
+        hintTypeMap.put(MUST_REDACT, true);
+        hintTypeMap.put(PUBLISHED_INFORMATION, true);
+        hintTypeMap.put(TEST_METHOD, true);
+        hintTypeMap.put(PII, false);
+        hintTypeMap.put(PURITY, false);
+        hintTypeMap.put(IMAGE, true);
+        hintTypeMap.put(OCR, true);
+        hintTypeMap.put(FORMULA, false);
+        hintTypeMap.put(LOGO, false);
+        hintTypeMap.put(SIGNATURE, false);
+        hintTypeMap.put(DOSSIER_REDACTIONS, false);
+        hintTypeMap.put(IMPORTED_REDACTION, false);
+        hintTypeMap.put(ROTATE_SIMPLE, false);
+
+        caseInSensitiveMap.put(VERTEBRATE, true);
+        caseInSensitiveMap.put(ADDRESS, false);
+        caseInSensitiveMap.put(AUTHOR, false);
+        caseInSensitiveMap.put(SPONSOR, false);
+        caseInSensitiveMap.put(NO_REDACTION_INDICATOR, true);
+        caseInSensitiveMap.put(REDACTION_INDICATOR, true);
+        caseInSensitiveMap.put(HINT_ONLY, true);
+        caseInSensitiveMap.put(MUST_REDACT, true);
+        caseInSensitiveMap.put(PUBLISHED_INFORMATION, true);
+        caseInSensitiveMap.put(TEST_METHOD, false);
+        caseInSensitiveMap.put(PII, false);
+        caseInSensitiveMap.put(PURITY, false);
+        caseInSensitiveMap.put(IMAGE, true);
+        caseInSensitiveMap.put(OCR, true);
+        caseInSensitiveMap.put(SIGNATURE, true);
+        caseInSensitiveMap.put(LOGO, true);
+        caseInSensitiveMap.put(FORMULA, true);
+        caseInSensitiveMap.put(DOSSIER_REDACTIONS, false);
+        caseInSensitiveMap.put(IMPORTED_REDACTION, false);
+        caseInSensitiveMap.put(ROTATE_SIMPLE, true);
+
+        recommendationTypeMap.put(VERTEBRATE, false);
+        recommendationTypeMap.put(ADDRESS, false);
+        recommendationTypeMap.put(AUTHOR, false);
+        recommendationTypeMap.put(SPONSOR, false);
+        recommendationTypeMap.put(NO_REDACTION_INDICATOR, false);
+        recommendationTypeMap.put(REDACTION_INDICATOR, false);
+        recommendationTypeMap.put(HINT_ONLY, false);
+        recommendationTypeMap.put(MUST_REDACT, false);
+        recommendationTypeMap.put(PUBLISHED_INFORMATION, false);
+        recommendationTypeMap.put(TEST_METHOD, false);
+        recommendationTypeMap.put(PII, false);
+        recommendationTypeMap.put(PURITY, false);
+        recommendationTypeMap.put(IMAGE, false);
+        recommendationTypeMap.put(OCR, false);
+        recommendationTypeMap.put(FORMULA, false);
+        recommendationTypeMap.put(SIGNATURE, false);
+        recommendationTypeMap.put(LOGO, false);
+        recommendationTypeMap.put(DOSSIER_REDACTIONS, false);
+        recommendationTypeMap.put(IMPORTED_REDACTION, false);
+        recommendationTypeMap.put(ROTATE_SIMPLE, false);
+
+        rankTypeMap.put(PURITY, 155);
+        rankTypeMap.put(PII, 150);
+        rankTypeMap.put(ADDRESS, 140);
+        rankTypeMap.put(AUTHOR, 130);
+        rankTypeMap.put(SPONSOR, 120);
+        rankTypeMap.put(VERTEBRATE, 110);
+        rankTypeMap.put(MUST_REDACT, 100);
+        rankTypeMap.put(REDACTION_INDICATOR, 90);
+        rankTypeMap.put(NO_REDACTION_INDICATOR, 80);
+        rankTypeMap.put(PUBLISHED_INFORMATION, 70);
+        rankTypeMap.put(TEST_METHOD, 60);
+        rankTypeMap.put(HINT_ONLY, 50);
+        rankTypeMap.put(IMAGE, 30);
+        rankTypeMap.put(OCR, 29);
+        rankTypeMap.put(LOGO, 28);
+        rankTypeMap.put(SIGNATURE, 27);
+        rankTypeMap.put(FORMULA, 26);
+        rankTypeMap.put(DOSSIER_REDACTIONS, 200);
+        rankTypeMap.put(IMPORTED_REDACTION, 200);
+        rankTypeMap.put(ROTATE_SIMPLE, 150);
+
+        colors.setSkippedColor("#cccccc");
+        colors.setRequestAddColor("#04b093");
+        colors.setRequestRemoveColor("#04b093");
+    }
+
+
+    @SneakyThrows
+    private void loadNerForTest() {
+
+        ClassPathResource responseJson = new ClassPathResource("files/ner_response.json");
+        storageService.storeObject(RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.NER_ENTITIES), responseJson.getInputStream());
+    }
+
+
+    private List<Type> getTypeResponse() {
+
+        return typeColorMap.entrySet()
+                .stream()
+                .map(typeColor -> Type.builder()
+                        .id(typeColor.getKey() + ":" + TEST_DOSSIER_TEMPLATE_ID)
+                        .type(typeColor.getKey())
+                        .dossierTemplateId(TEST_DOSSIER_TEMPLATE_ID)
+                        .hexColor(typeColor.getValue())
+                        .isHint(hintTypeMap.get(typeColor.getKey()))
+                        .isCaseInsensitive(caseInSensitiveMap.get(typeColor.getKey()))
+                        .isRecommendation(recommendationTypeMap.get(typeColor.getKey()))
+                        .rank(rankTypeMap.get(typeColor.getKey()))
+                        .build())
+
+                .collect(Collectors.toList());
+    }
+
+    protected List<File> getPathsRecursively(File path) {
+
+        List<File> result = new ArrayList<>();
+        if (path == null || path.listFiles() == null) {
+            return result;
+        }
+        for (File f : path.listFiles()) {
+            if (f.isFile()) {
+                result.add(f);
+            } else {
+                result.addAll(getPathsRecursively(f));
+            }
+        }
+        return result;
+
+    }
+
+    protected void loadOnlyDictionaryForSimpleFile() {
+
+        dictionary.clear();
+        dictionary.computeIfAbsent(ROTATE_SIMPLE, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/RotateTestFileSimple.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
+    }
+
+
+    private void loadDictionaryForTest() {
+
+        dictionary.computeIfAbsent(AUTHOR, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/CBI_author.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
+        dictionary.computeIfAbsent(SPONSOR, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/CBI_sponsor.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
+        dictionary.computeIfAbsent(VERTEBRATE, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/vertebrate.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
+        dictionary.computeIfAbsent(ADDRESS, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/CBI_address.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
+        dictionary.computeIfAbsent(NO_REDACTION_INDICATOR, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/no_redaction_indicator.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
+        dictionary.computeIfAbsent(REDACTION_INDICATOR, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/redaction_indicator.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
+        dictionary.computeIfAbsent(HINT_ONLY, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/hint_only.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
+        dictionary.computeIfAbsent(MUST_REDACT, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/must_redact.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
+        dictionary.computeIfAbsent(PUBLISHED_INFORMATION, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/published_information.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
+        dictionary.computeIfAbsent(TEST_METHOD, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/test_method.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
+        dictionary.computeIfAbsent(PII, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/PII.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
+        dictionary.computeIfAbsent(PURITY, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/purity.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
+        dictionary.computeIfAbsent(IMAGE, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/empty.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
+        dictionary.computeIfAbsent(OCR, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/empty.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
+        dictionary.computeIfAbsent(LOGO, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/empty.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
+        dictionary.computeIfAbsent(SIGNATURE, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/empty.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
+        dictionary.computeIfAbsent(FORMULA, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/empty.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
+        dossierDictionary.computeIfAbsent(DOSSIER_REDACTIONS, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/dossier_redactions.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
+        dossierDictionary.put(IMPORTED_REDACTION, new ArrayList<>());
+
+        falsePositive.computeIfAbsent(PII, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/PII_false_positive.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
+
+    }
+
+
+    @SneakyThrows
+    protected AnalyzeRequest prepareStorage(String file) {
+
+        return prepareStorage(file, "files/cv_service_empty_response.json");
+    }
+
+
+    @SneakyThrows
+    protected AnalyzeRequest prepareStorage(String file, String cvServiceResponseFile) {
+
+        ClassPathResource pdfFileResource = new ClassPathResource(file);
+        ClassPathResource cvServiceResponseFileResource = new ClassPathResource(cvServiceResponseFile);
+
+        return prepareStorage(pdfFileResource.getInputStream(), cvServiceResponseFileResource.getInputStream());
+    }
+
+
+    @SneakyThrows
+    protected AnalyzeRequest prepareStorage(InputStream fileStream, InputStream cvServiceResponseFileStream) {
+
+        AnalyzeRequest request = AnalyzeRequest.builder()
+                .dossierTemplateId(TEST_DOSSIER_TEMPLATE_ID)
+                .dossierId(TEST_DOSSIER_ID)
+                .fileId(TEST_FILE_ID)
+                .lastProcessed(OffsetDateTime.now())
+                .build();
+
+        storageService.storeObject(RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.TABLES), cvServiceResponseFileStream);
+        storageService.storeObject(RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.ORIGIN), fileStream);
+
+        return request;
+    }
+
+
+    @After
+    public void cleanupStorage() {
+
+        if (this.storageService instanceof FileSystemBackedStorageService) {
+            ((FileSystemBackedStorageService) this.storageService).clearStorage();
+        }
+    }
+
+
+    @Configuration
+    @EnableAutoConfiguration(exclude = {RabbitAutoConfiguration.class, StorageAutoConfiguration.class})
+    public static class TestConfiguration {
+
+        @Bean
+        public KieContainer kieContainer() {
+
+            KieServices kieServices = KieServices.Factory.get();
+
+            KieFileSystem kieFileSystem = kieServices.newKieFileSystem();
+            InputStream input = new ByteArrayInputStream(RULES.getBytes(StandardCharsets.UTF_8));
+            kieFileSystem.write("src/test/resources/drools/rules.drl", kieServices.getResources().newInputStreamResource(input));
+            KieBuilder kieBuilder = kieServices.newKieBuilder(kieFileSystem);
+            kieBuilder.buildAll();
+            KieModule kieModule = kieBuilder.getKieModule();
+
+            return kieServices.newKieContainer(kieModule.getReleaseId());
+        }
+
+        @Bean
+        @Primary
+        public StorageService inmemoryStorage() {
+
+            return new FileSystemBackedStorageService();
+        }
+
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DocumentGraphIntegrationTest.java

@@ -0,0 +1,65 @@
+package com.iqser.red.service.redaction.v1.server;
+
+import java.util.List;
+
+import org.junit.Test;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.core.io.ClassPathResource;
+
+import com.iqser.red.service.redaction.v1.server.document.graph.DocumentEntity;
+import com.iqser.red.service.redaction.v1.server.document.graph.TextBlockEntity;
+import com.iqser.red.service.redaction.v1.server.document.services.DocumentGraphFactory;
+import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
+import com.iqser.red.service.redaction.v1.server.redaction.service.DictionaryService;
+import com.iqser.red.service.redaction.v1.server.redaction.utils.SearchImplementation;
+import com.iqser.red.service.redaction.v1.server.segmentation.PdfSegmentationService;
+
+import lombok.SneakyThrows;
+
+public class DocumentGraphIntegrationTest extends AbstractTestWithDictionaries {
+
+    @Autowired
+    private DocumentGraphFactory documentGraphFactory;
+
+    @Autowired
+    private PdfSegmentationService segmentationService;
+
+    @Autowired
+    private DictionaryService dictionaryService;
+
+
+    @Test
+    @SneakyThrows
+    public void testBuildDocumentEntity() {
+
+        String filename = "files/new/crafted document";
+
+        prepareStorage(filename + ".pdf");
+        ClassPathResource fileResource = new ClassPathResource(filename + ".pdf");
+
+        var classifiedDoc = segmentationService.parseDocument(TEST_DOSSIER_ID, TEST_FILE_ID, fileResource.getInputStream(), null);
+        DocumentEntity document = documentGraphFactory.buildEntityFromDocument(classifiedDoc);
+
+        var start = System.currentTimeMillis();
+
+        TextBlockEntity textBlock = document.buildTextBlock();
+        dictionaryService.updateDictionary(TEST_DOSSIER_TEMPLATE_ID, TEST_DOSSIER_ID);
+        Dictionary dictionary = dictionaryService.getDeepCopyDictionary(TEST_DOSSIER_TEMPLATE_ID, TEST_DOSSIER_ID);
+
+        for (var model : dictionary.getDictionaryModels()) {
+            List<SearchImplementation.MatchPosition> entries = model.getEntriesSearch().getMatches(textBlock);
+            List<SearchImplementation.MatchPosition> falsePositives = model.getFalsePositiveSearch().getMatches(textBlock);
+            List<SearchImplementation.MatchPosition> falseRecommendations = model.getFalseRecommendationsSearch().getMatches(textBlock);
+
+            if (!entries.isEmpty()) {
+                var pos = entries.get(0);
+                System.out.println(textBlock.getPositions(pos.startIndex(), pos.endIndex()));
+            }
+
+        }
+
+        System.out.printf("Search took %fs", ((float) (System.currentTimeMillis() - start)) / 1000);
+
+    }
+
+}