RED-6009: Document Tree Structure

* some refactoring like renaming
* added migration poc

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/adapter/RedactionLogEntryAdapter.java

@@ -0,0 +1,164 @@
+package com.iqser.red.service.redaction.v1.server.layoutparsing.classification.adapter;
+
+import static java.lang.String.format;
+import static java.util.stream.Collectors.groupingBy;
+
+import java.awt.geom.Rectangle2D;
+import java.util.Collection;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import org.springframework.stereotype.Service;
+
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
+import com.iqser.red.service.redaction.v1.server.exception.NotFoundException;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.EntityType;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionEntity;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionPosition;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.Page;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.SemanticNode;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.document.services.EntityCreationService;
+import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.SearchImplementation;
+
+import lombok.RequiredArgsConstructor;
+
+@Service
+@RequiredArgsConstructor
+public class RedactionLogEntryAdapter {
+
+    private static final double MATCH_THRESHOLD = 1;
+    private final EntityCreationService entityCreationService;
+
+
+    public Stream<RedactionEntity> toRedactionEntity(RedactionLog redactionLog, SemanticNode node) {
+
+        List<Integer> pageNumbers = redactionLog.getRedactionLogEntry().stream().flatMap(entry -> entry.getPositions().stream().map(Rectangle::getPage)).distinct().toList();
+        if (!pageNumbers.stream().allMatch(node::isOnPage)) {
+            throw new IllegalArgumentException(format("SemanticNode %s does not contain these pages %s present in the redaction log",
+                    node,
+                    pageNumbers.stream().filter(pageNumber -> !node.isOnPage(pageNumber)).toList()));
+        }
+        Set<String> entryValues = redactionLog.getRedactionLogEntry().stream().map(RedactionLogEntry::getValue).map(String::toLowerCase).collect(Collectors.toSet());
+        SearchImplementation searchImplementation = new SearchImplementation(entryValues, true);
+
+        Map<String, List<RedactionEntity>> tempEntitiesByValue = findAllPossibleEntitiesAndGroupByValueIgnoringCase(node, searchImplementation);
+
+        assert allValuesFound(tempEntitiesByValue, entryValues);
+
+        List<RedactionEntity> entities = redactionLog.getRedactionLogEntry()
+                .stream()
+                .map(entry -> findClosestRedactionEntity(entry, tempEntitiesByValue.get(entry.getValue().toLowerCase(Locale.ROOT)), node))
+                .toList();
+        tempEntitiesByValue.values().stream().flatMap(Collection::stream).forEach(RedactionEntity::removeFromGraph);
+        return entities.stream();
+    }
+
+
+    private static boolean allValuesFound(Map<String, List<RedactionEntity>> entitiesByValue, Set<String> entryValues) {
+
+        return entitiesByValue.keySet().equals(entryValues);
+    }
+
+
+    private Map<String, List<RedactionEntity>> findAllPossibleEntitiesAndGroupByValueIgnoringCase(SemanticNode node, SearchImplementation searchImplementation) {
+
+        return searchImplementation.getBoundaries(node.getTextBlock(), node.getBoundary())
+                .stream()
+                .map(boundary -> entityCreationService.byBoundary(boundary, "temp", EntityType.ENTITY, node))
+                .collect(groupingBy(entity -> entity.getValue().toLowerCase(Locale.ROOT)));
+    }
+
+
+    private RedactionEntity findClosestRedactionEntity(RedactionLogEntry redactionLogEntry, List<RedactionEntity> entitiesWithSameValue, SemanticNode node) {
+
+        RedactionEntity closestEntity = entitiesWithSameValue.stream()
+                .filter(entity -> pagesMatch(entity, redactionLogEntry))
+                .min(Comparator.comparingDouble(entity -> calculateMinDistance(redactionLogEntry, entity)))
+                .orElseThrow(() -> new NotFoundException(format("No entity with similar position found for %s", redactionLogEntry)));
+
+        double distance = calculateMinDistance(redactionLogEntry, closestEntity);
+        if (distance > MATCH_THRESHOLD) {
+            throw new NotFoundException(format("Distance to closest found entity is %.2f for \n%s \n%s",
+                    distance,
+                    redactionLogEntry.getPositions(),
+                    closestEntity.getRedactionPositionsPerPage()));
+        }
+
+        return createCorrectEntity(redactionLogEntry, node, closestEntity);
+    }
+
+
+    private RedactionEntity createCorrectEntity(RedactionLogEntry redactionLogEntry, SemanticNode node, RedactionEntity closestEntity) {
+
+        RedactionEntity correctEntity = entityCreationService.byBoundary(closestEntity.getBoundary(),
+                redactionLogEntry.getType(),
+                redactionLogEntry.isRecommendation() ? EntityType.RECOMMENDATION : EntityType.ENTITY,
+                node);
+        correctEntity.setLegalBasis(redactionLogEntry.getLegalBasis());
+        correctEntity.setRedactionReason(redactionLogEntry.getReason());
+        correctEntity.addMatchedRule(redactionLogEntry.getMatchedRule());
+        correctEntity.setRedaction(redactionLogEntry.isRedacted());
+        correctEntity.setDictionaryEntry(redactionLogEntry.isDictionaryEntry());
+        correctEntity.setDossierDictionaryEntry(redactionLogEntry.isDossierDictionaryEntry());
+        return correctEntity;
+    }
+
+
+    private static boolean pagesMatch(RedactionEntity entity, RedactionLogEntry redactionLogEntry) {
+
+        Set<Integer> entityPageNumbers = entity.getRedactionPositionsPerPage().stream().map(RedactionPosition::getPage).map(Page::getNumber).collect(Collectors.toSet());
+        Set<Integer> redactionLogEntryPageNumbers = redactionLogEntry.getPositions().stream().map(Rectangle::getPage).collect(Collectors.toSet());
+        return entityPageNumbers.equals(redactionLogEntryPageNumbers);
+    }
+
+
+    private double calculateMinDistance(RedactionLogEntry redactionLogEntry, RedactionEntity entity) {
+
+        if (redactionLogEntry.getPositions().size() != countRectangles(entity)) {
+            return Double.MAX_VALUE;
+        }
+        return redactionLogEntry.getPositions().stream().mapToDouble(redactionLogEntryRectangle -> calculateMinDistancePerRectangle(entity, redactionLogEntryRectangle)).sum();
+    }
+
+
+    private static long countRectangles(RedactionEntity entity) {
+
+        return entity.getRedactionPositionsPerPage().stream().mapToLong(redactionPosition -> redactionPosition.getRectanglePerLine().size()).sum();
+    }
+
+
+    private double calculateMinDistancePerRectangle(RedactionEntity entity, Rectangle redactionLogEntryRectangle) {
+
+        return entity.getRedactionPositionsPerPage()
+                .stream()
+                .filter(redactionPosition -> redactionPosition.getPage().getNumber() == redactionLogEntryRectangle.getPage())
+                .map(RedactionPosition::getRectanglePerLine)
+                .flatMap(Collection::stream)
+                .mapToDouble(rectangle -> calculateDistance(rectangle, toRectangle2D(redactionLogEntryRectangle)))
+                .min()
+                .orElse(Double.MAX_VALUE);
+    }
+
+
+    private double calculateDistance(Rectangle2D rectangle, Rectangle2D rectangle2D) {
+
+        return Math.abs(rectangle.getMinX() - rectangle2D.getMinX()) //
+                + Math.abs(rectangle.getMinY() - rectangle2D.getMinY()) //
+                + Math.abs(rectangle.getMaxX() - rectangle2D.getMaxX()) //
+                + Math.abs(rectangle.getMaxY() - rectangle2D.getMaxY());
+    }
+
+
+    private Rectangle2D toRectangle2D(Rectangle rectangle) {
+
+        return new Rectangle2D.Float(rectangle.getTopLeft().getX(), rectangle.getTopLeft().getY() + rectangle.getHeight(), rectangle.getWidth(), -rectangle.getHeight());
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/model/AbstractPageBlock.java

@@ -2,7 +2,7 @@ package com.iqser.red.service.redaction.v1.server.layoutparsing.classification.m
 
 import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
-import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.ClassificationTextBlock;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.TextPageBlock;
 
 import lombok.AllArgsConstructor;
 import lombok.Data;
@@ -35,11 +35,11 @@ public abstract class AbstractPageBlock {
 
     public boolean isHeadline() {
 
-        return this instanceof ClassificationTextBlock && this.getClassification() != null && this.getClassification().isHeadline();
+        return this instanceof TextPageBlock && this.getClassification() != null && this.getClassification().isHeadline();
     }
 
 
-    public boolean containsBlock(ClassificationTextBlock other) {
+    public boolean containsBlock(TextPageBlock other) {
 
         return this.minX <= other.getPdfMinX() && this.maxX >= other.getPdfMaxX() && this.minY >= other.getPdfMinY() && this.maxY <= other.getPdfMaxY();
     }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/model/ClassificationFooter.java

@@ -2,7 +2,7 @@ package com.iqser.red.service.redaction.v1.server.layoutparsing.classification.m
 
 import java.util.List;
 
-import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.ClassificationTextBlock;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.TextPageBlock;
 
 import lombok.AllArgsConstructor;
 import lombok.Data;
@@ -11,6 +11,6 @@ import lombok.Data;
 @AllArgsConstructor
 public class ClassificationFooter {
 
-    private List<ClassificationTextBlock> textBlocks;
+    private List<TextPageBlock> textBlocks;
 
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/model/ClassificationHeader.java

@@ -2,7 +2,7 @@ package com.iqser.red.service.redaction.v1.server.layoutparsing.classification.m
 
 import java.util.List;
 
-import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.ClassificationTextBlock;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.TextPageBlock;
 
 import lombok.AllArgsConstructor;
 import lombok.Data;
@@ -11,6 +11,6 @@ import lombok.Data;
 @AllArgsConstructor
 public class ClassificationHeader {
 
-    private List<ClassificationTextBlock> textBlocks;
+    private List<TextPageBlock> textBlocks;
 
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/model/PageBlockType.java

@@ -5,6 +5,8 @@ public enum PageBlockType {
     H2,
     H3,
     H4,
+    H5,
+    H6,
     HEADER,
     FOOTER,
     TITLE,
@@ -22,13 +24,15 @@ public enum PageBlockType {
             case 1 -> PageBlockType.H1;
             case 2 -> PageBlockType.H2;
             case 3 -> PageBlockType.H3;
-            default -> PageBlockType.H4;
+            case 4 -> PageBlockType.H4;
+            case 5 -> PageBlockType.H5;
+            default -> PageBlockType.H6;
         };
     }
 
 
     public boolean isHeadline() {
 
-        return this.equals(H1) || this.equals(H2) || this.equals(H3) || this.equals(H4);
+        return this.equals(H1) || this.equals(H2) || this.equals(H3) || this.equals(H4) || this.equals(H5) || this.equals(H6);
     }
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/model/table/Cell.java

@@ -5,7 +5,7 @@ import java.util.ArrayList;
 import java.util.Iterator;
 import java.util.List;
 
-import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.ClassificationTextBlock;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.TextPageBlock;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.TextPositionSequence;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.TextNormalizationUtilities;
 
@@ -19,7 +19,7 @@ import lombok.NoArgsConstructor;
 @NoArgsConstructor
 public class Cell extends Rectangle {
 
-    private List<ClassificationTextBlock> textBlocks = new ArrayList<>();
+    private List<TextPageBlock> textBlocks = new ArrayList<>();
 
     private List<Cell> headerCells = new ArrayList<>();
 
@@ -36,7 +36,7 @@ public class Cell extends Rectangle {
     }
 
 
-    public void addTextBlock(ClassificationTextBlock textBlock) {
+    public void addTextBlock(TextPageBlock textBlock) {
 
         textBlocks.add(textBlock);
     }
@@ -47,11 +47,11 @@ public class Cell extends Rectangle {
 
         StringBuilder sb = new StringBuilder();
 
-        Iterator<ClassificationTextBlock> itty = textBlocks.iterator();
+        Iterator<TextPageBlock> itty = textBlocks.iterator();
         TextPositionSequence previous = null;
         while (itty.hasNext()) {
 
-            ClassificationTextBlock textBlock = itty.next();
+            TextPageBlock textBlock = itty.next();
 
             for (TextPositionSequence word : textBlock.getSequences()) {
                 if (previous != null) {

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/model/table/TablePageBlock.java

@@ -7,13 +7,12 @@ import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
 import java.util.TreeMap;
-import java.util.stream.Collectors;
 
 import org.apache.commons.collections4.CollectionUtils;
 
 import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.AbstractPageBlock;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.PageBlockType;
-import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.ClassificationTextBlock;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.TextPageBlock;
 
 import lombok.Getter;
 import lombok.Setter;
@@ -42,7 +41,6 @@ public class TablePageBlock extends AbstractPageBlock {
         maxY = area.getTop();
         classification = PageBlockType.TABLE;
         this.rotation = rotation;
-
     }
 
 
@@ -236,8 +234,8 @@ public class TablePageBlock extends AbstractPageBlock {
             uniqueY.add(c.getTop());
         });
 
-        var sortedUniqueX = uniqueX.stream().sorted().collect(Collectors.toList());
+        var sortedUniqueX = uniqueX.stream().sorted().toList();
-        var sortedUniqueY = uniqueY.stream().sorted().collect(Collectors.toList());
+        var sortedUniqueY = uniqueY.stream().sorted().toList();
 
         Float prevY = null;
         for (Float y : sortedUniqueY) {
@@ -251,9 +249,7 @@ public class TablePageBlock extends AbstractPageBlock {
                     var cell = new Cell(new Point2D.Float(prevX, prevY), new Point2D.Float(x, y));
 
                     var intersectionCell = cells.stream().filter(c -> cell.intersects(c) && cell.overlapRatio(c) > 0.1f).findFirst();
-                    if (intersectionCell.isPresent()) {
+                    intersectionCell.ifPresent(value -> cell.getTextBlocks().addAll(value.getTextBlocks()));
-                        cell.getTextBlocks().addAll(intersectionCell.get().getTextBlocks());
-                    }
                     if (cell.hasMinimumSize()) {
                         row.add(cell);
                     }
@@ -292,7 +288,7 @@ public class TablePageBlock extends AbstractPageBlock {
                     }
                     if (column != null && column.getTextBlocks() != null) {
                         boolean first = true;
-                        for (ClassificationTextBlock textBlock : column.getTextBlocks()) {
+                        for (TextPageBlock textBlock : column.getTextBlocks()) {
                             if (!first) {
                                 sb.append("\n");
                             }
@@ -324,7 +320,7 @@ public class TablePageBlock extends AbstractPageBlock {
                     sb.append(i == 0 ? "\n<th>" : "\n<td>");
                     if (column != null && column.getTextBlocks() != null) {
                         boolean first = true;
-                        for (ClassificationTextBlock textBlock : column.getTextBlocks()) {
+                        for (TextPageBlock textBlock : column.getTextBlocks()) {
                             if (!first) {
                                 sb.append("<br />");
                             }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/model/text/TextPageBlock.java

@@ -17,7 +17,7 @@ import lombok.NoArgsConstructor;
 @Builder
 @Data
 @NoArgsConstructor
-public class ClassificationTextBlock extends AbstractPageBlock {
+public class TextPageBlock extends AbstractPageBlock {
 
     @Builder.Default
     private List<TextPositionSequence> sequences = new ArrayList<>();
@@ -25,8 +25,6 @@ public class ClassificationTextBlock extends AbstractPageBlock {
     @JsonIgnore
     private int rotation;
 
-    private int indexOnPage;
-
     @JsonIgnore
     private String mostPopularWordFont;
 
@@ -176,9 +174,8 @@ public class ClassificationTextBlock extends AbstractPageBlock {
     }
 
 
-    public ClassificationTextBlock(float minX, float maxX, float minY, float maxY, List<TextPositionSequence> sequences, int rotation, int indexOnPage) {
+    public TextPageBlock(float minX, float maxX, float minY, float maxY, List<TextPositionSequence> sequences, int rotation) {
 
-        this.indexOnPage = indexOnPage;
         this.minX = minX;
         this.maxX = maxX;
         this.minY = minY;
@@ -188,23 +185,23 @@ public class ClassificationTextBlock extends AbstractPageBlock {
     }
 
 
-    public ClassificationTextBlock union(TextPositionSequence r) {
+    public TextPageBlock union(TextPositionSequence r) {
 
-        ClassificationTextBlock union = this.copy();
+        TextPageBlock union = this.copy();
         union.add(r);
         return union;
     }
 
 
-    public ClassificationTextBlock union(ClassificationTextBlock r) {
+    public TextPageBlock union(TextPageBlock r) {
 
-        ClassificationTextBlock union = this.copy();
+        TextPageBlock union = this.copy();
         union.add(r);
         return union;
     }
 
 
-    public void add(ClassificationTextBlock r) {
+    public void add(TextPageBlock r) {
 
         if (r.getMinX() < minX) {
             minX = r.getMinX();
@@ -239,9 +236,9 @@ public class ClassificationTextBlock extends AbstractPageBlock {
     }
 
 
-    public ClassificationTextBlock copy() {
+    public TextPageBlock copy() {
 
-        return new ClassificationTextBlock(minX, maxX, minY, maxY, sequences, rotation, indexOnPage);
+        return new TextPageBlock(minX, maxX, minY, maxY, sequences, rotation);
     }
 
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/model/text/UnclassifiedText.java

@@ -9,6 +9,6 @@ import lombok.Data;
 @AllArgsConstructor
 public class UnclassifiedText {
 
-    private List<ClassificationTextBlock> textBlocks;
+    private List<TextPageBlock> textBlocks;
 
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/service/BlockificationService.java

@@ -14,8 +14,8 @@ import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.mo
 import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.FloatFrequencyCounter;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.Orientation;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.table.Ruling;
-import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.ClassificationTextBlock;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.StringFrequencyCounter;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.TextPageBlock;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.TextPositionSequence;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.utils.RulingTextDirAdjustUtil;
 
@@ -64,7 +64,7 @@ public class BlockificationService {
                     prevOrientation = chunkBlockList.get(chunkBlockList.size() - 1).getOrientation();
                 }
 
-                ClassificationTextBlock cb1 = buildTextBlock(chunkWords, indexOnPage);
+                TextPageBlock cb1 = buildTextBlock(chunkWords, indexOnPage);
                 indexOnPage++;
 
                 chunkBlockList.add(cb1);
@@ -106,17 +106,17 @@ public class BlockificationService {
             }
         }
 
-        ClassificationTextBlock cb1 = buildTextBlock(chunkWords, indexOnPage);
+        TextPageBlock cb1 = buildTextBlock(chunkWords, indexOnPage);
         if (cb1 != null) {
             chunkBlockList.add(cb1);
         }
 
         Iterator<AbstractPageBlock> itty = chunkBlockList.iterator();
 
-        ClassificationTextBlock previousLeft = null;
+        TextPageBlock previousLeft = null;
-        ClassificationTextBlock previousRight = null;
+        TextPageBlock previousRight = null;
         while (itty.hasNext()) {
-            ClassificationTextBlock block = (ClassificationTextBlock) itty.next();
+            TextPageBlock block = (TextPageBlock) itty.next();
 
             if (previousLeft != null && block.getOrientation().equals(Orientation.LEFT)) {
                 if (previousLeft.getMinY() > block.getMinY() && block.getMaxY() + block.getMostPopularWordHeight() > previousLeft.getMinY()) {
@@ -142,9 +142,9 @@ public class BlockificationService {
         }
 
         itty = chunkBlockList.iterator();
-        ClassificationTextBlock previous = null;
+        TextPageBlock previous = null;
         while (itty.hasNext()) {
-            ClassificationTextBlock block = (ClassificationTextBlock) itty.next();
+            TextPageBlock block = (TextPageBlock) itty.next();
 
             if (previous != null && previous.getOrientation().equals(Orientation.LEFT) && block.getOrientation().equals(Orientation.LEFT) && equalsWithThreshold(block.getMaxY(),
                     previous.getMaxY()) || previous != null && previous.getOrientation().equals(Orientation.LEFT) && block.getOrientation()
@@ -167,9 +167,9 @@ public class BlockificationService {
     }
 
 
-    private ClassificationTextBlock buildTextBlock(List<TextPositionSequence> wordBlockList, int indexOnPage) {
+    private TextPageBlock buildTextBlock(List<TextPositionSequence> wordBlockList, int indexOnPage) {
 
-        ClassificationTextBlock textBlock = null;
+        TextPageBlock textBlock = null;
 
         FloatFrequencyCounter lineHeightFrequencyCounter = new FloatFrequencyCounter();
         FloatFrequencyCounter fontSizeFrequencyCounter = new FloatFrequencyCounter();
@@ -186,15 +186,14 @@ public class BlockificationService {
             styleFrequencyCounter.add(wordBlock.getFontStyle());
 
             if (textBlock == null) {
-                textBlock = new ClassificationTextBlock(wordBlock.getMinXDirAdj(),
+                textBlock = new TextPageBlock(wordBlock.getMinXDirAdj(),
                         wordBlock.getMaxXDirAdj(),
                         wordBlock.getMinYDirAdj(),
                         wordBlock.getMaxYDirAdj(),
                         wordBlockList,
-                        wordBlock.getRotation(),
+                        wordBlock.getRotation());
-                        indexOnPage);
             } else {
-                ClassificationTextBlock spatialEntity = textBlock.union(wordBlock);
+                TextPageBlock spatialEntity = textBlock.union(wordBlock);
                 textBlock.resize(spatialEntity.getMinX(), spatialEntity.getMinY(), spatialEntity.getWidth(), spatialEntity.getHeight());
             }
         }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/service/BodyTextFrameService.java

@@ -11,7 +11,7 @@ import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.mo
 import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.FloatFrequencyCounter;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.table.Cell;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.table.TablePageBlock;
-import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.ClassificationTextBlock;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.TextPageBlock;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.utils.PositionUtils;
 
 @Service
@@ -80,8 +80,8 @@ public class BodyTextFrameService {
 
             for (AbstractPageBlock container : page.getTextBlocks()) {
 
-                if (container instanceof ClassificationTextBlock) {
+                if (container instanceof TextPageBlock) {
-                    ClassificationTextBlock textBlock = (ClassificationTextBlock) container;
+                    TextPageBlock textBlock = (TextPageBlock) container;
                     if (textBlock.getMostPopularWordFont() == null || textBlock.getMostPopularWordStyle() == null) {
                         continue;
                     }
@@ -105,7 +105,7 @@ public class BodyTextFrameService {
                             if (cell == null || cell.getTextBlocks() == null) {
                                 continue;
                             }
-                            for (ClassificationTextBlock textBlock : cell.getTextBlocks()) {
+                            for (TextPageBlock textBlock : cell.getTextBlocks()) {
                                 expandRectangle(textBlock, page, expansionsRectangle);
                             }
                         }
@@ -120,7 +120,7 @@ public class BodyTextFrameService {
     }
 
 
-    private void expandRectangle(ClassificationTextBlock textBlock, ClassificationPage page, BodyTextFrameExpansionsRectangle expansionsRectangle) {
+    private void expandRectangle(TextPageBlock textBlock, ClassificationPage page, BodyTextFrameExpansionsRectangle expansionsRectangle) {
 
         if (page.getPageWidth() > page.getPageHeight() && page.getRotation() != 0) {
             if (textBlock.getPdfMinY() < expansionsRectangle.minX) {

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/service/ClassificationService.java

@@ -10,7 +10,7 @@ import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.mo
 import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.ClassificationDocument;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.ClassificationPage;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.PageBlockType;
-import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.ClassificationTextBlock;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.TextPageBlock;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.utils.PositionUtils;
 
 import lombok.RequiredArgsConstructor;
@@ -42,14 +42,14 @@ public class ClassificationService {
     public void classifyPage(ClassificationPage page, ClassificationDocument document, List<Float> headlineFontSizes) {
 
         for (AbstractPageBlock textBlock : page.getTextBlocks()) {
-            if (textBlock instanceof ClassificationTextBlock) {
+            if (textBlock instanceof TextPageBlock) {
-                classifyBlock((ClassificationTextBlock) textBlock, page, document, headlineFontSizes);
+                classifyBlock((TextPageBlock) textBlock, page, document, headlineFontSizes);
             }
         }
     }
 
 
-    public void classifyBlock(ClassificationTextBlock textBlock, ClassificationPage page, ClassificationDocument document, List<Float> headlineFontSizes) {
+    public void classifyBlock(TextPageBlock textBlock, ClassificationPage page, ClassificationDocument document, List<Float> headlineFontSizes) {
 
         var bodyTextFrame = page.getBodyTextFrame();
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/service/PdfSegmentationService.java

@@ -24,7 +24,7 @@ import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.mo
 import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.ClassificationPage;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.image.ClassifiedImage;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.table.CleanRulings;
-import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.ClassificationTextBlock;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.TextPageBlock;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.TextPositionSequence;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.parsing.PDFLinesTextStripper;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.utils.FileUtils;
@@ -157,11 +157,11 @@ public class PdfSegmentationService {
 
         // Collect all statistics for the page, except from blocks inside tables, as tables will always be added to BodyTextFrame.
         for (AbstractPageBlock textBlock : page.getTextBlocks()) {
-            if (textBlock instanceof ClassificationTextBlock) {
+            if (textBlock instanceof TextPageBlock) {
-                if (((ClassificationTextBlock) textBlock).getSequences() == null) {
+                if (((TextPageBlock) textBlock).getSequences() == null) {
                     continue;
                 }
-                for (TextPositionSequence word : ((ClassificationTextBlock) textBlock).getSequences()) {
+                for (TextPositionSequence word : ((TextPageBlock) textBlock).getSequences()) {
                     page.getTextHeightCounter().add(word.getTextHeight());
                     page.getFontCounter().add(word.getFont());
                     page.getFontSizeCounter().add(word.getFontSize());

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/service/SectionsBuilderService.java

@@ -20,7 +20,7 @@ import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.mo
 import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.image.ClassifiedImage;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.table.Cell;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.table.TablePageBlock;
-import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.ClassificationTextBlock;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.TextPageBlock;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.UnclassifiedText;
 
 import lombok.extern.slf4j.Slf4j;
@@ -42,9 +42,9 @@ public class SectionsBuilderService {
         String lastHeadline = "";
         TablePageBlock previousTable = null;
         for (ClassificationPage page : document.getPages()) {
-            List<ClassificationTextBlock> header = new ArrayList<>();
+            List<TextPageBlock> header = new ArrayList<>();
-            List<ClassificationTextBlock> footer = new ArrayList<>();
+            List<TextPageBlock> footer = new ArrayList<>();
-            List<ClassificationTextBlock> unclassifiedText = new ArrayList<>();
+            List<TextPageBlock> unclassifiedText = new ArrayList<>();
             for (AbstractPageBlock current : page.getTextBlocks()) {
 
                 if (current.getClassification() == null) {
@@ -54,17 +54,17 @@ public class SectionsBuilderService {
                 current.setPage(page.getPageNumber());
 
                 if (current.getClassification().equals(PageBlockType.HEADER)) {
-                    header.add((ClassificationTextBlock) current);
+                    header.add((TextPageBlock) current);
                     continue;
                 }
 
                 if (current.getClassification().equals(PageBlockType.FOOTER)) {
-                    footer.add((ClassificationTextBlock) current);
+                    footer.add((TextPageBlock) current);
                     continue;
                 }
 
                 if (current.getClassification().equals(PageBlockType.OTHER)) {
-                    unclassifiedText.add((ClassificationTextBlock) current);
+                    unclassifiedText.add((TextPageBlock) current);
                     continue;
                 }
 
@@ -258,7 +258,7 @@ public class SectionsBuilderService {
                 continue;
             }
 
-            ClassificationTextBlock wordBlock = (ClassificationTextBlock) container;
+            TextPageBlock wordBlock = (TextPageBlock) container;
             section.getPageBlocks().add(wordBlock);
         }
         return section;

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/service/TableExtractionService.java

@@ -20,7 +20,7 @@ import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.mo
 import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.table.Rectangle;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.table.Ruling;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.table.TablePageBlock;
-import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.ClassificationTextBlock;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.TextPageBlock;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.utils.Utils;
 
 @Service
@@ -83,10 +83,10 @@ public class TableExtractionService {
 
         List<Cell> cells = findCells(cleanRulings.getHorizontal(), cleanRulings.getVertical());
 
-        List<ClassificationTextBlock> toBeRemoved = new ArrayList<>();
+        List<TextPageBlock> toBeRemoved = new ArrayList<>();
 
         for (AbstractPageBlock abstractPageBlock : page.getTextBlocks()) {
-            ClassificationTextBlock textBlock = (ClassificationTextBlock) abstractPageBlock;
+            TextPageBlock textBlock = (TextPageBlock) abstractPageBlock;
             for (Cell cell : cells) {
                 if (cell.hasMinimumSize() && cell.intersects(textBlock.getPdfMinX(),
                         textBlock.getPdfMinY(),
@@ -122,7 +122,7 @@ public class TableExtractionService {
             Iterator<AbstractPageBlock> itty = page.getTextBlocks().iterator();
             while (itty.hasNext()) {
                 AbstractPageBlock textBlock = itty.next();
-                if (textBlock instanceof ClassificationTextBlock ? table.containsBlock((ClassificationTextBlock) textBlock) : table.contains(textBlock) && position == -1) {
+                if (textBlock instanceof TextPageBlock ? table.containsBlock((TextPageBlock) textBlock) : table.contains(textBlock) && position == -1) {
                     position = page.getTextBlocks().indexOf(textBlock);
                 }
             }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/utils/PositionUtils.java

@@ -1,7 +1,7 @@
 package com.iqser.red.service.redaction.v1.server.layoutparsing.classification.utils;
 
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
-import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.ClassificationTextBlock;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.TextPageBlock;
 
 import lombok.experimental.UtilityClass;
 
@@ -11,7 +11,7 @@ public final class PositionUtils {
 
     // TODO This currently uses pdf coord system. In the futher this should use java coord system.
     // Note: DirAdj (TextDirection Adjusted) can not be user for this.
-    public boolean isWithinBodyTextFrame(Rectangle btf, ClassificationTextBlock textBlock) {
+    public boolean isWithinBodyTextFrame(Rectangle btf, TextPageBlock textBlock) {
 
         if (btf == null || textBlock == null) {
             return false;
@@ -32,7 +32,7 @@ public final class PositionUtils {
 
     // TODO This currently uses pdf coord system. In the futher this should use java coord system.
     // Note: DirAdj (TextDirection Adjusted) can not be user for this.
-    public boolean isOverBodyTextFrame(Rectangle btf, ClassificationTextBlock textBlock, int rotation) {
+    public boolean isOverBodyTextFrame(Rectangle btf, TextPageBlock textBlock, int rotation) {
 
         if (btf == null || textBlock == null) {
             return false;
@@ -61,7 +61,7 @@ public final class PositionUtils {
 
     // TODO This currently uses pdf coord system. In the futher this should use java coord system.
     // Note: DirAdj (TextDirection Adjusted) can not be user for this.
-    public boolean isUnderBodyTextFrame(Rectangle btf, ClassificationTextBlock textBlock, int rotation) {
+    public boolean isUnderBodyTextFrame(Rectangle btf, TextPageBlock textBlock, int rotation) {
 
         if (btf == null || textBlock == null) {
             return false;
@@ -90,7 +90,7 @@ public final class PositionUtils {
 
     // TODO This currently uses pdf coord system. In the futher this should use java coord system.
     // Note: DirAdj (TextDirection Adjusted) can not be user for this.
-    public boolean isTouchingUnderBodyTextFrame(Rectangle btf, ClassificationTextBlock textBlock) {
+    public boolean isTouchingUnderBodyTextFrame(Rectangle btf, TextPageBlock textBlock) {
 
         //TODO Currently this is not working for rotated pages.
 
@@ -107,13 +107,13 @@ public final class PositionUtils {
     }
 
 
-    public float getHeightDifferenceBetweenChunkWordAndDocumentWord(ClassificationTextBlock textBlock, Float documentMostPopularWordHeight) {
+    public float getHeightDifferenceBetweenChunkWordAndDocumentWord(TextPageBlock textBlock, Float documentMostPopularWordHeight) {
 
         return textBlock.getMostPopularWordHeight() - documentMostPopularWordHeight;
     }
 
 
-    public Float getApproxLineCount(ClassificationTextBlock textBlock) {
+    public Float getApproxLineCount(TextPageBlock textBlock) {
 
         return textBlock.getHeight() / textBlock.getMostPopularWordHeight();
     }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/data/DocumentTreeData.java

@@ -10,7 +10,6 @@ import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.Do
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.Image;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.NodeType;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.Page;
-import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.Section;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.Table;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.TableCell;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.AtomicTextBlock;
@@ -38,9 +37,9 @@ public class DocumentTreeData {
         if (tocId.isEmpty()) {
             return root;
         }
-        EntryData entry = root.subEntries.get(tocId.get(0));
+        EntryData entry = root.children.get(tocId.get(0));
         for (int id : tocId.subList(1, tocId.size())) {
-            entry = entry.subEntries.get(id);
+            entry = entry.children.get(id);
         }
         return entry;
     }
@@ -48,7 +47,7 @@ public class DocumentTreeData {
 
     public Stream<EntryData> streamAllEntries() {
 
-        return Stream.concat(Stream.of(root), root.subEntries.stream()).flatMap(DocumentTreeData::flatten);
+        return Stream.concat(Stream.of(root), root.children.stream()).flatMap(DocumentTreeData::flatten);
     }
 
 
@@ -60,7 +59,7 @@ public class DocumentTreeData {
 
     private static Stream<EntryData> flatten(EntryData entry) {
 
-        return Stream.concat(Stream.of(entry), entry.subEntries.stream().flatMap(DocumentTreeData::flatten));
+        return Stream.concat(Stream.of(entry), entry.children.stream().flatMap(DocumentTreeData::flatten));
     }
 
 
@@ -75,7 +74,7 @@ public class DocumentTreeData {
         Long[] atomicBlockIds;
         Long[] pageNumbers;
         Map<String, String> properties;
-        List<EntryData> subEntries;
+        List<EntryData> children;
 
 
         public static EntryData fromEntry(DocumentTree.Entry entry) {
@@ -86,7 +85,6 @@ public class DocumentTreeData {
                 case TABLE -> PropertiesMapper.buildTableProperties((Table) entry.getNode());
                 case TABLE_CELL -> PropertiesMapper.buildTableCellProperties((TableCell) entry.getNode());
                 case IMAGE -> PropertiesMapper.buildImageProperties((Image) entry.getNode());
-                case SECTION -> PropertiesMapper.buildSectionProperties((Section) entry.getNode());
                 default -> new HashMap<>();
             };
             var treeId = entry.getTreeId().stream().mapToInt(Integer::intValue).toArray();

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/data/mapper/DocumentGraphMapper.java

@@ -1,8 +1,5 @@
 package com.iqser.red.service.redaction.v1.server.layoutparsing.document.data.mapper;
 
-import static com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.NodeType.FOOTER;
-import static com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.NodeType.HEADER;
-
 import java.util.Arrays;
 import java.util.HashSet;
 import java.util.LinkedList;
@@ -39,22 +36,18 @@ public class DocumentGraphMapper {
     public Document toDocumentGraph(DocumentData documentData) {
 
         Document document = new Document();
-        Context context = new Context(documentData,
+        DocumentTree documentTree = new DocumentTree(document);
-                new DocumentTree(document),
+        Context context = new Context(documentData, documentTree);
-                new LinkedList<>(),
-                new LinkedList<>(),
-                Arrays.stream(documentData.getAtomicTextBlocks()).toList(),
-                Arrays.stream(documentData.getAtomicPositionBlocks()).toList());
 
         context.pages.addAll(Arrays.stream(documentData.getPages()).map(DocumentGraphMapper::buildPage).toList());
 
-        context.documentTree.getRoot().getChildren().addAll(buildEntries(documentData.getDocumentTreeData().getRoot().getSubEntries(), context));
+        context.documentTree.getRoot().getChildren().addAll(buildEntries(documentData.getDocumentTreeData().getRoot().getChildren(), context));
 
         document.setDocumentTree(context.documentTree);
         document.setPages(new HashSet<>(context.pages));
         document.setNumberOfPages(documentData.getPages().length);
 
-        document.setTextBlock(document.buildTextBlock());
+        document.setTextBlock(document.getTextBlock());
         return document;
     }
 
@@ -68,40 +61,39 @@ public class DocumentGraphMapper {
             List<Page> pages = Arrays.stream(entryData.getPageNumbers()).map(pageNumber -> getPage(pageNumber, context)).toList();
 
             SemanticNode node = switch (entryData.getType()) {
-                case SECTION -> buildSection(context, entryData.getProperties());
+                case SECTION -> buildSection(context);
-                case PARAGRAPH -> buildParagraph(context, terminal);
+                case PARAGRAPH -> buildParagraph(context);
-                case HEADLINE -> buildHeadline(context, terminal);
+                case HEADLINE -> buildHeadline(context);
-                case HEADER -> buildHeader(context, terminal);
+                case HEADER -> buildHeader(context);
-                case FOOTER -> buildFooter(context, terminal);
+                case FOOTER -> buildFooter(context);
                 case TABLE -> buildTable(context, entryData.getProperties());
-                case TABLE_CELL -> buildTableCell(context, entryData.getProperties(), terminal);
+                case TABLE_CELL -> buildTableCell(context, entryData.getProperties());
                 case IMAGE -> buildImage(context, entryData.getProperties(), entryData.getPageNumbers());
                 default -> throw new UnsupportedOperationException("Not yet implemented for type " + entryData.getType());
             };
 
-            if (node.isLeaf()) {
+            if (entryData.getAtomicBlockIds().length > 0) {
                 TextBlock textBlock = toTextBlock(entryData.getAtomicBlockIds(), context, node);
                 node.setLeafTextBlock(textBlock);
             }
             List<Integer> treeId = Arrays.stream(entryData.getTreeId()).boxed().toList();
             node.setTreeId(treeId);
 
-            if (entryData.getType() == HEADER) {
+            switch (entryData.getType()) {
-                pages.forEach(page -> page.setHeader((Header) node));
+                case HEADER -> pages.forEach(page -> page.setHeader((Header) node));
-            } else if (entryData.getType() == FOOTER) {
+                case FOOTER -> pages.forEach(page -> page.setFooter((Footer) node));
-                pages.forEach(page -> page.setFooter((Footer) node));
+                default -> pages.forEach(page -> page.getMainBody().add(node));
-            } else {
-                pages.forEach(page -> page.getMainBody().add(node));
             }
-            newEntries.add(DocumentTree.Entry.builder().treeId(treeId).children(buildEntries(entryData.getSubEntries(), context)).node(node).build());
+
+            newEntries.add(DocumentTree.Entry.builder().treeId(treeId).children(buildEntries(entryData.getChildren(), context)).node(node).build());
         }
         return newEntries;
     }
 
 
-    private Headline buildHeadline(Context context, boolean terminal) {
+    private Headline buildHeadline(Context context) {
 
-        return Headline.builder().leaf(terminal).documentTree(context.documentTree()).build();
+        return Headline.builder().documentTree(context.documentTree).build();
     }
 
 
@@ -117,15 +109,15 @@ public class DocumentGraphMapper {
         Page page = getPage(pageNumbers[0], context);
         var builder = Image.builder();
         PropertiesMapper.parseImageProperties(properties, builder);
-        return builder.documentTree(context.documentTree()).page(page).build();
+        return builder.documentTree(context.documentTree).page(page).build();
     }
 
 
-    private TableCell buildTableCell(Context context, Map<String, String> properties, boolean terminal) {
+    private TableCell buildTableCell(Context context, Map<String, String> properties) {
 
         TableCell.TableCellBuilder builder = TableCell.builder();
         PropertiesMapper.parseTableCellProperties(properties, builder);
-        return builder.leaf(terminal).documentTree(context.documentTree()).build();
+        return builder.documentTree(context.documentTree).build();
     }
 
 
@@ -133,33 +125,31 @@ public class DocumentGraphMapper {
 
         Table.TableBuilder builder = Table.builder();
         PropertiesMapper.parseTableProperties(properties, builder);
-        return builder.documentTree(context.documentTree()).build();
+        return builder.documentTree(context.documentTree).build();
     }
 
 
-    private Footer buildFooter(Context context, boolean terminal) {
+    private Footer buildFooter(Context context) {
 
-        return Footer.builder().leaf(terminal).documentTree(context.documentTree()).build();
+        return Footer.builder().documentTree(context.documentTree).build();
     }
 
 
-    private Header buildHeader(Context context, boolean terminal) {
+    private Header buildHeader(Context context) {
 
-        return Header.builder().leaf(terminal).documentTree(context.documentTree()).build();
+        return Header.builder().documentTree(context.documentTree).build();
     }
 
 
-    private Section buildSection(Context context, Map<String, String> properties) {
+    private Section buildSection(Context context) {
 
-        var builder = Section.builder();
+        return Section.builder().documentTree(context.documentTree).build();
-        PropertiesMapper.parseSectionProperties(properties, builder);
-        return builder.documentTree(context.documentTree()).build();
     }
 
 
-    private Paragraph buildParagraph(Context context, boolean terminal) {
+    private Paragraph buildParagraph(Context context) {
 
-        return Paragraph.builder().leaf(terminal).documentTree(context.documentTree()).build();
+        return Paragraph.builder().documentTree(context.documentTree).build();
     }
 
 
@@ -193,13 +183,22 @@ public class DocumentGraphMapper {
     }
 
 
-    record Context(
+    static final class Context {
-            DocumentData layoutParsingModel,
+
-            DocumentTree documentTree,
+        private final DocumentTree documentTree;
-            List<Page> pages,
+        private final List<Page> pages;
-            List<Section> sections,
+        private final List<AtomicTextBlockData> atomicTextBlockData;
-            List<AtomicTextBlockData> atomicTextBlockData,
+        private final List<AtomicPositionBlockData> atomicPositionBlockData;
-            List<AtomicPositionBlockData> atomicPositionBlockData) {
+
+
+        Context(DocumentData documentData, DocumentTree documentTree) {
+
+            this.documentTree = documentTree;
+            this.pages = new LinkedList<>();
+            this.atomicTextBlockData = Arrays.stream(documentData.getAtomicTextBlocks()).toList();
+            this.atomicPositionBlockData = Arrays.stream(documentData.getAtomicPositionBlocks()).toList();
+
+        }
 
     }
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/data/mapper/PropertiesMapper.java

@@ -1,42 +1,61 @@
 package com.iqser.red.service.redaction.v1.server.layoutparsing.document.data.mapper;
 
-import static com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RectangleMapper.parseRectangle2D;
+import static java.lang.String.format;
 
+import java.awt.geom.Rectangle2D;
+import java.util.Arrays;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.Image;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.ImageType;
-import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.Section;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.Table;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.TableCell;
-import com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RectangleMapper;
 
 public class PropertiesMapper {
 
+    private static final String imageType = "imageType";
+    private static final String transparency = "transparency";
+    private static final String position = "position";
+    private static final String id = "id";
+    private static final String row = "row";
+    private static final String col = "col";
+    private static final String header = "header";
+    private static final String bBox = "bBox";
+    private static final String numberOfRows = "numberOfRows";
+    private static final String numberOfCols = "numberOfCols";
+
+
     public static Map<String, String> buildImageProperties(Image image) {
 
         Map<String, String> properties = new HashMap<>();
-        properties.put("imageType", image.getImageType().toString());
+        properties.put(imageType, image.getImageType().toString());
-        properties.put("transparency", String.valueOf(image.isTransparent()));
+        properties.put(transparency, String.valueOf(image.isTransparent()));
-        properties.put("position", RectangleMapper.toString(image.getPosition()));
+        properties.put(position, toString(image.getPosition()));
-        properties.put("id", image.getId());
+        properties.put(id, image.getId());
         return properties;
     }
 
 
+    private static String toString(Rectangle2D rectangle2D) {
+
+        return format("%f,%f,%f,%f", rectangle2D.getX(), rectangle2D.getY(), rectangle2D.getWidth(), rectangle2D.getHeight());
+    }
+
+
     public static Map<String, String> buildTableCellProperties(TableCell tableCell) {
 
         Map<String, String> properties = new HashMap<>();
-        properties.put("row", String.valueOf(tableCell.getRow()));
+        properties.put(row, String.valueOf(tableCell.getRow()));
-        properties.put("col", String.valueOf(tableCell.getCol()));
+        properties.put(col, String.valueOf(tableCell.getCol()));
-        properties.put("header", String.valueOf(tableCell.isHeader()));
+        properties.put(header, String.valueOf(tableCell.isHeader()));
 
         if (tableCell.getPages().size() > 1 || tableCell.getBBox().keySet().size() > 1) {
             throw new IllegalArgumentException("TableCell can only occur on a single page!");
         }
-        String bBoxString = RectangleMapper.toString(tableCell.getBBox().get(tableCell.getPages().stream().findFirst().get()));
+        String bBoxString = toString(tableCell.getBBox().get(tableCell.getPages().stream().findFirst().get()));
-        properties.put("bBox", bBoxString);
+        properties.put(bBox, bBoxString);
 
         return properties;
     }
@@ -45,60 +64,41 @@ public class PropertiesMapper {
     public static Map<String, String> buildTableProperties(Table table) {
 
         Map<String, String> properties = new HashMap<>();
-        properties.put("numberOfRows", String.valueOf(table.getNumberOfRows()));
+        properties.put(numberOfRows, String.valueOf(table.getNumberOfRows()));
-        properties.put("numberOfCols", String.valueOf(table.getNumberOfCols()));
+        properties.put(numberOfCols, String.valueOf(table.getNumberOfCols()));
-        return properties;
-    }
-
-
-    public static Map<String, String> buildSectionProperties(Section node) {
-
-        Map<String, String> properties = new HashMap<>();
-        properties.put("excludesTables", String.valueOf(node.isExcludesTables()));
         return properties;
     }
 
 
     public static void parseImageProperties(Map<String, String> properties, Image.ImageBuilder builder) {
 
-        builder.imageType(parseImageType(properties.get("imageType")));
+        builder.imageType(ImageType.fromString(properties.get(imageType)));
-        builder.transparent(Boolean.parseBoolean(properties.get("transparency")));
+        builder.transparent(Boolean.parseBoolean(properties.get(transparency)));
-        builder.position(parseRectangle2D(properties.get("position")));
+        builder.position(parseRectangle2D(properties.get(position)));
-        builder.id(properties.get("id"));
+        builder.id(properties.get(id));
     }
 
 
     public static void parseTableCellProperties(Map<String, String> properties, TableCell.TableCellBuilder builder) {
 
-        builder.row(Integer.parseInt(properties.get("row")));
+        builder.row(Integer.parseInt(properties.get(row)));
-        builder.col(Integer.parseInt(properties.get("col")));
+        builder.col(Integer.parseInt(properties.get(col)));
-        builder.header(Boolean.parseBoolean(properties.get("header")));
+        builder.header(Boolean.parseBoolean(properties.get(header)));
-        builder.bBox(parseRectangle2D(properties.get("bBox")));
+        builder.bBox(parseRectangle2D(properties.get(bBox)));
     }
 
 
     public static void parseTableProperties(Map<String, String> properties, Table.TableBuilder builder) {
 
-        builder.numberOfRows(Integer.parseInt(properties.get("numberOfRows")));
+        builder.numberOfRows(Integer.parseInt(properties.get(numberOfRows)));
-        builder.numberOfCols(Integer.parseInt(properties.get("numberOfCols")));
+        builder.numberOfCols(Integer.parseInt(properties.get(numberOfCols)));
     }
 
 
-    public static void parseSectionProperties(Map<String, String> properties, Section.SectionBuilder builder) {
+    private static Rectangle2D parseRectangle2D(String bBox) {
 
-        builder.excludesTables(Boolean.parseBoolean(properties.get("excludesTables")));
+        List<Float> floats = Arrays.stream(bBox.split(",")).map(Float::parseFloat).toList();
-    }
+        return new Rectangle2D.Float(floats.get(0), floats.get(1), floats.get(2), floats.get(3));
-
-
-    public static ImageType parseImageType(String imageType) {
-
-        return switch (imageType.toLowerCase()) {
-            case "logo" -> ImageType.LOGO;
-            case "formula" -> ImageType.FORMULA;
-            case "signature" -> ImageType.SIGNATURE;
-            case "ocr" -> ImageType.OCR;
-            default -> ImageType.OTHER;
-        };
     }
 
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/factory/DocumentGraphFactory.java

@@ -5,6 +5,7 @@ import static java.util.stream.Collectors.groupingBy;
 import static java.util.stream.Collectors.toList;
 
 import java.awt.geom.Rectangle2D;
+import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.LinkedList;
 import java.util.List;
@@ -18,7 +19,7 @@ import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.mo
 import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.ClassificationHeader;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.ClassificationPage;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.image.ClassifiedImage;
-import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.ClassificationTextBlock;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.TextPageBlock;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.DocumentTree;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.Document;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.Footer;
@@ -56,7 +57,7 @@ public class DocumentGraphFactory {
         documentGraph.setNumberOfPages(context.pages.size());
         documentGraph.setPages(context.pages.keySet());
         documentGraph.setDocumentTree(context.documentTree);
-        documentGraph.setTextBlock(documentGraph.buildTextBlock());
+        documentGraph.setTextBlock(documentGraph.getTextBlock());
         return documentGraph;
     }
 
@@ -67,10 +68,7 @@ public class DocumentGraphFactory {
     }
 
 
-    public void addParagraphOrHeadline(GenericSemanticNode parentNode,
+    public void addParagraphOrHeadline(GenericSemanticNode parentNode, TextPageBlock originalTextBlock, Context context, List<TextPageBlock> textBlocksToMerge) {
-                                       ClassificationTextBlock originalTextBlock,
-                                       Context context,
-                                       List<ClassificationTextBlock> textBlocksToMerge) {
 
         Page page = context.getPage(originalTextBlock.getPage());
 
@@ -83,7 +81,7 @@ public class DocumentGraphFactory {
 
         page.getMainBody().add(node);
 
-        List<ClassificationTextBlock> textBlocks = new LinkedList<>(textBlocksToMerge);
+        List<TextPageBlock> textBlocks = new ArrayList<>(textBlocksToMerge);
         textBlocks.add(originalTextBlock);
         AtomicTextBlock textBlock = context.textBlockFactory.buildAtomicTextBlock(TextPositionOperations.mergeAndSortTextPositionSequenceByYThenX(textBlocks), node, context, page);
         List<Integer> treeId = context.documentTree.createNewChildEntryAndReturnId(parentNode, node);
@@ -113,13 +111,13 @@ public class DocumentGraphFactory {
 
     private void addHeaderAndFooterToEachPage(ClassificationDocument document, Context context) {
 
-        Map<Integer, List<ClassificationTextBlock>> headers = document.getHeaders()
+        Map<Integer, List<TextPageBlock>> headers = document.getHeaders()
                 .stream()
                 .map(ClassificationHeader::getTextBlocks)
                 .flatMap(List::stream)
                 .collect(groupingBy(AbstractPageBlock::getPage, toList()));
 
-        Map<Integer, List<ClassificationTextBlock>> footers = document.getFooters()
+        Map<Integer, List<TextPageBlock>> footers = document.getFooters()
                 .stream()
                 .map(ClassificationFooter::getTextBlocks)
                 .flatMap(List::stream)
@@ -143,7 +141,7 @@ public class DocumentGraphFactory {
     }
 
 
-    private void addFooter(List<ClassificationTextBlock> textBlocks, Context context) {
+    private void addFooter(List<TextPageBlock> textBlocks, Context context) {
 
         Page page = context.getPage(textBlocks.get(0).getPage());
         Footer footer = Footer.builder().documentTree(context.getDocumentTree()).build();
@@ -158,7 +156,7 @@ public class DocumentGraphFactory {
     }
 
 
-    public void addHeader(List<ClassificationTextBlock> textBlocks, Context context) {
+    public void addHeader(List<TextPageBlock> textBlocks, Context context) {
 
         Page page = context.getPage(textBlocks.get(0).getPage());
         Header header = Header.builder().documentTree(context.getDocumentTree()).build();

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/factory/SearchTextWithTextPositionFactory.java

@@ -13,6 +13,12 @@ import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.mo
 public class SearchTextWithTextPositionFactory {
 
     public static final int HEIGHT_PADDING = 2;
+    // when checking for a hyphen linebreak, we need to check after a linebreak if the last hyphen was less than three symbols away.
+    // We detect a linebreak as either a "\n" character or if two adjacent symbol's position differ in y-coordinates by at least one character height.
+    // If there is a hyphen linebreak, the hyphen will be 1 position in front of a "\n" or 2 positions in front of the character which has a lower y-coordinate
+    // This is why, we need to initialize this to < -2, otherwise, if the very first symbol is a \n we would detect a hyphen linebreak that isn't there.
+    // Also, Integer.MIN_VALUE is a bad idea due to potential overflow during arithmetic operations. This is why the default should be -3.
+    public static final int MAX_HYPHEN_LINEBREAK_DISTANCE = 3;
 
 
     public static SearchTextWithTextPositionDto buildSearchTextToTextPositionModel(List<TextPositionSequence> sequences) {
@@ -30,8 +36,10 @@ public class SearchTextWithTextPositionFactory {
             for (int i = 0; i < word.getTextPositions().size(); ++i) {
 
                 currentTextPosition = word.getTextPositions().get(i);
-
+                if (isLineBreak(currentTextPosition, previousTextPosition)) {
-                removeHyphenLinebreaks(currentTextPosition, previousTextPosition, context);
+                    removeHyphenLinebreaks(context);
+                    context.lineBreaksStringIdx.add(context.stringIdx);
+                }
                 if (!isRepeatedWhitespace(currentTextPosition.getUnicode(), previousTextPosition.getUnicode())) {
                     if (isHyphen(currentTextPosition.getUnicode())) {
                         context.lastHyphenIdx = context.stringIdx;
@@ -76,24 +84,20 @@ public class SearchTextWithTextPositionFactory {
     }
 
 
-    private static void removeHyphenLinebreaks(RedTextPosition currentTextPosition, RedTextPosition previousTextPosition, Context context) {
+    private static void removeHyphenLinebreaks(Context context) {
 
-        if (isLineBreak(currentTextPosition, previousTextPosition)) {
+        if (lastHyphenDirectlyBeforeLineBreak(context)) {
-
+            context.stringBuilder.delete(context.lastHyphenIdx, context.stringBuilder.length());
-            if (lastHyphenDirectlyBeforeLineBreak(context)) {
+            context.stringIdxToPositionIdx = context.stringIdxToPositionIdx.subList(0, context.lastHyphenIdx);
-                context.stringBuilder.delete(context.lastHyphenIdx, context.stringBuilder.length());
+            context.stringIdx = context.lastHyphenIdx;
-                context.stringIdxToPositionIdx = context.stringIdxToPositionIdx.subList(0, context.lastHyphenIdx);
+            context.lastHyphenIdx = -MAX_HYPHEN_LINEBREAK_DISTANCE;
-                context.stringIdx = context.lastHyphenIdx;
-                context.lastHyphenIdx = -3;
-            }
-            context.lineBreaksStringIdx.add(context.stringIdx);
         }
     }
 
 
     private static boolean lastHyphenDirectlyBeforeLineBreak(Context context) {
 
-        return context.stringIdx - context.lastHyphenIdx < 3;
+        return context.stringIdx - context.lastHyphenIdx < MAX_HYPHEN_LINEBREAK_DISTANCE;
     }
 
 
@@ -170,12 +174,8 @@ public class SearchTextWithTextPositionFactory {
 
         int stringIdx;
         int positionIdx;
-        // when checking for a hyphen linebreak, we need to check after a linebreak if the last hyphen was less than three symbols away.
+
-        // We detect a linebreak as either a "\n" character or if two adjacent symbol's position differ in y-coordinates by at least one character height.
+        int lastHyphenIdx = -MAX_HYPHEN_LINEBREAK_DISTANCE;
-        // If there is a hyphen linebreak, the hyphen will be 1 position in front of a "\n" or 2 positions in front of the character which has a lower y-coordinate
-        // This is why, we need to initialize this to < -2, otherwise, if the very first symbol is a \n we would detect a hyphen linebreak that isn't there.
-        // Also, Integer.MIN_VALUE is a bad idea due to potential overflow during arithmetic operations. This is why the default should be -3.
-        int lastHyphenIdx = -3;
 
     }
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/factory/SectionNodeFactory.java

@@ -1,5 +1,6 @@
 package com.iqser.red.service.redaction.v1.server.layoutparsing.document.factory;
 
+import static java.lang.String.format;
 import static java.util.Collections.emptyList;
 import static java.util.stream.Collectors.groupingBy;
 
@@ -12,7 +13,7 @@ import java.util.Set;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.AbstractPageBlock;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.image.ClassifiedImage;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.table.TablePageBlock;
-import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.ClassificationTextBlock;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.TextPageBlock;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.GenericSemanticNode;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.Page;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.Section;
@@ -29,7 +30,7 @@ public class SectionNodeFactory {
             return;
         }
         Map<Integer, List<AbstractPageBlock>> blocksPerPage = pageBlocks.stream().collect(groupingBy(AbstractPageBlock::getPage));
-        Section section = Section.builder().entities(new HashSet<>()).documentTree(context.getDocumentTree()).build();
+        Section section = Section.builder().documentTree(context.getDocumentTree()).build();
 
         context.getSections().add(section);
         blocksPerPage.keySet().forEach(pageNumber -> addSectionNodeToPageNode(context, section, pageNumber));
@@ -41,9 +42,6 @@ public class SectionNodeFactory {
             splitPageBlocksIntoSubSections(pageBlocks).forEach(subSectionPageBlocks -> addSection(section, subSectionPageBlocks, emptyList(), context));
         } else {
             addTablesAndParagraphsAndHeadlinesToSection(pageBlocks, context, section);
-            if (listIsTextBlockOnly(pageBlocks)) {
-                section.setExcludesTables(true);
-            }
         }
 
         images.stream().distinct().forEach(image -> DocumentGraphFactory.addImage(section, image, context));
@@ -60,12 +58,6 @@ public class SectionNodeFactory {
     }
 
 
-    private boolean listIsTextBlockOnly(List<AbstractPageBlock> pageBlocks) {
-
-        return pageBlocks.stream().allMatch(abstractPageBlock -> abstractPageBlock instanceof ClassificationTextBlock);
-    }
-
-
     private void addFirstHeadlineDirectlyToSection(List<AbstractPageBlock> pageBlocks, DocumentGraphFactory.Context context, Section section) {
 
         if (pageBlocks.get(0).isHeadline()) {
@@ -87,16 +79,16 @@ public class SectionNodeFactory {
 
             remainingBlocks.removeAll(alreadyMerged);
 
-            if (abstractPageBlock instanceof ClassificationTextBlock) {
+            if (abstractPageBlock instanceof TextPageBlock) {
-                List<ClassificationTextBlock> textBlocks = findTextBlocksWithSameClassificationAndAlignsY(abstractPageBlock, remainingBlocks);
+                List<TextPageBlock> textBlocks = findTextBlocksWithSameClassificationAndAlignsY(abstractPageBlock, remainingBlocks);
                 alreadyMerged.addAll(textBlocks);
-                DocumentGraphFactory.addParagraphOrHeadline(section, (ClassificationTextBlock) abstractPageBlock, context, textBlocks);
+                DocumentGraphFactory.addParagraphOrHeadline(section, (TextPageBlock) abstractPageBlock, context, textBlocks);
-            }
+            } else if (abstractPageBlock instanceof TablePageBlock tablePageBlock) {
-            if (abstractPageBlock instanceof TablePageBlock tablePageBlock) {
-
                 List<TablePageBlock> tablesToMerge = TableMergingUtility.findConsecutiveTablesWithSameColCountAndSameHeaders(tablePageBlock, remainingBlocks);
                 alreadyMerged.addAll(tablesToMerge);
                 TableNodeFactory.addTable(section, tablesToMerge, context);
+            } else {
+                throw new RuntimeException(format("Unhandled AbstractPageBlockType %s!", abstractPageBlock.getClass()));
             }
         }
     }
@@ -104,8 +96,7 @@ public class SectionNodeFactory {
 
     private boolean containsTablesAndTextBlocks(List<AbstractPageBlock> pageBlocks) {
 
-        return pageBlocks.stream().anyMatch(pageBlock -> pageBlock instanceof TablePageBlock) && pageBlocks.stream()
+        return pageBlocks.stream().anyMatch(pageBlock -> pageBlock instanceof TablePageBlock) && pageBlocks.stream().anyMatch(pageBlock -> pageBlock instanceof TextPageBlock);
-                .anyMatch(pageBlock -> pageBlock instanceof ClassificationTextBlock);
     }
 
 
@@ -171,14 +162,14 @@ public class SectionNodeFactory {
     }
 
 
-    private List<ClassificationTextBlock> findTextBlocksWithSameClassificationAndAlignsY(AbstractPageBlock atc, List<AbstractPageBlock> pageBlocks) {
+    private List<TextPageBlock> findTextBlocksWithSameClassificationAndAlignsY(AbstractPageBlock atc, List<AbstractPageBlock> pageBlocks) {
 
         return pageBlocks.stream()
                 .filter(abstractTextContainer -> !abstractTextContainer.equals(atc))
                 .filter(abstractTextContainer -> abstractTextContainer.getPage() == atc.getPage())
-                .filter(abstractTextContainer -> abstractTextContainer instanceof ClassificationTextBlock)
+                .filter(abstractTextContainer -> abstractTextContainer instanceof TextPageBlock)
                 .filter(abstractTextContainer -> abstractTextContainer.intersectsY(atc))
-                .map(abstractTextContainer -> (ClassificationTextBlock) abstractTextContainer)
+                .map(abstractTextContainer -> (TextPageBlock) abstractTextContainer)
                 .toList();
     }
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/factory/TableNodeFactory.java

@@ -36,11 +36,11 @@ public class TableNodeFactory {
 
         pages.forEach(page -> addTableToPage(page, parentNode, table));
 
-        List<Integer> tocId = context.getDocumentTree().createNewChildEntryAndReturnId(parentNode, table);
+        List<Integer> treeId = context.getDocumentTree().createNewChildEntryAndReturnId(parentNode, table);
-        table.setTreeId(tocId);
+        table.setTreeId(treeId);
         addTableCells(mergedRows, table, context);
 
-        ifTableHasNoHeadersAssumeFirstRowAreHeaders(table);
+        ifTableHasNoHeadersSetFirstRowAsHeaders(table);
     }
 
 
@@ -75,7 +75,7 @@ public class TableNodeFactory {
     }
 
 
-    private void ifTableHasNoHeadersAssumeFirstRowAreHeaders(Table table) {
+    private void ifTableHasNoHeadersSetFirstRowAsHeaders(Table table) {
 
         if (table.streamHeaders().findAny().isEmpty()) {
             table.streamRow(0).forEach(tableCellNode -> tableCellNode.setHeader(true));
@@ -101,35 +101,24 @@ public class TableNodeFactory {
         TableCell tableCell = TableCell.builder().documentTree(context.getDocumentTree()).row(rowIndex).col(colIndex).header(cell.isHeaderCell()).bBox(cell.getBounds2D()).build();
         page.getMainBody().add(tableCell);
 
+        List<Integer> treeId = context.getDocumentTree().createNewTableChildEntryAndReturnId(tableNode, tableCell);
+        tableCell.setTreeId(treeId);
+
         TextBlock textBlock;
-
-        List<Integer> tocId = context.getDocumentTree().createNewTableChildEntryAndReturnId(tableNode, tableCell);
-        tableCell.setTreeId(tocId);
-
         if (cell.getTextBlocks().isEmpty()) {
             tableCell.setLeafTextBlock(context.getTextBlockFactory().emptyTextBlock(tableNode, context, page));
-            tableCell.setLeaf(true);
-
         } else if (cell.getTextBlocks().size() == 1) {
             textBlock = context.getTextBlockFactory().buildAtomicTextBlock(cell.getTextBlocks().get(0).getSequences(), tableCell, context, page);
             tableCell.setLeafTextBlock(textBlock);
-            tableCell.setLeaf(true);
-
         } else if (firstTextBlockIsHeadline(cell)) {
             SectionNodeFactory.addSection(tableCell, cell.getTextBlocks().stream().map(tb -> (AbstractPageBlock) tb).toList(), emptyList(), context);
-            tableCell.setLeaf(false);
-
         } else if (cellAreaIsSmallerThanPageAreaTimesThreshold(cell, page)) {
             List<TextPositionSequence> sequences = TextPositionOperations.mergeAndSortTextPositionSequenceByYThenX(cell.getTextBlocks());
             textBlock = context.getTextBlockFactory().buildAtomicTextBlock(sequences, tableCell, context, page);
             tableCell.setLeafTextBlock(textBlock);
-            tableCell.setLeaf(true);
-
         } else {
             cell.getTextBlocks().forEach(tb -> DocumentGraphFactory.addParagraphOrHeadline(tableCell, tb, context, emptyList()));
-            tableCell.setLeaf(false);
         }
-
     }
 
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/Boundary.java

@@ -6,9 +6,13 @@ import java.util.Collection;
 import java.util.LinkedList;
 import java.util.List;
 
+import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.TextBlock;
+
+import lombok.EqualsAndHashCode;
 import lombok.Setter;
 
 @Setter
+@EqualsAndHashCode
 public class Boundary implements Comparable<Boundary> {
 
     private int start;
@@ -135,17 +139,25 @@ public class Boundary implements Comparable<Boundary> {
     }
 
 
-    @Override
+    /**
-    public int hashCode() {
+     * shrinks the boundary, such that textBlock.subSequence(boundary) returns a string without whitespaces.
+     *
+     * @param textBlock TextBlock to check whitespaces against
+     * @return boundary
+     */
+    public Boundary trim(TextBlock textBlock) {
 
-        return toString().hashCode();
+        int trimmedStart = this.start;
-    }
+        while (Character.isWhitespace(textBlock.charAt(trimmedStart))) {
+            trimmedStart++;
+        }
 
+        int trimmedEnd = this.end;
+        while (Character.isWhitespace(textBlock.charAt(trimmedEnd - 1))) {
+            trimmedEnd--;
+        }
 
-    @Override
+        return new Boundary(trimmedStart, Math.max(trimmedEnd, trimmedStart));
-    public boolean equals(Object object) {
-
-        return hashCode() == object.hashCode();
     }
 
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/DocumentTree.java

@@ -2,13 +2,11 @@ package com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph;
 
 import static java.lang.String.format;
 
-import java.nio.charset.StandardCharsets;
 import java.util.Collections;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.stream.Stream;
 
-import com.google.common.hash.Hashing;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.Document;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.GenericSemanticNode;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.NodeType;
@@ -22,10 +20,12 @@ import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;
+import lombok.EqualsAndHashCode;
 import lombok.Getter;
 import lombok.experimental.FieldDefaults;
 
 @Data
+@EqualsAndHashCode
 public class DocumentTree {
 
     private final Entry root;
@@ -39,7 +39,7 @@ public class DocumentTree {
 
     public TextBlock buildTextBlock() {
 
-        return streamAllEntriesInOrder().map(Entry::getNode).filter(SemanticNode::isLeaf).map(SemanticNode::getLeafTextBlock).collect(new TextBlockCollector());
+        return allEntriesInOrder().map(Entry::getNode).filter(SemanticNode::isLeaf).map(SemanticNode::getLeafTextBlock).collect(new TextBlockCollector());
     }
 
 
@@ -71,13 +71,13 @@ public class DocumentTree {
     private List<Integer> createNewChildEntryAndReturnIdImpl(List<Integer> parentId, SemanticNode node) {
 
         if (!entryExists(parentId)) {
-            throw new UnsupportedOperationException(format("parentId %s does not exist!", parentId));
+            throw new IllegalArgumentException(format("parentId %s does not exist!", parentId));
         }
 
         Entry parent = getEntryById(parentId);
         List<Integer> newId = new LinkedList<>(parentId);
         newId.add(parent.children.size());
-        parent.children.add(Entry.builder().treeId(newId).node(node).children(new LinkedList<>()).build());
+        parent.children.add(Entry.builder().treeId(newId).node(node).build());
 
         return newId;
     }
@@ -111,13 +111,13 @@ public class DocumentTree {
     }
 
 
-    public Stream<SemanticNode> streamChildNodes(List<Integer> treeId) {
+    public Stream<SemanticNode> childNodes(List<Integer> treeId) {
 
         return getEntryById(treeId).children.stream().map(Entry::getNode);
     }
 
 
-    public Stream<SemanticNode> streamChildNodesOfType(List<Integer> treeId, NodeType nodeType) {
+    public Stream<SemanticNode> childNodesOfType(List<Integer> treeId, NodeType nodeType) {
 
         return getEntryById(treeId).children.stream().filter(entry -> entry.node.getType().equals(nodeType)).map(Entry::getNode);
     }
@@ -148,19 +148,19 @@ public class DocumentTree {
     }
 
 
-    public Stream<Entry> streamMainEntries() {
+    public Stream<Entry> mainEntries() {
 
         return root.children.stream();
     }
 
 
-    public Stream<Entry> streamAllEntriesInOrder() {
+    public Stream<Entry> allEntriesInOrder() {
 
         return Stream.of(root).flatMap(DocumentTree::flatten);
     }
 
 
-    public Stream<Entry> streamAllSubEntriesInOrder(List<Integer> parentId) {
+    public Stream<Entry> allSubEntriesInOrder(List<Integer> parentId) {
 
         return getEntryById(parentId).children.stream().flatMap(DocumentTree::flatten);
     }
@@ -169,7 +169,7 @@ public class DocumentTree {
     @Override
     public String toString() {
 
-        return String.join("\n", streamAllEntriesInOrder().map(Entry::toString).toList());
+        return String.join("\n", allEntriesInOrder().map(Entry::toString).toList());
     }
 
 
@@ -196,7 +196,8 @@ public class DocumentTree {
 
         List<Integer> treeId;
         SemanticNode node;
-        List<Entry> children;
+        @Builder.Default
+        List<Entry> children = new LinkedList<>();
 
 
         @Override
@@ -211,20 +212,6 @@ public class DocumentTree {
             return node.getType();
         }
 
-
-        @Override
-        public int hashCode() {
-
-            return Hashing.murmur3_32_fixed().hashString(toString(), StandardCharsets.UTF_8).hashCode();
-        }
-
-
-        @Override
-        public boolean equals(Object o) {
-
-            return o instanceof Entry && o.hashCode() == this.hashCode();
-        }
-
     }
 
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/entity/RedactionEntity.java

@@ -1,7 +1,6 @@
 package com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity;
 
 import java.awt.geom.Rectangle2D;
-import java.nio.charset.StandardCharsets;
 import java.util.Collection;
 import java.util.Comparator;
 import java.util.Deque;
@@ -11,7 +10,6 @@ import java.util.List;
 import java.util.Map;
 import java.util.Set;
 
-import com.google.common.hash.Hashing;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.Boundary;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.Page;
@@ -22,17 +20,22 @@ import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;
+import lombok.EqualsAndHashCode;
 import lombok.experimental.FieldDefaults;
 
 @Data
 @Builder
 @AllArgsConstructor
 @FieldDefaults(level = AccessLevel.PRIVATE)
+@EqualsAndHashCode(onlyExplicitlyIncluded = true)
 public class RedactionEntity {
 
     // initial values
+    @EqualsAndHashCode.Include
     final Boundary boundary;
+    @EqualsAndHashCode.Include
     final String type;
+    @EqualsAndHashCode.Include
     final EntityType entityType;
 
     // empty defaults
@@ -51,6 +54,7 @@ public class RedactionEntity {
     String legalBasis;
 
     // inferred on graph insertion
+    @EqualsAndHashCode.Include
     String value;
     String textBefore;
     String textAfter;
@@ -64,22 +68,7 @@ public class RedactionEntity {
 
     public static RedactionEntity initialEntityNode(Boundary boundary, String type, EntityType entityType) {
 
-        return RedactionEntity.builder()
+        return RedactionEntity.builder().type(type).entityType(entityType).boundary(boundary).engines(new HashSet<>()).references(new HashSet<>()).build();
-                .type(type)
-                .entityType(entityType)
-                .boundary(boundary)
-                .redaction(false)
-                .removed(false)
-                .ignored(false)
-                .resized(false)
-                .skipRemoveEntitiesContainedInLarger(false)
-                .dictionaryEntry(false)
-                .dossierDictionaryEntry(false)
-                .engines(new HashSet<>())
-                .references(new HashSet<>())
-                .redactionReason("")
-                .legalBasis("")
-                .build();
     }
 
 
@@ -134,7 +123,7 @@ public class RedactionEntity {
     public int getMatchedRule() {
 
         if (matchedRules.isEmpty()) {
-            return -1;
+            return 0;
         }
         return matchedRules.getLast();
     }
@@ -143,7 +132,7 @@ public class RedactionEntity {
     public List<RedactionPosition> getRedactionPositionsPerPage() {
 
         if (redactionPositionsPerPage == null || redactionPositionsPerPage.isEmpty()) {
-            Map<Page, List<Rectangle2D>> rectanglesPerLinePerPage = deepestFullyContainingNode.buildTextBlock().getPositionsPerPage(boundary);
+            Map<Page, List<Rectangle2D>> rectanglesPerLinePerPage = deepestFullyContainingNode.getTextBlock().getPositionsPerPage(boundary);
 
             Page firstPage = rectanglesPerLinePerPage.keySet()
                     .stream()
@@ -237,18 +226,4 @@ public class RedactionEntity {
         return sb.toString();
     }
 
-
-    @Override
-    public int hashCode() {
-
-        return Hashing.murmur3_128().hashString(toString(), StandardCharsets.UTF_8).hashCode();
-    }
-
-
-    @Override
-    public boolean equals(Object o) {
-
-        return o.hashCode() == hashCode();
-    }
-
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/entity/RedactionPosition.java

@@ -17,6 +17,8 @@ public class RedactionPosition {
 
     final String id;
     Page page;
+    // Each entry in this list corresponds to an entry in the redaction log, this means:
+    // An entity might be represented by multiple redaction log entries
     List<Rectangle2D> rectanglePerLine;
 
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/Document.java

@@ -45,9 +45,12 @@ public class Document implements GenericSemanticNode {
     }
 
 
-    public TextBlock buildTextBlock() {
+    public TextBlock getTextBlock() {
 
-        return streamTerminalTextBlocksInOrder().collect(new TextBlockCollector());
+        if (textBlock == null) {
+            textBlock = streamTerminalTextBlocksInOrder().collect(new TextBlockCollector());
+        }
+        return textBlock;
     }
 
 
@@ -86,7 +89,7 @@ public class Document implements GenericSemanticNode {
 
     private Stream<SemanticNode> streamAllNodes() {
 
-        return documentTree.streamAllEntriesInOrder().map(DocumentTree.Entry::getNode);
+        return documentTree.allEntriesInOrder().map(DocumentTree.Entry::getNode);
     }
 
 
@@ -99,7 +102,7 @@ public class Document implements GenericSemanticNode {
     @Override
     public String toString() {
 
-        return NodeType.DOCUMENT + ": " + buildTextBlock().buildSummary();
+        return NodeType.DOCUMENT + ": " + this.getTextBlock().buildSummary();
     }
 
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/Footer.java

@@ -26,9 +26,6 @@ public class Footer implements GenericSemanticNode {
     List<Integer> treeId;
     TextBlock leafTextBlock;
 
-    @Builder.Default
-    boolean leaf = true;
-
     @EqualsAndHashCode.Exclude
     DocumentTree documentTree;
 
@@ -45,7 +42,14 @@ public class Footer implements GenericSemanticNode {
 
 
     @Override
-    public TextBlock buildTextBlock() {
+    public boolean isLeaf() {
+
+        return true;
+    }
+
+
+    @Override
+    public TextBlock getTextBlock() {
 
         return leafTextBlock;
     }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/Header.java

@@ -26,9 +26,6 @@ public class Header implements GenericSemanticNode {
     List<Integer> treeId;
     TextBlock leafTextBlock;
 
-    @Builder.Default
-    boolean leaf = true;
-
     @EqualsAndHashCode.Exclude
     DocumentTree documentTree;
 
@@ -37,6 +34,13 @@ public class Header implements GenericSemanticNode {
     Set<RedactionEntity> entities = new HashSet<>();
 
 
+    @Override
+    public boolean isLeaf() {
+
+        return true;
+    }
+
+
     @Override
     public NodeType getType() {
 
@@ -45,7 +49,7 @@ public class Header implements GenericSemanticNode {
 
 
     @Override
-    public TextBlock buildTextBlock() {
+    public TextBlock getTextBlock() {
 
         return leafTextBlock;
     }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/Headline.java

@@ -26,9 +26,6 @@ public class Headline implements GenericSemanticNode {
     List<Integer> treeId;
     TextBlock leafTextBlock;
 
-    @Builder.Default
-    final boolean leaf = true;
-
     @EqualsAndHashCode.Exclude
     DocumentTree documentTree;
 
@@ -45,7 +42,14 @@ public class Headline implements GenericSemanticNode {
 
 
     @Override
-    public TextBlock buildTextBlock() {
+    public boolean isLeaf() {
+
+        return true;
+    }
+
+
+    @Override
+    public TextBlock getTextBlock() {
 
         return leafTextBlock;
     }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/Image.java

@@ -63,7 +63,7 @@ public class Image implements GenericSemanticNode {
 
 
     @Override
-    public TextBlock buildTextBlock() {
+    public TextBlock getTextBlock() {
 
         return streamAllSubNodes().filter(SemanticNode::isLeaf).map(SemanticNode::getLeafTextBlock).collect(new TextBlockCollector());
     }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/ImageType.java

@@ -5,5 +5,17 @@ public enum ImageType {
     FORMULA,
     SIGNATURE,
     OTHER,
-    OCR
+    OCR;
+
+
+    public static ImageType fromString(String imageType) {
+
+        return switch (imageType.toLowerCase()) {
+            case "logo" -> ImageType.LOGO;
+            case "formula" -> ImageType.FORMULA;
+            case "signature" -> ImageType.SIGNATURE;
+            case "ocr" -> ImageType.OCR;
+            default -> ImageType.OTHER;
+        };
+    }
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/NodeType.java

@@ -1,5 +1,7 @@
 package com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes;
 
+import java.util.Locale;
+
 public enum NodeType {
     DOCUMENT,
     SECTION,
@@ -9,5 +11,11 @@ public enum NodeType {
     TABLE_CELL,
     IMAGE,
     HEADER,
-    FOOTER
+    FOOTER;
+
+
+    public String toString() {
+
+        return this.name().charAt(0) + this.name().substring(1).toLowerCase(Locale.ROOT);
+    }
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/Paragraph.java

@@ -24,9 +24,6 @@ public class Paragraph implements GenericSemanticNode {
     List<Integer> treeId;
     TextBlock leafTextBlock;
 
-    @Builder.Default
-    boolean leaf = true;
-
     @EqualsAndHashCode.Exclude
     DocumentTree documentTree;
 
@@ -43,7 +40,14 @@ public class Paragraph implements GenericSemanticNode {
 
 
     @Override
-    public TextBlock buildTextBlock() {
+    public boolean isLeaf() {
+
+        return true;
+    }
+
+
+    @Override
+    public TextBlock getTextBlock() {
 
         return leafTextBlock;
     }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/Section.java

@@ -29,7 +29,6 @@ public class Section implements GenericSemanticNode {
     TextBlock textBlock;
     @EqualsAndHashCode.Exclude
     DocumentTree documentTree;
-    boolean excludesTables;
 
     @Builder.Default
     @EqualsAndHashCode.Exclude
@@ -43,8 +42,14 @@ public class Section implements GenericSemanticNode {
     }
 
 
+    public boolean hasTables() {
+
+        return streamAllSubNodesOfType(NodeType.TABLE).findAny().isPresent();
+    }
+
+
     @Override
-    public TextBlock buildTextBlock() {
+    public TextBlock getTextBlock() {
 
         if (textBlock == null) {
             textBlock = streamAllSubNodes().filter(SemanticNode::isLeaf).map(SemanticNode::getLeafTextBlock).collect(new TextBlockCollector());
@@ -56,7 +61,7 @@ public class Section implements GenericSemanticNode {
     @Override
     public String toString() {
 
-        return treeId.toString() + ": " + NodeType.SECTION + ": " + buildTextBlock().buildSummary();
+        return treeId.toString() + ": " + NodeType.SECTION + ": " + this.getTextBlock().buildSummary();
     }
 
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/SemanticNode.java

@@ -34,7 +34,7 @@ public interface SemanticNode {
      *
      * @return TextBlock containing all AtomicTextBlocks that are located under this Node.
      */
-    TextBlock buildTextBlock();
+    TextBlock getTextBlock();
 
 
     /**
@@ -53,7 +53,7 @@ public interface SemanticNode {
      */
     default Set<Page> getPages() {
 
-        return buildTextBlock().getPages();
+        return getTextBlock().getPages();
     }
 
 
@@ -67,7 +67,13 @@ public interface SemanticNode {
         if (!getBoundary().contains(boundary)) {
             throw new IllegalArgumentException(format("%s which was used to query for pages is not contained in the %s of this node!", boundary, getBoundary()));
         }
-        return buildTextBlock().getPages(boundary);
+        return getTextBlock().getPages(boundary);
+    }
+
+
+    default boolean isOnPage(int pageNumber) {
+
+        return getPages().stream().anyMatch(page -> page.getNumber() == pageNumber);
     }
 
 
@@ -218,9 +224,9 @@ public interface SemanticNode {
      */
     default Integer getNumberOnPage() {
 
-        TextBlock textBlock = buildTextBlock();
+        TextBlock textBlock = getTextBlock();
         if (!textBlock.getAtomicTextBlocks().isEmpty()) {
-            return buildTextBlock().getAtomicTextBlocks().get(0).getNumberOnPage();
+            return getTextBlock().getAtomicTextBlocks().get(0).getNumberOnPage();
         } else {
             return -1;
         }
@@ -234,7 +240,7 @@ public interface SemanticNode {
      */
     default boolean hasText() {
 
-        return !buildTextBlock().isEmpty();
+        return !getTextBlock().isEmpty();
     }
 
 
@@ -246,7 +252,7 @@ public interface SemanticNode {
      */
     default boolean containsString(String string) {
 
-        return buildTextBlock().getSearchText().contains(string);
+        return getTextBlock().getSearchText().contains(string);
     }
 
 
@@ -270,7 +276,7 @@ public interface SemanticNode {
      */
     default boolean containsStringIgnoreCase(String string) {
 
-        return buildTextBlock().getSearchText().toLowerCase().contains(string.toLowerCase());
+        return getTextBlock().getSearchText().toLowerCase().contains(string.toLowerCase());
     }
 
 
@@ -306,7 +312,7 @@ public interface SemanticNode {
      */
     default void addThisToEntityIfIntersects(RedactionEntity redactionEntity) {
 
-        TextBlock textBlock = buildTextBlock();
+        TextBlock textBlock = getTextBlock();
         if (textBlock.getBoundary().intersects(redactionEntity.getBoundary())) {
             if (textBlock.containsBoundary(redactionEntity.getBoundary())) {
                 redactionEntity.setDeepestFullyContainingNode(this);
@@ -326,7 +332,7 @@ public interface SemanticNode {
      */
     default Stream<SemanticNode> streamChildren() {
 
-        return getDocumentTree().streamChildNodes(getTreeId());
+        return getDocumentTree().childNodes(getTreeId());
     }
 
 
@@ -337,7 +343,7 @@ public interface SemanticNode {
      */
     default Stream<SemanticNode> streamChildrenOfType(NodeType nodeType) {
 
-        return getDocumentTree().streamChildNodesOfType(getTreeId(), nodeType);
+        return getDocumentTree().childNodesOfType(getTreeId(), nodeType);
     }
 
 
@@ -348,7 +354,7 @@ public interface SemanticNode {
      */
     default Stream<SemanticNode> streamAllSubNodes() {
 
-        return getDocumentTree().streamAllSubEntriesInOrder(getTreeId()).map(DocumentTree.Entry::getNode);
+        return getDocumentTree().allSubEntriesInOrder(getTreeId()).map(DocumentTree.Entry::getNode);
     }
 
 
@@ -359,7 +365,7 @@ public interface SemanticNode {
      */
     default Stream<SemanticNode> streamAllSubNodesOfType(NodeType nodeType) {
 
-        return getDocumentTree().streamAllSubEntriesInOrder(getTreeId()).filter(entry -> entry.getType().equals(nodeType)).map(DocumentTree.Entry::getNode);
+        return getDocumentTree().allSubEntriesInOrder(getTreeId()).filter(entry -> entry.getType().equals(nodeType)).map(DocumentTree.Entry::getNode);
     }
 
 
@@ -370,7 +376,7 @@ public interface SemanticNode {
      */
     default Boundary getBoundary() {
 
-        return buildTextBlock().getBoundary();
+        return getTextBlock().getBoundary();
     }
 
 
@@ -429,7 +435,7 @@ public interface SemanticNode {
      */
     private Map<Page, Rectangle2D> getBBoxFromLeafTextBlock(Map<Page, Rectangle2D> bBoxPerPage) {
 
-        Map<Page, List<AtomicTextBlock>> atomicTextBlockPerPage = buildTextBlock().getAtomicTextBlocks().stream().collect(Collectors.groupingBy(AtomicTextBlock::getPage));
+        Map<Page, List<AtomicTextBlock>> atomicTextBlockPerPage = getTextBlock().getAtomicTextBlocks().stream().collect(Collectors.groupingBy(AtomicTextBlock::getPage));
         atomicTextBlockPerPage.forEach((page, atbs) -> bBoxPerPage.put(page, RectangleTransformations.bBoxUnionAtomicTextBlock(atbs)));
         return bBoxPerPage;
     }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/Table.java

@@ -66,7 +66,7 @@ public class Table implements SemanticNode {
      */
     public boolean rowContainsStringsIgnoreCase(Integer row, List<String> strings) {
 
-        String rowText = streamRow(row).map(TableCell::buildTextBlock).collect(new TextBlockCollector()).getSearchText().toLowerCase();
+        String rowText = streamRow(row).map(TableCell::getTextBlock).collect(new TextBlockCollector()).getSearchText().toLowerCase();
         return strings.stream().map(String::toLowerCase).allMatch(rowText::contains);
     }
 
@@ -95,11 +95,9 @@ public class Table implements SemanticNode {
      */
     public Stream<RedactionEntity> streamEntitiesWhereRowHasHeaderAndAnyValue(String header, List<String> values) {
 
-        List<Integer> vertebrateStudyCols = streamHeaders().filter(headerNode -> headerNode.containsString(header)).map(TableCell::getCol).toList();
+        List<Integer> colsWithHeader = streamHeaders().filter(headerNode -> headerNode.containsString(header)).map(TableCell::getCol).toList();
-        return streamTableCells().filter(tableCellNode -> vertebrateStudyCols.stream()
+        return streamTableCells().filter(tableCellNode -> colsWithHeader.stream()
-                        .anyMatch(vertebrateStudyCol -> getCell(tableCellNode.getRow(), vertebrateStudyCol).containsAnyString(values)))
+                .anyMatch(colWithHeader -> getCell(tableCellNode.getRow(), colWithHeader).containsAnyString(values))).map(TableCell::getEntities).flatMap(Collection::stream);
-                .map(TableCell::getEntities)
-                .flatMap(Collection::stream);
     }
 
 
@@ -158,7 +156,7 @@ public class Table implements SemanticNode {
      */
     public Stream<TableCell> streamTableCellsWithHeader(String header) {
 
-        return streamHeaders().filter(tableCellNode -> tableCellNode.buildTextBlock().getSearchText().contains(header))
+        return streamHeaders().filter(tableCellNode -> tableCellNode.getTextBlock().getSearchText().contains(header))
                 .map(TableCell::getCol)
                 .flatMap(this::streamCol)
                 .filter(tableCellNode -> !tableCellNode.isHeader());
@@ -221,7 +219,7 @@ public class Table implements SemanticNode {
      */
     public boolean hasHeader(String header) {
 
-        return streamHeaders().anyMatch(tableCellNode -> tableCellNode.buildTextBlock().getSearchText().strip().equals(header));
+        return streamHeaders().anyMatch(tableCellNode -> tableCellNode.getTextBlock().getSearchText().strip().equals(header));
     }
 
 
@@ -279,7 +277,7 @@ public class Table implements SemanticNode {
 
 
     @Override
-    public TextBlock buildTextBlock() {
+    public TextBlock getTextBlock() {
 
         if (textBlock == null) {
             textBlock = streamAllSubNodes().filter(SemanticNode::isLeaf).map(SemanticNode::getLeafTextBlock).collect(new TextBlockCollector());
@@ -291,7 +289,7 @@ public class Table implements SemanticNode {
     @Override
     public String toString() {
 
-        return treeId.toString() + ": " + NodeType.TABLE + ": #cols: " + numberOfCols + ", #rows: " + numberOfRows + ", " + buildTextBlock().buildSummary();
+        return treeId.toString() + ": " + NodeType.TABLE + ": #cols: " + numberOfCols + ", #rows: " + numberOfRows + ", " + this.getTextBlock().buildSummary();
     }
 
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/TableCell.java

@@ -32,8 +32,6 @@ public class TableCell implements GenericSemanticNode {
 
     Rectangle2D bBox;
 
-    @Builder.Default
-    boolean leaf = true;
     TextBlock leafTextBlock;
 
     TextBlock textBlock;
@@ -63,9 +61,16 @@ public class TableCell implements GenericSemanticNode {
 
 
     @Override
-    public TextBlock buildTextBlock() {
+    public boolean isLeaf() {
 
-        if (leaf) {
+        return getDocumentTree().getEntryById(getTreeId()).getChildren().isEmpty();
+    }
+
+
+    @Override
+    public TextBlock getTextBlock() {
+
+        if (isLeaf()) {
             return leafTextBlock;
         }
 
@@ -79,7 +84,7 @@ public class TableCell implements GenericSemanticNode {
     @Override
     public String toString() {
 
-        return treeId + ": " + NodeType.TABLE_CELL + ": " + buildTextBlock().buildSummary();
+        return treeId + ": " + NodeType.TABLE_CELL + ": " + this.getTextBlock().buildSummary();
     }
 
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/textblock/ConcatenatedTextBlock.java

@@ -57,6 +57,7 @@ public class ConcatenatedTextBlock implements TextBlock {
         }
         this.atomicTextBlocks.addAll(textBlock.getAtomicTextBlocks());
         boundary.setEnd(textBlock.getBoundary().end());
+        this.searchText = null;
         return this;
     }
 
@@ -160,7 +161,8 @@ public class ConcatenatedTextBlock implements TextBlock {
         }
 
         AtomicTextBlock lastTextBlock = textBlocks.get(textBlocks.size() - 1);
-        rectanglesPerLinePerPage = mergeEntityPositionsWithSamePageNode(rectanglesPerLinePerPage, lastTextBlock.getPositionsPerPage(lastTextBlock.getBoundary()));
+        rectanglesPerLinePerPage = mergeEntityPositionsWithSamePageNode(rectanglesPerLinePerPage,
+                lastTextBlock.getPositionsPerPage(new Boundary(lastTextBlock.getBoundary().start(), stringBoundary.end())));
 
         return rectanglesPerLinePerPage;
     }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/services/EntityCreationService.java

@@ -3,7 +3,6 @@ package com.iqser.red.service.redaction.v1.server.layoutparsing.document.service
 import static com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RedactionSearchUtility.getExpandedEndByRegex;
 import static com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RedactionSearchUtility.getExpandedStartByRegex;
 import static com.iqser.red.service.redaction.v1.server.redaction.utils.SeparatorUtils.boundaryIsSurroundedBySeparators;
-import static com.iqser.red.service.redaction.v1.server.redaction.utils.SeparatorUtils.isWhiteSpacesOrSeparatorsOnly;
 
 import java.util.Collection;
 import java.util.Collections;
@@ -46,7 +45,7 @@ public class EntityCreationService {
 
     public Stream<RedactionEntity> betweenStrings(String start, String stop, String type, EntityType entityType, SemanticNode node) {
 
-        TextBlock textBlock = node.buildTextBlock();
+        TextBlock textBlock = node.getTextBlock();
         List<Boundary> startBoundaries = RedactionSearchUtility.findBoundariesByString(start, textBlock);
         List<Boundary> stopBoundaries = RedactionSearchUtility.findBoundariesByString(stop, textBlock);
 
@@ -62,22 +61,25 @@ public class EntityCreationService {
             stopBoundaries.remove(optionalFirstStopBoundary.get());
             entityBoundaries.add(new Boundary(startBoundary.end(), optionalFirstStopBoundary.get().start()));
         }
-        return entityBoundaries.stream().filter(boundary -> isValidEntityBoundary(textBlock, boundary)).map(boundary -> byBoundary(boundary, type, entityType, node));
+        return entityBoundaries.stream()
+                .map(boundary -> boundary.trim(textBlock))
+                .filter(boundary -> isValidEntityBoundary(textBlock, boundary))
+                .map(boundary -> byBoundary(boundary, type, entityType, node));
     }
 
 
     public Stream<RedactionEntity> bySearchImplementation(SearchImplementation searchImplementation, String type, EntityType entityType, SemanticNode node) {
 
-        return searchImplementation.getBoundaries(node.buildTextBlock(), node.getBoundary())
+        return searchImplementation.getBoundaries(node.getTextBlock(), node.getBoundary())
                 .stream()
-                .filter(boundary -> isValidEntityBoundary(node.buildTextBlock(), boundary))
+                .filter(boundary -> isValidEntityBoundary(node.getTextBlock(), boundary))
                 .map(bounds -> byBoundary(bounds, type, entityType, node));
     }
 
 
     public Stream<RedactionEntity> lineAfterStrings(List<String> strings, String type, EntityType entityType, SemanticNode node) {
 
-        TextBlock textBlock = node.buildTextBlock();
+        TextBlock textBlock = node.getTextBlock();
         SearchImplementation searchImplementation = new SearchImplementation(strings, false);
         return searchImplementation.getBoundaries(textBlock, node.getBoundary())
                 .stream()
@@ -89,7 +91,7 @@ public class EntityCreationService {
 
     public Stream<RedactionEntity> lineAfterString(String string, String type, EntityType entityType, SemanticNode node) {
 
-        TextBlock textBlock = node.buildTextBlock();
+        TextBlock textBlock = node.getTextBlock();
         return RedactionSearchUtility.findBoundariesByString(string, textBlock)
                 .stream()
                 .map(boundary -> toLineAfterBoundary(textBlock, boundary))
@@ -112,13 +114,13 @@ public class EntityCreationService {
 
     public Stream<RedactionEntity> byRegex(String regexPattern, String type, EntityType entityType, int group, SemanticNode node) {
 
-        return RedactionSearchUtility.findBoundariesByRegex(regexPattern, group, node.buildTextBlock()).stream().map(boundary -> byBoundary(boundary, type, entityType, node));
+        return RedactionSearchUtility.findBoundariesByRegex(regexPattern, group, node.getTextBlock()).stream().map(boundary -> byBoundary(boundary, type, entityType, node));
     }
 
 
     public Stream<RedactionEntity> byRegexIgnoreCase(String regexPattern, String type, EntityType entityType, int group, SemanticNode node) {
 
-        return RedactionSearchUtility.findBoundariesByRegexCaseInsensitive(regexPattern, group, node.buildTextBlock())
+        return RedactionSearchUtility.findBoundariesByRegexCaseInsensitive(regexPattern, group, node.getTextBlock())
                 .stream()
                 .map(boundary -> byBoundary(boundary, type, entityType, node));
     }
@@ -126,13 +128,13 @@ public class EntityCreationService {
 
     public Stream<RedactionEntity> byString(String keyword, String type, EntityType entityType, SemanticNode node) {
 
-        return RedactionSearchUtility.findBoundariesByString(keyword, node.buildTextBlock()).stream().map(boundary -> byBoundary(boundary, type, entityType, node));
+        return RedactionSearchUtility.findBoundariesByString(keyword, node.getTextBlock()).stream().map(boundary -> byBoundary(boundary, type, entityType, node));
     }
 
 
     public RedactionEntity bySemanticNode(SemanticNode node, String type, EntityType entityType) {
 
-        Boundary boundary = node.buildTextBlock().getBoundary();
+        Boundary boundary = node.getTextBlock().getBoundary();
         if (boundary.length() > 0) {
             boundary = new Boundary(boundary.start(), boundary.end() - 1);
         }
@@ -150,7 +152,7 @@ public class EntityCreationService {
     public RedactionEntity bySuffixExpansionRegex(RedactionEntity entity, String regexPattern) {
 
         int expandedEnd = getExpandedEndByRegex(entity, regexPattern);
-        expandedEnd = truncateEndIfLineBreakIsBetween(entity.getBoundary().end(), expandedEnd, entity.getDeepestFullyContainingNode().buildTextBlock());
+        expandedEnd = truncateEndIfLineBreakIsBetween(entity.getBoundary().end(), expandedEnd, entity.getDeepestFullyContainingNode().getTextBlock());
         return byBoundary(new Boundary(entity.getBoundary().start(), expandedEnd), entity.getType(), entity.getEntityType(), entity.getDeepestFullyContainingNode());
     }
 
@@ -166,7 +168,8 @@ public class EntityCreationService {
 
     public RedactionEntity byBoundary(Boundary boundary, String type, EntityType entityType, SemanticNode node) {
 
-        RedactionEntity entity = RedactionEntity.initialEntityNode(boundary, type, entityType);
+        Boundary trimmedBoundary = boundary.trim(node.getTextBlock());
+        RedactionEntity entity = RedactionEntity.initialEntityNode(trimmedBoundary, type, entityType);
         addEntityToGraph(entity, node);
         return entity;
     }
@@ -175,10 +178,13 @@ public class EntityCreationService {
     public RedactionEntity byEntities(List<RedactionEntity> entitiesToMerge, String type, EntityType entityType, SemanticNode node) {
 
         if (!allEntitiesIntersectAndHaveSameTypes(entitiesToMerge)) {
-            throw new IllegalArgumentException("Provided entities can not be merged!");
+            throw new IllegalArgumentException("Provided entities can not be merged, since they do not intersect or are not the same type!" + entitiesToMerge);
         }
-        if (entitiesToMerge.size() < 2) {
+        if (entitiesToMerge.isEmpty()) {
-            throw new IllegalArgumentException("more than 2 entities are required to merge!");
+            throw new IllegalArgumentException("No Entities to merge!");
+        }
+        if (entitiesToMerge.size() == 1) {
+            return entitiesToMerge.get(0);
         }
 
         RedactionEntity mergedEntity = RedactionEntity.initialEntityNode(Boundary.merge(entitiesToMerge.stream().map(RedactionEntity::getBoundary).toList()), type, entityType);
@@ -224,7 +230,7 @@ public class EntityCreationService {
 
     private boolean isValidEntityBoundary(TextBlock textBlock, Boundary boundary) {
 
-        return boundaryIsSurroundedBySeparators(textBlock, boundary) && !isWhiteSpacesOrSeparatorsOnly(textBlock, boundary);
+        return boundaryIsSurroundedBySeparators(textBlock, boundary);
     }
 
 
@@ -237,7 +243,7 @@ public class EntityCreationService {
             documentTree.getRoot().getNode().getEntities().add(entity);
         } catch (NoSuchElementException e) {
             entity.setDeepestFullyContainingNode(documentTree.getRoot().getNode());
-            entityEnrichmentService.enrichEntity(entity, entity.getDeepestFullyContainingNode().buildTextBlock());
+            entityEnrichmentService.enrichEntity(entity, entity.getDeepestFullyContainingNode().getTextBlock());
             entity.addIntersectingNode(documentTree.getRoot().getNode());
             addToPages(entity);
             addEntityToNodeEntitySets(entity);
@@ -247,19 +253,18 @@ public class EntityCreationService {
 
     private void addEntityToGraph(RedactionEntity entity, DocumentTree documentTree) {
 
-        SemanticNode containingNode = documentTree.streamChildNodes(Collections.emptyList())
+        SemanticNode containingNode = documentTree.childNodes(Collections.emptyList())
-                .filter(node -> node.buildTextBlock().containsBoundary(entity.getBoundary()))
+                .filter(node -> node.getTextBlock().containsBoundary(entity.getBoundary()))
                 .findFirst()
                 .orElseThrow(() -> new NoSuchElementException("No containing Node found!"));
 
         containingNode.addThisToEntityIfIntersects(entity);
 
-        TextBlock textBlock = entity.getDeepestFullyContainingNode().buildTextBlock();
+        TextBlock textBlock = entity.getDeepestFullyContainingNode().getTextBlock();
         entityEnrichmentService.enrichEntity(entity, textBlock);
 
         addToPages(entity);
         addEntityToNodeEntitySets(entity);
-
     }
 
 
@@ -297,7 +302,7 @@ public class EntityCreationService {
 
     private static Boundary toLineAfterBoundary(TextBlock textBlock, Boundary boundary) {
 
-        return new Boundary(boundary.end(), textBlock.getNextLinebreak(boundary.end()));
+        return new Boundary(boundary.end(), textBlock.getNextLinebreak(boundary.end())).trim(textBlock);
     }
 
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/services/EntityEnrichmentService.java

@@ -34,7 +34,7 @@ public class EntityEnrichmentService {
         if (!textAfter.isBlank()) {
             List<String> wordsAfter = splitToWordsAndRemoveEmptyWords(textAfter);
             int numberOfWordsAfter = Math.min(wordsAfter.size(), redactionServiceSettings.getNumberOfSurroundingWords());
-            if (wordsAfter.size() > 0) {
+            if (!wordsAfter.isEmpty()) {
                 return concatWordsAfter(wordsAfter.subList(0, numberOfWordsAfter), textAfter.startsWith(" "));
             }
         }
@@ -49,7 +49,7 @@ public class EntityEnrichmentService {
         if (!textBefore.isBlank()) {
             List<String> wordsBefore = splitToWordsAndRemoveEmptyWords(textBefore);
             int numberOfWordsBefore = Math.min(wordsBefore.size(), redactionServiceSettings.getNumberOfSurroundingWords());
-            if (wordsBefore.size() > 0) {
+            if (!wordsBefore.isEmpty()) {
                 return concatWordsBefore(wordsBefore.subList(wordsBefore.size() - numberOfWordsBefore, wordsBefore.size()), textBefore.endsWith(" "));
             }
         }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/services/ManualRedactionApplicationService.java

@@ -1,5 +1,6 @@
 package com.iqser.red.service.redaction.v1.server.layoutparsing.document.services;
 
+import java.awt.geom.Rectangle2D;
 import java.util.NoSuchElementException;
 
 import org.springframework.stereotype.Service;
@@ -8,7 +9,6 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionEntity;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionPosition;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.SemanticNode;
-import com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RectangleMapper;
 
 import lombok.RequiredArgsConstructor;
 
@@ -27,7 +27,7 @@ public class ManualRedactionApplicationService {
                 .findFirst()
                 .orElseThrow(() -> new NoSuchElementException("No redaction position with matching annotation id found!"));
 
-        redactionPositionToBeResized.setRectanglePerLine(manualResizeRedaction.getPositions().stream().map(RectangleMapper::toRectangle2D).toList());
+        redactionPositionToBeResized.setRectanglePerLine(manualResizeRedaction.getPositions().stream().map(ManualRedactionApplicationService::toRectangle2D).toList());
 
         int newStartOffset;
         if (manualResizeRedaction.getValue().length() > entityToBeResized.getValue().length()) {
@@ -46,4 +46,10 @@ public class ManualRedactionApplicationService {
         entityCreationService.addEntityToGraph(entityToBeResized, nodeToInsertInto);
     }
 
+
+    private static Rectangle2D toRectangle2D(com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle rect) {
+
+        return new Rectangle2D.Double(rect.getTopLeftX() - rect.getWidth(), rect.getTopLeftY() - rect.getHeight(), rect.getWidth(), rect.getHeight());
+    }
+
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/utils/PdfVisualisationUtility.java

@@ -33,7 +33,7 @@ public class PdfVisualisationUtility {
 
     public void drawDocumentGraph(PDDocument document, Document documentGraph) {
 
-        documentGraph.getDocumentTree().streamAllEntriesInOrder().forEach(entry -> drawNode(document, entry));
+        documentGraph.getDocumentTree().allEntriesInOrder().forEach(entry -> drawNode(document, entry));
     }
 
 
@@ -129,7 +129,7 @@ public class PdfVisualisationUtility {
         rectanglesPerPage.forEach((page, rectangle2D) -> {
             Rectangle2D paddedRectangle2D = rectangle2D;
             if (entry.getType() == NodeType.SECTION) {
-                paddedRectangle2D = RectangleTransformations.pad(rectangle2D, 10, 10);
+                paddedRectangle2D = pad(rectangle2D, 10, 10);
             }
             drawRectangle2DList(document, page.getNumber(), List.of(paddedRectangle2D), options);
             drawText(buildString(entry), document, new Point2D.Double(paddedRectangle2D.getMinX(), paddedRectangle2D.getMaxY() + 2), page.getNumber(), options);
@@ -137,6 +137,12 @@ public class PdfVisualisationUtility {
     }
 
 
+    private static Rectangle2D pad(Rectangle2D rectangle2D, int deltaX, int deltaY) {
+
+        return new Rectangle2D.Double(rectangle2D.getMinX() - deltaX, rectangle2D.getMinY() - deltaY, rectangle2D.getWidth() + 2 * deltaX, rectangle2D.getHeight() + 2 * deltaY);
+    }
+
+
     private String buildString(DocumentTree.Entry entry) {
 
         return entry.getNode().getNumberOnPage() + ": " + entry.getTreeId() + ": " + entry.getType().toString();

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/utils/RectangleMapper.java

@@ -1,59 +0,0 @@
-package com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils;
-
-import static java.lang.String.format;
-
-import java.awt.geom.Rectangle2D;
-import java.util.Arrays;
-import java.util.List;
-
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Point;
-import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
-
-public class RectangleMapper {
-    //TODO: Move functions into using services
-
-
-    public static Rectangle2D toRectangle2D(Rectangle redactionLogRectangle) {
-
-        return new Rectangle2D.Double(redactionLogRectangle.getTopLeft().getX(),
-                redactionLogRectangle.getTopLeft().getY(),
-                redactionLogRectangle.getWidth(),
-                redactionLogRectangle.getHeight());
-    }
-
-
-    public static Rectangle toRedactionLogRectangle(Rectangle2D rectangle2D, int pageNumber) {
-
-        return new Rectangle(new Point((float) rectangle2D.getMinX(), (float) rectangle2D.getMinY()), (float) rectangle2D.getWidth(), (float) rectangle2D.getHeight(), pageNumber);
-    }
-
-
-    public static com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle toAnnotationRectangle(Rectangle2D rectangle2D, int pageNumber) {
-
-        return new com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle((float) rectangle2D.getMaxX(),
-                (float) rectangle2D.getMaxY(),
-                (float) rectangle2D.getWidth(),
-                (float) rectangle2D.getHeight(),
-                pageNumber);
-    }
-
-
-    public static String toString(Rectangle2D rectangle2D) {
-
-        return format("%f,%f,%f,%f", rectangle2D.getX(), rectangle2D.getY(), rectangle2D.getWidth(), rectangle2D.getHeight());
-    }
-
-
-    public static Rectangle2D parseRectangle2D(String bBox) {
-
-        List<Float> floats = Arrays.stream(bBox.split(",")).map(Float::parseFloat).toList();
-        return new Rectangle2D.Float(floats.get(0), floats.get(1), floats.get(2), floats.get(3));
-    }
-
-
-    public static Rectangle2D toRectangle2D(com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle rect) {
-
-        return new Rectangle2D.Double(rect.getTopLeftX() - rect.getWidth(), rect.getTopLeftY() - rect.getHeight(), rect.getWidth(), rect.getHeight());
-    }
-
-}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/utils/RectangleTransformations.java

@@ -15,30 +15,12 @@ import java.util.stream.Collector;
 
 import org.apache.pdfbox.pdmodel.common.PDRectangle;
 
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Point;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
-import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.AbstractPageBlock;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.AtomicTextBlock;
 
 public class RectangleTransformations {
 
-    public static Rectangle2D pad(Rectangle2D rectangle2D, int deltaX, int deltaY) {
-
-        return new Rectangle2D.Double(rectangle2D.getMinX() - deltaX, rectangle2D.getMinY() - deltaY, rectangle2D.getWidth() + 2 * deltaX, rectangle2D.getHeight() + 2 * deltaY);
-    }
-
-
-    public static Rectangle2D bBoxUnionAbstractTextContainer(List<AbstractPageBlock> abstractPageBlocks) {
-
-        return abstractPageBlocks.stream().map(RectangleTransformations::toRectangle2D).collect(new Rectangle2DUnion());
-    }
-
-
-    public static Rectangle2D toRectangle2D(AbstractPageBlock abstractPageBlock) {
-
-        return new Rectangle2D.Float(abstractPageBlock.getMinX(), abstractPageBlock.getMinY(), abstractPageBlock.getWidth(), abstractPageBlock.getHeight());
-    }
-
-
     public static PDRectangle toPDRectangleUnion(List<Rectangle> rectangles) {
 
         Rectangle2D rectangle2D = RectangleTransformations.bBoxUnionRectangle(rectangles);
@@ -60,7 +42,25 @@ public class RectangleTransformations {
 
     public static Rectangle2D bBoxUnionRectangle(List<Rectangle> rectangles) {
 
-        return rectangles.stream().map(RectangleMapper::toRectangle2D).collect(new Rectangle2DUnion());
+        return rectangles.stream().map(RectangleTransformations::toRectangle2D).collect(new Rectangle2DUnion());
+    }
+
+
+    public static Rectangle2D toRectangle2D(Rectangle redactionLogRectangle) {
+
+        return new Rectangle2D.Double(redactionLogRectangle.getTopLeft().getX(),
+                redactionLogRectangle.getTopLeft().getY() + redactionLogRectangle.getHeight(),
+                redactionLogRectangle.getWidth(),
+                -redactionLogRectangle.getHeight());
+    }
+
+
+    public static Rectangle toRedactionLogRectangle(Rectangle2D rectangle2D, int pageNumber) {
+
+        return new Rectangle(new Point((float) rectangle2D.getMinX(), (float) (rectangle2D.getMinY() + rectangle2D.getHeight())),
+                (float) rectangle2D.getWidth(),
+                -(float) rectangle2D.getHeight(),
+                pageNumber);
     }
 
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/utils/RedactionSearchUtility.java

@@ -1,18 +1,20 @@
 package com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils;
 
+import static java.lang.String.format;
+
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Locale;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.Boundary;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionEntity;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.TextBlock;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.Patterns;
+
 import lombok.experimental.UtilityClass;
 
-import java.util.LinkedList;
-import java.util.List;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-import static java.lang.String.format;
-
 @UtilityClass
 public class RedactionSearchUtility {
 
@@ -115,4 +117,11 @@ public class RedactionSearchUtility {
         return boundaries;
     }
 
+
+    public static List<Boundary> findBoundariesByStringIgnoreCase(String searchString, TextBlock textBlock) {
+
+        String searchStringLowerCase = searchString.toLowerCase(Locale.ROOT);
+        return findBoundariesByString(searchStringLowerCase, textBlock);
+    }
+
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/utils/TextPositionOperations.java

@@ -3,12 +3,12 @@ package com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils;
 import java.util.Comparator;
 import java.util.List;
 
-import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.ClassificationTextBlock;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.TextPageBlock;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.TextPositionSequence;
 
 public class TextPositionOperations {
 
-    public static List<TextPositionSequence> mergeAndSortTextPositionSequenceByYThenX(List<ClassificationTextBlock> textBlocks) {
+    public static List<TextPositionSequence> mergeAndSortTextPositionSequenceByYThenX(List<TextPageBlock> textBlocks) {
 
         return textBlocks.stream()//
                 .flatMap(tb -> tb.getSequences().stream())//

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/adapter/NerEntitiesAdapter.java

@@ -152,8 +152,8 @@ public class NerEntitiesAdapter {
 
     private static Stream<EntityRecognitionEntity> addOffsetsAndFlatten(List<Integer> stringOffsetsForMainSections, NerEntitiesModel nerEntitiesModel) {
 
-        nerEntitiesModel.getData().forEach((key, value) -> value.forEach(entityRecognitionEntity -> {
+        nerEntitiesModel.getData().forEach((sectionNumber, listOfNerEntities) -> listOfNerEntities.forEach(entityRecognitionEntity -> {
-            int newStartOffset = entityRecognitionEntity.getStartOffset() + stringOffsetsForMainSections.get(key);
+            int newStartOffset = entityRecognitionEntity.getStartOffset() + stringOffsetsForMainSections.get(sectionNumber);
             entityRecognitionEntity.setStartOffset(newStartOffset);
             entityRecognitionEntity.setEndOffset(newStartOffset + entityRecognitionEntity.getValue().length());
         }));
@@ -163,7 +163,7 @@ public class NerEntitiesAdapter {
 
     private static List<Integer> getStringStartOffsetsForMainSections(Document document) {
 
-        return document.getMainSections().stream().map(Section::buildTextBlock).map(TextBlock::getBoundary).map(Boundary::start).toList();
+        return document.getMainSections().stream().map(Section::getTextBlock).map(TextBlock::getBoundary).map(Boundary::start).toList();
     }
 
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/dictionary/Dictionary.java

@@ -90,6 +90,9 @@ public class Dictionary {
 
     public void addLocalDictionaryEntry(String type, String value, boolean alsoAddLastname) {
 
+        if (value.isBlank()) {
+            return;
+        }
         if (localAccessMap.get(type) == null) {
             throw new IllegalArgumentException(format("DictionaryModel of type %s does not exist", type));
         }
@@ -99,7 +102,7 @@ public class Dictionary {
         if (StringUtils.isEmpty(value)) {
             throw new IllegalArgumentException(format("%s is not a valid dictionary entry", value));
         }
-        localAccessMap.get(type).getLocalEntries().add(value);
+        localAccessMap.get(type).getLocalEntries().add(value.trim());
         if (alsoAddLastname) {
             String lastname = value.split(" ")[0];
             localAccessMap.get(type).getLocalEntries().add(lastname);

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/AnalyzeService.java

@@ -345,7 +345,7 @@ public class AnalyzeService {
 
     private SimplifiedSectionText toSimplifiedSectionText(Section section) {
 
-        return SimplifiedSectionText.builder().sectionNumber(section.getTreeId().get(0)).text(section.buildTextBlock().getSearchText()).build();
+        return SimplifiedSectionText.builder().sectionNumber(section.getTreeId().get(0)).text(section.getTextBlock().getSearchText()).build();
     }
 
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ManualRedactionSurroundingTextService.java

@@ -22,7 +22,6 @@ import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.en
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.Document;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.SemanticNode;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.services.EntityCreationService;
-import com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RectangleMapper;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RedactionSearchUtility;
 import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
 
@@ -43,10 +42,10 @@ public class ManualRedactionSurroundingTextService {
     public AnalyzeResult addSurroundingText(String dossierId, String fileId, ManualRedactions manualRedactions) {
 
         long startTime = System.currentTimeMillis();
-        Document text = DocumentGraphMapper.toDocumentGraph(redactionStorageService.getDocumentData(dossierId, fileId));
+        Document document = DocumentGraphMapper.toDocumentGraph(redactionStorageService.getDocumentData(dossierId, fileId));
         List<ManualRedactionEntry> processedAddRedactions = new ArrayList<>();
 
-        for (SemanticNode node : text.streamChildren().toList()) {
+        for (SemanticNode node : document.streamChildren().toList()) {
 
             if (manualRedactions.getEntriesToAdd().isEmpty()) {
                 break;
@@ -74,7 +73,7 @@ public class ManualRedactionSurroundingTextService {
 
     private Pair<String, String> findSurroundingText(SemanticNode node, String value, List<Rectangle> toFindPositions) {
 
-        Set<RedactionEntity> entities = RedactionSearchUtility.findBoundariesByString(value, node.buildTextBlock())
+        Set<RedactionEntity> entities = RedactionSearchUtility.findBoundariesByString(value, node.getTextBlock())
                 .stream()
                 .map(boundary -> entityCreationService.byBoundary(boundary, "searchHelper", EntityType.RECOMMENDATION, node))
                 .collect(Collectors.toSet());
@@ -87,7 +86,7 @@ public class ManualRedactionSurroundingTextService {
     private boolean sectionContainsEntry(SemanticNode semanticNode, List<Rectangle> positions) {
 
         for (Rectangle position : positions) {
-            if (semanticNode.containsRectangle(RectangleMapper.toRectangle2D(position), position.getPage())) {
+            if (semanticNode.containsRectangle(ManualRedactionSurroundingTextService.toRectangle2D(position), position.getPage())) {
                 return true;
             }
         }
@@ -110,7 +109,13 @@ public class ManualRedactionSurroundingTextService {
 
     private static boolean toFindPositionsIntersectRectangle(List<Rectangle> toFindPositions, Rectangle2D rectangle2D) {
 
-        return toFindPositions.stream().map(RectangleMapper::toRectangle2D).anyMatch(toFindRectangle -> toFindRectangle.intersects(rectangle2D));
+        return toFindPositions.stream().map(ManualRedactionSurroundingTextService::toRectangle2D).anyMatch(toFindRectangle -> toFindRectangle.intersects(rectangle2D));
+    }
+
+
+    private static Rectangle2D toRectangle2D(Rectangle rect) {
+
+        return new Rectangle2D.Double(rect.getTopLeftX() - rect.getWidth(), rect.getTopLeftY() - rect.getHeight(), rect.getWidth(), rect.getHeight());
     }
 
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java

@@ -1,7 +1,5 @@
 package com.iqser.red.service.redaction.v1.server.redaction.service;
 
-import static com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RectangleMapper.toRedactionLogRectangle;
-
 import java.util.ArrayList;
 import java.util.HashSet;
 import java.util.List;
@@ -17,6 +15,7 @@ import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.en
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.Document;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.Image;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.ImageType;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RectangleTransformations;
 
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
@@ -67,7 +66,7 @@ public class RedactionLogCreatorService {
 
             List<Rectangle> rectanglesPerLine = redactionPosition.getRectanglePerLine()
                     .stream()
-                    .map(rectangle2D -> toRedactionLogRectangle(rectangle2D, redactionPosition.getPage().getNumber()))
+                    .map(rectangle2D -> RectangleTransformations.toRedactionLogRectangle(rectangle2D, redactionPosition.getPage().getNumber()))
                     .toList();
 
             redactionLogEntry.setPositions(rectanglesPerLine);
@@ -85,7 +84,7 @@ public class RedactionLogCreatorService {
                 .stream()
                 .filter(redactionEntity -> !redactionEntity.isRemoved() && !redactionEntity.isIgnored())
                 .forEach(ref -> ref.getRedactionPositionsPerPage().forEach(pos -> referenceIds.add(pos.getId())));
-        int sectionNumber = entity.getDeepestFullyContainingNode().getTreeId().isEmpty() ? -1 : entity.getDeepestFullyContainingNode().getTreeId().get(0);
+        int sectionNumber = entity.getDeepestFullyContainingNode().getTreeId().isEmpty() ? 0 : entity.getDeepestFullyContainingNode().getTreeId().get(0);
 
         return RedactionLogEntry.builder()
                 .color(getColor(entity.getType(), dossierTemplateId, entity.isRedaction()))
@@ -127,9 +126,9 @@ public class RedactionLogCreatorService {
                 .isHint(dictionaryService.isHint(image.getImageType().toString(), dossierTemplateId))
                 .isDictionaryEntry(false)
                 .isRecommendation(false)
-                .positions(List.of(toRedactionLogRectangle(image.getPosition(), image.getPage().getNumber())))
+                .positions(List.of(RectangleTransformations.toRedactionLogRectangle(image.getPosition(), image.getPage().getNumber())))
                 .sectionNumber(image.getTreeId().get(0))
-                .section(image.getParent().buildTextBlock().getSearchText())
+                .section(image.getParent().getTextBlock().getSearchText())
                 .imageHasTransparency(image.isTransparent())
                 .build();
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/SectionFinderService.java

@@ -50,7 +50,7 @@ class SectionFinderService {
                 true);
 
         document.streamChildren().forEach(mainNode -> {
-            if (dictionaryIncrementsSearch.atLeastOneMatches(mainNode.buildTextBlock().getSearchText())) {
+            if (dictionaryIncrementsSearch.atLeastOneMatches(mainNode.getTextBlock().getSearchText())) {
                 sectionsToReanalyse.add(mainNode.getTreeId().get(0));
             }
         });

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/SectionGridCreatorService.java

@@ -13,7 +13,7 @@ import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.mo
 import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.ClassificationSection;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.table.Cell;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.table.TablePageBlock;
-import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.ClassificationTextBlock;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.TextPageBlock;
 
 import lombok.RequiredArgsConstructor;
 
@@ -41,9 +41,9 @@ public class SectionGridCreatorService {
                     continue;
                 }
 
-                if (textBlock instanceof ClassificationTextBlock) {
+                if (textBlock instanceof TextPageBlock) {
 
-                    ClassificationTextBlock tb = (ClassificationTextBlock) textBlock;
+                    TextPageBlock tb = (TextPageBlock) textBlock;
                     classifiedDoc.getSectionGrid()
                             .getRectanglesPerPage()
                             .computeIfAbsent(page, (x) -> new ArrayList<>())

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/IdBuilder.java

@@ -2,8 +2,10 @@ package com.iqser.red.service.redaction.v1.server.redaction.utils;
 
 import java.awt.geom.Rectangle2D;
 import java.nio.charset.StandardCharsets;
+import java.util.Comparator;
 import java.util.List;
 import java.util.Set;
+import java.util.stream.Collectors;
 
 import com.google.common.hash.HashFunction;
 import com.google.common.hash.Hashing;
@@ -19,9 +21,19 @@ public final class IdBuilder {
 
     public String buildId(Set<Page> pages, List<Rectangle2D> rectanglesPerLine) {
 
+        return buildId(pages.stream().map(Page::getNumber).collect(Collectors.toList()), rectanglesPerLine);
+    }
+
+
+    public String buildId(List<Integer> pageNumbers, List<Rectangle2D> rectanglesPerLine) {
+
         StringBuilder sb = new StringBuilder();
-        pages.forEach(pageNode -> sb.append(pageNode.getNumber()));
+        List<Integer> sortedPageNumbers = pageNumbers.stream().sorted(Comparator.comparingInt(Integer::intValue)).toList();
-        rectanglesPerLine.forEach(rectangle2D -> sb.append(rectangle2D.getX()).append(rectangle2D.getY()).append(rectangle2D.getWidth()).append(rectangle2D.getHeight()));
+        sortedPageNumbers.forEach(sb::append);
+        rectanglesPerLine.forEach(rectangle2D -> sb.append(Math.round(rectangle2D.getX()))
+                .append(Math.round(rectangle2D.getY()))
+                .append(Math.round(rectangle2D.getWidth()))
+                .append(Math.round(rectangle2D.getHeight())));
 
         return hashFunction.hashString(sb.toString(), StandardCharsets.UTF_8).toString();
     }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/SeparatorUtils.java

@@ -47,7 +47,7 @@ public final class SeparatorUtils {
 
     public static boolean boundaryIsSurroundedBySeparators(TextBlock textBlock, Boundary boundary) {
 
-        return validateStart(textBlock, boundary) && validateEnd(textBlock, boundary);
+        return validateStart(textBlock, boundary) && validateEnd(textBlock, boundary) && !isWhiteSpacesOrSeparatorsOnly(textBlock, boundary);
     }
 
 
@@ -55,7 +55,7 @@ public final class SeparatorUtils {
 
         return boundary.end() == textBlock.getBoundary().end() ||//
                 SeparatorUtils.isSeparator(textBlock.charAt(boundary.end())) ||//
-                SeparatorUtils.isSeparator(textBlock.charAt(boundary.end() - 1));
+                SeparatorUtils.isJapaneseSeparator(textBlock.charAt(boundary.end() - 1));
     }
 
 
@@ -63,7 +63,7 @@ public final class SeparatorUtils {
 
         return boundary.start() == textBlock.getBoundary().start() ||//
                 SeparatorUtils.isSeparator(textBlock.charAt(boundary.start() - 1)) ||//
-                SeparatorUtils.isSeparator(textBlock.charAt(boundary.start()));
+                SeparatorUtils.isJapaneseSeparator(textBlock.charAt(boundary.start()));
     }
 
 }

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/HeadlinesGoldStandardIntegrationTest.java

@@ -104,7 +104,7 @@ public class HeadlinesGoldStandardIntegrationTest {
         var foundHeadlines = documentGraph.streamAllSubNodes()
                 .map(SemanticNode::getHeadline)
                 .distinct()
-                .map(headlineNode -> new Headline(headlineNode.getPages().stream().findFirst().get().getNumber(), headlineNode.buildTextBlock().getSearchText().stripTrailing()))
+                .map(headlineNode -> new Headline(headlineNode.getPages().stream().findFirst().get().getNumber(), headlineNode.getTextBlock().getSearchText().stripTrailing()))
                 .toList();
 
         Set<Headline> correct = new HashSet<>();

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java

@@ -5,6 +5,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.mockito.Mockito.when;
 import static org.wildfly.common.Assert.assertTrue;
 
+import java.awt.geom.Rectangle2D;
 import java.io.File;
 import java.io.FileOutputStream;
 import java.io.IOException;
@@ -16,6 +17,7 @@ import java.nio.file.Paths;
 import java.time.OffsetDateTime;
 import java.time.ZoneOffset;
 import java.util.ArrayList;
+import java.util.Comparator;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -58,6 +60,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSON
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.configuration.Colors;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.ChangeType;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
 import com.iqser.red.service.redaction.v1.model.StructureAnalyzeRequest;
@@ -70,7 +73,6 @@ import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.en
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.Document;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.Section;
 import com.iqser.red.service.redaction.v1.server.layoutparsing.document.services.EntityCreationService;
-import com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RectangleMapper;
 import com.iqser.red.service.redaction.v1.server.multitenancy.TenantContext;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.OsUtils;
 import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
@@ -224,7 +226,6 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
         System.out.println("Finished structure analysis");
         AnalyzeResult result = analyzeService.analyze(request);
         System.out.println("Finished analysis");
-
         var redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID);
 
         AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder().dossierId(TEST_DOSSIER_ID).fileId(TEST_FILE_ID).build());
@@ -379,7 +380,7 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
                     continue loop;
                 }
                 if (redactionLogEntry.getSectionNumber() == section.getTreeId().get(0)) {
-                    String value = section.buildTextBlock().subSequence(new Boundary(redactionLogEntry.getStartOffset(), redactionLogEntry.getEndOffset())).toString();
+                    String value = section.getTextBlock().subSequence(new Boundary(redactionLogEntry.getStartOffset(), redactionLogEntry.getEndOffset())).toString();
                     if (redactionLogEntry.getValue().equalsIgnoreCase(value)) {
                         correctFound++;
                     } else {
@@ -528,7 +529,7 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
                 .stream()
                 .filter(e -> !e.isImage())
                 .map(redactionLogEntry -> new Boundary(redactionLogEntry.getStartOffset(), redactionLogEntry.getEndOffset()))
-                .map(boundary -> documentGraph.buildTextBlock().subSequence(boundary).toString())
+                .map(boundary -> documentGraph.getTextBlock().subSequence(boundary).toString())
                 .toList();
         List<String> valuesInRedactionLog = redactionLog.getRedactionLogEntry().stream().filter(e -> !e.isImage()).map(RedactionLogEntry::getValue).toList();
 
@@ -690,20 +691,27 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
         analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
         AnalyzeResult result = analyzeService.analyze(request);
 
+        String testEntityValue1 = "Desiree";
+        String testEntityValue2 = "Melanie";
         RedactionLog redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID);
-        assertTrue(redactionLog.getRedactionLogEntry().stream().anyMatch(entry -> entry.getValue().equals("Desiree et al")));
+        assertEquals(2, redactionLog.getRedactionLogEntry().stream().filter(entry -> entry.getValue().equals(testEntityValue1)).count());
-        assertTrue(redactionLog.getRedactionLogEntry().stream().anyMatch(entry -> entry.getValue().equals("Melanie et al.")));
+        assertEquals(2, redactionLog.getRedactionLogEntry().stream().filter(entry -> entry.getValue().equals(testEntityValue2)).count());
 
         Document document = DocumentGraphMapper.toDocumentGraph(redactionStorageService.getDocumentData(TEST_DOSSIER_ID, TEST_FILE_ID));
         String expandedEntityKeyword = "Lorem ipsum dolor sit amet, consectetur adipiscing elit Desiree et al sed do eiusmod tempor incididunt ut labore et dolore magna aliqua Melanie et al. Reference No 12345 Lorem ipsum.";
         RedactionEntity expandedEntity = entityCreationService.byString(expandedEntityKeyword, "PII", EntityType.ENTITY, document).findFirst().get();
 
-        String idToResize = redactionLog.getRedactionLogEntry().stream().filter(entry -> entry.getValue().equals("Desiree et al")).findFirst().get().getId();
+        String idToResize = redactionLog.getRedactionLogEntry()
+                .stream()
+                .filter(entry -> entry.getValue().equals(testEntityValue1))
+                .max(Comparator.comparingInt(RedactionLogEntry::getStartOffset))
+                .get()
+                .getId();
         List<Rectangle> resizedPositions = expandedEntity.getRedactionPositionsPerPage()
                 .get(0)
                 .getRectanglePerLine()
                 .stream()
-                .map(rectangle2D -> RectangleMapper.toAnnotationRectangle(rectangle2D, 3))
+                .map(rectangle2D -> toAnnotationRectangle(rectangle2D, 3))
                 .toList();
         ManualResizeRedaction manualResizeRedaction = ManualResizeRedaction.builder()
                 .annotationId(idToResize)
@@ -724,9 +732,21 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
         try (FileOutputStream fileOutputStream = new FileOutputStream(tmpFile)) {
             fileOutputStream.write(annotateResponse.getDocument());
         }
-        assertTrue(redactionLog.getRedactionLogEntry().stream().noneMatch(entry -> entry.getValue().equals("Desiree et al")));
+        RedactionLogEntry resizedEntry = redactionLog.getRedactionLogEntry().stream().filter(entry -> entry.getValue().equals(expandedEntityKeyword)).findFirst().get();
-        assertTrue(redactionLog.getRedactionLogEntry().stream().noneMatch(entry -> entry.getValue().equals("Melanie et al.") && !entry.lastChangeIsRemoved()));
+        assertTrue(resizedEntry.getChanges().get(resizedEntry.getChanges().size() - 1).getType().equals(ChangeType.CHANGED));
-        assertTrue(redactionLog.getRedactionLogEntry().stream().anyMatch(entry -> entry.getValue().equals(expandedEntityKeyword)));
+        assertEquals(idToResize, resizedEntry.getId());
+        assertEquals(1, redactionLog.getRedactionLogEntry().stream().filter(entry -> entry.getValue().equals(testEntityValue1)).count());
+        assertEquals(1, redactionLog.getRedactionLogEntry().stream().filter(entry -> entry.getValue().equals(testEntityValue2) && !entry.lastChangeIsRemoved()).count());
+    }
+
+
+    private static com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle toAnnotationRectangle(Rectangle2D rectangle2D, int pageNumber) {
+
+        return new com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle((float) rectangle2D.getMaxX(),
+                (float) rectangle2D.getMaxY() - (float) rectangle2D.getHeight(),
+                (float) rectangle2D.getWidth(),
+                -(float) rectangle2D.getHeight(),
+                pageNumber);
     }
 
 

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationV2Test.java

@@ -3,7 +3,6 @@ package com.iqser.red.service.redaction.v1.server;
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.mockito.Mockito.when;
 
-import java.util.Arrays;
 import java.util.List;
 
 import org.junit.jupiter.api.BeforeEach;
@@ -35,7 +34,6 @@ import lombok.SneakyThrows;
 @ExtendWith(SpringExtension.class)
 @SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
 @Import(RedactionIntegrationV2Test.RedactionIntegrationTestConfiguration.class)
-
 public class RedactionIntegrationV2Test extends AbstractRedactionIntegrationTest {
 
     private static final String RULES = loadFromClassPath("drools/rules_v2.drl");
@@ -150,7 +148,7 @@ public class RedactionIntegrationV2Test extends AbstractRedactionIntegrationTest
 
         String entryAuthorDictionary = "Evans P.G.";
         dictionary.put(DICTIONARY_AUTHOR, List.of(entryAuthorDictionary));
-        falsePositive.put(DICTIONARY_PII, Arrays.asList("Dr. Alan Miller COMPLETION DATE:"));
+        falsePositive.put(DICTIONARY_PII, List.of("Dr. Alan Miller COMPLETION DATE:"));
 
         analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
         analyzeService.analyze(request);

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/annotate/AnnotationService.java

@@ -37,7 +37,6 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlo
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.SectionGrid;
 import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.SectionRectangle;
 import com.iqser.red.service.redaction.v1.server.exception.RedactionException;
-import com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RectangleMapper;
 import com.iqser.red.service.redaction.v1.server.redaction.service.DictionaryService;
 import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
 
@@ -178,7 +177,16 @@ public class AnnotationService {
 
     public static List<Double> toQuadPoints(List<Rectangle> rectangles) {
 
-        return rectangles.stream().map(RectangleMapper::toRectangle2D).flatMap(AnnotationService::toQuadPoints).toList();
+        return rectangles.stream().map(AnnotationService::toRectangle2D).flatMap(AnnotationService::toQuadPoints).toList();
+    }
+
+
+    private static Rectangle2D toRectangle2D(Rectangle redactionLogRectangle) {
+
+        return new Rectangle2D.Double(redactionLogRectangle.getTopLeft().getX(),
+                redactionLogRectangle.getTopLeft().getY(),
+                redactionLogRectangle.getWidth(),
+                redactionLogRectangle.getHeight());
     }
 
 

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentEntityInsertionIntegrationTest.java

@@ -67,7 +67,7 @@ public class DocumentEntityInsertionIntegrationTest extends BuildDocumentIntegra
         assertEquals("’s Donut ←", redactionEntity.getTextAfter());
         assertEquals(searchTerm, redactionEntity.getValue());
         assertEquals("Rule 5: Do not redact genitive CBI_authors (Entries based on Dict) ",
-                redactionEntity.getDeepestFullyContainingNode().getHeadline().buildTextBlock().getSearchText());
+                redactionEntity.getDeepestFullyContainingNode().getHeadline().getTextBlock().getSearchText());
         assertEquals(3, redactionEntity.getIntersectingNodes().size());
         assertEquals(5, redactionEntity.getDeepestFullyContainingNode().getNumberOnPage());
         assertInstanceOf(Paragraph.class, redactionEntity.getDeepestFullyContainingNode());
@@ -86,7 +86,7 @@ public class DocumentEntityInsertionIntegrationTest extends BuildDocumentIntegra
         assertEquals("", redactionEntity.getTextBefore());
         assertEquals(" Purity Hint", redactionEntity.getTextAfter());
         assertEquals(searchTerm, redactionEntity.getValue());
-        assertEquals("Rule 39: Purity Hint ", redactionEntity.getDeepestFullyContainingNode().getHeadline().buildTextBlock().getSearchText());
+        assertEquals("Rule 39: Purity Hint ", redactionEntity.getDeepestFullyContainingNode().getHeadline().getTextBlock().getSearchText());
         assertEquals(3, redactionEntity.getIntersectingNodes().size());
         assertEquals(6, redactionEntity.getDeepestFullyContainingNode().getNumberOnPage());
         assertInstanceOf(Headline.class, redactionEntity.getDeepestFullyContainingNode());
@@ -105,7 +105,7 @@ public class DocumentEntityInsertionIntegrationTest extends BuildDocumentIntegra
         assertEquals("", redactionEntity.getTextBefore());
         assertEquals("", redactionEntity.getTextAfter());
         assertEquals(searchTerm, redactionEntity.getValue());
-        assertEquals("Rule 6-11 (Authors Table) ", redactionEntity.getDeepestFullyContainingNode().getHeadline().buildTextBlock().getSearchText());
+        assertEquals("Rule 6-11 (Authors Table) ", redactionEntity.getDeepestFullyContainingNode().getHeadline().getTextBlock().getSearchText());
         assertEquals(5, redactionEntity.getIntersectingNodes().size());
         assertEquals(15, redactionEntity.getDeepestFullyContainingNode().getNumberOnPage());
         assertInstanceOf(TableCell.class, redactionEntity.getDeepestFullyContainingNode());
@@ -131,14 +131,14 @@ public class DocumentEntityInsertionIntegrationTest extends BuildDocumentIntegra
 
         Document document = buildGraph("files/new/crafted document");
         Table table = (Table) document.getDocumentTree()//
-                .streamAllEntriesInOrder()//
+                .allEntriesInOrder()//
                 .filter(entry -> entry.getType().equals(NodeType.TABLE))//
                 .map(DocumentTree.Entry::getNode)//
                 .findFirst().orElseThrow();
         assertEquals(5, table.getNumberOfCols());
         assertEquals(4, table.getNumberOfRows());
         assertEquals(5, table.streamHeaders().toList().size());
-        CharSequence firstHeader = table.streamHeadersForCell(1, 1).map(TableCell::buildTextBlock).map(TextBlock::getSearchText).findFirst().orElseThrow();
+        CharSequence firstHeader = table.streamHeadersForCell(1, 1).map(TableCell::getTextBlock).map(TextBlock::getSearchText).findFirst().orElseThrow();
         assertEquals("Author(s)", firstHeader.toString().stripTrailing());
     }
 
@@ -161,7 +161,7 @@ public class DocumentEntityInsertionIntegrationTest extends BuildDocumentIntegra
 
         Document document = buildGraph("files/Metolachlor/S-Metolachlor_RAR_01_Volume_1_2018-09-06");
         Table table = (Table) document.getDocumentTree()
-                .streamAllEntriesInOrder()
+                .allEntriesInOrder()
                 .filter(entry -> entry.getNode().getPages().stream().anyMatch(page -> page.getNumber() == 22))
                 .filter(entry -> entry.getType().equals(NodeType.TABLE))
                 .map(DocumentTree.Entry::getNode)
@@ -170,7 +170,7 @@ public class DocumentEntityInsertionIntegrationTest extends BuildDocumentIntegra
         assertEquals(5, table.getNumberOfCols());
         assertEquals(14, table.getNumberOfRows());
         assertEquals(10, table.streamHeaders().toList().size());
-        List<String> twoHeaders = table.streamHeadersForCell(2, 1).map(TableCell::buildTextBlock).map(TextBlock::getSearchText).toList();
+        List<String> twoHeaders = table.streamHeadersForCell(2, 1).map(TableCell::getTextBlock).map(TextBlock::getSearchText).toList();
         assertEquals(2, twoHeaders.size());
         assertEquals("Component of residue definition: S-Metolachlor", twoHeaders.get(0).stripTrailing());
         assertEquals("Method type", twoHeaders.get(1).stripTrailing());
@@ -187,7 +187,7 @@ public class DocumentEntityInsertionIntegrationTest extends BuildDocumentIntegra
         assertEquals("except Cranberry; Vegetable, ", redactionEntity.getTextBefore());
         assertEquals(", Group 9;", redactionEntity.getTextAfter());
         assertEquals("1.1.4 Evaluations carried out under other regulatory contexts ",
-                redactionEntity.getDeepestFullyContainingNode().getHeadline().buildTextBlock().getSearchText());
+                redactionEntity.getDeepestFullyContainingNode().getHeadline().getTextBlock().getSearchText());
         assertEquals(searchTerm, redactionEntity.getValue());
         assertEquals(3, redactionEntity.getIntersectingNodes().size());
         assertEquals(5, redactionEntity.getDeepestFullyContainingNode().getNumberOnPage());
@@ -215,7 +215,7 @@ public class DocumentEntityInsertionIntegrationTest extends BuildDocumentIntegra
         assertEquals("2.6.1 Summary of ", redactionEntity.getTextBefore());
         assertEquals(" and excretion in", redactionEntity.getTextAfter());
         assertEquals("2.6.1 Summary of absorption, distribution, metabolism and excretion in mammals ",
-                redactionEntity.getDeepestFullyContainingNode().getHeadline().buildTextBlock().getSearchText());
+                redactionEntity.getDeepestFullyContainingNode().getHeadline().getTextBlock().getSearchText());
         assertEquals(searchTerm, redactionEntity.getValue());
         assertEquals(3, redactionEntity.getIntersectingNodes().size());
         assertEquals(4, redactionEntity.getDeepestFullyContainingNode().getNumberOnPage());
@@ -238,7 +238,7 @@ public class DocumentEntityInsertionIntegrationTest extends BuildDocumentIntegra
         assertEquals(searchTerm, redactionEntity.getValue());
         assertEquals(4, redactionEntity.getIntersectingNodes().size());
         assertEquals("Table 2.7-1: List of substances and metabolites and related structural formula ",
-                redactionEntity.getDeepestFullyContainingNode().getHeadline().buildTextBlock().getSearchText());
+                redactionEntity.getDeepestFullyContainingNode().getHeadline().getTextBlock().getSearchText());
         assertTrue(redactionEntity.getPages().stream().allMatch(pageNode -> pageNode.getNumber() == 54));
         assertEquals(26, redactionEntity.getDeepestFullyContainingNode().getNumberOnPage());
 
@@ -252,7 +252,7 @@ public class DocumentEntityInsertionIntegrationTest extends BuildDocumentIntegra
     private static void assertSameOffsetInAllIntersectingNodes(String searchTerm, RedactionEntity redactionEntity) {
 
         List<Integer> paragraphStart = redactionEntity.getIntersectingNodes().stream()//
-                .map(SemanticNode::buildTextBlock)//
+                .map(SemanticNode::getTextBlock)//
                 .map(textBlock -> textBlock.indexOf(searchTerm, redactionEntity.getDeepestFullyContainingNode().getBoundary().start()))//
                 .toList();
 

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentVisualizationIntegrationTest.java

@@ -22,10 +22,10 @@ public class DocumentVisualizationIntegrationTest extends BuildDocumentIntegrati
     @Disabled
     public void visualizeMetolachlor() {
 
-        String filename = "files/Metolachlor/S-Metolachlor_RAR_01_Volume_1_2018-09-06";
+        String filename = "files/Trinexapac/93 Trinexapac-ethyl_RAR_03_Volume_3CA_B-1_2017-03-31";
 
         Document document = buildGraph(filename);
-        TextBlock textBlock = document.buildTextBlock();
+        TextBlock textBlock = document.getTextBlock();
 
         visualizeSemanticNodes(filename, document, textBlock);
     }
@@ -39,7 +39,7 @@ public class DocumentVisualizationIntegrationTest extends BuildDocumentIntegrati
         String filename = "files/Fludioxonil/56 Fludioxonil_RAR_12_Volume_3CA_B-7_2018-02-21";
 
         Document document = buildGraph(filename);
-        TextBlock textBlock = document.buildTextBlock();
+        TextBlock textBlock = document.getTextBlock();
 
         visualizeSemanticNodes(filename, document, textBlock);
     }
@@ -53,7 +53,7 @@ public class DocumentVisualizationIntegrationTest extends BuildDocumentIntegrati
         String filename = "files/new/test1S1T1";
 
         Document document = buildGraph(filename);
-        TextBlock textBlock = document.buildTextBlock();
+        TextBlock textBlock = document.getTextBlock();
 
         visualizeSemanticNodes(filename, document, textBlock);
     }
@@ -67,7 +67,7 @@ public class DocumentVisualizationIntegrationTest extends BuildDocumentIntegrati
         String filename = "files/new/crafted document";
 
         Document document = buildGraph(filename);
-        TextBlock textBlock = document.buildTextBlock();
+        TextBlock textBlock = document.getTextBlock();
 
         visualizeSemanticNodes(filename, document, textBlock);
     }

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/MigrationPocTest.java

@@ -0,0 +1,162 @@
+package com.iqser.red.service.redaction.v1.server.document.graph;
+
+import static java.util.stream.Collectors.toMap;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.Mockito.when;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import org.drools.io.ClassPathResource;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.springframework.beans.factory.annotation.Autowired;
+
+import com.google.common.base.Functions;
+import com.google.common.collect.Sets;
+import com.iqser.red.commons.jackson.ObjectMapperFactory;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeResult;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
+import com.iqser.red.service.redaction.v1.model.StructureAnalyzeRequest;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.adapter.RedactionLogEntryAdapter;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.document.data.DocumentData;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.document.data.mapper.DocumentGraphMapper;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionEntity;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.Document;
+import com.iqser.red.service.redaction.v1.server.multitenancy.TenantContext;
+import com.iqser.red.service.redaction.v1.server.redaction.service.RedactionLogCreatorService;
+
+import lombok.SneakyThrows;
+
+public class MigrationPocTest extends BuildDocumentIntegrationTest {
+
+    private static final String RULES = loadFromClassPath("drools/rules.drl");
+    
+    @Autowired
+    private RedactionLogEntryAdapter redactionLogAdapter;
+    @Autowired
+    private RedactionLogCreatorService redactionLogCreatorService;
+
+
+    @BeforeEach
+    public void stubClients() {
+
+        TenantContext.setTenantId("redaction");
+        when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L);
+        when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(JSONPrimitive.of(RULES));
+        loadDictionaryForTest();
+        loadTypeForTest();
+        loadNerForTest();
+        when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L);
+        when(dictionaryClient.getAllTypesForDossierTemplate(TEST_DOSSIER_TEMPLATE_ID, false)).thenReturn(getTypeResponse());
+
+        when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L);
+        when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, false)).thenReturn(List.of(Type.builder()
+                .id(DOSSIER_REDACTIONS_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID)
+                .type(DOSSIER_REDACTIONS_INDICATOR)
+                .dossierTemplateId(TEST_DOSSIER_ID)
+                .hexColor("#ffe187")
+                .isHint(hintTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
+                .isCaseInsensitive(caseInSensitiveMap.get(DOSSIER_REDACTIONS_INDICATOR))
+                .isRecommendation(recommendationTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
+                .rank(rankTypeMap.get(DOSSIER_REDACTIONS_INDICATOR))
+                .build()));
+
+        mockDictionaryCalls(null);
+
+        when(dictionaryClient.getColors(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(colors);
+    }
+
+
+    @Test
+    @SneakyThrows
+    public void testMigration() {
+
+        AnalyzeRequest request = uploadFileToStorage("files/new/crafted document.pdf");
+        analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
+        AnalyzeResult result = analyzeService.analyze(request);
+        var newRedactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID);
+
+        var originalRedactionLog = getOriginalRedactionLog();
+
+        // IMPORTANT: always use the graph which is mapped from the DocumentData, since rounding errors occur during storage.
+        Document document = DocumentGraphMapper.toDocumentGraph(DocumentData.fromDocument(buildGraph("files/new/crafted document.pdf")));
+        Set<RedactionEntity> migratedEntities = redactionLogAdapter.toRedactionEntity(originalRedactionLog, document).collect(Collectors.toSet());
+
+        var migratedRedactionLogEntries = redactionLogCreatorService.createRedactionLog(document, TEST_DOSSIER_TEMPLATE_ID);
+
+        Map<String, RedactionLogEntry> migratedIds = migratedRedactionLogEntries.stream().collect(toMap(RedactionLogEntry::getId, Functions.identity()));
+        Map<String, RedactionLogEntry> newIds = newRedactionLog.getRedactionLogEntry().stream().collect(toMap(RedactionLogEntry::getId, Functions.identity()));
+
+        logPrecision(migratedIds, newIds);
+        logRecall(migratedIds, newIds);
+
+        assertEquals(originalRedactionLog.getRedactionLogEntry().size(), migratedEntities.size());
+    }
+
+
+    private static void logPrecision(Map<String, RedactionLogEntry> migratedIds, Map<String, RedactionLogEntry> newIds) {
+
+        var precision = computePrecision(migratedIds, newIds);
+        System.out.printf("precision %.2f\n", precision);
+        System.out.println("New Entries");
+        getAddedEntries(migratedIds, newIds).forEach(System.out::println);
+        assertTrue(precision > 0.9);
+        System.out.println();
+    }
+
+
+    private static void logRecall(Map<String, RedactionLogEntry> migratedIds, Map<String, RedactionLogEntry> newIds) {
+
+        var recall = computeRecall(migratedIds, newIds);
+        System.out.printf("recall %.2f\n", recall);
+        System.out.println("Missing entries");
+        getMissingEntries(migratedIds, newIds).forEach(System.out::println);
+        assertTrue(recall > 0.9);
+        System.out.println();
+    }
+
+
+    private static Stream<RedactionLogEntry> getMissingEntries(Map<String, RedactionLogEntry> migratedIds, Map<String, RedactionLogEntry> newIds) {
+
+        return migratedIds.entrySet().stream().filter(entry -> !newIds.containsKey(entry.getKey())).map(Map.Entry::getValue);
+    }
+
+
+    private static Stream<RedactionLogEntry> getAddedEntries(Map<String, RedactionLogEntry> migratedIds, Map<String, RedactionLogEntry> newIds) {
+
+        return newIds.entrySet().stream().filter(entry -> !migratedIds.containsKey(entry.getKey())).map(Map.Entry::getValue);
+    }
+
+
+    private static double computePrecision(Map<String, RedactionLogEntry> migratedIds, Map<String, RedactionLogEntry> newIds) {
+
+        return (double) Sets.intersection(newIds.keySet(), migratedIds.keySet()).size() / (double) newIds.size();
+    }
+
+
+    private static double computeRecall(Map<String, RedactionLogEntry> migratedIds, Map<String, RedactionLogEntry> newIds) {
+
+        return (double) Sets.intersection(newIds.keySet(), migratedIds.keySet()).size() / (double) migratedIds.size();
+    }
+
+
+    private static RedactionLog getOriginalRedactionLog() throws IOException {
+
+        RedactionLog originalRedactionLog;
+        try (var inputStream = new ClassPathResource("files/migration/legacy_redactionlog.json").getInputStream()) {
+            originalRedactionLog = ObjectMapperFactory.create().readValue(inputStream, RedactionLog.class);
+        }
+        return originalRedactionLog;
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/services/DocumentPerformanceIntegrationTest.java

@@ -178,7 +178,7 @@ public class DocumentPerformanceIntegrationTest extends BuildDocumentIntegration
         Section section = document.getMainSections().get(8);
         start = System.currentTimeMillis();
         for (int i = 0; i < n; i++) {
-            section.buildTextBlock();
+            section.getTextBlock();
         }
         durationMillis = ((float) (System.currentTimeMillis() - start));
         System.out.printf("%d calls of buildTextBlock() on section took %f s, average is %f ms\n", n, durationMillis / 1000, durationMillis / n);
@@ -186,7 +186,7 @@ public class DocumentPerformanceIntegrationTest extends BuildDocumentIntegration
         SemanticNode paragraph = document.getDocumentTree().getEntryById(List.of(8, 1)).getNode();
         start = System.currentTimeMillis();
         for (int i = 0; i < n; i++) {
-            paragraph.buildTextBlock();
+            paragraph.getTextBlock();
         }
         durationMillis = ((float) (System.currentTimeMillis() - start));
         System.out.printf("%d calls of buildTextBlock() on paragraph took %f s, average is %f ms\n", n, durationMillis / 1000, durationMillis / n);

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/adapter/NerEntitiesAdapterTest.java

@@ -130,7 +130,7 @@ class NerEntitiesAdapterTest extends BuildDocumentIntegrationTest {
 
         List<NerEntities.NerEntity> cbiAuthors = nerEntities.streamEntitiesOfType("CBI_author").toList();
         Stream<NerEntities.NerEntity> cbiAddress = nerEntitiesAdapter.combineNerEntitiesToCbiAddressDefaults(nerEntities)
-                .map(boundary -> new NerEntities.NerEntity(document.buildTextBlock().subSequence(boundary).toString(), boundary, "CBI_address"));
+                .map(boundary -> new NerEntities.NerEntity(document.getTextBlock().subSequence(boundary).toString(), boundary, "CBI_address"));
 
         return Stream.concat(cbiAuthors.stream(), cbiAddress).toList();
     }

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/manual_redaction_rules.drl

@@ -3,7 +3,6 @@ package drools
 import static java.lang.String.format;
 import static com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RedactionSearchUtility.anyMatch;
 import static com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RedactionSearchUtility.exactMatch;
-import static com.iqser.red.service.redaction.v1.server.layoutparsing.document.data.mapper.PropertiesMapper.parseImageType;
 
 import java.util.List;
 import java.util.LinkedList;
@@ -93,7 +92,7 @@ rule "Apply image recategorization"
         ManualImageRecategorization($id: annotationId, status == AnnotationStatus.APPROVED, $imageType: type)
         $image: Image($id == id)
     then
-        $image.setImageType(parseImageType($imageType));
+        $image.setImageType(ImageType.fromString($imageType));
     end
 
 

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/prod_syngenta_new.drl

@@ -3,7 +3,6 @@ package drools
 import static java.lang.String.format;
 import static com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RedactionSearchUtility.anyMatch;
 import static com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RedactionSearchUtility.exactMatch;
-import static com.iqser.red.service.redaction.v1.server.layoutparsing.document.data.mapper.PropertiesMapper.parseImageType;
 
 import java.util.List;
 import java.util.LinkedList;
@@ -619,7 +618,7 @@ rule "Apply image recategorization"
         ManualImageRecategorization($id: annotationId, status == AnnotationStatus.APPROVED, $imageType: type)
         $image: Image($id == id)
     then
-        $image.setImageType(parseImageType($imageType));
+        $image.setImageType(ImageType.fromString($imageType));
     end
 
 // --------------------------------------- merging rules -------------------------------------------------------------------

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl

@@ -3,7 +3,6 @@ package drools
 import static java.lang.String.format;
 import static com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RedactionSearchUtility.anyMatch;
 import static com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RedactionSearchUtility.exactMatch;
-import static com.iqser.red.service.redaction.v1.server.layoutparsing.document.data.mapper.PropertiesMapper.parseImageType;
 
 import java.util.List;
 import java.util.LinkedList;
@@ -52,6 +51,15 @@ query "getFileAttributes"
 
 // --------------------------------------- CBI rules -------------------------------------------------------------------
 
+rule "5: Add FALSE_POSITIVE Entity for genitive CBI_author"
+    when
+        $entity: RedactionEntity(type == "CBI_author", anyMatch(textAfter, "['’’'ʼˈ´`‘′ʻ’']s"), entityType == EntityType.ENTITY)
+    then
+        RedactionEntity falsePositive = entityCreationService.byBoundary($entity.getBoundary(), "CBI_author", EntityType.FALSE_POSITIVE, document);
+        falsePositive.addMatchedRule(5);
+        insert(falsePositive);
+    end
+
 rule "0: Expand CBI_author entities with firstname initials"
     no-loop true
     when
@@ -80,7 +88,7 @@ rule "0: Expand CBI_author and PII entities with salutation prefix"
 rule "1: Redacted because Section contains Vertebrate"
     when
         $section: Section(hasEntitiesOfType("vertebrate"),
-                             (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
+                         (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
     then
        $section.getEntitiesOfType(List.of("CBI_author", "CBI_address"))
             .forEach(redactionEntity -> {
@@ -94,7 +102,7 @@ rule "1: Redacted because Section contains Vertebrate"
 rule "2: Not Redacted because Section contains no Vertebrate"
     when
         $section: Section(!hasEntitiesOfType("vertebrate"),
-                             (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
+                         (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
     then
        $section.getEntitiesOfType(List.of("CBI_author", "CBI_address"))
         .forEach(redactionEntity -> {
@@ -107,8 +115,8 @@ rule "2: Not Redacted because Section contains no Vertebrate"
 rule "3: Do not redact Names and Addresses if no redaction Indicator is contained"
     when
         $section: Section(hasEntitiesOfType("vertebrate"),
-                              hasEntitiesOfType("no_redaction_indicator"),
+                          hasEntitiesOfType("no_redaction_indicator"),
-                             (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
+                         (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
     then
         $section.getEntitiesOfType(List.of("CBI_author", "CBI_address"))
             .forEach(redactionEntity -> {
@@ -121,9 +129,9 @@ rule "3: Do not redact Names and Addresses if no redaction Indicator is containe
 rule "4: Redact Names and Addresses if no_redaction_indicator and redaction_indicator is contained"
     when
         $section: Section(hasEntitiesOfType("vertebrate"),
-                              hasEntitiesOfType("no_redaction_indicator"),
+                          hasEntitiesOfType("no_redaction_indicator"),
-                              hasEntitiesOfType("redaction_indicator"),
+                          hasEntitiesOfType("redaction_indicator"),
-                             (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
+                         (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
     then
         $section.getEntitiesOfType(List.of("CBI_author", "CBI_address"))
             .forEach(redactionEntity -> {
@@ -137,8 +145,8 @@ rule "4: Redact Names and Addresses if no_redaction_indicator and redaction_indi
 rule "5: Do not redact Names and Addresses if published information found"
     when
         $section: Section(hasEntitiesOfType("vertebrate"),
-                              hasEntitiesOfType("published_information"),
+                          hasEntitiesOfType("published_information"),
-                             (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
+                         (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
     then
         List<RedactionEntity>  publishedInformationEntities = $section.getEntitiesOfType("published_information");
         $section.getEntitiesOfType(List.of("CBI_author", "CBI_address"))
@@ -193,7 +201,7 @@ rule "7: Redact CBI_author, if its row contains a cell with header \"Vertebrate
 rule "8: Redact if must_redact entity is found"
     when
         $section: Section(hasEntitiesOfType("must_redact"),
-                             (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
+                         (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
     then
        $section.getEntitiesOfType(List.of("CBI_author", "CBI_address"))
             .forEach(redactionEntity -> {
@@ -227,7 +235,7 @@ rule "10: Redact row if row contains \"determination of residues\" and livestock
                                         "ruminants")
         $residueKeyword: String() from List.of("determination of residues", "determination of total residues")
         $table: Table(containsStringIgnoreCase($residueKeyword)
-                            && containsStringIgnoreCase($keyword))
+                    && containsStringIgnoreCase($keyword))
     then
         entityCreationService.byString($keyword, "must_redact", EntityType.ENTITY, $table)
             .forEach(keywordEntity -> insert(keywordEntity));
@@ -244,11 +252,11 @@ rule "10: Redact row if row contains \"determination of residues\" and livestock
 
 rule "11: Redact if CTL/* or BL/*  was found"
     when
-        $section: Section(excludesTables, (containsString("CTL/") || containsString("BL/")))
+        $section: Section(!hasTables(), (containsString("CTL/") || containsString("BL/")))
     then
-        entityCreationService.byString("CTL/", "must_redact", EntityType.ENTITY, $section)
+        entityCreationService.byString("CTL", "must_redact", EntityType.ENTITY, $section)
             .forEach(mustRedactEntity -> insert(mustRedactEntity));
-        entityCreationService.byString("BL/", "must_redact", EntityType.ENTITY, $section)
+        entityCreationService.byString("BL", "must_redact", EntityType.ENTITY, $section)
             .forEach(mustRedactEntity -> insert(mustRedactEntity));
 
         $section.getEntitiesOfType(List.of("CBI_author", "CBI_address"))
@@ -265,7 +273,7 @@ rule "12: Add CBI_author with \"et al.\" Regex"
     when
          $section: Section(containsString("et al."))
     then
-         entityCreationService.byRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, $section)
+         entityCreationService.byRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, 1, $section)
             .forEach(entity -> {
                 entity.setRedaction(true);
                 entity.setRedactionReason("Author found by \"et al\" regex");
@@ -278,7 +286,7 @@ rule "12: Add CBI_author with \"et al.\" Regex"
 
 rule "13: Add recommendation for Addresses in Test Organism sections"
     when
-        $section: Section(excludesTables, containsString("Species") && containsString("Source") && !containsString("Species:") && !containsString("Source:"))
+        $section: Section(!hasTables(), containsString("Species") && containsString("Source") && !containsString("Species:") && !containsString("Source:"))
     then
         entityCreationService.lineAfterString("Source",  "CBI_address", EntityType.RECOMMENDATION, $section)
             .forEach(redactionEntity -> {
@@ -291,7 +299,7 @@ rule "13: Add recommendation for Addresses in Test Organism sections"
 rule "14: Add recommendation for Addresses in Test Animals sections"
 
     when
-        $section: Section(excludesTables, containsString("Species:"), containsString("Source:"))
+        $section: Section(!hasTables(), containsString("Species:"), containsString("Source:"))
     then
         entityCreationService.lineAfterString("Source:",  "CBI_address", EntityType.RECOMMENDATION, $section)
             .forEach(redactionEntity -> {
@@ -385,7 +393,7 @@ rule "18: redact line between contact keywords"
 rule "19: Redact AUTHOR(S)"
     when
         FileAttribute(placeholder == "{fileattributes.vertebrateStudy}", value == "true")
-        $section: Section(excludesTables, containsString("AUTHOR(S):"), containsString("COMPLETION DATE:"))
+        $section: Section(!hasTables(), containsString("AUTHOR(S):"), containsString("COMPLETION DATE:"))
     then
         entityCreationService.betweenStrings("AUTHOR(S):", "COMPLETION DATE:", "PII", EntityType.ENTITY, $section)
          .forEach(authorEntity -> {
@@ -397,9 +405,27 @@ rule "19: Redact AUTHOR(S)"
          });
     end
 
-rule "20: Redact PERFORMING LABORATORY"
+rule "20: Redact PERFORMING LABORATORY (Non vertebrate study)"
+    agenda-group "LOCAL_DICTIONARY_ADDS"
     when
-        $section: Section(excludesTables, containsString("PERFORMING LABORATORY:"))
+        not FileAttribute(label == "Vertebrate Study", value == "Yes")
+        $section: Section(!hasTables(), containsString("PERFORMING LABORATORY:"))
+    then
+        entityCreationService.betweenStrings("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", EntityType.ENTITY, $section)
+         .forEach(laboratoryEntity -> {
+             laboratoryEntity.setRedaction(false);
+             laboratoryEntity.addMatchedRule(31);
+             laboratoryEntity.addEngine(Engine.RULE);
+             laboratoryEntity.setRedactionReason("PERFORMING LABORATORY was found for non vertebrate study");
+             dictionary.addLocalDictionaryEntry(laboratoryEntity);
+             insert(laboratoryEntity);
+         });
+    end
+
+rule "20: Redact PERFORMING LABORATORY"
+    agenda-group "LOCAL_DICTIONARY_ADDS"
+    when
+        $section: Section(!hasTables(), containsString("PERFORMING LABORATORY:"))
     then
         entityCreationService.betweenStrings("PERFORMING LABORATORY:", "COMPLETION DATE:", "PII", EntityType.ENTITY, $section)
          .forEach(authorEntity -> {
@@ -407,13 +433,14 @@ rule "20: Redact PERFORMING LABORATORY"
              authorEntity.addMatchedRule(20);
              authorEntity.setRedactionReason("PERFORMING LABORATORY was found");
              authorEntity.setLegalBasis("Reg (EC) No 1107/2009 Art. 63 (2e)");
+             dictionary.addLocalDictionaryEntry(authorEntity);
              insert(authorEntity);
          });
     end
 
 rule "21: Redact On behalf of Sequani Ltd.:"
     when
-        $section: Section(excludesTables, containsString("On behalf of Sequani Ltd.: Name Title"))
+        $section: Section(!hasTables(), containsString("On behalf of Sequani Ltd.: Name Title"))
     then
         entityCreationService.betweenStrings("On behalf of Sequani Ltd.: Name Title", "On behalf of", "PII", EntityType.ENTITY, $section)
          .forEach(authorEntity -> {
@@ -427,7 +454,7 @@ rule "21: Redact On behalf of Sequani Ltd.:"
 
 rule "22: Redact On behalf of Syngenta Ltd.:"
     when
-        $section: Section(excludesTables, containsString("On behalf of Syngenta Ltd.: Name Title"))
+        $section: Section(!hasTables(), containsString("On behalf of Syngenta Ltd.: Name Title"))
     then
         entityCreationService.betweenStrings("On behalf of Syngenta Ltd.: Name Title", "Study dates", "PII", EntityType.ENTITY, $section)
          .forEach(authorEntity -> {
@@ -439,6 +466,22 @@ rule "22: Redact On behalf of Syngenta Ltd.:"
          });
     end
 
+
+rule "25: Redact Purity"
+    when
+        $section: Section(containsStringIgnoreCase("purity"))
+    then
+	    entityCreationService.byRegex("\\bPurity:\\s*(<?>?\\s*\\d{1,2}(?:\\.\\d{1,2})?\\s*%)", "purity", EntityType.ENTITY, 1, $section)
+	        .forEach(entity -> {
+                entity.addMatchedRule(25);
+                entity.addEngine(Engine.RULE);
+                entity.setRedaction(true);
+                entity.setRedactionReason("Purity found");
+                entity.setLegalBasis("Reg (EC) No 1107/2009 Art. 63 (2a)");
+	        });
+    end
+
+
 rule "26: Redact signatures"
     when
         $signature: Image(imageType == ImageType.SIGNATURE)
@@ -504,11 +547,12 @@ rule "101: Redact CAS Number"
     end
 
 rule "102: Guidelines FileAttributes"
+    salience 999
     when
-        $section: Section(excludesTables, (containsString("DATA REQUIREMENT(S):") || containsString("TEST GUIDELINE(S):")) && (containsString("OECD") || containsString("EPA") || containsString("OPPTS")))
+        $section: Section(!hasTables(), (containsString("DATA REQUIREMENT(S):") || containsString("TEST GUIDELINE(S):")) && (containsString("OECD") || containsString("EPA") || containsString("OPPTS")))
     then
-        RedactionSearchUtility.findBoundariesByRegex("OECD (No\\.? )?\\d{3}( \\(\\d{4}\\))?", $section.buildTextBlock()).stream()
+        RedactionSearchUtility.findBoundariesByRegex("OECD (No\\.? )?\\d{3}( \\(\\d{4}\\))?", $section.getTextBlock()).stream()
-            .map(boundary -> $section.buildTextBlock().subSequence(boundary).toString())
+            .map(boundary -> $section.getTextBlock().subSequence(boundary).toString())
             .map(value -> FileAttribute.builder().label("OECD Number").value(value).build())
             .forEach(fileAttribute -> insert(fileAttribute));
     end
@@ -589,7 +633,7 @@ rule "Apply image recategorization"
         ManualImageRecategorization($id: annotationId, status == AnnotationStatus.APPROVED, $imageType: type)
         $image: Image($id == id)
     then
-        $image.setImageType(parseImageType($imageType));
+        $image.setImageType(ImageType.fromString($imageType));
     end
 
 // --------------------------------------- merging rules -------------------------------------------------------------------
@@ -653,13 +697,13 @@ rule "remove Entity of type RECOMMENDATION when contained by ENTITY"
     salience 256
     when
         $entity: RedactionEntity(entityType == EntityType.ENTITY)
-        $recommendation: RedactionEntity(containedBy($entity), entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger)
+        $recommendation: RedactionEntity(intersects($entity), entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger)
     then
         $recommendation.removeFromGraph();
         retract($recommendation);
     end
 
-rule "remove Entity of lower rank, when equal boundaries and entityType"
+rule "remove Entity of lower rank, when boundaries interect and entityType"
     salience 32
     when
         $higherRank: RedactionEntity($type: type, $entityType: entityType)

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_backup.drl

@@ -0,0 +1,697 @@
+package drools
+
+import static java.lang.String.format;
+import static com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RedactionSearchUtility.anyMatch;
+import static com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RedactionSearchUtility.exactMatch;
+import static com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RedactionSearchUtility.exactMatch;
+
+import java.util.List;
+import java.util.LinkedList;
+import java.util.HashSet;
+
+import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.*;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.*;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.*;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.*;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.EntityType;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.ImageType;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;
+import java.util.Set
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.document.services.EntityCreationService;
+import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.Dictionary;
+import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.DictionaryModel;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualImageRecategorization;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.document.services.ManualRedactionApplicationService;
+import com.iqser.red.service.redaction.v1.server.client.model.EntityRecognitionEntity;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.Boundary;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionEntity;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.Boundary;
+import com.iqser.red.service.redaction.v1.server.redaction.adapter.NerEntitiesAdapter;
+import com.iqser.red.service.redaction.v1.server.redaction.adapter.NerEntities;
+import java.util.stream.Collectors;
+import java.util.Collection;
+import java.util.stream.Stream;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RedactionSearchUtility;
+
+global Document document
+global EntityCreationService entityCreationService
+global ManualRedactionApplicationService manualRedactionApplicationService
+global NerEntitiesAdapter nerEntitiesAdapter
+global Dictionary dictionary
+
+// --------------------------------------- queries -------------------------------------------------------------------
+
+query "getFileAttributes"
+    $fileAttribute: FileAttribute()
+    end
+
+// --------------------------------------- CBI rules -------------------------------------------------------------------
+
+rule "0: Expand CBI_author entities with firstname initials"
+    no-loop true
+    when
+        $entityToExpand: RedactionEntity(type == "CBI_author",
+          value.matches("[^\\s]+"),
+          textAfter.startsWith(" "),
+          anyMatch(textAfter, "(,? [A-Z]\\.?( ?[A-Z]\\.?)?( ?[A-Z]\\.?)?\\b\\.?)")
+          )
+    then
+        RedactionEntity expandedEntity = entityCreationService.bySuffixExpansionRegex($entityToExpand, "(,? [A-Z]\\.?( ?[A-Z]\\.?)?( ?[A-Z]\\.?)?\\b\\.?)");
+        expandedEntity.addMatchedRule(0);
+        $entityToExpand.removeFromGraph();
+        retract($entityToExpand);
+        insert(expandedEntity);
+    end
+
+rule "0: Expand CBI_author and PII entities with salutation prefix"
+    when
+        $entityToExpand: RedactionEntity((type == "CBI_author" || type == "PII"), anyMatch(textBefore, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*"))
+    then
+        RedactionEntity expandedEntity = entityCreationService.byPrefixExpansionRegex($entityToExpand, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*");
+        expandedEntity.addMatchedRule(0);
+        insert(expandedEntity);
+    end
+
+rule "1: Redacted because Section contains Vertebrate"
+    when
+        $section: Section(hasEntitiesOfType("vertebrate"),
+                             (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
+    then
+       $section.getEntitiesOfType(List.of("CBI_author", "CBI_address"))
+            .forEach(redactionEntity -> {
+                redactionEntity.setRedaction(true);
+                redactionEntity.addMatchedRule(1);
+                redactionEntity.setRedactionReason("Vertebrate Found in this section");
+                redactionEntity.setLegalBasis("Reg (EC) No 1107/2009 Art. 63 (2g)");
+            });
+    end
+
+rule "2: Not Redacted because Section contains no Vertebrate"
+    when
+        $section: Section(!hasEntitiesOfType("vertebrate"),
+                             (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
+    then
+       $section.getEntitiesOfType(List.of("CBI_author", "CBI_address"))
+        .forEach(redactionEntity -> {
+            redactionEntity.setRedaction(false);
+            redactionEntity.addMatchedRule(2);
+            redactionEntity.setRedactionReason("No Vertebrate Found in this section");
+        });
+    end
+
+rule "3: Do not redact Names and Addresses if no redaction Indicator is contained"
+    when
+        $section: Section(hasEntitiesOfType("vertebrate"),
+                              hasEntitiesOfType("no_redaction_indicator"),
+                             (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
+    then
+        $section.getEntitiesOfType(List.of("CBI_author", "CBI_address"))
+            .forEach(redactionEntity -> {
+                redactionEntity.setRedaction(false);
+                redactionEntity.addMatchedRule(3);
+                redactionEntity.setRedactionReason("Vertebrate and a no-redaction-indicator found in this section");
+            });
+    end
+
+rule "4: Redact Names and Addresses if no_redaction_indicator and redaction_indicator is contained"
+    when
+        $section: Section(hasEntitiesOfType("vertebrate"),
+                              hasEntitiesOfType("no_redaction_indicator"),
+                              hasEntitiesOfType("redaction_indicator"),
+                             (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
+    then
+        $section.getEntitiesOfType(List.of("CBI_author", "CBI_address"))
+            .forEach(redactionEntity -> {
+                redactionEntity.setRedaction(true);
+                redactionEntity.addMatchedRule(4);
+                redactionEntity.setRedactionReason("Vertebrate and a no-redaction-indicator, but also redaction-indicator, found in this section");
+                redactionEntity.setLegalBasis("Reg (EC) No 1107/2009 Art. 63 (2g)");
+            });
+    end
+
+rule "5: Do not redact Names and Addresses if published information found"
+
+    when
+        $section: Section(hasEntitiesOfType("vertebrate"),
+                              hasEntitiesOfType("published_information"),
+                             (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
+    then
+        List<RedactionEntity>  publishedInformationEntities = $section.getEntitiesOfType("published_information");
+        $section.getEntitiesOfType(List.of("CBI_author", "CBI_address"))
+            .forEach(redactionEntity -> {
+                redactionEntity.setRedaction(false);
+                redactionEntity.setRedactionReason("Vertebrate but also Published Information found in this section");
+                redactionEntity.addReferences(publishedInformationEntities);
+            });
+    end
+
+rule "6.0: Add all Cell's with Header Author(s) as CBI_author"
+    when
+        $table: Table(hasHeader("Author(s)"))
+    then
+        $table.streamTableCellsWithHeader("Author(s)")
+            .map(tableCell -> entityCreationService.bySemanticNode(tableCell, "CBI_author", EntityType.ENTITY))
+            .forEach(redactionEntity -> {
+                redactionEntity.addMatchedRule(6);
+                redactionEntity.setRedactionReason("Author(s) header found");
+                insert(redactionEntity);
+            });
+    end
+
+rule "6.1: Dont redact CBI_author, if its row contains a cell with header \"Vertebrate study Y/N\" and value No"
+    when
+        $table: Table(hasRowWithHeaderAndValue("Vertebrate study Y/N", "N") || hasRowWithHeaderAndValue("Vertebrate study Y/N", "No"))
+    then
+        $table.streamEntitiesWhereRowHasHeaderAndAnyValue("Vertebrate study Y/N", List.of("N", "No"))
+            .filter(redactionEntity -> redactionEntity.isAnyType(List.of("CBI_author", "CBI_address")))
+            .forEach(authorEntity -> {
+                authorEntity.setRedaction(false);
+                authorEntity.setRedactionReason("Not redacted because it's row does not belong to a vertebrate study");
+                authorEntity.setLegalBasis("");
+                authorEntity.addMatchedRule(6);
+            });
+    end
+
+rule "7: Redact CBI_author, if its row contains a cell with header \"Vertebrate study Y/N\" and value Yes"
+    when
+        $table: Table(hasRowWithHeaderAndValue("Vertebrate study Y/N", "Y") || hasRowWithHeaderAndValue("Vertebrate study Y/N", "Yes"))
+    then
+        $table.streamEntitiesWhereRowHasHeaderAndAnyValue("Vertebrate study Y/N", List.of("Y", "Yes"))
+            .filter(redactionEntity -> redactionEntity.isAnyType(List.of("CBI_author", "CBI_address")))
+            .forEach(authorEntity -> {
+                authorEntity.setRedaction(true);
+                authorEntity.setRedactionReason("Redacted because it's row belongs to a vertebrate study");
+                authorEntity.setLegalBasis("Reg (EC) No 1107/2009 Art. 63 (2g)");
+                authorEntity.addMatchedRule(7);
+            });
+    end
+
+rule "8: Redact if must_redact entity is found"
+    when
+        $section: Section(hasEntitiesOfType("must_redact"),
+                             (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
+    then
+       $section.getEntitiesOfType(List.of("CBI_author", "CBI_address"))
+            .forEach(redactionEntity -> {
+                redactionEntity.setRedaction(true);
+                redactionEntity.setRedactionReason("must_redact entry was found.");
+                redactionEntity.setLegalBasis("Reg (EC) No 1107/2009 Art. 63 (2g)");
+                redactionEntity.addMatchedRule(8);
+            });
+    end
+
+rule "9: Redact CBI_sponsor entities if preceded by \" batches produced at\""
+    when
+        $sponsorEntity: RedactionEntity(type == "CBI_sponsor", textBefore.contains("batches produced at"))
+    then
+        $sponsorEntity.setRedaction(true);
+        $sponsorEntity.setRedactionReason("Redacted because it represents a sponsor company");
+        $sponsorEntity.setLegalBasis("Reg (EC) No 1107/2009 Art. 63 (2g)");
+        $sponsorEntity.addMatchedRule(9);
+    end
+
+rule "10: Redact row if row contains \"determination of residues\" and livestock keyword"
+    when
+        $keyword: String() from List.of("livestock",
+                                        "live stock",
+                                        "tissue",
+                                        "tissues",
+                                        "liver",
+                                        "muscle",
+                                        "bovine",
+                                        "ruminant",
+                                        "ruminants")
+        $residueKeyword: String() from List.of("determination of residues", "determination of total residues")
+        $table: Table(containsStringIgnoreCase($residueKeyword)
+                            && containsStringIgnoreCase($keyword))
+    then
+        entityCreationService.byString($keyword, "must_redact", EntityType.ENTITY, $table)
+            .forEach(keywordEntity -> insert(keywordEntity));
+
+        $table.streamEntitiesWhereRowContainsStringsIgnoreCase(List.of($keyword, $residueKeyword))
+            .filter(redactionEntity -> redactionEntity.isAnyType(List.of("CBI_author", "CBI_address")))
+            .forEach(redactionEntity -> {
+                redactionEntity.setRedaction(true);
+                redactionEntity.setRedactionReason("Determination of residues and keyword \"" + $keyword + "\" was found.");
+                redactionEntity.setLegalBasis("Reg (EC) No 1107/2009 Art. 63 (2g)");
+                redactionEntity.addMatchedRule(10);
+            });
+    end
+
+rule "11: Redact if CTL/* or BL/*  was found"
+    when
+        $section: Section(excludesTables, (containsString("CTL/") || containsString("BL/")))
+    then
+        entityCreationService.byString("CTL/", "must_redact", EntityType.ENTITY, $section)
+            .forEach(mustRedactEntity -> insert(mustRedactEntity));
+        entityCreationService.byString("BL/", "must_redact", EntityType.ENTITY, $section)
+            .forEach(mustRedactEntity -> insert(mustRedactEntity));
+
+        $section.getEntitiesOfType(List.of("CBI_author", "CBI_address"))
+            .forEach(redactionEntity -> {
+                redactionEntity.setRedaction(true);
+                redactionEntity.setRedactionReason("Laboratory for vertebrate studies found");
+                redactionEntity.setLegalBasis("Reg (EC) No 1107/2009 Art. 63 (2g)");
+                redactionEntity.addMatchedRule(11);
+            });
+    end
+
+rule "12: Add CBI_author with \"et al.\" Regex"
+    agenda-group "LOCAL_DICTIONARY_ADDS"
+    when
+         $section: Section(containsString("et al."))
+    then
+         entityCreationService.byRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, $section)
+            .forEach(entity -> {
+                entity.setRedaction(true);
+                entity.setRedactionReason("Author found by \"et al\" regex");
+                entity.setLegalBasis("Reg (EC) No 1107/2009 Art. 63 (2g)");
+                entity.addMatchedRule(12);
+                insert(entity);
+                dictionary.addLocalDictionaryEntry("CBI_author", entity.getValue(), false);
+            });
+    end
+
+rule "13: Add recommendation for Addresses in Test Organism sections"
+    when
+        $section: Section(excludesTables, containsString("Species") && containsString("Source") && !containsString("Species:") && !containsString("Source:"))
+    then
+        entityCreationService.lineAfterString("Source",  "CBI_address", EntityType.RECOMMENDATION, $section)
+            .forEach(redactionEntity -> {
+                redactionEntity.setRedactionReason("Line after \"Source\" in Test Organism Section");
+                redactionEntity.addMatchedRule(13);
+                insert(redactionEntity);
+            });
+    end
+
+rule "14: Add recommendation for Addresses in Test Animals sections"
+
+    when
+        $section: Section(excludesTables, containsString("Species:"), containsString("Source:"))
+    then
+        entityCreationService.lineAfterString("Source:",  "CBI_address", EntityType.RECOMMENDATION, $section)
+            .forEach(redactionEntity -> {
+                redactionEntity.setRedactionReason("Line after \"Source:\" in Test Animals Section");
+                redactionEntity.addMatchedRule(14);
+                insert(redactionEntity);
+            });
+    end
+
+// --------------------------------------- PII rules -------------------------------------------------------------------
+
+rule "15: Redact all PII"
+    when
+        $pii: RedactionEntity(type == "PII", redaction == false)
+    then
+        $pii.setRedaction(true);
+        $pii.setRedactionReason("PII found");
+        $pii.setLegalBasis("Reg (EC) No 1107/2009 Art. 63 (2e)");
+        $pii.addMatchedRule(15);
+    end
+
+rule "16: Redact Emails by RegEx (Non vertebrate study)"
+    when
+        $section: Section(containsString("@"))
+    then
+        entityCreationService.byRegex("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", "PII", EntityType.ENTITY, $section)
+            .forEach(emailEntity -> {
+                emailEntity.setRedaction(true);
+                emailEntity.setRedactionReason("Found by Email Regex");
+                emailEntity.setLegalBasis("Reg (EC) No 1107/2009 Art. 63 (2e)");
+                emailEntity.addMatchedRule(16);
+                insert(emailEntity);
+            });
+    end
+
+rule "17: Redact line after contact information keywords"
+    agenda-group "LOCAL_DICTIONARY_ADDS"
+    when
+        $contactKeyword: String() from List.of("Contact point:",
+                                         "Contact:",
+                                         "Alternative contact:",
+                                         "European contact:",
+                                         "No:",
+                                         "Contact:",
+                                         "Tel.:",
+                                         "Tel:",
+                                         "Telephone number:",
+                                         "Telephone No:",
+                                         "Telephone:",
+                                         "Phone No.",
+                                         "Phone:",
+                                         "Fax number:",
+                                         "Fax:",
+                                         "E-mail:",
+                                         "Email:",
+                                         "e-mail:",
+                                         "E-mail address:")
+        $section: Section(containsString($contactKeyword))
+    then
+        entityCreationService.lineAfterString($contactKeyword, "PII", EntityType.ENTITY, $section)
+            .forEach(contactEntity -> {
+                contactEntity.setRedaction(true);
+                contactEntity.addMatchedRule(17);
+                contactEntity.setRedactionReason("Found after \"" + $contactKeyword + "\" contact keyword");
+                contactEntity.setLegalBasis("Reg (EC) No 1107/2009 Art. 63 (2e)");
+                insert(contactEntity);
+                dictionary.addLocalDictionaryEntry("PII", contactEntity.getValue(), false);
+            });
+    end
+
+
+rule "18: redact line between contact keywords"
+    agenda-group "LOCAL_DICTIONARY_ADDS"
+    when
+        $section: Section((containsString("No:") && containsString("Fax")) || (containsString("Contact:") && containsString("Tel")))
+    then
+        Stream.concat(
+            entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section),
+            entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section)
+        )
+            .forEach(contactEntity -> {
+                contactEntity.setRedaction(true);
+                contactEntity.addMatchedRule(18);
+                contactEntity.setRedactionReason("Found between contact keywords");
+                contactEntity.setLegalBasis("Reg (EC) No 1107/2009 Art. 63 (2e)");
+                insert(contactEntity);
+                dictionary.addLocalDictionaryEntry("PII", contactEntity.getValue(), false);
+            });
+    end
+
+rule "19: Redact AUTHOR(S)"
+    when
+        FileAttribute(placeholder == "{fileattributes.vertebrateStudy}", value == "true")
+        $section: Section(excludesTables, containsString("AUTHOR(S):"), containsString("COMPLETION DATE:"))
+    then
+        entityCreationService.betweenStrings("AUTHOR(S):", "COMPLETION DATE:", "PII", EntityType.ENTITY, $section)
+         .forEach(authorEntity -> {
+             authorEntity.setRedaction(true);
+             authorEntity.addMatchedRule(19);
+             authorEntity.setRedactionReason("AUTHOR(S) was found");
+             authorEntity.setLegalBasis("Reg (EC) No 1107/2009 Art. 63 (2e)");
+             insert(authorEntity);
+         });
+    end
+
+rule "20: Redact PERFORMING LABORATORY"
+    when
+        $section: Section(excludesTables, containsString("PERFORMING LABORATORY:"))
+    then
+        entityCreationService.betweenStrings("PERFORMING LABORATORY:", "COMPLETION DATE:", "PII", EntityType.ENTITY, $section)
+         .forEach(authorEntity -> {
+             authorEntity.setRedaction(true);
+             authorEntity.addMatchedRule(20);
+             authorEntity.setRedactionReason("PERFORMING LABORATORY was found");
+             authorEntity.setLegalBasis("Reg (EC) No 1107/2009 Art. 63 (2e)");
+             insert(authorEntity);
+         });
+    end
+
+rule "21: Redact On behalf of Sequani Ltd.:"
+    when
+        $section: Section(excludesTables, containsString("On behalf of Sequani Ltd.: Name Title"))
+    then
+        entityCreationService.betweenStrings("On behalf of Sequani Ltd.: Name Title", "On behalf of", "PII", EntityType.ENTITY, $section)
+         .forEach(authorEntity -> {
+             authorEntity.setRedaction(true);
+             authorEntity.addMatchedRule(21);
+             authorEntity.setRedactionReason("On behalf of Sequani Ltd.: Name Title was found");
+             authorEntity.setLegalBasis("Reg (EC) No 1107/2009 Art. 63 (2e)");
+             insert(authorEntity);
+         });
+    end
+
+rule "22: Redact On behalf of Syngenta Ltd.:"
+    when
+        $section: Section(excludesTables, containsString("On behalf of Syngenta Ltd.: Name Title"))
+    then
+        entityCreationService.betweenStrings("On behalf of Syngenta Ltd.: Name Title", "Study dates", "PII", EntityType.ENTITY, $section)
+         .forEach(authorEntity -> {
+             authorEntity.setRedaction(true);
+             authorEntity.addMatchedRule(21);
+             authorEntity.setRedactionReason("On behalf of Syngenta Ltd.: Name Title was found");
+             authorEntity.setLegalBasis("Reg (EC) No 1107/2009 Art. 63 (2e)");
+             insert(authorEntity);
+         });
+    end
+
+rule "26: Redact signatures"
+    when
+        $signature: Image(imageType == ImageType.SIGNATURE)
+    then
+        $signature.setRedaction(true);
+        $signature.setMatchedRule(26);
+        $signature.setRedactionReason("Signature Found");
+        $signature.setLegalBasis("Reg (EC) No 1107/2009 Art. 63 (2g)");
+    end
+
+rule "27: Redact formulas"
+    when
+        $formula: Image(imageType == ImageType.FORMULA)
+    then
+        $formula.setRedaction(true);
+        $formula.setMatchedRule(27);
+        $formula.setRedactionReason("Formula Found");
+        $formula.setLegalBasis("Reg (EC) No 1107/2009 Art. 63 (2g)");
+    end
+
+rule "28: Redact logos"
+    when
+        $logo: Image(imageType == ImageType.LOGO)
+    then
+        $logo.setRedaction(true);
+        $logo.setMatchedRule(28);
+        $logo.setRedactionReason("Logo Found");
+        $logo.setLegalBasis("Reg (EC) No 1107/2009 Art. 63 (2g)");
+    end
+
+rule "29: Redact Dossier Redactions"
+     when
+         $dossierRedaction: RedactionEntity(type == "dossier_redactions")
+     then
+        $dossierRedaction.setRedaction(true);
+        $dossierRedaction.addMatchedRule(29);
+        $dossierRedaction.setRedactionReason("Dossier Redaction found");
+        $dossierRedaction.setLegalBasis("Article 39(1)(2) of Regulation (EC) No 178/2002");
+     end
+
+rule "30: Remove Dossier redactions if file is confidential"
+     when
+        FileAttribute(label == "Confidentiality", value == "confidential")
+        $dossierRedaction: RedactionEntity(type == "dossier_redactions")
+     then
+        $dossierRedaction.removeFromGraph();
+        retract($dossierRedaction)
+     end
+
+rule "101: Redact CAS Number"
+    when
+        $table: Table(hasHeader("Sample #"))
+    then
+        $table.streamTableCellsWithHeader("Sample #")
+            .map(tableCell -> entityCreationService.bySemanticNode(tableCell, "PII", EntityType.ENTITY))
+            .forEach(redactionEntity -> {
+                redactionEntity.setRedaction(true);
+                redactionEntity.addMatchedRule(101);
+                redactionEntity.setRedactionReason("Sample # found in Header");
+                redactionEntity.setLegalBasis("Reg (EC) No 1107/2009 Art. 63 (2g)");
+                insert(redactionEntity);
+            });
+    end
+
+rule "102: Guidelines FileAttributes"
+    when
+        $section: Section(excludesTables, (containsString("DATA REQUIREMENT(S):") || containsString("TEST GUIDELINE(S):")) && (containsString("OECD") || containsString("EPA") || containsString("OPPTS")))
+    then
+        RedactionSearchUtility.findBoundariesByRegex("OECD (No\\.? )?\\d{3}( \\(\\d{4}\\))?", $section.getTextBlock()).stream()
+            .map(boundary -> $section.getTextBlock().subSequence(boundary).toString())
+            .map(value -> FileAttribute.builder().label("OECD Number").value(value).build())
+            .forEach(fileAttribute -> insert(fileAttribute));
+    end
+
+// --------------------------------------- NER Entities rules -------------------------------------------------------------------
+
+rule "add NER Entities of type CBI_author"
+    salience 999
+    when
+        nerEntities: NerEntities(hasEntitiesOfType("CBI_author"))
+    then
+        nerEntities.streamEntitiesOfType("CBI_author")
+            .map(nerEntity -> entityCreationService.byNerEntity(nerEntity, EntityType.RECOMMENDATION, document))
+            .forEach(entity -> insert(entity));
+    end
+
+rule "combine and add NER Entities as CBI_address"
+    salience 999
+    when
+        nerEntities: NerEntities(hasEntitiesOfType("ORG") || hasEntitiesOfType("STREET") || hasEntitiesOfType("CITY"))
+    then
+        nerEntitiesAdapter.combineNerEntitiesToCbiAddressDefaults(nerEntities)
+            .map(boundary -> entityCreationService.byBoundary(boundary, "CBI_address", EntityType.RECOMMENDATION, document))
+            .forEach(entity -> {
+                entity.addEngine(Engine.NER);
+                insert(entity);
+            });
+    end
+
+// --------------------------------------- manual redaction rules -------------------------------------------------------------------
+
+rule "Apply manual resize redaction"
+    salience 128
+    when
+        $resizeRedaction: ManualResizeRedaction($id: annotationId)
+        $entityToBeResized: RedactionEntity(matchesAnnotationId($id))
+    then
+        manualRedactionApplicationService.resizeEntityAndReinsert($entityToBeResized, $resizeRedaction);
+        retract($resizeRedaction);
+        update($entityToBeResized);
+    end
+
+rule "Apply id removals that are valid and not in forced redactions to Entity"
+    salience 128
+    when
+        IdRemoval(status == AnnotationStatus.APPROVED, !removeFromDictionary, requestDate != null, $id: annotationId)
+        not ManualForceRedaction($id == annotationId, status == AnnotationStatus.APPROVED, requestDate != null)
+        $entityToBeRemoved: RedactionEntity(matchesAnnotationId($id))
+    then
+        $entityToBeRemoved.removeFromGraph();
+        retract($entityToBeRemoved);
+    end
+
+rule "Apply id removals that are valid and not in forced redactions to Image"
+    salience 128
+    when
+        IdRemoval(status == AnnotationStatus.APPROVED, !removeFromDictionary, requestDate != null, $id: annotationId)
+        not ManualForceRedaction($id == annotationId, status == AnnotationStatus.APPROVED, requestDate != null)
+        $entityToBeRemoved: Image($id == id)
+    then
+        $entityToBeRemoved.setIgnored(true);
+    end
+
+rule "Apply force redaction"
+    salience 128
+    when
+        ManualForceRedaction($id: annotationId, status == AnnotationStatus.APPROVED, requestDate != null, $legalBasis: legalBasis)
+        $entityToForce: RedactionEntity(matchesAnnotationId($id))
+    then
+        $entityToForce.setLegalBasis($legalBasis);
+        $entityToForce.setRedaction(true);
+        $entityToForce.setSkipRemoveEntitiesContainedInLarger(true);
+    end
+
+rule "Apply image recategorization"
+    salience 128
+    when
+        ManualImageRecategorization($id: annotationId, status == AnnotationStatus.APPROVED, $imageType: type)
+        $image: Image($id == id)
+    then
+        $image.setImageType(ImageType.fromString($imageType));
+    end
+
+// --------------------------------------- merging rules -------------------------------------------------------------------
+
+rule "remove Entity contained by Entity of same type"
+    salience 65
+    when
+        $larger: RedactionEntity($type: type, $entityType: entityType)
+        $contained: RedactionEntity(containedBy($larger), type == $type, entityType == $entityType, this != $larger, !resized, !skipRemoveEntitiesContainedInLarger)
+    then
+        $contained.removeFromGraph();
+        retract($contained);
+    end
+
+rule "merge intersecting Entities of same type"
+    salience 64
+    when
+        $first: RedactionEntity($type: type, $entityType: entityType, !resized, !skipRemoveEntitiesContainedInLarger)
+        $second: RedactionEntity(intersects($first), type == $type, entityType == $entityType, this != $first, !resized, !skipRemoveEntitiesContainedInLarger)
+    then
+        $first.removeFromGraph();
+        $second.removeFromGraph();
+        RedactionEntity mergedEntity = entityCreationService.byEntities(List.of($first, $second), $type, $entityType, document);
+        retract($first);
+        retract($second);
+        insert(mergedEntity);
+    end
+
+rule "remove Entity of type ENTITY when contained by FALSE_POSITIVE"
+    salience 64
+    when
+        $falsePositive: RedactionEntity($type: type, entityType == EntityType.FALSE_POSITIVE)
+        $entity: RedactionEntity(containedBy($falsePositive), type == $type, entityType == EntityType.ENTITY, !resized, !skipRemoveEntitiesContainedInLarger)
+    then
+        $entity.removeFromGraph();
+        retract($entity)
+    end
+
+rule "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"
+    salience 64
+    when
+        $falseRecommendation: RedactionEntity($type: type, entityType == EntityType.FALSE_RECOMMENDATION)
+        $recommendation: RedactionEntity(containedBy($falseRecommendation), type == $type, entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger)
+    then
+        $recommendation.removeFromGraph();
+        retract($recommendation);
+    end
+
+rule "remove Entity of type RECOMMENDATION when intersected by ENTITY with same type"
+    salience 256
+    when
+        $entity: RedactionEntity($type: type, entityType == EntityType.ENTITY)
+        $recommendation: RedactionEntity(intersects($entity), type == $type, entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger)
+    then
+        $entity.addEngines($recommendation.getEngines());
+        $recommendation.removeFromGraph();
+        retract($recommendation);
+    end
+
+rule "remove Entity of type RECOMMENDATION when contained by ENTITY"
+    salience 256
+    when
+        $entity: RedactionEntity(entityType == EntityType.ENTITY)
+        $recommendation: RedactionEntity(containedBy($entity), entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger)
+    then
+        $recommendation.removeFromGraph();
+        retract($recommendation);
+    end
+
+rule "remove Entity of lower rank, when equal boundaries and entityType"
+    salience 32
+    when
+        $higherRank: RedactionEntity($type: type, $entityType: entityType, $boundary: boundary)
+        $lowerRank: RedactionEntity($boundary == boundary, type != $type, entityType == $entityType, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !redaction)
+    then
+        $lowerRank.removeFromGraph();
+        retract($lowerRank);
+    end
+
+// --------------------------------------- FileAttribute Rules -------------------------------------------------------------------
+
+rule "remove duplicate FileAttributes"
+    salience 64
+    when
+        $fileAttribute: FileAttribute($label: label, $value: value)
+        $duplicate: FileAttribute(this != $fileAttribute, label == $label, value == $value)
+    then
+        retract($duplicate);
+    end
+
+// --------------------------------------- local dictionary search -------------------------------------------------------------------
+
+rule "run local dictionary search"
+    agenda-group "LOCAL_DICTIONARY_ADDS"
+    salience -999
+    when
+        DictionaryModel(!localEntries.isEmpty(), $type: type, $searchImplementation: localSearch) from dictionary.getDictionaryModels()
+    then
+        entityCreationService.bySearchImplementation($searchImplementation, $type, EntityType.RECOMMENDATION, document)
+            .forEach(entity -> {
+                entity.addEngine(Engine.RULE);
+                insert(entity);
+            });
+    end

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_v2.drl

@@ -3,7 +3,6 @@ package drools
 import static java.lang.String.format;
 import static com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RedactionSearchUtility.anyMatch;
 import static com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RedactionSearchUtility.exactMatch;
-import static com.iqser.red.service.redaction.v1.server.layoutparsing.document.data.mapper.PropertiesMapper.parseImageType;
 
 import java.util.List;
 import java.util.LinkedList;

redaction-service-v1/redaction-service-server-v1/src/test/resources/performance/dictionaries/EFSA_sanitisation_GFL_v1/rules.drl

@@ -508,8 +508,8 @@ rule "102: Guidelines FileAttributes"
     when
         $section: Section(excludesTables, (containsString("DATA REQUIREMENT(S):") || containsString("TEST GUIDELINE(S):")) && (containsString("OECD") || containsString("EPA") || containsString("OPPTS")))
     then
-        RedactionSearchUtility.findBoundariesByRegex("OECD (No\\.? )?\\d{3}( \\(\\d{4}\\))?", $section.buildTextBlock()).stream()
+        RedactionSearchUtility.findBoundariesByRegex("OECD (No\\.? )?\\d{3}( \\(\\d{4}\\))?", $section.getTextBlock()).stream()
-            .map(boundary -> $section.buildTextBlock().subSequence(boundary).toString())
+            .map(boundary -> $section.getTextBlock().subSequence(boundary).toString())
             .map(value -> FileAttribute.builder().label("OECD Number").value(value).build())
             .forEach(fileAttribute -> insert(fileAttribute));
     end