RED-5381: Use new codestyle

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/Rectangle.java

@@ -14,4 +14,5 @@ public class Rectangle {
     private float height;
 
     private int page;
+
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/model/Page.java

@@ -1,15 +1,18 @@
 package com.iqser.red.service.redaction.v1.server.classification.model;
 
-import com.iqser.red.service.redaction.v1.server.redaction.model.PdfImage;
-import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
-import com.iqser.red.service.redaction.v1.server.tableextraction.model.Rectangle;
-import lombok.Data;
-import lombok.NonNull;
-import lombok.RequiredArgsConstructor;
-
 import java.util.ArrayList;
 import java.util.List;
 
+import org.apache.pdfbox.pdmodel.common.PDRectangle;
+
+import com.iqser.red.service.redaction.v1.model.Rectangle;
+import com.iqser.red.service.redaction.v1.server.redaction.model.PdfImage;
+import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
+
+import lombok.Data;
+import lombok.NonNull;
+import lombok.RequiredArgsConstructor;
+
 @Data
 @RequiredArgsConstructor
 public class Page {
@@ -32,6 +35,7 @@ public class Page {
     private StringFrequencyCounter fontStyleCounter = new StringFrequencyCounter();
 
     private double cropBoxArea;
+    private PDRectangle cropBox;
 
 
     public boolean isRotated() {

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/model/TextBlock.java

@@ -1,19 +1,21 @@
 package com.iqser.red.service.redaction.v1.server.classification.model;
 
+import java.util.ArrayList;
+import java.util.List;
+
 import com.dslplatform.json.CompiledJson;
 import com.dslplatform.json.JsonAttribute;
 import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.iqser.red.service.redaction.v1.server.parsing.model.TextDirection;
 import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.TextNormalizationUtilities;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
+
 import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;
 import lombok.NoArgsConstructor;
 
-import java.util.ArrayList;
-import java.util.List;
-
 @AllArgsConstructor
 @Builder
 @Data
@@ -45,12 +47,100 @@ public class TextBlock extends AbstractTextContainer {
     @JsonIgnore
     private float highestFontSize;
 
-
     @JsonIgnore
     private String classification;
 
 
+    private TextDirection getDir() {
+
+        return sequences.get(0).getDir();
+    }
+
+
+    private float getPageHeight() {
+
+        return sequences.get(0).getPageHeight();
+    }
+
+
+    private float getPageWidth() {
+
+        return sequences.get(0).getPageWidth();
+    }
+
+
+    @JsonIgnore
+    @JsonAttribute(ignore = true)
+    public float getX1() {
+
+        if (getDir().getDegrees() == 90) {
+            return minY;
+        } else if (getDir().getDegrees() == 180) {
+            return getPageWidth() - maxX;
+
+        } else if (getDir().getDegrees() == 270) {
+
+            return getPageWidth() - maxY;
+        } else {
+            return minX;
+        }
+    }
+
+
+    @JsonIgnore
+    @JsonAttribute(ignore = true)
+    public float getX2() {
+
+        if (getDir().getDegrees() == 90) {
+            return maxY;
+        } else if (getDir().getDegrees() == 180) {
+            return getPageWidth() - minX;
+        } else if (getDir().getDegrees() == 270) {
+            return getPageWidth() - minY;
+
+        } else {
+            return maxX;
+        }
+    }
+
+
+    @JsonIgnore
+    @JsonAttribute(ignore = true)
+    public float getY1() {
+
+        if (getDir().getDegrees() == 90) {
+            return minX;
+        } else if (getDir().getDegrees() == 180) {
+            return maxY;
+
+        } else if (getDir().getDegrees() == 270) {
+            return getPageHeight() - maxX;
+
+        } else {
+            return getPageHeight() - maxY;
+        }
+    }
+
+
+    @JsonIgnore
+    @JsonAttribute(ignore = true)
+    public float getY2() {
+
+        if (getDir().getDegrees() == 90) {
+            return maxX;
+        } else if (getDir().getDegrees() == 180) {
+
+            return minY;
+        } else if (getDir().getDegrees() == 270) {
+            return getPageHeight() - minX;
+        } else {
+            return getPageHeight() - minY;
+        }
+    }
+
+
     public TextBlock(float minX, float maxX, float minY, float maxY, List<TextPositionSequence> sequences, int rotation) {
+
         this.minX = minX;
         this.maxX = maxX;
         this.minY = minY;
@@ -59,19 +149,25 @@ public class TextBlock extends AbstractTextContainer {
         this.rotation = rotation;
     }
 
+
     public TextBlock union(TextPositionSequence r) {
+
         TextBlock union = this.copy();
         union.add(r);
         return union;
     }
 
+
     public TextBlock union(TextBlock r) {
+
         TextBlock union = this.copy();
         union.add(r);
         return union;
     }
 
+
     public void add(TextBlock r) {
+
         if (r.getMinX() < minX) {
             minX = r.getMinX();
         }
@@ -87,7 +183,9 @@ public class TextBlock extends AbstractTextContainer {
         sequences.addAll(r.getSequences());
     }
 
+
     public void add(TextPositionSequence r) {
+
         if (r.getX1() < minX) {
             minX = r.getX1();
         }
@@ -102,15 +200,21 @@ public class TextBlock extends AbstractTextContainer {
         }
     }
 
+
     public TextBlock copy() {
+
         return new TextBlock(minX, maxX, minY, maxY, sequences, rotation);
     }
 
+
     public void resize(float x1, float y1, float width, float height) {
+
         set(x1, y1, x1 + width, y1 + height);
     }
 
+
     public void set(float x1, float y1, float x2, float y2) {
+
         this.minX = Math.min(x1, x2);
         this.maxX = Math.max(x1, x2);
         this.minY = Math.min(y1, y2);
@@ -136,6 +240,7 @@ public class TextBlock extends AbstractTextContainer {
 
     }
 
+
     @Override
     @JsonIgnore
     @JsonAttribute(ignore = true)

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/service/BlockificationService.java

@@ -7,18 +7,21 @@ import java.util.Comparator;
 import java.util.Iterator;
 import java.util.List;
 
+import org.apache.pdfbox.pdmodel.common.PDRectangle;
 import org.springframework.stereotype.Service;
 
+import com.iqser.red.service.redaction.v1.model.Point;
+import com.iqser.red.service.redaction.v1.model.Rectangle;
 import com.iqser.red.service.redaction.v1.server.classification.model.FloatFrequencyCounter;
 import com.iqser.red.service.redaction.v1.server.classification.model.Orientation;
 import com.iqser.red.service.redaction.v1.server.classification.model.Page;
 import com.iqser.red.service.redaction.v1.server.classification.model.StringFrequencyCounter;
 import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
+import com.iqser.red.service.redaction.v1.server.classification.utils.CoordSystemHelper;
 import com.iqser.red.service.redaction.v1.server.classification.utils.PositionUtils;
 import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.Cell;
-import com.iqser.red.service.redaction.v1.server.tableextraction.model.Rectangle;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.Ruling;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;
 
@@ -29,10 +32,9 @@ public class BlockificationService {
     static final float THRESHOLD = 1f;
 
 
-    public Page blockify(List<TextPositionSequence> textPositions, List<Ruling> horizontalRulingLines,
+    public Page blockify(List<TextPositionSequence> textPositions, List<Ruling> horizontalRulingLines, List<Ruling> verticalRulingLines, int rotation, PDRectangle cropbox) {
-                         List<Ruling> verticalRulingLines) {
 
-        sortRotatedSequences(textPositions);
+//        sortRotatedSequences(textPositions);
 
         List<TextPositionSequence> chunkWords = new ArrayList<>();
         List<AbstractTextContainer> chunkBlockList1 = new ArrayList<>();
@@ -44,18 +46,45 @@ public class BlockificationService {
         Float splitX1 = null;
         for (TextPositionSequence word : textPositions) {
 
-            boolean lineSeparation = minY - word.getY2() > word.getHeight() * 1.25;
+            boolean lineSeparation = word.getY1() - maxY > word.getHeight() * 1.25;
-            boolean startFromTop = word.getY1() > maxY + word.getHeight();
+            boolean startFromTop = prev != null && word.getY1() < prev.getY1() - prev.getTextHeight();
             boolean splitByX = prev != null && maxX + 50 < word.getX1() && prev.getY1() == word.getY1();
+            boolean xIsBeforeFirstX = prev != null && word.getX1() < minX;
             boolean newLineAfterSplit = prev != null && word.getY1() != prev.getY1() && wasSplitted && splitX1 != word.getX1();
-            boolean splittedByRuling =
+            boolean splittedByRuling = isSplittedByRuling(maxX,
-                    isSplittedByRuling(maxX, minY, word.getX1(), word.getY1(), verticalRulingLines) ||
+                    minY,
-                    isSplittedByRuling(minX, minY, word.getX1(), word.getY2(), horizontalRulingLines)
+                    word.getX1(),
+                    word.getY1(),
+                    verticalRulingLines,
+                    word.getDir().getDegrees(),
+                    cropbox.getWidth(),
+                    cropbox.getHeight()) || isSplittedByRuling(minX,
+                    minY,
+                    word.getX1(),
+                    word.getY2(),
+                    horizontalRulingLines,
+                    word.getDir().getDegrees(),
+                    cropbox.getWidth(),
+                    cropbox.getHeight())
 
-                    || isSplittedByRuling(maxX, minY, word.getX1(), word.getY1(), horizontalRulingLines)
+                    || isSplittedByRuling(maxX,
-                    || isSplittedByRuling(minX, minY, word.getX1(), word.getY2(), verticalRulingLines);
+                    minY,
+                    word.getX1(),
+                    word.getY1(),
+                    horizontalRulingLines,
+                    word.getDir().getDegrees(),
+                    cropbox.getWidth(),
+                    cropbox.getHeight()) || isSplittedByRuling(minX,
+                    minY,
+                    word.getX1(),
+                    word.getY2(),
+                    verticalRulingLines,
+                    word.getDir().getDegrees(),
+                    cropbox.getWidth(),
+                    cropbox.getHeight());
+            boolean splitByDir = prev != null && !prev.getDir().equals(word.getDir());
 
-            if (prev != null && (lineSeparation || startFromTop || splitByX || newLineAfterSplit || splittedByRuling)) {
+            if (prev != null && (lineSeparation || startFromTop || xIsBeforeFirstX || splitByX || splitByDir || splittedByRuling)) {
 
                 Orientation prevOrientation = null;
                 if (!chunkBlockList1.isEmpty()) {
@@ -115,8 +144,7 @@ public class BlockificationService {
             TextBlock block = (TextBlock) itty.next();
 
             if (previousLeft != null && block.getOrientation().equals(Orientation.LEFT)) {
-                if (previousLeft.getMinY() > block.getMinY() && block.getMaxY() + block.getMostPopularWordHeight() > previousLeft
+                if (previousLeft.getMinY() > block.getMinY() && block.getMaxY() + block.getMostPopularWordHeight() > previousLeft.getMinY()) {
-                        .getMinY()) {
                     previousLeft.add(block);
                     itty.remove();
                     continue;
@@ -124,8 +152,7 @@ public class BlockificationService {
             }
 
             if (previousRight != null && block.getOrientation().equals(Orientation.RIGHT)) {
-                if (previousRight.getMinY() > block.getMinY() && block.getMaxY() + block.getMostPopularWordHeight() > previousRight
+                if (previousRight.getMinY() > block.getMinY() && block.getMaxY() + block.getMostPopularWordHeight() > previousRight.getMinY()) {
-                        .getMinY()) {
                     previousRight.add(block);
                     itty.remove();
                     continue;
@@ -144,10 +171,8 @@ public class BlockificationService {
         while (itty.hasNext()) {
             TextBlock block = (TextBlock) itty.next();
 
-            if (previous != null && previous.getOrientation().equals(Orientation.LEFT) && block.getOrientation()
+            if (previous != null && previous.getOrientation().equals(Orientation.LEFT) && block.getOrientation().equals(Orientation.LEFT) && equalsWithThreshold(block.getMaxY(),
-                    .equals(Orientation.LEFT) && equalsWithThreshold(block.getMaxY(), previous.getMaxY()) || previous != null && previous
+                    previous.getMaxY()) || previous != null && previous.getOrientation().equals(Orientation.LEFT) && block.getOrientation()
-                    .getOrientation()
-                    .equals(Orientation.LEFT) && block.getOrientation()
                     .equals(Orientation.RIGHT) && equalsWithThreshold(block.getMaxY(), previous.getMaxY())) {
                 previous.add(block);
                 itty.remove();
@@ -186,12 +211,10 @@ public class BlockificationService {
             styleFrequencyCounter.add(wordBlock.getFontStyle());
 
             if (textBlock == null) {
-                textBlock = new TextBlock(wordBlock.getX1(), wordBlock.getX2(), wordBlock.getY1(), wordBlock.getY2(), wordBlockList, wordBlock
+                textBlock = new TextBlock(wordBlock.getX1(), wordBlock.getX2(), wordBlock.getY1(), wordBlock.getY2(), wordBlockList, wordBlock.getRotation());
-                        .getRotation());
             } else {
                 TextBlock spatialEntity = textBlock.union(wordBlock);
-                textBlock.resize(spatialEntity.getMinX(), spatialEntity.getMinY(), spatialEntity.getWidth(), spatialEntity
+                textBlock.resize(spatialEntity.getMinX(), spatialEntity.getMinY(), spatialEntity.getWidth(), spatialEntity.getHeight());
-                        .getHeight());
             }
         }
 
@@ -204,22 +227,25 @@ public class BlockificationService {
             textBlock.setHighestFontSize(fontSizeFrequencyCounter.getHighest());
         }
 
-        if (textBlock != null && textBlock.getSequences() != null && textBlock.getSequences()
+        if (textBlock != null && textBlock.getSequences() != null && textBlock.getSequences().stream().map(t -> round(t.getY1(), 3)).collect(toSet()).size() == 1) {
-                .stream()
-                .map(t -> round(t.getY1(), 3))
-                .collect(toSet())
-                .size() == 1) {
             textBlock.getSequences().sort(Comparator.comparing(TextPositionSequence::getX1));
         }
         return textBlock;
     }
 
 
-    private boolean isSplittedByRuling(float previousX2, float previousY1, float currentX1, float currentY1,
+    private boolean isSplittedByRuling(float previousX2,
-                                       List<Ruling> rulingLines) {
+                                       float previousY1,
+                                       float currentX1,
+                                       float currentY1,
+                                       List<Ruling> rulingLines,
+                                       float rotation,
+                                       float pageWidth,
+                                       float pageHeight) {
 
         for (Ruling ruling : rulingLines) {
-            if (ruling.intersectsLine(previousX2, previousY1, currentX1, currentY1)) {
+            var line = CoordSystemHelper.convertToDirAdj(ruling, rotation, pageWidth, pageHeight);
+            if (line.intersectsLine(previousX2, previousY1, currentX1, currentY1)) {
                 return true;
             }
         }
@@ -227,8 +253,7 @@ public class BlockificationService {
     }
 
 
-    public Rectangle calculateBodyTextFrame(List<Page> pages, FloatFrequencyCounter documentFontSizeCounter,
+    public Rectangle calculateBodyTextFrame(List<Page> pages, FloatFrequencyCounter documentFontSizeCounter, boolean landscape) {
-                                            boolean landscape) {
 
         float minX = 10000;
         float maxX = -100;
@@ -257,19 +282,33 @@ public class BlockificationService {
                     if (documentFontSizeCounter.getMostPopular() != null) {
                         if (textBlock.getMostPopularWordFontSize() >= documentFontSizeCounter.getMostPopular()) {
 
-                            if (textBlock.getMinX() < minX) {
+                            if (page.getCropBox().getWidth() > page.getCropBox().getHeight() && page.getRotation() != 0) {
-                                minX = textBlock.getMinX();
+                                if (textBlock.getY1() < minX) {
+                                    minX = textBlock.getY1();
                                 }
-                            if (textBlock.getMaxX() > maxX) {
+                                if (textBlock.getY2() > maxX) {
-                                maxX = textBlock.getMaxX();
+                                    maxX = textBlock.getY2();
                                 }
-                            if (textBlock.getMinY() < minY) {
+                                if (textBlock.getX1() < minY) {
-                                minY = textBlock.getMinY();
+                                    minY = textBlock.getX1();
+                                }
+                                if (textBlock.getX2() > maxY) {
+                                    maxY = textBlock.getX2();
+                                }
+                            } else {
+                                if (textBlock.getX1() < minX) {
+                                    minX = textBlock.getX1();
+                                }
+                                if (textBlock.getX2() > maxX) {
+                                    maxX = textBlock.getX2();
+                                }
+                                if (textBlock.getY1() < minY) {
+                                    minY = textBlock.getY1();
+                                }
+                                if (textBlock.getY2() > maxY) {
+                                    maxY = textBlock.getY2();
                                 }
-                            if (textBlock.getMaxY() > maxY) {
-                                maxY = textBlock.getMaxY();
                             }
-
                         }
                     }
                 }
@@ -283,45 +322,42 @@ public class BlockificationService {
                                 continue;
                             }
                             for (TextBlock textBlock : cell.getTextBlocks()) {
-                                if (textBlock.getMinX() < minX) {
+                                if (page.getCropBox().getWidth() > page.getCropBox().getHeight() && page.getRotation() != 0) {
-                                    minX = textBlock.getMinX();
+                                    if (textBlock.getY1() < minX) {
+                                        minX = textBlock.getMinY();
                                     }
-                                if (textBlock.getMaxX() > maxX) {
+                                    if (textBlock.getY2() > maxX) {
-                                    maxX = textBlock.getMaxX();
+                                        maxX = textBlock.getY2();
                                     }
-                                if (textBlock.getMinY() < minY) {
+                                    if (textBlock.getX1() < minY) {
-                                    minY = textBlock.getMinY();
+                                        minY = textBlock.getX1();
                                     }
-                                if (textBlock.getMaxY() > maxY) {
+                                    if (textBlock.getX2() > maxY) {
-                                    maxY = textBlock.getMaxY();
+                                        maxY = textBlock.getX2();
+                                    }
+                                } else {
+                                    if (textBlock.getX1() < minX) {
+                                        minX = textBlock.getX1();
+                                    }
+                                    if (textBlock.getX2() > maxX) {
+                                        maxX = textBlock.getX2();
+                                    }
+                                    if (textBlock.getY1() < minY) {
+                                        minY = textBlock.getY1();
+                                    }
+                                    if (textBlock.getY2() > maxY) {
+                                        maxY = textBlock.getY2();
                                     }
                                 }
                             }
                         }
                     }
                 }
+            }
+            System.out.println("Page: " + page.getPageNumber() + " Landscape: " + page.isLandscape() + " MinX: " + minX + " MaxX: " + maxX + " MinY: " + minY + " MaxY: " + maxY);
 
         }
-        return new Rectangle(minY, minX, maxX - minX, maxY - minY);
+        return new Rectangle(new Point(minX, minY), maxX - minX, maxY - minY, 0);
-    }
-
-
-    private void sortRotatedSequences(List<TextPositionSequence> sequences) {
-
-        List<TextPositionSequence> rotatedWords = new ArrayList<>();
-        Iterator<TextPositionSequence> itty = sequences.iterator();
-        while (itty.hasNext()) {
-            var pos = itty.next();
-            if (pos.getTextPositions().get(0).getDir() == 270) {
-                rotatedWords.add(pos);
-                itty.remove();
-            }
-        }
-
-        if (!rotatedWords.isEmpty() && !sequences.isEmpty()) {
-            rotatedWords.sort(Comparator.comparing(TextPositionSequence::getX1));
-        }
-        sequences.addAll(rotatedWords);
     }
 
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/service/ClassificationService.java

@@ -1,17 +1,20 @@
 package com.iqser.red.service.redaction.v1.server.classification.service;
 
+import java.util.List;
+import java.util.regex.Pattern;
+
+import org.springframework.stereotype.Service;
+
+import com.iqser.red.service.redaction.v1.model.Point;
+import com.iqser.red.service.redaction.v1.model.Rectangle;
 import com.iqser.red.service.redaction.v1.server.classification.model.Document;
 import com.iqser.red.service.redaction.v1.server.classification.model.Page;
 import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
 import com.iqser.red.service.redaction.v1.server.classification.utils.PositionUtils;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
-import com.iqser.red.service.redaction.v1.server.tableextraction.model.Rectangle;
+
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
-import org.springframework.stereotype.Service;
-
-import java.util.List;
-import java.util.regex.Pattern;
 
 @Slf4j
 @Service
@@ -32,7 +35,23 @@ public class ClassificationService {
 
         for (Page page : document.getPages()) {
             Rectangle btf = page.isLandscape() ? landscapeBodyTextFrame : bodyTextFrame;
+
+            if (page.getCropBox().getWidth() > page.getCropBox().getHeight() && page.getRotation() == 270) {
+                btf = new Rectangle(new Point(btf.getTopLeft().getY(), page.getCropBox().getHeight() - btf.getTopLeft().getX() - btf.getWidth()),
+                        btf.getHeight(),
+                        btf.getWidth(),
+                        0);
+            } else if (page.getCropBox().getWidth() > page.getCropBox().getHeight() && page.getRotation() != 0) {
+                btf = new Rectangle(new Point(btf.getTopLeft().getY(), btf.getTopLeft().getX()), btf.getHeight(), btf.getWidth(), 0);
+            } else if (page.getRotation() == 180) {
+                btf = new Rectangle(new Point(btf.getTopLeft().getX(), page.getCropBox().getHeight() - btf.getTopLeft().getY() - btf.getHeight()),
+                        btf.getWidth(),
+                        btf.getHeight(),
+                        0);
+            }
+
             page.setBodyTextFrame(btf);
+
             classifyPage(btf, page, document, headlineFontSizes);
         }
     }
@@ -48,35 +67,43 @@ public class ClassificationService {
     }
 
 
-    public void classifyBlock(TextBlock textBlock, Rectangle bodyTextFrame, Page page, Document document,
+    public void classifyBlock(TextBlock textBlock, Rectangle bodyTextFrame, Page page, Document document, List<Float> headlineFontSizes) {
-                              List<Float> headlineFontSizes) {
 
-        if (document.getFontSizeCounter().getMostPopular() == null) {
+        System.out.println("Page: " + page.getPageNumber() + " rotation " + page.getRotation() + " B " + textBlock.getSequences().get(0).getDir());
-            textBlock.setClassification("Other");
+
-            return;
+//        if (document.getFontSizeCounter().getMostPopular() == null) {
-        }
+//            textBlock.setClassification("Other");
-        if (PositionUtils.isOverBodyTextFrame(bodyTextFrame, textBlock, page.isRotated()) && (document.getFontSizeCounter()
+//            return;
-                .getMostPopular() == null || textBlock.getHighestFontSize() <= document.getFontSizeCounter()
+//        }
-                .getMostPopular())) {
+        if (PositionUtils.isOverBodyTextFrame(bodyTextFrame, textBlock, page.getRotation()))
+//            && (document.getFontSizeCounter()
+//                .getMostPopular() == null || textBlock.getHighestFontSize() <= document.getFontSizeCounter()
+//                .getMostPopular()))
+        {
             textBlock.setClassification("Header");
 
-        } else if (PositionUtils.isUnderBodyTextFrame(bodyTextFrame, textBlock) && (document.getFontSizeCounter()
+        } else if (PositionUtils.isUnderBodyTextFrame(bodyTextFrame, textBlock, page.getRotation()))
-                .getMostPopular() == null || textBlock.getHighestFontSize() <= document.getFontSizeCounter()
+
-                .getMostPopular())) {
+//                   && (document.getFontSizeCounter()
+//                .getMostPopular() == null || textBlock.getHighestFontSize() <= document.getFontSizeCounter()
+//                .getMostPopular()))
+        {
             textBlock.setClassification("Footer");
-        } else if (page.getPageNumber() == 1 && (!PositionUtils.isTouchingUnderBodyTextFrame(bodyTextFrame, textBlock) && PositionUtils
+        } else if (page.getPageNumber() == 1 && (!PositionUtils.isTouchingUnderBodyTextFrame(bodyTextFrame,
-                .getHeightDifferenceBetweenChunkWordAndDocumentWord(textBlock, document.getTextHeightCounter()
+                textBlock) && PositionUtils.getHeightDifferenceBetweenChunkWordAndDocumentWord(textBlock,
-                        .getMostPopular()) > 2.5 && textBlock.getHighestFontSize() > document.getFontSizeCounter()
+                document.getTextHeightCounter().getMostPopular()) > 2.5 && textBlock.getHighestFontSize() > document.getFontSizeCounter().getMostPopular() || page.getTextBlocks()
-                .getMostPopular() || page.getTextBlocks().size() == 1)) {
+                .size() == 1)) {
             if (!Pattern.matches("[0-9]+", textBlock.toString())) {
                 textBlock.setClassification("Title");
             }
-        } else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordFontSize() > document
+        } else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordFontSize() > document.getFontSizeCounter()
-                .getFontSizeCounter()
+                .getMostPopular() && PositionUtils.getApproxLineCount(textBlock) < 4.9 && (textBlock.getMostPopularWordStyle().equals("bold") || !document.getFontStyleCounter()
-                .getMostPopular() && PositionUtils.getApproxLineCount(textBlock) < 4.9 && (textBlock.getMostPopularWordStyle()
+                .getCountPerValue()
-                .equals("bold") || !document.getFontStyleCounter().getCountPerValue().containsKey("bold") && textBlock.getMostPopularWordFontSize() > document
+                .containsKey("bold") && textBlock.getMostPopularWordFontSize() > document.getFontSizeCounter().getMostPopular() + 1) && textBlock.getSequences()
-                .getFontSizeCounter()
+                .get(0)
-                .getMostPopular() + 1) && textBlock.getSequences().get(0).getTextPositions().get(0).getFontSizeInPt() >= textBlock.getMostPopularWordFontSize()) {
+                .getTextPositions()
+                .get(0)
+                .getFontSizeInPt() >= textBlock.getMostPopularWordFontSize()) {
 
             for (int i = 1; i <= headlineFontSizes.size(); i++) {
                 if (textBlock.getMostPopularWordFontSize() == headlineFontSizes.get(i - 1)) {
@@ -84,28 +111,25 @@ public class ClassificationService {
                     document.setHeadlines(true);
                 }
             }
-        } else if (!textBlock.getText().startsWith("Table ") && !textBlock.getText()
+        } else if (!textBlock.getText().startsWith("Table ") && !textBlock.getText().startsWith("Figure ") && PositionUtils.isWithinBodyTextFrame(bodyTextFrame,
-                .startsWith("Figure ") && PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordStyle()
+                textBlock) && textBlock.getMostPopularWordStyle().equals("bold") && !document.getFontStyleCounter()
-                .equals("bold") && !document.getFontStyleCounter()
                 .getMostPopular()
-                .equals("bold") && PositionUtils.getApproxLineCount(textBlock) < 2.9 && textBlock.getSequences().get(0).getTextPositions().get(0).getFontSizeInPt() >= textBlock.getMostPopularWordFontSize()) {
+                .equals("bold") && PositionUtils.getApproxLineCount(textBlock) < 2.9 && textBlock.getSequences()
+                .get(0)
+                .getTextPositions()
+                .get(0)
+                .getFontSizeInPt() >= textBlock.getMostPopularWordFontSize()) {
             textBlock.setClassification("H " + (headlineFontSizes.size() + 1));
             document.setHeadlines(true);
-        } else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordFontSize() == document
+        } else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordFontSize() == document.getFontSizeCounter()
-                .getFontSizeCounter()
+                .getMostPopular() && textBlock.getMostPopularWordStyle().equals("bold") && !document.getFontStyleCounter().getMostPopular().equals("bold")) {
-                .getMostPopular() && textBlock.getMostPopularWordStyle()
-                .equals("bold") && !document.getFontStyleCounter().getMostPopular().equals("bold")) {
             textBlock.setClassification("TextBlock Bold");
         } else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordFont()
                 .equals(document.getFontCounter().getMostPopular()) && textBlock.getMostPopularWordStyle()
-                .equals(document.getFontStyleCounter()
+                .equals(document.getFontStyleCounter().getMostPopular()) && textBlock.getMostPopularWordFontSize() == document.getFontSizeCounter().getMostPopular()) {
-                        .getMostPopular()) && textBlock.getMostPopularWordFontSize() == document.getFontSizeCounter()
-                .getMostPopular()) {
             textBlock.setClassification("TextBlock");
-        } else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordFontSize() == document
+        } else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordFontSize() == document.getFontSizeCounter()
-                .getFontSizeCounter()
+                .getMostPopular() && textBlock.getMostPopularWordStyle().equals("italic") && !document.getFontStyleCounter()
-                .getMostPopular() && textBlock.getMostPopularWordStyle()
-                .equals("italic") && !document.getFontStyleCounter()
                 .getMostPopular()
                 .equals("italic") && PositionUtils.getApproxLineCount(textBlock) < 2.9) {
             textBlock.setClassification("TextBlock Italic");

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/utils/CoordSystemHelper.java

@@ -0,0 +1,75 @@
+package com.iqser.red.service.redaction.v1.server.classification.utils;
+
+import java.awt.geom.Line2D;
+import java.awt.geom.Point2D;
+
+import com.iqser.red.service.redaction.v1.model.Point;
+import com.iqser.red.service.redaction.v1.model.Rectangle;
+import com.iqser.red.service.redaction.v1.server.tableextraction.model.Ruling;
+
+import lombok.experimental.UtilityClass;
+
+@UtilityClass
+public class CoordSystemHelper {
+
+    public Rectangle convertToDirAdj(Rectangle rectangle, float rotation, float pageWidth, float pageHeight) {
+
+        var topLeft = convertPoint(rectangle.getTopLeft().getX(), rectangle.getTopLeft().getY() + rectangle.getHeight(), rotation, pageWidth, pageHeight);
+        var bottomRight = convertPoint(rectangle.getTopLeft().getX() + rectangle.getWidth(), rectangle.getTopLeft().getY(), rotation, pageWidth, pageHeight);
+
+        return new Rectangle(new Point((float) topLeft.getX(), (float) topLeft.getY()),
+                (float) (bottomRight.getX() - topLeft.getX()),
+                (float) (bottomRight.getY() - topLeft.getY()),
+                0);
+    }
+
+
+    public Line2D.Float convertToDirAdj(Ruling ruling, float rotation, float pageWidth, float pageHeight) {
+
+        return new Line2D.Float(convertPoint(ruling.x1, ruling.y1, rotation, pageWidth, pageHeight), convertPoint(ruling.x2, ruling.y2, rotation, pageWidth, pageHeight));
+    }
+
+
+    private Point2D convertPoint(float x, float y, float rotation, float pageWidth, float pageHeight) {
+
+        var xAdj = getXRot(x, y, rotation, pageWidth, pageHeight);
+        var yAdj = 0f;
+        if (rotation == 0 || rotation == 180) {
+            yAdj = pageHeight - getYLowerLeftRot(x, y, rotation, pageWidth, pageHeight);
+        } else {
+            yAdj = pageWidth - getYLowerLeftRot(x, y, rotation, pageWidth, pageHeight);
+        }
+        return new Point2D.Float(xAdj, yAdj);
+    }
+
+
+    private float getXRot(float x, float y, float rotation, float pageWidth, float pageHeight) {
+
+        if (rotation == 0) {
+            return x;
+        } else if (rotation == 90) {
+            return y;
+        } else if (rotation == 180) {
+            return pageWidth - x;
+        } else if (rotation == 270) {
+            return pageHeight - y;
+        }
+        return 0;
+    }
+
+
+    private float getYLowerLeftRot(float x, float y, float rotation, float pageWidth, float pageHeight) {
+
+        if (rotation == 0) {
+            return y;
+        } else if (rotation == 90) {
+            return pageWidth - x;
+        } else if (rotation == 180) {
+            return pageHeight - y;
+        } else if (rotation == 270) {
+            return x;
+        }
+        return 0;
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/utils/PositionUtils.java

@@ -1,14 +1,14 @@
 package com.iqser.red.service.redaction.v1.server.classification.utils;
 
+import com.iqser.red.service.redaction.v1.model.Rectangle;
 import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
-import com.iqser.red.service.redaction.v1.server.tableextraction.model.Rectangle;
+
 import lombok.experimental.UtilityClass;
 
 @UtilityClass
 @SuppressWarnings("all")
 public class PositionUtils {
 
-
     public boolean isWithinBodyTextFrame(Rectangle btf, TextBlock textBlock) {
 
         //TODO Currently this is not working for rotated pages.
@@ -19,10 +19,9 @@ public class PositionUtils {
 
         double threshold = textBlock.getMostPopularWordHeight() * 3;
 
-        if (textBlock.getMinX() + threshold > btf.getX() &&
+        if (textBlock.getMinX() + threshold > btf.getTopLeft().getX() && textBlock.getMaxX() - threshold < btf.getTopLeft()
-                textBlock.getMaxX() - threshold < btf.getX() + btf.getWidth() &&
+                .getX() + btf.getWidth() && textBlock.getMinY() + threshold > btf.getTopLeft().getY() && textBlock.getMaxY() - threshold < btf.getTopLeft()
-                textBlock.getMinY() + threshold > btf.getY() &&
+                .getY() + btf.getHeight()) {
-                textBlock.getMaxY() - threshold < btf.getY() + btf.getHeight()) {
             return true;
         } else {
             return false;
@@ -31,16 +30,25 @@ public class PositionUtils {
     }
 
 
-    public boolean isOverBodyTextFrame(Rectangle btf, TextBlock textBlock, boolean rotated) {
+    public boolean isOverBodyTextFrame(Rectangle btf, TextBlock textBlock, int rotation) {
 
         if (btf == null || textBlock == null) {
             return false;
         }
 
-        if (rotated && textBlock.getMinX() < btf.getX()) {
+        if (rotation == 90 && textBlock.getX2() < btf.getTopLeft().getX()) {
-            // Its very strange, P{0,0} is on top left in this case, instead of lower left.
             return true;
-        } else if (!rotated && textBlock.getMinY() > btf.getY() + btf.getHeight()) {
+        }
+
+        if (rotation == 180 && textBlock.getY2() < btf.getTopLeft().getY()) {
+            return true;
+        }
+
+        if (rotation == 270 && textBlock.getX1() > btf.getTopLeft().getX() + btf.getWidth()) {
+            return true;
+        }
+
+        if (rotation == 0 && textBlock.getY1() > btf.getTopLeft().getY() + btf.getHeight()) {
             return true;
         } else {
             return false;
@@ -49,15 +57,25 @@ public class PositionUtils {
     }
 
 
-    public boolean isUnderBodyTextFrame(Rectangle btf, TextBlock textBlock) {
+    public boolean isUnderBodyTextFrame(Rectangle btf, TextBlock textBlock, int rotation) {
-
-        //TODO Currently this is not working for rotated pages.
 
         if (btf == null || textBlock == null) {
             return false;
         }
 
-        if (textBlock.getMaxY() < btf.getY()) {
+        if (rotation == 90 && textBlock.getX1() > btf.getTopLeft().getX() + btf.getWidth()) {
+            return true;
+        }
+
+        if (rotation == 180 && textBlock.getY1() > btf.getTopLeft().getY() + btf.getHeight()) {
+            return true;
+        }
+
+        if (rotation == 270 && textBlock.getX2() < btf.getTopLeft().getX()) {
+            return true;
+        }
+
+        if (rotation == 0 && textBlock.getY2() < btf.getTopLeft().getY()) {
             return true;
         } else {
             return false;
@@ -74,7 +92,7 @@ public class PositionUtils {
             return false;
         }
 
-        if (textBlock.getMinY() < btf.getY()) {
+        if (textBlock.getMinY() < btf.getTopLeft().getY()) {
             return true;
         } else {
             return false;
@@ -84,11 +102,14 @@ public class PositionUtils {
 
 
     public float getHeightDifferenceBetweenChunkWordAndDocumentWord(TextBlock textBlock, Float documentMostPopularWordHeight) {
+
         return textBlock.getMostPopularWordHeight() - documentMostPopularWordHeight;
     }
 
 
     public Float getApproxLineCount(TextBlock textBlock) {
+
         return textBlock.getHeight() / textBlock.getMostPopularWordHeight();
     }
+
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/parsing/model/TextPositionSequence.java

@@ -147,30 +147,6 @@ public class TextPositionSequence implements CharSequence {
     }
 
 
-    @JsonIgnore
-    @JsonAttribute(ignore = true)
-    public float getX1() {
-
-        if (rotation == 90) {
-            return textPositions.get(0).getYDirAdj() - getTextHeight();
-        } else {
-            return textPositions.get(0).getXDirAdj();
-        }
-    }
-
-
-    @JsonIgnore
-    @JsonAttribute(ignore = true)
-    public float getX2() {
-
-        if (rotation == 90) {
-            return textPositions.get(0).getYDirAdj();
-        } else {
-            return textPositions.get(textPositions.size() - 1).getXDirAdj() + textPositions.get(textPositions.size() - 1).getWidthDirAdj() + HEIGHT_PADDING;
-        }
-    }
-
-
     @JsonIgnore
     @JsonAttribute(ignore = true)
     public float getRotationAdjustedY() {
@@ -179,15 +155,30 @@ public class TextPositionSequence implements CharSequence {
     }
 
 
+    @JsonIgnore
+    @JsonAttribute(ignore = true)
+    public float getX1() {
+
+        return textPositions.get(0).getXDirAdj();
+
+    }
+
+
+    @JsonIgnore
+    @JsonAttribute(ignore = true)
+    public float getX2() {
+
+        return textPositions.get(textPositions.size() - 1).getXDirAdj() + textPositions.get(textPositions.size() - 1).getWidthDirAdj() + HEIGHT_PADDING;
+
+    }
+
+
     @JsonIgnore
     @JsonAttribute(ignore = true)
     public float getY1() {
 
-        if (rotation == 90) {
+        return textPositions.get(0).getYDirAdj() - getTextHeight();
-            return textPositions.get(0).getXDirAdj();
+
-        } else {
-            return pageHeight - textPositions.get(0).getYDirAdj();
-        }
     }
 
 
@@ -195,11 +186,8 @@ public class TextPositionSequence implements CharSequence {
     @JsonAttribute(ignore = true)
     public float getY2() {
 
-        if (rotation == 90) {
+        return textPositions.get(0).getYDirAdj();
-            return textPositions.get(textPositions.size() - 1).getXDirAdj() + getTextHeight() - HEIGHT_PADDING;
+
-        } else {
-            return pageHeight - textPositions.get(0).getYDirAdj() + getTextHeight();
-        }
     }
 
 
@@ -304,7 +292,10 @@ public class TextPositionSequence implements CharSequence {
         topRight = transform.transform(topRight, null);
 
         return new Rectangle( //
-                new Point((float) bottomLeft.getX(), (float) bottomLeft.getY()), (float) (topRight.getX() - bottomLeft.getX()), (float) (topRight.getY() - bottomLeft.getY()), page);
+                new Point((float) bottomLeft.getX(), (float) bottomLeft.getY()),
+                (float) (topRight.getX() - bottomLeft.getX()),
+                (float) (topRight.getY() - bottomLeft.getY()),
+                page);
     }
 
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/segmentation/PdfSegmentationService.java

@@ -99,21 +99,27 @@ public class PdfSegmentationService {
                     stripper.getText(pdDocument);
 
                     PDRectangle pdr = pdPage.getMediaBox();
-                    boolean isLandscape = pdr.getWidth() > pdr.getHeight();
 
                     int rotation = pdPage.getRotation();
-                    boolean isRotated = rotation != 0 && rotation != 360;
+                    boolean isLandscape = pdr.getWidth() > pdr.getHeight() && (rotation == 0 || rotation == 180) || pdr.getHeight() > pdr.getWidth() && (rotation == 90 || rotation == 270);
-
-                    CleanRulings cleanRulings = rulingCleaningService.getCleanRulings(pdfTableCells.get(pageNumber), stripper.getRulings(), stripper.getMinCharWidth(), stripper.getMaxCharHeight());
-                    Page page = blockificationService.blockify(stripper.getTextPositionSequences(), cleanRulings.getHorizontal(), cleanRulings.getVertical());
 
                     PDRectangle cropbox = pdPage.getCropBox();
+                    CleanRulings cleanRulings = rulingCleaningService.getCleanRulings(pdfTableCells.get(pageNumber),
+                            stripper.getRulings(),
+                            stripper.getMinCharWidth(),
+                            stripper.getMaxCharHeight());
+                    Page page = blockificationService.blockify(stripper.getTextPositionSequences(), cleanRulings.getHorizontal(), cleanRulings.getVertical(), rotation, cropbox);
+
                     float cropboxArea = cropbox.getHeight() * cropbox.getWidth();
                     page.setCropBoxArea(cropboxArea);
 
+                    System.out.println("CropBox: w:" + cropbox.getWidth() + " h:" + cropbox.getHeight() + "MediaBox: w:" + pdPage.getMediaBox()
+                            .getWidth() + " h:" + pdPage.getMediaBox().getHeight());
+
                     page.setRotation(rotation);
-                    page.setLandscape(isLandscape || isRotated);
+                    page.setLandscape(isLandscape);
                     page.setPageNumber(pageNumber);
+                    page.setCropBox(cropbox);
 
                     tableExtractionService.extractTables(cleanRulings, page);
                     buildPageStatistics(page);

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/tableextraction/model/AbstractTextContainer.java

@@ -4,6 +4,8 @@ import com.dslplatform.json.JsonAttribute;
 import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.iqser.red.service.redaction.v1.model.Rectangle;
 import com.iqser.red.service.redaction.v1.server.classification.model.Orientation;
+import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
+
 import lombok.AllArgsConstructor;
 import lombok.Data;
 import lombok.NoArgsConstructor;
@@ -29,25 +31,41 @@ public abstract class AbstractTextContainer {
     @JsonIgnore
     private Orientation orientation = Orientation.NONE;
 
+
     public abstract String getText();
 
+
+    public boolean containsBlock(TextBlock other) {
+
+        return this.minX <= other.getX1() && this.maxX >= other.getX2() && this.minY >= other.getY1() && this.maxY <= other.getY2();
+    }
+
+
     public boolean contains(AbstractTextContainer other) {
+
         return this.minX <= other.minX && this.maxX >= other.maxX && this.minY >= other.minY && this.maxY <= other.maxY;
     }
 
+
     public boolean contains(Rectangle other) {
-        return page == other.getPage() && this.minX <= other.getTopLeft().getX() && this.maxX >= other.getTopLeft().getX() + other.getWidth() && this.minY <= other.getTopLeft().getY() && this.maxY >= other.getTopLeft().getY() + other.getHeight();
+
+        return page == other.getPage() && this.minX <= other.getTopLeft().getX() && this.maxX >= other.getTopLeft().getX() + other.getWidth() && this.minY <= other.getTopLeft()
+                .getY() && this.maxY >= other.getTopLeft().getY() + other.getHeight();
     }
 
+
     @JsonIgnore
     @JsonAttribute(ignore = true)
     public float getHeight() {
+
         return maxY - minY;
     }
 
+
     @JsonIgnore
     @JsonAttribute(ignore = true)
     public float getWidth() {
+
         return maxX - minX;
     }
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/tableextraction/model/Rectangle.java

@@ -31,6 +31,10 @@ public class Rectangle extends Rectangle2D.Float {
         }
     };
 
+
+
+
+
     public Rectangle() {
         super();
     }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/tableextraction/service/TableExtractionService.java

@@ -1,14 +1,27 @@
 package com.iqser.red.service.redaction.v1.server.tableextraction.service;
 
+import java.awt.geom.Point2D;
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import org.springframework.stereotype.Service;
+
 import com.iqser.red.service.redaction.v1.server.classification.model.Page;
 import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
-import com.iqser.red.service.redaction.v1.server.tableextraction.model.*;
+import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
+import com.iqser.red.service.redaction.v1.server.tableextraction.model.Cell;
+import com.iqser.red.service.redaction.v1.server.tableextraction.model.CleanRulings;
+import com.iqser.red.service.redaction.v1.server.tableextraction.model.Rectangle;
+import com.iqser.red.service.redaction.v1.server.tableextraction.model.Ruling;
+import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;
 import com.iqser.red.service.redaction.v1.server.tableextraction.utils.Utils;
-import org.springframework.stereotype.Service;
-
-import java.awt.geom.Point2D;
-import java.util.*;
-import java.util.stream.Collectors;
 
 @Service
 public class TableExtractionService {
@@ -52,6 +65,7 @@ public class TableExtractionService {
         return rv;
     };
 
+
     public void extractTables(CleanRulings cleanRulings, Page page) {
 
         List<Cell> cells = findCells(cleanRulings.getHorizontal(), cleanRulings.getVertical());
@@ -61,7 +75,7 @@ public class TableExtractionService {
         for (AbstractTextContainer abstractTextContainer : page.getTextBlocks()) {
             TextBlock textBlock = (TextBlock) abstractTextContainer;
             for (Cell cell : cells) {
-                if (cell.intersects(textBlock.getMinX(), textBlock.getMinY(), textBlock.getWidth(), textBlock.getHeight())) {
+                if (cell.intersects(textBlock.getX1(), textBlock.getY1(), textBlock.getX2() - textBlock.getX1(), textBlock.getY2() - textBlock.getY1())) {
                     cell.addTextBlock(textBlock);
                     toBeRemoved.add(textBlock);
                     break;
@@ -72,9 +86,7 @@ public class TableExtractionService {
         cells = new ArrayList<>(new HashSet<>(cells));
         Utils.sort(cells, Rectangle.ILL_DEFINED_ORDER);
 
-        List<Rectangle> spreadsheetAreas = findSpreadsheetsFromCells(cells).stream()
+        List<Rectangle> spreadsheetAreas = findSpreadsheetsFromCells(cells).stream().filter(r -> r.getWidth() > 0f && r.getHeight() > 0f).collect(Collectors.toList());
-                .filter(r -> r.getWidth() > 0f && r.getHeight() > 0f)
-                .collect(Collectors.toList());
 
         List<Table> tables = new ArrayList<>();
         for (Rectangle area : spreadsheetAreas) {
@@ -94,7 +106,7 @@ public class TableExtractionService {
             Iterator<AbstractTextContainer> itty = page.getTextBlocks().iterator();
             while (itty.hasNext()) {
                 AbstractTextContainer textBlock = itty.next();
-                if (table.contains(textBlock) && position == -1) {
+                if (textBlock instanceof TextBlock ? table.containsBlock((TextBlock) textBlock) : table.contains(textBlock) && position == -1) {
                     position = page.getTextBlocks().indexOf(textBlock);
                 }
             }
@@ -106,6 +118,7 @@ public class TableExtractionService {
         page.getTextBlocks().removeAll(toBeRemoved);
     }
 
+
     public List<Cell> findCells(List<Ruling> horizontalRulingLines, List<Ruling> verticalRulingLines) {
 
         List<Cell> cellsFound = new ArrayList<>();
@@ -142,8 +155,8 @@ public class TableExtractionService {
                         continue;
                     }
                     Point2D btmRight = new Point2D.Float((float) yPoint.getX(), (float) xPoint.getY());
-                    if (intersectionPoints.containsKey(btmRight) && intersectionPoints.get(btmRight)[0].equals(intersectionPoints
+                    if (intersectionPoints.containsKey(btmRight) && intersectionPoints.get(btmRight)[0].equals(intersectionPoints.get(xPoint)[0]) && intersectionPoints.get(btmRight)[1].equals(
-                            .get(xPoint)[0]) && intersectionPoints.get(btmRight)[1].equals(intersectionPoints.get(yPoint)[1])) {
+                            intersectionPoints.get(yPoint)[1])) {
                         cellsFound.add(new Cell(topLeft, btmRight));
                         break outer;
                     }
@@ -158,6 +171,7 @@ public class TableExtractionService {
         return cellsFound;
     }
 
+
     private List<Rectangle> findSpreadsheetsFromCells(List<? extends Rectangle> cells) {
         // via: http://stackoverflow.com/questions/13746284/merging-multiple-adjacent-rectangles-into-one-polygon
         List<Rectangle> rectangles = new ArrayList<>();
@@ -257,8 +271,10 @@ public class TableExtractionService {
         return rectangles;
     }
 
+
     private enum Direction {
-        HORIZONTAL, VERTICAL
+        HORIZONTAL,
+        VERTICAL
     }
 
     static class PolygonVertex {
@@ -297,8 +313,7 @@ public class TableExtractionService {
         @Override
         public String toString() {
 
-            return String.format("%s[point=%s,direction=%s]", this.getClass()
+            return String.format("%s[point=%s,direction=%s]", this.getClass().getName(), this.point.toString(), this.direction.toString());
-                    .getName(), this.point.toString(), this.direction.toString());
         }
 
     }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/visualization/service/PdfVisualisationService.java

@@ -1,5 +1,15 @@
 package com.iqser.red.service.redaction.v1.server.visualization.service;
 
+import java.awt.Color;
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.pdfbox.pdmodel.PDDocument;
+import org.apache.pdfbox.pdmodel.PDPage;
+import org.apache.pdfbox.pdmodel.PDPageContentStream;
+import org.apache.pdfbox.pdmodel.font.PDType1Font;
+import org.springframework.stereotype.Service;
+
 import com.iqser.red.service.redaction.v1.server.classification.model.Document;
 import com.iqser.red.service.redaction.v1.server.classification.model.Page;
 import com.iqser.red.service.redaction.v1.server.classification.model.Paragraph;
@@ -7,24 +17,15 @@ import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.Cell;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;
+
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
-import org.apache.pdfbox.pdmodel.PDDocument;
-import org.apache.pdfbox.pdmodel.PDPage;
-import org.apache.pdfbox.pdmodel.PDPageContentStream;
-import org.apache.pdfbox.pdmodel.font.PDType1Font;
-import org.springframework.stereotype.Service;
-
-import java.awt.Color;
-import java.io.IOException;
-import java.util.List;
 
 @Slf4j
 @Service
 @RequiredArgsConstructor
 public class PdfVisualisationService {
 
-
     public void visualizeParagraphs(Document classifiedDoc, PDDocument document) throws IOException {
 
         for (int page = 1; page <= document.getNumberOfPages(); page++) {
@@ -66,7 +67,6 @@ public class PdfVisualisationService {
             PDPage pdPage = document.getPage(page - 1);
             PDPageContentStream contentStream = new PDPageContentStream(document, pdPage, PDPageContentStream.AppendMode.APPEND, true);
 
-
             for (AbstractTextContainer textBlock : analyzedPage.getTextBlocks()) {
                 if (textBlock == null) {
                     continue;
@@ -79,7 +79,16 @@ public class PdfVisualisationService {
             }
 
             contentStream.setStrokingColor(Color.YELLOW);
-            contentStream.addRect((float) analyzedPage.getBodyTextFrame().getX(), (float) analyzedPage.getBodyTextFrame().getY(), (float) analyzedPage.getBodyTextFrame().getWidth(), (float) analyzedPage.getBodyTextFrame().getHeight());
+//            var width = analyzedPage.getBodyTextFrame().getX2(pdPage.getRotation(), pdPage.getCropBox().getWidth(), pdPage.getCropBox().getHeight()) -  analyzedPage.getBodyTextFrame().getX1(pdPage.getRotation(), pdPage.getCropBox().getWidth(), pdPage.getCropBox().getHeight());
+//            var height = analyzedPage.getBodyTextFrame().getY2(pdPage.getRotation(), pdPage.getCropBox().getWidth(), pdPage.getCropBox().getHeight()) -  analyzedPage.getBodyTextFrame().getY1(pdPage.getRotation(), pdPage.getCropBox().getWidth(), pdPage.getCropBox().getHeight());
+//
+//            contentStream.addRect(analyzedPage.getBodyTextFrame().getX1(pdPage.getRotation(), pdPage.getCropBox().getWidth(), pdPage.getCropBox().getHeight()),  analyzedPage.getBodyTextFrame().getY1(pdPage.getRotation(), pdPage.getCropBox().getWidth(), pdPage.getCropBox().getHeight()), width, height);
+
+            contentStream.addRect(analyzedPage.getBodyTextFrame().getTopLeft().getX(),
+                    analyzedPage.getBodyTextFrame().getTopLeft().getY(),
+                    analyzedPage.getBodyTextFrame().getWidth(),
+                    analyzedPage.getBodyTextFrame().getHeight());
+
             contentStream.stroke();
 
             contentStream.close();
@@ -91,25 +100,49 @@ public class PdfVisualisationService {
 
         contentStream.setStrokingColor(Color.RED);
 
-        contentStream.addRect(textBlock.getMinX(), textBlock.getMinY(), textBlock.getWidth(), textBlock.getHeight());
+        contentStream.addRect(textBlock.getX1(), textBlock.getY1(), textBlock.getX2() - textBlock.getX1(), textBlock.getY2() - textBlock.getY1());
         contentStream.stroke();
 
         if (textBlock.getClassification() != null) {
             contentStream.beginText();
 
             contentStream.setNonStrokingColor(Color.BLUE);
-            contentStream.setFont(PDType1Font.TIMES_ROMAN, 12f);
+            contentStream.setFont(PDType1Font.TIMES_ROMAN, 9f);
 
-            contentStream.newLineAtOffset(textBlock.getMinX(), textBlock.getMaxY());
+            contentStream.newLineAtOffset(textBlock.getX1(), textBlock.getY2() + 2);
-
+            contentStream.showText(textBlock.getClassification() + textBlock.getOrientation() + "-->" + textBlock.getSequences().get(0).getDir());
-            contentStream.showText(textBlock.getClassification() + textBlock.getOrientation());
 
             contentStream.endText();
+
+//            contentStream.setNonStrokingColor(Color.BLUE);
+//            contentStream.setFont(PDType1Font.TIMES_ROMAN, 2f);
+//
+//
+//
+//            contentStream.beginText();
+//            contentStream.newLineAtOffset(textBlock.getX1(), textBlock.getY1());
+//            contentStream.showText("MinX,MinY(" + textBlock.getX1() + "," + textBlock.getY1() + ")");
+//            contentStream.endText();
+//            contentStream.beginText();
+//            contentStream.newLineAtOffset(textBlock.getX2(), textBlock.getY1());
+//            contentStream.showText("MaxX,MinY(" + textBlock.getX2() + "," + textBlock.getY1() + ")");
+//            contentStream.endText();
+//            contentStream.beginText();
+//            contentStream.newLineAtOffset(textBlock.getX1(), textBlock.getY2());
+//            contentStream.showText("MinX,MaxY(" + textBlock.getX1() + "," + textBlock.getY2() + ")");
+//            contentStream.endText();
+//            contentStream.beginText();
+//            contentStream.newLineAtOffset(textBlock.getX2(), textBlock.getY2());
+//            contentStream.showText("MaxX,MaxY(" + textBlock.getX2() + "," + textBlock.getY2() + ")");
+//
+//
+//            contentStream.endText();
         }
     }
 
 
     private void visualizeTable(Table table, PDPageContentStream contentStream) throws IOException {
+
         for (List<Cell> row : table.getRows()) {
             for (Cell cell : row) {
 
@@ -120,7 +153,7 @@ public class PdfVisualisationService {
 
                     contentStream.setStrokingColor(Color.GREEN);
                     for (TextBlock textBlock : cell.getTextBlocks()) {
-                        contentStream.addRect(textBlock.getMinX(), textBlock.getMinY(), textBlock.getWidth(), textBlock.getHeight());
+                        contentStream.addRect(textBlock.getX1(), textBlock.getY1(), textBlock.getX2() - textBlock.getX1(), textBlock.getY2() - textBlock.getY1());
                         contentStream.stroke();
                     }
                 }
@@ -140,4 +173,5 @@ public class PdfVisualisationService {
             contentStream.endText();
         }
     }
+
 }

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java

@@ -183,8 +183,7 @@ public class RedactionIntegrationTest {
 
             KieFileSystem kieFileSystem = kieServices.newKieFileSystem();
             InputStream input = new ByteArrayInputStream(RULES.getBytes(StandardCharsets.UTF_8));
-            kieFileSystem.write("src/test/resources/drools/rules.drl", kieServices.getResources()
+            kieFileSystem.write("src/test/resources/drools/rules.drl", kieServices.getResources().newInputStreamResource(input));
-                    .newInputStreamResource(input));
             KieBuilder kieBuilder = kieServices.newKieBuilder(kieFileSystem);
             kieBuilder.buildAll();
             KieModule kieModule = kieBuilder.getKieModule();
@@ -245,16 +244,25 @@ public class RedactionIntegrationTest {
 
     private void mockDictionaryCalls(Long version) {
 
-        when(dictionaryClient.getDictionaryForType(VERTEBRATE + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(VERTEBRATE, false));
+        when(dictionaryClient.getDictionaryForType(VERTEBRATE + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(VERTEBRATE,
+                false));
         when(dictionaryClient.getDictionaryForType(ADDRESS + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(ADDRESS, false));
         when(dictionaryClient.getDictionaryForType(AUTHOR + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(AUTHOR, false));
         when(dictionaryClient.getDictionaryForType(SPONSOR + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(SPONSOR, false));
-        when(dictionaryClient.getDictionaryForType(NO_REDACTION_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(NO_REDACTION_INDICATOR, false));
+        when(dictionaryClient.getDictionaryForType(NO_REDACTION_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(
-        when(dictionaryClient.getDictionaryForType(REDACTION_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(REDACTION_INDICATOR, false));
+                NO_REDACTION_INDICATOR,
+                false));
+        when(dictionaryClient.getDictionaryForType(REDACTION_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(
+                REDACTION_INDICATOR,
+                false));
         when(dictionaryClient.getDictionaryForType(HINT_ONLY + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(HINT_ONLY, false));
-        when(dictionaryClient.getDictionaryForType(MUST_REDACT + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(MUST_REDACT, false));
+        when(dictionaryClient.getDictionaryForType(MUST_REDACT + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(MUST_REDACT,
-        when(dictionaryClient.getDictionaryForType(PUBLISHED_INFORMATION + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(PUBLISHED_INFORMATION, false));
+                false));
-        when(dictionaryClient.getDictionaryForType(TEST_METHOD + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(TEST_METHOD, false));
+        when(dictionaryClient.getDictionaryForType(PUBLISHED_INFORMATION + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(
+                PUBLISHED_INFORMATION,
+                false));
+        when(dictionaryClient.getDictionaryForType(TEST_METHOD + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(TEST_METHOD,
+                false));
         when(dictionaryClient.getDictionaryForType(PII + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(PII, false));
         when(dictionaryClient.getDictionaryForType(PURITY + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(PURITY, false));
         when(dictionaryClient.getDictionaryForType(IMAGE + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(IMAGE, false));
@@ -262,9 +270,14 @@ public class RedactionIntegrationTest {
         when(dictionaryClient.getDictionaryForType(LOGO + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(LOGO, false));
         when(dictionaryClient.getDictionaryForType(SIGNATURE + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(SIGNATURE, false));
         when(dictionaryClient.getDictionaryForType(FORMULA + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(FORMULA, false));
-        when(dictionaryClient.getDictionaryForType(ROTATE_SIMPLE + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(ROTATE_SIMPLE, false));
+        when(dictionaryClient.getDictionaryForType(ROTATE_SIMPLE + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(ROTATE_SIMPLE,
-        when(dictionaryClient.getDictionaryForType(DOSSIER_REDACTIONS + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(DOSSIER_REDACTIONS, true));
+                false));
-        when(dictionaryClient.getDictionaryForType(IMPORTED_REDACTION + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(IMPORTED_REDACTION, true));
+        when(dictionaryClient.getDictionaryForType(DOSSIER_REDACTIONS + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(
+                DOSSIER_REDACTIONS,
+                true));
+        when(dictionaryClient.getDictionaryForType(IMPORTED_REDACTION + ":" + TEST_DOSSIER_TEMPLATE_ID, version)).then((Answer<Type>) invocation -> getDictionaryResponse(
+                IMPORTED_REDACTION,
+                true));
 
     }
 
@@ -312,10 +325,7 @@ public class RedactionIntegrationTest {
         dictionary.get(AUTHOR).add("Drinking water");
         when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(1L);
 
-        AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder()
+        AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder().dossierId(TEST_DOSSIER_ID).fileId(TEST_FILE_ID).build());
-                .dossierId(TEST_DOSSIER_ID)
-                .fileId(TEST_FILE_ID)
-                .build());
 
         try (FileOutputStream fileOutputStream = new FileOutputStream(OsUtils.getTemporaryDirectory() + "/Annotated3.pdf")) {
             fileOutputStream.write(annotateResponse.getDocument());
@@ -345,16 +355,12 @@ public class RedactionIntegrationTest {
 
         var redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID);
 
-        var values = redactionLog.getRedactionLogEntry()
+        var values = redactionLog.getRedactionLogEntry().stream().map(RedactionLogEntry::getValue).collect(Collectors.toList());
-                .stream()
-                .map(RedactionLogEntry::getValue)
-                .collect(Collectors.toList());
 
         assertThat(values).containsExactlyInAnyOrder("Lastname M.", "Doe", "Doe J.", "M. Mustermann", "Mustermann M.", "F. Lastname");
     }
 
 
-
     @Test
     public void titleExtraction() throws IOException {
 
@@ -367,10 +373,7 @@ public class RedactionIntegrationTest {
 
         var text = redactionStorageService.getText(TEST_DOSSIER_ID, TEST_FILE_ID);
 
-        AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder()
+        AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder().dossierId(TEST_DOSSIER_ID).fileId(TEST_FILE_ID).build());
-                .dossierId(TEST_DOSSIER_ID)
-                .fileId(TEST_FILE_ID)
-                .build());
 
         String outputFileName = OsUtils.getTemporaryDirectory() + "/Annotated.pdf";
 
@@ -381,8 +384,6 @@ public class RedactionIntegrationTest {
     }
 
 
-
-
     @Test
     @Ignore
     @SneakyThrows
@@ -417,18 +418,10 @@ public class RedactionIntegrationTest {
                 .fileId(TEST_FILE_ID)
                 .build());
 
-        var cbiAddressBeforeHintRemoval = redactionLog.getRedactionLogEntry()
+        var cbiAddressBeforeHintRemoval = redactionLog.getRedactionLogEntry().stream().filter(re -> re.getType().equalsIgnoreCase("CBI_Address")).findAny().get();
-                .stream()
-                .filter(re -> re.getType().equalsIgnoreCase("CBI_Address"))
-                .findAny()
-                .get();
         assertThat(cbiAddressBeforeHintRemoval.isRedacted()).isFalse();
 
-        var cbiAddressAfterHintRemoval = mergedRedactionLog.getRedactionLogEntry()
+        var cbiAddressAfterHintRemoval = mergedRedactionLog.getRedactionLogEntry().stream().filter(re -> re.getType().equalsIgnoreCase("CBI_Address")).findAny().get();
-                .stream()
-                .filter(re -> re.getType().equalsIgnoreCase("CBI_Address"))
-                .findAny()
-                .get();
         assertThat(cbiAddressAfterHintRemoval.isRedacted()).isTrue();
     }
 
@@ -526,8 +519,7 @@ public class RedactionIntegrationTest {
                     continue loop;
                 }
                 if (redactionLogEntry.getSectionNumber() == sectionText.getSectionNumber()) {
-                    String value = sectionText.getText()
+                    String value = sectionText.getText().substring(redactionLogEntry.getStartOffset(), redactionLogEntry.getEndOffset());
-                            .substring(redactionLogEntry.getStartOffset(), redactionLogEntry.getEndOffset());
                     if (redactionLogEntry.getValue().equalsIgnoreCase(value)) {
                         correctFound++;
                     } else {
@@ -573,10 +565,7 @@ public class RedactionIntegrationTest {
         end = System.currentTimeMillis();
         System.out.println("reanalysis analysis duration: " + (end - start));
 
-        AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder()
+        AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder().dossierId(TEST_DOSSIER_ID).fileId(TEST_FILE_ID).build());
-                .dossierId(TEST_DOSSIER_ID)
-                .fileId(TEST_FILE_ID)
-                .build());
 
         try (FileOutputStream fileOutputStream = new FileOutputStream(outputFileName)) {
             fileOutputStream.write(annotateResponse.getDocument());
@@ -627,19 +616,11 @@ public class RedactionIntegrationTest {
 
         var redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID);
 
-        var changes = redactionLog.getRedactionLogEntry()
+        var changes = redactionLog.getRedactionLogEntry().stream().filter(entry -> entry.getValue() != null && entry.getValue().equals("report")).findFirst().get().getChanges();
-                .stream()
-                .filter(entry -> entry.getValue() != null && entry.getValue().equals("report"))
-                .findFirst()
-                .get()
-                .getChanges();
 
         assertThat(changes.size()).isEqualTo(2);
 
-        AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder()
+        AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder().dossierId(TEST_DOSSIER_ID).fileId(TEST_FILE_ID).build());
-                .dossierId(TEST_DOSSIER_ID)
-                .fileId(TEST_FILE_ID)
-                .build());
 
         try (FileOutputStream fileOutputStream = new FileOutputStream(outputFileName)) {
             fileOutputStream.write(annotateResponse.getDocument());
@@ -690,8 +671,7 @@ public class RedactionIntegrationTest {
                     continue loop;
                 }
                 if (redactionLogEntry.getSectionNumber() == sectionText.getSectionNumber()) {
-                    String value = sectionText.getText()
+                    String value = sectionText.getText().substring(redactionLogEntry.getStartOffset(), redactionLogEntry.getEndOffset());
-                            .substring(redactionLogEntry.getStartOffset(), redactionLogEntry.getEndOffset());
                     if (redactionLogEntry.getValue().equalsIgnoreCase(value)) {
                         correctFound++;
                     } else {
@@ -740,10 +720,7 @@ public class RedactionIntegrationTest {
         end = System.currentTimeMillis();
         System.out.println("reanalysis analysis duration: " + (end - start));
 
-        AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder()
+        AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder().dossierId(TEST_DOSSIER_ID).fileId(TEST_FILE_ID).build());
-                .dossierId(TEST_DOSSIER_ID)
-                .fileId(TEST_FILE_ID)
-                .build());
 
         try (FileOutputStream fileOutputStream = new FileOutputStream(outputFileName)) {
             fileOutputStream.write(annotateResponse.getDocument());
@@ -840,10 +817,7 @@ public class RedactionIntegrationTest {
         analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
         AnalyzeResult result = analyzeService.analyze(request);
 
-        AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder()
+        AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder().dossierId(TEST_DOSSIER_ID).fileId(TEST_FILE_ID).build());
-                .dossierId(TEST_DOSSIER_ID)
-                .fileId(TEST_FILE_ID)
-                .build());
 
         try (FileOutputStream fileOutputStream = new FileOutputStream(OsUtils.getTemporaryDirectory() + "/Annotated.pdf")) {
             fileOutputStream.write(annotateResponse.getDocument());
@@ -867,10 +841,7 @@ public class RedactionIntegrationTest {
         analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
         AnalyzeResult result = analyzeService.analyze(request);
 
-        AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder()
+        AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder().dossierId(TEST_DOSSIER_ID).fileId(TEST_FILE_ID).build());
-                .dossierId(TEST_DOSSIER_ID)
-                .fileId(TEST_FILE_ID)
-                .build());
 
         try (FileOutputStream fileOutputStream = new FileOutputStream(OsUtils.getTemporaryDirectory() + "/Annotated.pdf")) {
             fileOutputStream.write(annotateResponse.getDocument());
@@ -894,10 +865,7 @@ public class RedactionIntegrationTest {
         analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
         AnalyzeResult result = analyzeService.analyze(request);
 
-        AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder()
+        AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder().dossierId(TEST_DOSSIER_ID).fileId(TEST_FILE_ID).build());
-                                                                                      .dossierId(TEST_DOSSIER_ID)
-                                                                                      .fileId(TEST_FILE_ID)
-                                                                                      .build());
 
         try (FileOutputStream fileOutputStream = new FileOutputStream(OsUtils.getTemporaryDirectory() + "/Annotated.pdf")) {
             fileOutputStream.write(annotateResponse.getDocument());
@@ -918,10 +886,7 @@ public class RedactionIntegrationTest {
         analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
         AnalyzeResult result = analyzeService.analyze(request);
 
-        AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder()
+        AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder().dossierId(TEST_DOSSIER_ID).fileId(TEST_FILE_ID).build());
-                .dossierId(TEST_DOSSIER_ID)
-                .fileId(TEST_FILE_ID)
-                .build());
 
         try (FileOutputStream fileOutputStream = new FileOutputStream(OsUtils.getTemporaryDirectory() + "/Annotated.pdf")) {
             fileOutputStream.write(annotateResponse.getDocument());
@@ -943,10 +908,7 @@ public class RedactionIntegrationTest {
         analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
         AnalyzeResult result = analyzeService.analyze(request);
 
-        AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder()
+        AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder().dossierId(TEST_DOSSIER_ID).fileId(TEST_FILE_ID).build());
-                .dossierId(TEST_DOSSIER_ID)
-                .fileId(TEST_FILE_ID)
-                .build());
 
         try (FileOutputStream fileOutputStream = new FileOutputStream(OsUtils.getTemporaryDirectory() + "/Annotated.pdf")) {
             fileOutputStream.write(annotateResponse.getDocument());
@@ -957,8 +919,10 @@ public class RedactionIntegrationTest {
         System.out.println("numberOfPages: " + result.getNumberOfPages());
     }
 
+
     @Test
     public void testRotationsSimple() throws IOException {
+
         loadOnlyDictionaryForSimpleFile();
         mockDictionaryCalls(null);
         mockDictionaryCalls(0L);
@@ -970,10 +934,7 @@ public class RedactionIntegrationTest {
         analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
         AnalyzeResult result = analyzeService.analyze(request);
 
-        AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder()
+        AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder().dossierId(TEST_DOSSIER_ID).fileId(TEST_FILE_ID).build());
-                .dossierId(TEST_DOSSIER_ID)
-                .fileId(TEST_FILE_ID)
-                .build());
 
         try (FileOutputStream fileOutputStream = new FileOutputStream(OsUtils.getTemporaryDirectory() + "/AnnotatedSimple.pdf")) {
             fileOutputStream.write(annotateResponse.getDocument());
@@ -995,10 +956,7 @@ public class RedactionIntegrationTest {
         analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
         AnalyzeResult result = analyzeService.analyze(request);
 
-        AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder()
+        AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder().dossierId(TEST_DOSSIER_ID).fileId(TEST_FILE_ID).build());
-                .dossierId(TEST_DOSSIER_ID)
-                .fileId(TEST_FILE_ID)
-                .build());
 
         try (FileOutputStream fileOutputStream = new FileOutputStream(OsUtils.getTemporaryDirectory() + "/Annotated.pdf")) {
             fileOutputStream.write(annotateResponse.getDocument());
@@ -1014,7 +972,6 @@ public class RedactionIntegrationTest {
     }
 
 
-
     @Test
     public void testFindDictionaryEntryInResizedEntryPosition() throws IOException {
 
@@ -1034,7 +991,6 @@ public class RedactionIntegrationTest {
 
         ManualRedactions manualRedactions = new ManualRedactions();
 
-
         manualRedactions.getResizeRedactions()
                 .add(ManualResizeRedaction.builder()
                         .annotationId("ca2b437e2480a4b5966cb8386020d454")
@@ -1084,16 +1040,8 @@ public class RedactionIntegrationTest {
 
         String manualAddId = UUID.randomUUID().toString();
 
-        Comment comment = Comment.builder()
+        Comment comment = Comment.builder().date(OffsetDateTime.now()).user("TEST_USER").text("This is a comment test").build();
-                .date(OffsetDateTime.now())
+        manualRedactions.setIdsToRemove(Set.of(IdRemoval.builder().annotationId("5b940b2cb401ed9f5be6fc24f6e77bcf").fileId("fileId").status(AnnotationStatus.DECLINED).build()));
-                .user("TEST_USER")
-                .text("This is a comment test")
-                .build();
-        manualRedactions.setIdsToRemove(Set.of(IdRemoval.builder()
-                .annotationId("5b940b2cb401ed9f5be6fc24f6e77bcf")
-                .fileId("fileId")
-                .status(AnnotationStatus.DECLINED)
-                .build()));
         manualRedactions.setForceRedactions(Set.of(ManualForceRedaction.builder()
                 .annotationId("675eba69b0c2917de55462c817adaa05")
                 .fileId("fileId")
@@ -1112,19 +1060,8 @@ public class RedactionIntegrationTest {
         manualRedactionEntry.setType("name");
         manualRedactionEntry.setValue("O'Loughlin C.K.");
         manualRedactionEntry.setReason("Manual Redaction");
-        manualRedactionEntry.setPositions(List.of(Rectangle.builder()
+        manualRedactionEntry.setPositions(List.of(Rectangle.builder().topLeftX(375.61096f).topLeftY(241.282f).width(7.648041f).height(43.72262f).page(1).build(),
-                .topLeftX(375.61096f)
+                Rectangle.builder().topLeftX(384.83517f).topLeftY(241.282f).width(7.648041f).height(17.043358f).page(1).build()));
-                .topLeftY(241.282f)
-                .width(7.648041f)
-                .height(43.72262f)
-                .page(1)
-                .build(), Rectangle.builder()
-                .topLeftX(384.83517f)
-                .topLeftY(241.282f)
-                .width(7.648041f)
-                .height(17.043358f)
-                .page(1)
-                .build()));
 
         AnalyzeRequest request = prepareStorage(pdfFile);
         request.setManualRedactions(manualRedactions);
@@ -1132,11 +1069,7 @@ public class RedactionIntegrationTest {
         AnalyzeResult result = analyzeService.analyze(request);
 
         manualRedactions.getEntriesToAdd().add(manualRedactionEntry);
-        manualRedactions.setIdsToRemove(Set.of(IdRemoval.builder()
+        manualRedactions.setIdsToRemove(Set.of(IdRemoval.builder().annotationId("5b940b2cb401ed9f5be6fc24f6e77bcf").fileId("fileId").status(AnnotationStatus.APPROVED).build()));
-                .annotationId("5b940b2cb401ed9f5be6fc24f6e77bcf")
-                .fileId("fileId")
-                .status(AnnotationStatus.APPROVED)
-                .build()));
         manualRedactions.setLegalBasisChanges((Set.of(ManualLegalBasisChange.builder()
                 .annotationId("675eba69b0c2917de55462c817adaa05")
                 .fileId("fileId")
@@ -1148,10 +1081,7 @@ public class RedactionIntegrationTest {
 
         var redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID);
 
-        AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder()
+        AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder().dossierId(TEST_DOSSIER_ID).fileId(TEST_FILE_ID).build());
-                .dossierId(TEST_DOSSIER_ID)
-                .fileId(TEST_FILE_ID)
-                .build());
 
         try (FileOutputStream fileOutputStream = new FileOutputStream(OsUtils.getTemporaryDirectory() + "/Annotated.pdf")) {
             fileOutputStream.write(annotateResponse.getDocument());
@@ -1168,7 +1098,7 @@ public class RedactionIntegrationTest {
 
         System.out.println("classificationTest");
 
-        AnalyzeRequest request = prepareStorage("files/Metolachlor/S-Metolachlor_RAR_01_Volume_1_2018-09-06.pdf");
+        AnalyzeRequest request = prepareStorage("files/new/VV-511309_OCR.pdf");
 
         RedactionRequest redactionRequest = RedactionRequest.builder()
                 .dossierId(request.getDossierId())
@@ -1183,6 +1113,7 @@ public class RedactionIntegrationTest {
         }
     }
 
+
     @Test
     public void classificationTestWithCvTableService() throws IOException {
 
@@ -1295,10 +1226,7 @@ public class RedactionIntegrationTest {
         analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
         AnalyzeResult result = analyzeService.analyze(request);
 
-        AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder()
+        AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder().dossierId(TEST_DOSSIER_ID).fileId(TEST_FILE_ID).build());
-                .dossierId(TEST_DOSSIER_ID)
-                .fileId(TEST_FILE_ID)
-                .build());
 
         try (FileOutputStream fileOutputStream = new FileOutputStream(OsUtils.getTemporaryDirectory() + "/Annotated.pdf")) {
             fileOutputStream.write(annotateResponse.getDocument());
@@ -1320,11 +1248,7 @@ public class RedactionIntegrationTest {
 
         String manualAddId = UUID.randomUUID().toString();
 
-        Comment comment = Comment.builder()
+        Comment comment = Comment.builder().date(OffsetDateTime.now()).user("TEST_USER").text("This is a comment test").build();
-                .date(OffsetDateTime.now())
-                .user("TEST_USER")
-                .text("This is a comment test")
-                .build();
         manualRedactions.setIdsToRemove(Set.of(IdRemoval.builder()
                 .annotationId("5b940b2cb401ed9f5be6fc24f6e77bcf")
                 .fileId("fileId")
@@ -1354,19 +1278,8 @@ public class RedactionIntegrationTest {
         manualRedactionEntry.setReason("Manual Redaction");
         manualRedactionEntry.setProcessedDate(OffsetDateTime.now());
         manualRedactionEntry.setRequestDate(OffsetDateTime.now());
-        manualRedactionEntry.setPositions(List.of(Rectangle.builder()
+        manualRedactionEntry.setPositions(List.of(Rectangle.builder().topLeftX(375.61096f).topLeftY(241.282f).width(7.648041f).height(43.72262f).page(1).build(),
-                .topLeftX(375.61096f)
+                Rectangle.builder().topLeftX(384.83517f).topLeftY(241.282f).width(7.648041f).height(17.043358f).page(1).build()));
-                .topLeftY(241.282f)
-                .width(7.648041f)
-                .height(43.72262f)
-                .page(1)
-                .build(), Rectangle.builder()
-                .topLeftX(384.83517f)
-                .topLeftY(241.282f)
-                .width(7.648041f)
-                .height(17.043358f)
-                .page(1)
-                .build()));
 
 //        manualRedactions.getEntriesToAdd().add(manualRedactionEntry);
 
@@ -1397,31 +1310,10 @@ public class RedactionIntegrationTest {
                 .processedDate(OffsetDateTime.now())
                 .requestDate(OffsetDateTime.now())
                 .value("Syngenta Crop Protection AG, Basel, Switzerland RCC Ltd., Itingen, Switzerland")
-                .positions(List.of(Rectangle.builder()
+                .positions(List.of(Rectangle.builder().topLeftX(289.44595f).topLeftY(327.567f).width(7.648041f).height(82.51475f).page(1).build(),
-                        .topLeftX(289.44595f)
+                        Rectangle.builder().topLeftX(298.67056f).topLeftY(327.567f).width(7.648041f).height(75.32377f).page(1).build(),
-                        .topLeftY(327.567f)
+                        Rectangle.builder().topLeftX(307.89517f).topLeftY(327.567f).width(7.648041f).height(61.670967f).page(1).build(),
-                        .width(7.648041f)
+                        Rectangle.builder().topLeftX(316.99985f).topLeftY(327.567f).width(7.648041f).height(38.104286f).page(1).build()))
-                        .height(82.51475f)
-                        .page(1)
-                        .build(), Rectangle.builder()
-                        .topLeftX(298.67056f)
-                        .topLeftY(327.567f)
-                        .width(7.648041f)
-                        .height(75.32377f)
-                        .page(1)
-                        .build(), Rectangle.builder()
-                        .topLeftX(307.89517f)
-                        .topLeftY(327.567f)
-                        .width(7.648041f)
-                        .height(61.670967f)
-                        .page(1)
-                        .build(), Rectangle.builder()
-                        .topLeftX(316.99985f)
-                        .topLeftY(327.567f)
-                        .width(7.648041f)
-                        .height(38.104286f)
-                        .page(1)
-                        .build()))
                 .status(AnnotationStatus.APPROVED)
                 .build()));
 
@@ -1451,10 +1343,7 @@ public class RedactionIntegrationTest {
         analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
         AnalyzeResult result = analyzeService.analyze(request);
 
-        AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder()
+        AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder().dossierId(TEST_DOSSIER_ID).fileId(TEST_FILE_ID).build());
-                .dossierId(TEST_DOSSIER_ID)
-                .fileId(TEST_FILE_ID)
-                .build());
 
         try (FileOutputStream fileOutputStream = new FileOutputStream(OsUtils.getTemporaryDirectory() + "/Annotated.pdf")) {
             fileOutputStream.write(annotateResponse.getDocument());
@@ -1481,13 +1370,7 @@ public class RedactionIntegrationTest {
         manualRedactionEntry.setType("CBI_author");
         manualRedactionEntry.setValue("rabbits");
         manualRedactionEntry.setReason("Manual Redaction");
-        manualRedactionEntry.setPositions(List.of(Rectangle.builder()
+        manualRedactionEntry.setPositions(List.of(Rectangle.builder().topLeftX(70.944f).topLeftY(670.1595f).width(30.07296f).height(10.048125f).page(1).build()));
-                .topLeftX(70.944f)
-                .topLeftY(670.1595f)
-                .width(30.07296f)
-                .height(10.048125f)
-                .page(1)
-                .build()));
 
         ManualRedactionEntry manualRedactionEntry2 = new ManualRedactionEntry();
         manualRedactionEntry2.setAnnotationId(UUID.randomUUID().toString());
@@ -1496,13 +1379,7 @@ public class RedactionIntegrationTest {
         manualRedactionEntry2.setType("CBI_author");
         manualRedactionEntry2.setValue("rabbits");
         manualRedactionEntry2.setReason("Manual Redaction");
-        manualRedactionEntry2.setPositions(List.of(Rectangle.builder()
+        manualRedactionEntry2.setPositions(List.of(Rectangle.builder().topLeftX(470.5204f).topLeftY(746.1195f).width(29.96256f).height(10.048125f).page(1).build()));
-                .topLeftX(470.5204f)
-                .topLeftY(746.1195f)
-                .width(29.96256f)
-                .height(10.048125f)
-                .page(1)
-                .build()));
 
         ManualRedactionEntry manualRedactionEntry3 = new ManualRedactionEntry();
         manualRedactionEntry3.setAnnotationId(UUID.randomUUID().toString());
@@ -1511,13 +1388,7 @@ public class RedactionIntegrationTest {
         manualRedactionEntry3.setType("CBI_author");
         manualRedactionEntry3.setValue("AOEL");
         manualRedactionEntry3.setReason("Manual Redaction");
-        manualRedactionEntry3.setPositions(List.of(Rectangle.builder()
+        manualRedactionEntry3.setPositions(List.of(Rectangle.builder().topLeftX(355.53775f).topLeftY(266.1895f).width(29.32224f).height(10.048125f).page(1).build()));
-                .topLeftX(355.53775f)
-                .topLeftY(266.1895f)
-                .width(29.32224f)
-                .height(10.048125f)
-                .page(1)
-                .build()));
 
         manualRedactions.getEntriesToAdd().add(manualRedactionEntry);
         manualRedactions.getEntriesToAdd().add(manualRedactionEntry2);
@@ -1539,8 +1410,7 @@ public class RedactionIntegrationTest {
             fileOutputStream.write(annotateResponse.getDocument());
         }
 
-        var surroundingTextResult = manualRedactionSurroundingTextService.addSurroundingText(TEST_DOSSIER_ID, TEST_FILE_ID, manualRedactions)
+        var surroundingTextResult = manualRedactionSurroundingTextService.addSurroundingText(TEST_DOSSIER_ID, TEST_FILE_ID, manualRedactions).getManualRedactions();
-                .getManualRedactions();
         surroundingTextResult.getEntriesToAdd().forEach(addEntry -> {
             assertThat(addEntry.getTextAfter()).isNotEmpty();
         });
@@ -1551,112 +1421,54 @@ public class RedactionIntegrationTest {
     private void loadDictionaryForTest() {
 
         dictionary.computeIfAbsent(AUTHOR, v -> new ArrayList<>())
-                .addAll(ResourceLoader.load("dictionaries/CBI_author.txt")
+                .addAll(ResourceLoader.load("dictionaries/CBI_author.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
-                        .stream()
-                        .map(this::cleanDictionaryEntry)
-                        .collect(Collectors.toSet()));
         dictionary.computeIfAbsent(SPONSOR, v -> new ArrayList<>())
-                .addAll(ResourceLoader.load("dictionaries/CBI_sponsor.txt")
+                .addAll(ResourceLoader.load("dictionaries/CBI_sponsor.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
-                        .stream()
-                        .map(this::cleanDictionaryEntry)
-                        .collect(Collectors.toSet()));
         dictionary.computeIfAbsent(VERTEBRATE, v -> new ArrayList<>())
-                .addAll(ResourceLoader.load("dictionaries/vertebrate.txt")
+                .addAll(ResourceLoader.load("dictionaries/vertebrate.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
-                        .stream()
-                        .map(this::cleanDictionaryEntry)
-                        .collect(Collectors.toSet()));
         dictionary.computeIfAbsent(ADDRESS, v -> new ArrayList<>())
-                .addAll(ResourceLoader.load("dictionaries/CBI_address.txt")
+                .addAll(ResourceLoader.load("dictionaries/CBI_address.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
-                        .stream()
-                        .map(this::cleanDictionaryEntry)
-                        .collect(Collectors.toSet()));
         dictionary.computeIfAbsent(NO_REDACTION_INDICATOR, v -> new ArrayList<>())
-                .addAll(ResourceLoader.load("dictionaries/no_redaction_indicator.txt")
+                .addAll(ResourceLoader.load("dictionaries/no_redaction_indicator.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
-                        .stream()
-                        .map(this::cleanDictionaryEntry)
-                        .collect(Collectors.toSet()));
         dictionary.computeIfAbsent(REDACTION_INDICATOR, v -> new ArrayList<>())
-                .addAll(ResourceLoader.load("dictionaries/redaction_indicator.txt")
+                .addAll(ResourceLoader.load("dictionaries/redaction_indicator.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
-                        .stream()
-                        .map(this::cleanDictionaryEntry)
-                        .collect(Collectors.toSet()));
         dictionary.computeIfAbsent(HINT_ONLY, v -> new ArrayList<>())
-                .addAll(ResourceLoader.load("dictionaries/hint_only.txt")
+                .addAll(ResourceLoader.load("dictionaries/hint_only.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
-                        .stream()
-                        .map(this::cleanDictionaryEntry)
-                        .collect(Collectors.toSet()));
         dictionary.computeIfAbsent(MUST_REDACT, v -> new ArrayList<>())
-                .addAll(ResourceLoader.load("dictionaries/must_redact.txt")
+                .addAll(ResourceLoader.load("dictionaries/must_redact.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
-                        .stream()
-                        .map(this::cleanDictionaryEntry)
-                        .collect(Collectors.toSet()));
         dictionary.computeIfAbsent(PUBLISHED_INFORMATION, v -> new ArrayList<>())
-                .addAll(ResourceLoader.load("dictionaries/published_information.txt")
+                .addAll(ResourceLoader.load("dictionaries/published_information.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
-                        .stream()
-                        .map(this::cleanDictionaryEntry)
-                        .collect(Collectors.toSet()));
         dictionary.computeIfAbsent(TEST_METHOD, v -> new ArrayList<>())
-                .addAll(ResourceLoader.load("dictionaries/test_method.txt")
+                .addAll(ResourceLoader.load("dictionaries/test_method.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
-                        .stream()
-                        .map(this::cleanDictionaryEntry)
-                        .collect(Collectors.toSet()));
         dictionary.computeIfAbsent(PII, v -> new ArrayList<>())
-                .addAll(ResourceLoader.load("dictionaries/PII.txt")
+                .addAll(ResourceLoader.load("dictionaries/PII.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
-                        .stream()
-                        .map(this::cleanDictionaryEntry)
-                        .collect(Collectors.toSet()));
         dictionary.computeIfAbsent(PURITY, v -> new ArrayList<>())
-                .addAll(ResourceLoader.load("dictionaries/purity.txt")
+                .addAll(ResourceLoader.load("dictionaries/purity.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
-                        .stream()
-                        .map(this::cleanDictionaryEntry)
-                        .collect(Collectors.toSet()));
         dictionary.computeIfAbsent(IMAGE, v -> new ArrayList<>())
-                .addAll(ResourceLoader.load("dictionaries/empty.txt")
+                .addAll(ResourceLoader.load("dictionaries/empty.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
-                        .stream()
-                        .map(this::cleanDictionaryEntry)
-                        .collect(Collectors.toSet()));
         dictionary.computeIfAbsent(OCR, v -> new ArrayList<>())
-                .addAll(ResourceLoader.load("dictionaries/empty.txt")
+                .addAll(ResourceLoader.load("dictionaries/empty.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
-                        .stream()
-                        .map(this::cleanDictionaryEntry)
-                        .collect(Collectors.toSet()));
         dictionary.computeIfAbsent(LOGO, v -> new ArrayList<>())
-                .addAll(ResourceLoader.load("dictionaries/empty.txt")
+                .addAll(ResourceLoader.load("dictionaries/empty.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
-                        .stream()
-                        .map(this::cleanDictionaryEntry)
-                        .collect(Collectors.toSet()));
         dictionary.computeIfAbsent(SIGNATURE, v -> new ArrayList<>())
-                .addAll(ResourceLoader.load("dictionaries/empty.txt")
+                .addAll(ResourceLoader.load("dictionaries/empty.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
-                        .stream()
-                        .map(this::cleanDictionaryEntry)
-                        .collect(Collectors.toSet()));
         dictionary.computeIfAbsent(FORMULA, v -> new ArrayList<>())
-                .addAll(ResourceLoader.load("dictionaries/empty.txt")
+                .addAll(ResourceLoader.load("dictionaries/empty.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
-                        .stream()
-                        .map(this::cleanDictionaryEntry)
-                        .collect(Collectors.toSet()));
         dossierDictionary.computeIfAbsent(DOSSIER_REDACTIONS, v -> new ArrayList<>())
-                .addAll(ResourceLoader.load("dictionaries/dossier_redactions.txt")
+                .addAll(ResourceLoader.load("dictionaries/dossier_redactions.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
-                        .stream()
-                        .map(this::cleanDictionaryEntry)
-                        .collect(Collectors.toSet()));
         dossierDictionary.put(IMPORTED_REDACTION, new ArrayList<>());
 
         falsePositive.computeIfAbsent(PII, v -> new ArrayList<>())
-                .addAll(ResourceLoader.load("dictionaries/PII_false_positive.txt")
+                .addAll(ResourceLoader.load("dictionaries/PII_false_positive.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
-                        .stream()
-                        .map(this::cleanDictionaryEntry)
-                        .collect(Collectors.toSet()));
 
     }
 
+
     private void loadOnlyDictionaryForSimpleFile() {
+
         dictionary.clear();
         dictionary.computeIfAbsent(ROTATE_SIMPLE, v -> new ArrayList<>())
-                .addAll(ResourceLoader.load("dictionaries/RotateTestFileSimple.txt")
+                .addAll(ResourceLoader.load("dictionaries/RotateTestFileSimple.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
-                        .stream()
-                        .map(this::cleanDictionaryEntry)
-                        .collect(Collectors.toSet()));
     }
 
 
@@ -1851,6 +1663,7 @@ public class RedactionIntegrationTest {
                 .build();
     }
 
+
     private String cleanDictionaryEntry(String entry) {
 
         return TextNormalizationUtilities.removeHyphenLineBreaks(entry).replaceAll("\\n", " ");
@@ -1858,16 +1671,14 @@ public class RedactionIntegrationTest {
 
 
     private List<DictionaryEntry> toDictionaryEntry(List<String> entries) {
-        if (entries == null)
+
+        if (entries == null) {
             entries = Collections.emptyList();
+        }
 
         List<DictionaryEntry> dictionaryEntries = new ArrayList<>();
         entries.forEach(entry -> {
-            dictionaryEntries.add(DictionaryEntry.builder()
+            dictionaryEntries.add(DictionaryEntry.builder().value(entry).version(reanlysisVersions.getOrDefault(entry, 0L)).deleted(deleted.contains(entry)).build());
-                    .value(entry)
-                    .version(reanlysisVersions.getOrDefault(entry, 0L))
-                    .deleted(deleted.contains(entry))
-                    .build());
         });
         return dictionaryEntries;
     }
@@ -1880,17 +1691,15 @@ public class RedactionIntegrationTest {
         ClassPathResource importedRedactions = new ClassPathResource("files/ImportedRedactions/RotateTestFile_without_highlights.IMPORTED_REDACTIONS.json");
 
         AnalyzeRequest request = prepareStorage("files/ImportedRedactions/RotateTestFile_without_highlights.pdf");
-        storageService.storeObject(RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.IMPORTED_REDACTIONS), importedRedactions.getInputStream());
+        storageService.storeObject(RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.IMPORTED_REDACTIONS),
+                importedRedactions.getInputStream());
 
         analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
         AnalyzeResult result = analyzeService.analyze(request);
 
         var redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID);
 
-        AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder()
+        AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder().dossierId(TEST_DOSSIER_ID).fileId(TEST_FILE_ID).build());
-                .dossierId(TEST_DOSSIER_ID)
-                .fileId(TEST_FILE_ID)
-                .build());
 
         redactionLog.getRedactionLogEntry().forEach(entry -> {
             if (entry.getValue() == null) {
@@ -1928,20 +1737,14 @@ public class RedactionIntegrationTest {
         analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
         analyzeService.analyze(request);
 
-        AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder()
+        AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder().dossierId(TEST_DOSSIER_ID).fileId(TEST_FILE_ID).build());
-                .dossierId(TEST_DOSSIER_ID)
-                .fileId(TEST_FILE_ID)
-                .build());
 
         try (FileOutputStream fileOutputStream = new FileOutputStream(outputFileName)) {
             fileOutputStream.write(annotateResponse.getDocument());
         }
 
         var redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID);
-        var values = redactionLog.getRedactionLogEntry()
+        var values = redactionLog.getRedactionLogEntry().stream().map(RedactionLogEntry::getValue).collect(Collectors.toList());
-                .stream()
-                .map(RedactionLogEntry::getValue)
-                .collect(Collectors.toList());
 
         assertThat(values).contains("Mrs. Robinson");
         assertThat(values).contains("Mr. Bojangles");
@@ -1973,6 +1776,7 @@ public class RedactionIntegrationTest {
         return prepareStorage(file, "files/cv_service_empty_response.json");
     }
 
+
     @SneakyThrows
     private AnalyzeRequest prepareStorage(String file, String cvServiceResponseFile) {