hotfix: Tests for layoutParsing

redaction-service-v1/redaction-service-api-v1/src/main/java/com/iqser/red/service/redaction/v1/model/RedactionResult.java

@@ -12,6 +12,7 @@ import lombok.NoArgsConstructor;
 public class RedactionResult {
 
     private byte[] document;
+    private byte[] jsonDoc;
     private int numberOfPages;
 
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/service/BlockificationService.java

@@ -4,8 +4,8 @@ import static java.util.stream.Collectors.toSet;
 
 import java.util.ArrayList;
 import java.util.Comparator;
-import java.util.Iterator;
 import java.util.List;
+import java.util.regex.Matcher;
 
 import org.springframework.stereotype.Service;
 
@@ -16,6 +16,7 @@ import com.iqser.red.service.redaction.v1.server.classification.model.StringFreq
 import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
 import com.iqser.red.service.redaction.v1.server.classification.utils.RulingTextDirAdjustUtil;
 import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
+import com.iqser.red.service.redaction.v1.server.redaction.utils.Patterns;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.Ruling;
 
@@ -43,19 +44,34 @@ public class BlockificationService {
         float minX = 1000, maxX = 0, minY = 1000, maxY = 0;
         TextPositionSequence prev = null;
 
+
+        var pattern = Patterns.getCompiledPattern("\\b(?:[1-9]|1\\d|20|[ivxlc]+)\\s*(?:[.)])", true);
+        var pattern2 = Patterns.getCompiledPattern(".*\\d$", true);
+        var pattern3 = Patterns.getCompiledPattern("^(\\d{1,1}\\.){1,3}\\d{1,2}\\.?\\s[a-z]{1,2}\\/[a-z]{1,2}.*", false);
+
+
+
         boolean wasSplitted = false;
         Float splitX1 = null;
         for (TextPositionSequence word : textPositions) {
 
+            Matcher matcher = pattern.matcher(word.toString());
+            Matcher matcher2 = pattern2.matcher(word.toString());
+            Matcher matcher3 = pattern3.matcher(word.toString());
+
             boolean lineSeparation = word.getMinYDirAdj() - maxY > word.getHeight() * 1.25;
             boolean startFromTop = prev != null && word.getMinYDirAdj() < prev.getMinYDirAdj() - prev.getTextHeight();
             boolean splitByX = prev != null && maxX + 50 < word.getMinXDirAdj() && prev.getMinYDirAdj() == word.getMinYDirAdj();
-            boolean xIsBeforeFirstX = prev != null && word.getMinXDirAdj() < minX;
+            boolean xIsBeforeFirstX = prev != null &&  word.getMinXDirAdj() - minX < -5;
             boolean newLineAfterSplit = prev != null && word.getMinYDirAdj() != prev.getMinYDirAdj() && wasSplitted && splitX1 != word.getMinXDirAdj();
             boolean isSplitByRuling = isSplitByRuling(minX, minY, maxX, maxY, word, horizontalRulingLines, verticalRulingLines);
             boolean splitByDir = prev != null && !prev.getDir().equals(word.getDir());
+            boolean splitOnFontDifferent = prev != null && (!word.getFont().equals(prev.getFont()) || !word.getFontStyle().equals(prev.getFontStyle()) || word.getFontSize() !=prev.getFontSize());
+            boolean newline = prev != null && Math.abs(word.getMinYDirAdj()- prev.getMinYDirAdj()) > word.getHeight();
+            boolean isNumericalIdentifier = matcher.matches();
 
-            if (prev != null && (lineSeparation || startFromTop || splitByX || splitByDir || isSplitByRuling)) {
+            if (prev != null && (prev.isParagraphStart() || xIsBeforeFirstX || splitByX || lineSeparation || startFromTop || isSplitByRuling || (newline && splitOnFontDifferent) || (isNumericalIdentifier && newline))) {
+//            if (prev != null && (lineSeparation || startFromTop || splitByX || splitByDir || isSplitByRuling)) {
 
                 Orientation prevOrientation = null;
                 if (!chunkBlockList1.isEmpty()) {
@@ -107,51 +123,51 @@ public class BlockificationService {
             chunkBlockList1.add(cb1);
         }
 
-        Iterator<AbstractTextContainer> itty = chunkBlockList1.iterator();
+//        Iterator<AbstractTextContainer> itty = chunkBlockList1.iterator();
 
-        TextBlock previousLeft = null;
-        TextBlock previousRight = null;
-        while (itty.hasNext()) {
-            TextBlock block = (TextBlock) itty.next();
-
-            if (previousLeft != null && block.getOrientation().equals(Orientation.LEFT)) {
-                if (previousLeft.getMinY() > block.getMinY() && block.getMaxY() + block.getMostPopularWordHeight() > previousLeft.getMinY()) {
-                    previousLeft.add(block);
-                    itty.remove();
-                    continue;
-                }
-            }
-
-            if (previousRight != null && block.getOrientation().equals(Orientation.RIGHT)) {
-                if (previousRight.getMinY() > block.getMinY() && block.getMaxY() + block.getMostPopularWordHeight() > previousRight.getMinY()) {
-                    previousRight.add(block);
-                    itty.remove();
-                    continue;
-                }
-            }
-
-            if (block.getOrientation().equals(Orientation.LEFT)) {
-                previousLeft = block;
-            } else if (block.getOrientation().equals(Orientation.RIGHT)) {
-                previousRight = block;
-            }
-        }
-
-        itty = chunkBlockList1.iterator();
-        TextBlock previous = null;
-        while (itty.hasNext()) {
-            TextBlock block = (TextBlock) itty.next();
-
-            if (previous != null && previous.getOrientation().equals(Orientation.LEFT) && block.getOrientation().equals(Orientation.LEFT) && equalsWithThreshold(block.getMaxY(),
-                    previous.getMaxY()) || previous != null && previous.getOrientation().equals(Orientation.LEFT) && block.getOrientation()
-                    .equals(Orientation.RIGHT) && equalsWithThreshold(block.getMaxY(), previous.getMaxY())) {
-                previous.add(block);
-                itty.remove();
-                continue;
-            }
-
-            previous = block;
-        }
+//        TextBlock previousLeft = null;
+//        TextBlock previousRight = null;
+//        while (itty.hasNext()) {
+//            TextBlock block = (TextBlock) itty.next();
+//
+//            if (previousLeft != null && block.getOrientation().equals(Orientation.LEFT)) {
+//                if (previousLeft.getMinY() > block.getMinY() && block.getMaxY() + block.getMostPopularWordHeight() > previousLeft.getMinY()) {
+//                    previousLeft.add(block);
+//                    itty.remove();
+//                    continue;
+//                }
+//            }
+//
+//            if (previousRight != null && block.getOrientation().equals(Orientation.RIGHT)) {
+//                if (previousRight.getMinY() > block.getMinY() && block.getMaxY() + block.getMostPopularWordHeight() > previousRight.getMinY()) {
+//                    previousRight.add(block);
+//                    itty.remove();
+//                    continue;
+//                }
+//            }
+//
+//            if (block.getOrientation().equals(Orientation.LEFT)) {
+//                previousLeft = block;
+//            } else if (block.getOrientation().equals(Orientation.RIGHT)) {
+//                previousRight = block;
+//            }
+//        }
+//
+//        itty = chunkBlockList1.iterator();
+//        TextBlock previous = null;
+//        while (itty.hasNext()) {
+//            TextBlock block = (TextBlock) itty.next();
+//
+//            if (previous != null && previous.getOrientation().equals(Orientation.LEFT) && block.getOrientation().equals(Orientation.LEFT) && equalsWithThreshold(block.getMaxY(),
+//                    previous.getMaxY()) || previous != null && previous.getOrientation().equals(Orientation.LEFT) && block.getOrientation()
+//                    .equals(Orientation.RIGHT) && equalsWithThreshold(block.getMaxY(), previous.getMaxY())) {
+//                previous.add(block);
+//                itty.remove();
+//                continue;
+//            }
+//
+//            previous = block;
+//        }
 
         return new Page(chunkBlockList1);
     }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/service/BodyTextFrameService.java

@@ -88,7 +88,7 @@ public class BodyTextFrameService {
                         continue;
                     }
 
-                    if (documentFontSizeCounter.getMostPopular() != null && textBlock.getMostPopularWordFontSize() >= documentFontSizeCounter.getMostPopular()) {
+                    if (documentFontSizeCounter.getMostPopular() != null ) {
 
                         expandRectangle(textBlock, page, expansionsRectangle);
                     }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/service/ClassificationService.java

@@ -52,65 +52,67 @@ public class ClassificationService {
 
         var bodyTextFrame = page.getBodyTextFrame();
 
-        if (document.getFontSizeCounter().getMostPopular() == null) {
-            textBlock.setClassification("Other");
-            return;
-        }
-        if (PositionUtils.isOverBodyTextFrame(bodyTextFrame, textBlock, page.getRotation()) && (document.getFontSizeCounter()
-                .getMostPopular() == null || textBlock.getHighestFontSize() <= document.getFontSizeCounter().getMostPopular())) {
-            textBlock.setClassification("Header");
+        textBlock.setClassification("TextBlock");
+//        return;
 
-        } else if (PositionUtils.isUnderBodyTextFrame(bodyTextFrame, textBlock, page.getRotation()) && (document.getFontSizeCounter()
-                .getMostPopular() == null || textBlock.getHighestFontSize() <= document.getFontSizeCounter().getMostPopular())) {
-            textBlock.setClassification("Footer");
-        } else if (page.getPageNumber() == 1 && (PositionUtils.getHeightDifferenceBetweenChunkWordAndDocumentWord(textBlock,
-                document.getTextHeightCounter().getMostPopular()) > 2.5 && textBlock.getHighestFontSize() > document.getFontSizeCounter().getMostPopular() || page.getTextBlocks()
-                .size() == 1)) {
-            if (!Pattern.matches("[0-9]+", textBlock.toString())) {
-                textBlock.setClassification("Title");
-            }
-        } else if (textBlock.getMostPopularWordFontSize() > document.getFontSizeCounter()
-                .getMostPopular() && PositionUtils.getApproxLineCount(textBlock) < 4.9 && (textBlock.getMostPopularWordStyle().equals("bold") || !document.getFontStyleCounter()
-                .getCountPerValue()
-                .containsKey("bold") && textBlock.getMostPopularWordFontSize() > document.getFontSizeCounter().getMostPopular() + 1) && textBlock.getSequences()
-                .get(0)
-                .getTextPositions()
-                .get(0)
-                .getFontSizeInPt() >= textBlock.getMostPopularWordFontSize()) {
-
-            for (int i = 1; i <= headlineFontSizes.size(); i++) {
-                if (textBlock.getMostPopularWordFontSize() == headlineFontSizes.get(i - 1)) {
-                    textBlock.setClassification("H " + i);
-                    document.setHeadlines(true);
-                }
-            }
-        } else if (!textBlock.getText().startsWith("Table ") && !textBlock.getText().startsWith("Figure ") && PositionUtils.isWithinBodyTextFrame(bodyTextFrame,
-                textBlock) && textBlock.getMostPopularWordStyle().equals("bold") && !document.getFontStyleCounter()
-                .getMostPopular()
-                .equals("bold") && PositionUtils.getApproxLineCount(textBlock) < 2.9 && textBlock.getSequences()
-                .get(0)
-                .getTextPositions()
-                .get(0)
-                .getFontSizeInPt() >= textBlock.getMostPopularWordFontSize()) {
-            textBlock.setClassification("H " + (headlineFontSizes.size() + 1));
-            document.setHeadlines(true);
-        } else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordFontSize() == document.getFontSizeCounter()
-                .getMostPopular() && textBlock.getMostPopularWordStyle().equals("bold") && !document.getFontStyleCounter().getMostPopular().equals("bold")) {
-            textBlock.setClassification("TextBlock Bold");
-        } else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordFont()
-                .equals(document.getFontCounter().getMostPopular()) && textBlock.getMostPopularWordStyle()
-                .equals(document.getFontStyleCounter().getMostPopular()) && textBlock.getMostPopularWordFontSize() == document.getFontSizeCounter().getMostPopular()) {
-            textBlock.setClassification("TextBlock");
-        } else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordFontSize() == document.getFontSizeCounter()
-                .getMostPopular() && textBlock.getMostPopularWordStyle().equals("italic") && !document.getFontStyleCounter()
-                .getMostPopular()
-                .equals("italic") && PositionUtils.getApproxLineCount(textBlock) < 2.9) {
-            textBlock.setClassification("TextBlock Italic");
-        } else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock)) {
-            textBlock.setClassification("TextBlock Unknown");
-        } else {
-            textBlock.setClassification("Other");
-        }
+//        if (document.getFontSizeCounter().getMostPopular() == null) {
+//            textBlock.setClassification("Other");
+//            return;
+//        }
+//        if (PositionUtils.isOverBodyTextFrame(bodyTextFrame, textBlock, page.getRotation()) ) {
+//            textBlock.setClassification("Header");
+//
+//        } else if (PositionUtils.isUnderBodyTextFrame(bodyTextFrame, textBlock, page.getRotation()) && (document.getFontSizeCounter()
+//                .getMostPopular() == null || textBlock.getHighestFontSize() <= document.getFontSizeCounter().getMostPopular())) {
+//            textBlock.setClassification("Footer");
+//        } else if (page.getPageNumber() == 1 && (PositionUtils.getHeightDifferenceBetweenChunkWordAndDocumentWord(textBlock,
+//                document.getTextHeightCounter().getMostPopular()) > 2.5 && textBlock.getHighestFontSize() > document.getFontSizeCounter().getMostPopular() || page.getTextBlocks()
+//                .size() == 1)) {
+//            if (!Pattern.matches("[0-9]+", textBlock.toString())) {
+//                textBlock.setClassification("Title");
+//            }
+//        } else if (textBlock.getMostPopularWordFontSize() > document.getFontSizeCounter()
+//                .getMostPopular() && PositionUtils.getApproxLineCount(textBlock) < 4.9 && (textBlock.getMostPopularWordStyle().equals("bold") || !document.getFontStyleCounter()
+//                .getCountPerValue()
+//                .containsKey("bold") && textBlock.getMostPopularWordFontSize() > document.getFontSizeCounter().getMostPopular() + 1) && textBlock.getSequences()
+//                .get(0)
+//                .getTextPositions()
+//                .get(0)
+//                .getFontSizeInPt() >= textBlock.getMostPopularWordFontSize()) {
+//
+//            for (int i = 1; i <= headlineFontSizes.size(); i++) {
+//                if (textBlock.getMostPopularWordFontSize() == headlineFontSizes.get(i - 1)) {
+//                    textBlock.setClassification("H " + i);
+//                    document.setHeadlines(true);
+//                }
+//            }
+//        } else if (!textBlock.getText().startsWith("Table ") && !textBlock.getText().startsWith("Figure ") && PositionUtils.isWithinBodyTextFrame(bodyTextFrame,
+//                textBlock) && textBlock.getMostPopularWordStyle().equals("bold") && !document.getFontStyleCounter()
+//                .getMostPopular()
+//                .equals("bold") && PositionUtils.getApproxLineCount(textBlock) < 2.9 && textBlock.getSequences()
+//                .get(0)
+//                .getTextPositions()
+//                .get(0)
+//                .getFontSizeInPt() >= textBlock.getMostPopularWordFontSize()) {
+//            textBlock.setClassification("H " + (headlineFontSizes.size() + 1));
+//            document.setHeadlines(true);
+//        } else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordFontSize() == document.getFontSizeCounter()
+//                .getMostPopular() && textBlock.getMostPopularWordStyle().equals("bold") && !document.getFontStyleCounter().getMostPopular().equals("bold")) {
+//            textBlock.setClassification("TextBlock Bold");
+//        } else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordFont()
+//                .equals(document.getFontCounter().getMostPopular()) && textBlock.getMostPopularWordStyle()
+//                .equals(document.getFontStyleCounter().getMostPopular()) && textBlock.getMostPopularWordFontSize() == document.getFontSizeCounter().getMostPopular()) {
+//            textBlock.setClassification("TextBlock");
+//        } else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordFontSize() == document.getFontSizeCounter()
+//                .getMostPopular() && textBlock.getMostPopularWordStyle().equals("italic") && !document.getFontStyleCounter()
+//                .getMostPopular()
+//                .equals("italic") && PositionUtils.getApproxLineCount(textBlock) < 2.9) {
+//            textBlock.setClassification("TextBlock Italic");
+//        } else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock)) {
+//            textBlock.setClassification("TextBlock Unknown");
+//        } else {
+//            textBlock.setClassification("Other");
+//        }
     }
 
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/Document.java

@@ -0,0 +1,13 @@
+package com.iqser.red.service.redaction.v1.server.controller;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import lombok.Data;
+
+@Data
+public class Document {
+
+    private List<Paragraph> paragraphs = new ArrayList<>();
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/Paragraph.java

@@ -0,0 +1,21 @@
+package com.iqser.red.service.redaction.v1.server.controller;
+
+import com.iqser.red.service.redaction.v1.model.Rectangle;
+import com.iqser.red.service.redaction.v1.server.classification.model.Orientation;
+import com.iqser.red.service.redaction.v1.server.parsing.model.TextDirection;
+
+import lombok.Builder;
+import lombok.Data;
+
+@Builder
+@Data
+public class Paragraph {
+        private int paragraphNum;
+        private String text;
+        private String textStyle;
+        private String classification;
+        private int page;
+        private Rectangle boundingBox;
+        private Orientation orientation;
+        private TextDirection textDirection;
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RedactionController.java

@@ -1,21 +1,27 @@
 package com.iqser.red.service.redaction.v1.server.controller;
 
 import java.io.ByteArrayOutputStream;
+import java.io.FileOutputStream;
 import java.io.IOException;
+import java.util.stream.Collectors;
 
 import org.apache.pdfbox.pdmodel.PDDocument;
 import org.springframework.web.bind.annotation.PathVariable;
 import org.springframework.web.bind.annotation.RequestBody;
 import org.springframework.web.bind.annotation.RestController;
 
+import com.fasterxml.jackson.databind.ObjectMapper;
 import com.iqser.red.service.persistence.service.v1.api.model.annotations.ManualRedactions;
 import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.dossier.file.FileType;
+import com.iqser.red.service.redaction.v1.model.Point;
+import com.iqser.red.service.redaction.v1.model.Rectangle;
 import com.iqser.red.service.redaction.v1.model.RedactionLog;
 import com.iqser.red.service.redaction.v1.model.RedactionRequest;
 import com.iqser.red.service.redaction.v1.model.RedactionResult;
 import com.iqser.red.service.redaction.v1.resources.RedactionResource;
 import com.iqser.red.service.redaction.v1.server.classification.model.Document;
 import com.iqser.red.service.redaction.v1.server.classification.model.Page;
+import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
 import com.iqser.red.service.redaction.v1.server.exception.RedactionException;
 import com.iqser.red.service.redaction.v1.server.exception.RulesValidationException;
 import com.iqser.red.service.redaction.v1.server.redaction.service.DroolsExecutionService;
@@ -41,6 +47,7 @@ public class RedactionController implements RedactionResource {
     private final RedactionStorageService redactionStorageService;
     private final RedactionLogMergeService redactionLogMergeService;
     private final ManualRedactionSurroundingTextService manualRedactionSurroundingTextService;
+    private final ObjectMapper objectMapper;
 
 
     @Override
@@ -58,9 +65,66 @@ public class RedactionController implements RedactionResource {
             try (PDDocument pdDocument = PDDocument.load(storedObjectStream)) {
                 pdDocument.setAllSecurityToBeRemoved(true);
 
+                com.iqser.red.service.redaction.v1.server.controller.Document document = new com.iqser.red.service.redaction.v1.server.controller.Document();
+
+                classifiedDoc.getPages().forEach(page -> {
+
+                    for (int i = 0; i < page.getTextBlocks().size(); i++) {
+                        var t = page.getTextBlocks().get(i);
+
+
+                        if(t instanceof TextBlock){
+
+                            var textBlock = (TextBlock) t;
+
+                            document.getParagraphs()
+                                    .add(Paragraph.builder()
+                                            .text(textBlock.getText())
+                                            .boundingBox(new Rectangle(new Point(textBlock.getMinX(), textBlock.getMinY()), textBlock.getWidth(), textBlock.getHeight(), textBlock.getPage()))
+                                            .textStyle(textBlock.getMostPopularWordStyle())
+                                            .paragraphNum(i)
+                                            .orientation(textBlock.getOrientation())
+                                            .classification(textBlock.getClassification())
+                                            .textDirection(textBlock.getSequences().get(0).getDir())
+                                            .page(textBlock.getPage())
+                                            .build());
+                        } else {
+
+                            var table = (Table) t;
+                            if(table == null){
+                                continue;
+                            }
+
+                            for (var row: table.getRows()){
+
+                                StringBuilder sb = new StringBuilder();
+
+                                row.forEach(cell -> {
+                                    sb.append(cell.getTextBlocks().stream().map(a -> a.getText()).collect(Collectors.joining("|")));
+                                });
+
+                                document.getParagraphs()
+                                        .add(Paragraph.builder()
+                                                .text(sb.toString())
+                                                .boundingBox(new Rectangle(new Point(table.getMinX(), table.getMinY()), table.getWidth(), table.getHeight(), table.getPage()))
+                                                .paragraphNum(i)
+                                                .orientation(table.getOrientation())
+                                                .classification("Table Row")
+                                                .page(table.getPage())
+                                                .build());
+                            }
+                        }
+
+
+
+                    }
+
+                });
+
+
                 pdfVisualisationService.visualizeClassifications(classifiedDoc, pdDocument);
 
-                return convert(pdDocument, classifiedDoc.getPages().size());
+                return convert(pdDocument, classifiedDoc.getPages().size(), objectMapper.writeValueAsBytes(document));
 
             } catch (IOException e) {
                 throw new RedactionException(e);
@@ -89,7 +153,7 @@ public class RedactionController implements RedactionResource {
                 pdDocument.setAllSecurityToBeRemoved(true);
 
                 pdfVisualisationService.visualizeParagraphs(classifiedDoc, pdDocument);
-                return convert(pdDocument, classifiedDoc.getPages().size());
+                return convert(pdDocument, classifiedDoc.getPages().size(), null);
 
             } catch (IOException e) {
                 throw new RedactionException(e);
@@ -149,11 +213,12 @@ public class RedactionController implements RedactionResource {
     }
 
 
-    private RedactionResult convert(PDDocument document, int numberOfPages) throws IOException {
+    private RedactionResult convert(PDDocument document, int numberOfPages, byte[] json) throws IOException {
 
         try (ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream()) {
             document.save(byteArrayOutputStream);
-            return RedactionResult.builder().document(byteArrayOutputStream.toByteArray()).numberOfPages(numberOfPages).build();
+            return RedactionResult.builder().document(byteArrayOutputStream.toByteArray()).numberOfPages(numberOfPages)
+                    .jsonDoc(json).build();
         }
 
     }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/parsing/PDFAreaTextStripper.java

@@ -1,82 +0,0 @@
-package com.iqser.red.service.redaction.v1.server.parsing;
-
-import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
-
-import lombok.Getter;
-import lombok.Setter;
-
-import org.apache.pdfbox.text.PDFTextStripperByArea;
-import org.apache.pdfbox.text.TextPosition;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-
-public class PDFAreaTextStripper extends PDFTextStripperByArea {
-
-    @Getter
-    private List<TextPositionSequence> textPositionSequences = new ArrayList<>();
-
-    @Setter
-    private int pageNumber;
-
-
-    public PDFAreaTextStripper() throws IOException {
-
-    }
-
-
-    @Override
-    public void writeString(String text, List<TextPosition> textPositions) throws IOException {
-
-        int startIndex = 0;
-        for (int i = 0; i <= textPositions.size() - 1; i++) {
-
-            if (i == 0 && (textPositions.get(i).getUnicode().equals(" ") || textPositions.get(i).getUnicode().equals("\u00A0"))) {
-                startIndex++;
-                continue;
-            }
-
-            // Strange but sometimes this is happening, for example: Metolachlor2.pdf
-            if (i > 0 && textPositions.get(i).getX() < textPositions.get(i - 1).getX()) {
-                List<TextPosition> sublist = textPositions.subList(startIndex, i);
-                if (!(sublist.isEmpty() || sublist.size() == 1 && (sublist.get(0).getUnicode().equals(" ") || sublist.get(0).getUnicode().equals("\u00A0")))) {
-                    textPositionSequences.add(new TextPositionSequence(sublist, pageNumber));
-                }
-                startIndex = i;
-            }
-
-            if (textPositions.get(i).getRotation() == 0 && i > 0 && textPositions.get(i).getX() > textPositions.get(i - 1).getEndX() + 1) {
-                List<TextPosition> sublist = textPositions.subList(startIndex, i);
-                if (!(sublist.isEmpty() || sublist.size() == 1 && (sublist.get(0).getUnicode().equals(" ") || sublist.get(0).getUnicode().equals("\u00A0")))) {
-                    textPositionSequences.add(new TextPositionSequence(sublist, pageNumber));
-                }
-                startIndex = i;
-            }
-
-            if (i > 0 && (textPositions.get(i).getUnicode().equals(" ") || textPositions.get(i).getUnicode().equals("\u00A0")) && i <= textPositions.size() - 2) {
-                List<TextPosition> sublist = textPositions.subList(startIndex, i);
-                if (!(sublist.isEmpty() || sublist.size() == 1 && (sublist.get(0).getUnicode().equals(" ") || sublist.get(0).getUnicode().equals("\u00A0")))) {
-                    textPositionSequences.add(new TextPositionSequence(sublist, pageNumber));
-                }
-                startIndex = i + 1;
-            }
-        }
-
-        List<TextPosition> sublist = textPositions.subList(startIndex, textPositions.size());
-        if (!sublist.isEmpty() && (sublist.get(sublist.size() - 1).getUnicode().equals(" ") || sublist.get(sublist.size() - 1).getUnicode().equals("\u00A0"))) {
-            sublist = sublist.subList(0, sublist.size() - 1);
-        }
-        if (!(sublist.isEmpty() || sublist.size() == 1 && (sublist.get(0).getUnicode().equals(" ") || sublist.get(0).getUnicode().equals("\u00A0")))) {
-            textPositionSequences.add(new TextPositionSequence(sublist, pageNumber));
-        }
-        super.writeString(text);
-    }
-
-
-    public void clearPositions() {
-
-        textPositionSequences = new ArrayList<>();
-    }
-
-}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/parsing/PDFLinesTextStripper.java

@@ -195,7 +195,7 @@ public class PDFLinesTextStripper extends PDFTextStripper {
 
 
     @Override
-    public void writeString(String text, List<TextPosition> textPositions) throws IOException {
+    public void writeString(String text, List<TextPosition> textPositions, boolean isParagraphStart) throws IOException {
 
         int startIndex = 0;
         RedTextPosition previous = null;
@@ -235,7 +235,7 @@ public class PDFLinesTextStripper extends PDFTextStripper {
                 if (!(sublist.isEmpty() || sublist.size() == 1 && (sublist.get(0).getUnicode().equals(" ") || sublist.get(0).getUnicode().equals("\u00A0") || sublist.get(0)
                         .getUnicode()
                         .equals("\t")))) {
-                    textPositionSequences.add(new TextPositionSequence(sublist, pageNumber));
+                    textPositionSequences.add(new TextPositionSequence(sublist, pageNumber, i == textPositions.size() -1 && isParagraphStart));
                 }
                 startIndex = i;
             }
@@ -245,7 +245,7 @@ public class PDFLinesTextStripper extends PDFTextStripper {
                 if (!(sublist.isEmpty() || sublist.size() == 1 && (sublist.get(0).getUnicode().equals(" ") || sublist.get(0).getUnicode().equals("\u00A0") || sublist.get(0)
                         .getUnicode()
                         .equals("\t")))) {
-                    textPositionSequences.add(new TextPositionSequence(sublist, pageNumber));
+                    textPositionSequences.add(new TextPositionSequence(sublist, pageNumber, i == textPositions.size() -1 && isParagraphStart));
                 }
                 startIndex = i;
             }
@@ -265,7 +265,7 @@ public class PDFLinesTextStripper extends PDFTextStripper {
                             textPositionSequences.get(textPositionSequences.size() - 1).add(t);
                         }
                     } else {
-                        textPositionSequences.add(new TextPositionSequence(sublist, pageNumber));
+                        textPositionSequences.add(new TextPositionSequence(sublist, pageNumber, i == textPositions.size() -1 && isParagraphStart));
                     }
                 }
                 startIndex = i + 1;
@@ -288,7 +288,7 @@ public class PDFLinesTextStripper extends PDFTextStripper {
                     textPositionSequences.get(textPositionSequences.size() - 1).add(t);
                 }
             } else {
-                textPositionSequences.add(new TextPositionSequence(sublist, pageNumber));
+                textPositionSequences.add(new TextPositionSequence(sublist, pageNumber, isParagraphStart));
             }
         }
         super.writeString(text);

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/parsing/PDFTextStripper.java

@@ -27,6 +27,7 @@ import java.text.Bidi;
 import java.text.Normalizer;
 import java.util.ArrayList;
 import java.util.Collections;
+import java.util.Comparator;
 import java.util.HashMap;
 import java.util.Iterator;
 import java.util.LinkedList;
@@ -240,10 +241,10 @@ public class PDFTextStripper extends LegacyPDFStreamEngine {
         document = doc;
         output = outputStream;
         if (getAddMoreFormatting()) {
-            paragraphEnd = lineSeparator;
+            paragraphEnd = "\n----ParagraphEnd----\n\n";
             pageStart = lineSeparator;
-            articleStart = lineSeparator;
-            articleEnd = lineSeparator;
+            articleStart = "\n----ArticelStart----\n\n";
+            articleEnd = "\n----ArticelEnd----\n\n";
         }
         startDocument(document);
         processPages(document.getPages());
@@ -594,9 +595,14 @@ public class PDFTextStripper extends LegacyPDFStreamEngine {
                     // but this caused a lot of regression test failures. So, I'm leaving it be for
                     // now
                     if (!overlap(positionY, positionHeight, maxYForLine, maxHeightForLine)) {
-                        writeLine(normalize(line));
-                        line.clear();
+                        var normalized = normalize(line);
+//                        normalized.stream().filter(l -> System.out.println(l.getText().contains("Plenarprotokoll 20/24")).findFirst().isPresent()
+
+
                         lastLineStartPosition = handleLineSeparation(current, lastPosition, lastLineStartPosition, maxHeightForLine);
+                        writeLine(normalized, current.isParagraphStart);
+                        line.clear();
+
                         expectedStartOfNextWordX = EXPECTED_START_OF_NEXT_WORD_X_RESET_VALUE;
                         maxYForLine = MAX_Y_FOR_LINE_RESET_VALUE;
                         maxHeightForLine = MAX_HEIGHT_FOR_LINE_RESET_VALUE;
@@ -630,7 +636,24 @@ public class PDFTextStripper extends LegacyPDFStreamEngine {
                     if (startOfPage && lastPosition == null) {
                         writeParagraphStart();// not sure this is correct for RTL?
                     }
+
                     line.add(new LineItem(position));
+
+//                    Collections.sort(line, new Comparator<LineItem>() {
+//
+//                        @Override
+//                        public int compare(LineItem str1, LineItem str2) {
+//                            if(null == str1.getTextPosition()) {
+//                                return 0;
+//                            }
+//                            else if(null == str2.getTextPosition()) {
+//                                return 0;
+//                            }
+//                            return Float.compare(str1.getTextPosition().getX(), str2.getTextPosition().getX());
+//                        }
+//                    });
+
+//                    line.sort(Comparator.comparing(a -> a.getTextPosition() != null && a.getTextPosition().getX()));
                 }
                 maxHeightForLine = Math.max(maxHeightForLine, positionHeight);
                 minYTopForLine = Math.min(minYTopForLine, positionY - positionHeight);
@@ -646,7 +669,7 @@ public class PDFTextStripper extends LegacyPDFStreamEngine {
             }
             // print the final line
             if (line.size() > 0) {
-                writeLine(normalize(line));
+                writeLine(normalize(line), false);
                 writeParagraphEnd();
             }
             endArticle();
@@ -703,7 +726,7 @@ public class PDFTextStripper extends LegacyPDFStreamEngine {
      * @param textPositions The TextPositions belonging to the text.
      * @throws IOException If there is an error when writing the text.
      */
-    protected void writeString(String text, List<TextPosition> textPositions) throws IOException {
+    protected void writeString(String text, List<TextPosition> textPositions, boolean isParagraphEnd) throws IOException {
 
         writeString(text);
     }
@@ -1385,6 +1408,7 @@ public class PDFTextStripper extends LegacyPDFStreamEngine {
             } else {
                 writeLineSeparator();
                 writeParagraphSeparator();
+                lastLineStartPosition.setEndParagraphWritten();
             }
         } else {
             writeLineSeparator();
@@ -1428,7 +1452,11 @@ public class PDFTextStripper extends LegacyPDFStreamEngine {
             float newXVal = multiplyFloat(getIndentThreshold(), position.getTextPosition().getWidthOfSpace());
             float positionWidth = multiplyFloat(0.25f, position.getTextPosition().getWidth());
 
-            if (yGap > newYVal) {
+//            if(xGap < 0){
+//                result = true;
+//            }
+//            else
+                if (yGap > newYVal) {
                 result = true;
             } else if (xGap > newXVal) {
                 // text is indented, but try to screen for hanging indent
@@ -1636,12 +1664,13 @@ public class PDFTextStripper extends LegacyPDFStreamEngine {
      * @param line a list with the words of the given line
      * @throws IOException if something went wrong
      */
-    private void writeLine(List<WordWithTextPositions> line) throws IOException {
+    private void writeLine(List<WordWithTextPositions> line, boolean isParagraphEnd) throws IOException {
 
         int numberOfStrings = line.size();
         for (int i = 0; i < numberOfStrings; i++) {
             WordWithTextPositions word = line.get(i);
-            writeString(word.getText(), word.getTextPositions());
+            word.getTextPositions().sort(Comparator.comparing(TextPosition::getX));
+            writeString(word.getText(), word.getTextPositions(), isParagraphEnd && i == numberOfStrings - 1);
             if (i < numberOfStrings - 1) {
                 writeWordSeparator();
             }
@@ -1963,6 +1992,8 @@ public class PDFTextStripper extends LegacyPDFStreamEngine {
         private boolean isHangingIndent = false;
         private boolean isArticleStart = false;
 
+        private boolean endParagraphWritten = false;
+
         private TextPosition position = null;
 
 
@@ -2024,6 +2055,16 @@ public class PDFTextStripper extends LegacyPDFStreamEngine {
         }
 
 
+        public boolean isEndParagraphWritten() {
+
+            return endParagraphWritten;
+        }
+
+        public void setEndParagraphWritten(){
+            endParagraphWritten = true;
+        }
+
+
         /**
          * Sets the isArticleStart() flag to true.
          */

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/parsing/model/TextPositionSequence.java

@@ -39,6 +39,7 @@ public class TextPositionSequence implements CharSequence {
     private int rotation;
     private float pageHeight;
     private float pageWidth;
+    private boolean isParagraphStart;
 
 
     public TextPositionSequence(int page) {
@@ -47,7 +48,7 @@ public class TextPositionSequence implements CharSequence {
     }
 
 
-    public TextPositionSequence(List<TextPosition> textPositions, int page) {
+    public TextPositionSequence(List<TextPosition> textPositions, int page, boolean isParagraphStart) {
 
         this.textPositions = textPositions.stream().map(RedTextPosition::fromTextPosition).collect(Collectors.toList());
         this.page = page;
@@ -55,6 +56,7 @@ public class TextPositionSequence implements CharSequence {
         this.rotation = textPositions.get(0).getRotation();
         this.pageHeight = textPositions.get(0).getPageHeight();
         this.pageWidth = textPositions.get(0).getPageWidth();
+        this.isParagraphStart = isParagraphStart;
     }
 
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/segmentation/PdfSegmentationService.java

@@ -10,6 +10,7 @@ import java.nio.file.attribute.PosixFilePermission;
 import java.nio.file.attribute.PosixFilePermissions;
 import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -20,6 +21,7 @@ import org.apache.pdfbox.io.MemoryUsageSetting;
 import org.apache.pdfbox.pdmodel.PDDocument;
 import org.apache.pdfbox.pdmodel.PDPage;
 import org.apache.pdfbox.pdmodel.common.PDRectangle;
+import org.apache.pdfbox.util.QuickSort;
 import org.springframework.stereotype.Service;
 
 import com.iqser.red.service.redaction.v1.server.classification.model.Document;
@@ -96,7 +98,18 @@ public class PdfSegmentationService {
                     stripper.setStartPage(pageNumber);
                     stripper.setEndPage(pageNumber);
                     stripper.setPdpage(pdPage);
-                    stripper.getText(pdDocument);
+                    stripper.setAddMoreFormatting(true);
+                    stripper.setShouldSeparateByBeads(true);
+//                    stripper.setDropThreshold(0.1f);
+                    var text = stripper.getText(pdDocument);
+
+                    System.out.println(text);
+//                    stripper.getTextPositionSequences().forEach(a -> {
+//                        System.out.println(a);
+//                        if(a.isParagraphEnd()){
+//                            System.out.println("------------------------");
+//                        }
+//                    });
 
                     PDRectangle pdr = pdPage.getMediaBox();
 
@@ -110,13 +123,35 @@ public class PdfSegmentationService {
                             stripper.getMaxCharHeight());
                     Page page = blockificationService.blockify(stripper.getTextPositionSequences(), cleanRulings.getHorizontal(), cleanRulings.getVertical());
 
+
+//                    QuickSort.sort(page.getTextBlocks(), new TextBlockComparator());
+//
+//                    List<AbstractTextContainer> combined = new ArrayList<>();
+//                    Iterator<AbstractTextContainer> itty = page.getTextBlocks().iterator();
+//                    TextBlock prev = null;
+//                    while (itty.hasNext()){
+//                        TextBlock current = (TextBlock) itty.next();
+//
+//                        if(prev != null && (((Math.abs(prev.getMaxX() - current.getMaxX()) < 1)|| Math.abs(prev.getMinX() - current.getMinX()) < 1) && current.getMaxY() > prev.getMaxY() && current.getMinY() - prev.getMaxY() < 3)){
+//                            combined.remove(prev);
+//                            current = prev.union(current);
+//                        }
+//
+//                        combined.add(current);
+//
+//                        prev= current;
+//                    }
+//
+//                    page.setTextBlocks(combined);
+
+
                     page.setRotation(rotation);
                     page.setLandscape(isLandscape);
                     page.setPageNumber(pageNumber);
                     page.setPageWidth(cropbox.getWidth());
                     page.setPageHeight(cropbox.getHeight());
 
-                    tableExtractionService.extractTables(cleanRulings, page);
+//                    tableExtractionService.extractTables(cleanRulings, page);
                     buildPageStatistics(page);
                     increaseDocumentStatistics(page, document);
 
@@ -131,8 +166,8 @@ public class PdfSegmentationService {
                 document.setPages(pages);
 
                 classificationService.classifyDocument(document);
-                sectionsBuilderService.buildSections(document);
-                sectionsBuilderService.addImagesToSections(document);
+//                sectionsBuilderService.buildSections(document);
+//                sectionsBuilderService.addImagesToSections(document);
 
                 IOUtils.close(pdDocument);
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/segmentation/TextBlockComparator.java

@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.iqser.red.service.redaction.v1.server.segmentation;
+
+import java.util.Comparator;
+
+import org.apache.pdfbox.text.TextPosition;
+
+import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
+import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
+
+/**
+ * This class is a comparator for TextPosition operators.  It handles
+ * pages with text in different directions by grouping the text based
+ * on direction and sorting in that direction. This allows continuous text
+ * in a given direction to be more easily grouped together.  
+ *
+ * @author Ben Litchfield
+ */
+public class TextBlockComparator implements Comparator<AbstractTextContainer>
+{
+    @Override
+    public int compare(AbstractTextContainer pos1, AbstractTextContainer pos2)
+    {
+
+        
+        // get the text direction adjusted coordinates
+        float x1 = pos1.getMinX();
+        float x2 = pos2.getMinX();
+        
+        float pos1YBottom = pos1.getMaxY();
+        float pos2YBottom = pos2.getMaxY();
+
+        // note that the coordinates have been adjusted so 0,0 is in upper left
+        float pos1YTop = pos1YBottom - pos1.getHeight();
+        float pos2YTop = pos2YBottom - pos2.getHeight();
+
+        float yDifference = Math.abs(pos1YBottom - pos2YBottom);
+
+        // we will do a simple tolerance comparison
+        if (yDifference < .1 ||
+            pos2YBottom >= pos1YTop && pos2YBottom <= pos1YBottom ||
+            pos1YBottom >= pos2YTop && pos1YBottom <= pos2YBottom)
+        {
+            return Float.compare(x1, x2);
+        }
+        else if (pos1YBottom < pos2YBottom)
+        {
+            return -1;
+        }
+        else
+        {
+            return 1;
+        }
+    }
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/visualization/service/PdfVisualisationService.java

@@ -82,14 +82,14 @@ public class PdfVisualisationService {
                 }
             }
 
-            contentStream.setStrokingColor(Color.YELLOW);
-            contentStream.addRect(analyzedPage.getBodyTextFrame().getTopLeft().getX(),
-                    analyzedPage.getBodyTextFrame().getTopLeft().getY(),
-                    analyzedPage.getBodyTextFrame().getWidth(),
-                    analyzedPage.getBodyTextFrame().getHeight());
-
-            contentStream.stroke();
-
+//            contentStream.setStrokingColor(Color.YELLOW);
+//            contentStream.addRect(analyzedPage.getBodyTextFrame().getTopLeft().getX(),
+//                    analyzedPage.getBodyTextFrame().getTopLeft().getY(),
+//                    analyzedPage.getBodyTextFrame().getWidth(),
+//                    analyzedPage.getBodyTextFrame().getHeight());
+//
+//            contentStream.stroke();
+//
             contentStream.close();
         }
     }
@@ -103,15 +103,15 @@ public class PdfVisualisationService {
         contentStream.stroke();
 
         if (textBlock.getClassification() != null) {
-            contentStream.beginText();
-
-            contentStream.setNonStrokingColor(Color.BLUE);
-            contentStream.setFont(PDType1Font.TIMES_ROMAN, 9f);
-
-            contentStream.newLineAtOffset(textBlock.getPdfMinX(), textBlock.getPdfMaxY() + 2);
-            contentStream.showText(textBlock.getClassification() + textBlock.getOrientation() + "-->" + textBlock.getSequences().get(0).getDir());
-
-            contentStream.endText();
+//            contentStream.beginText();
+//
+//            contentStream.setNonStrokingColor(Color.BLUE);
+//            contentStream.setFont(PDType1Font.TIMES_ROMAN, 9f);
+//
+//            contentStream.newLineAtOffset(textBlock.getPdfMinX(), textBlock.getPdfMaxY() + 2);
+//            contentStream.showText(textBlock.getClassification() + textBlock.getOrientation() + "-->" + textBlock.getSequences().get(0).getDir());
+//
+//            contentStream.endText();
 
             if (DRAW_POSITIONS) {
                 drawPositions(contentStream, textBlock);

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java

@@ -6,12 +6,17 @@ import static org.mockito.Mockito.when;
 import java.io.BufferedReader;
 import java.io.ByteArrayInputStream;
 import java.io.File;
+import java.io.FileInputStream;
 import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.InputStreamReader;
 import java.net.URL;
 import java.nio.charset.StandardCharsets;
+import java.nio.file.DirectoryStream;
+import java.nio.file.FileSystems;
+import java.nio.file.Files;
+import java.nio.file.Path;
 import java.time.OffsetDateTime;
 import java.time.ZoneOffset;
 import java.util.ArrayList;
@@ -24,6 +29,7 @@ import java.util.Set;
 import java.util.UUID;
 import java.util.stream.Collectors;
 
+import org.apache.commons.lang3.StringUtils;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Ignore;
@@ -1111,29 +1117,69 @@ public class RedactionIntegrationTest {
 
         System.out.println("classificationTest");
 
-        AnalyzeRequest request = prepareStorage("files/new/RotateTestFile.pdf");
+        String sourcePath ="/tmp/Input";
+        String targetPath = "/tmp/Output";
 
-        RedactionRequest redactionRequest = RedactionRequest.builder()
-                .dossierId(request.getDossierId())
-                .fileId(request.getFileId())
-                .dossierTemplateId(request.getDossierTemplateId())
-                .build();
+        var files = getFileNames(new HashSet<>(), FileSystems.getDefault().getPath(sourcePath), sourcePath);
 
-        RedactionResult result = redactionController.classify(redactionRequest);
+        for(var file : files){
+            AnalyzeRequest request = prepareStorage(file);
 
-        try (FileOutputStream fileOutputStream = new FileOutputStream(OsUtils.getTemporaryDirectory() + "/Classified.pdf")) {
-            fileOutputStream.write(result.getDocument());
+            RedactionRequest redactionRequest = RedactionRequest.builder()
+                    .dossierId(request.getDossierId())
+                    .fileId(request.getFileId())
+                    .dossierTemplateId(request.getDossierTemplateId())
+                    .build();
+
+            RedactionResult result = redactionController.classify(redactionRequest);
+
+            String filename = file.substring(file.lastIndexOf("/"));
+
+            try (FileOutputStream fileOutputStream = new FileOutputStream(targetPath + filename)) {
+                fileOutputStream.write(result.getDocument());
+            }
+
+            try (FileOutputStream fileOutputStream = new FileOutputStream(targetPath + filename + ".json")) {
+                fileOutputStream.write(result.getJsonDoc());
+            }
         }
+
+
+
     }
 
 
+    @SneakyThrows
+    private Set<String> getFileNames(Set<String> fileNames, Path dir, String resourcePath) {
+
+        try (DirectoryStream<Path> stream = Files.newDirectoryStream(dir)) {
+            for (Path path : stream) {
+                if (path.toFile().isDirectory()) {
+                    getFileNames(fileNames, path, resourcePath);
+                } else if (StringUtils.endsWith(path.toAbsolutePath().toString(), ".pdf")) {
+                    String absolutePath = cleanPath(path.toAbsolutePath().toString());
+                    fileNames.add(absolutePath);
+                }
+            }
+        }
+        return fileNames;
+    }
+
+    private String cleanPath(String path) {
+
+        return StringUtils.replace(path, "\\", "/");
+    }
+
+
+
+
     @Test
     public void classificationTestWithCvTableService() throws IOException {
 
         System.out.println("classificationTest");
 
         String tableServiceResponseFile = "files/cv_table_response_VV-511309.json";
-        AnalyzeRequest request = prepareStorage("files/new/VV-511309_OCR.pdf", tableServiceResponseFile);
+        AnalyzeRequest request = prepareStorage("/Users/deiflaender/git/RED/redaction-service/redaction-service-v1/redaction-service-server-v1/src/test/resources/files/new/Plenarprotokoll 1 (keine Druchsache!) (1).pdf");
 
         RedactionRequest redactionRequest = RedactionRequest.builder()
                 .dossierId(request.getDossierId())
@@ -1174,7 +1220,7 @@ public class RedactionIntegrationTest {
     public void htmlTablesTest() throws IOException {
 
         System.out.println("htmlTablesTest");
-        AnalyzeRequest request = prepareStorage("files/Minimal Examples/Single Table.pdf");
+        AnalyzeRequest request = prepareStorage("/Users/deiflaender/git/RED/redaction-service/redaction-service-v1/redaction-service-server-v1/src/test/resources/files/Minimal Examples/Single Table1.pdf");
 
         RedactionRequest redactionRequest = RedactionRequest.builder()
                 .dossierId(request.getDossierId())
@@ -1195,7 +1241,7 @@ public class RedactionIntegrationTest {
 
         System.out.println("htmlTableRotationTest");
 
-        AnalyzeRequest request = prepareStorage("files/Metolachlor/S-Metolachlor_RAR_02_Volume_2_2018-09-06.pdf");
+        AnalyzeRequest request = prepareStorage("/Users/deiflaender/git/RED/redaction-service/redaction-service-v1/redaction-service-server-v1/src/test/resources/files/new/Plenarprotokoll 1 (keine Druchsache!) (1).pdf");
 
         RedactionRequest redactionRequest = RedactionRequest.builder()
                 .dossierId(request.getDossierId())
@@ -1208,6 +1254,12 @@ public class RedactionIntegrationTest {
         try (FileOutputStream fileOutputStream = new FileOutputStream(OsUtils.getTemporaryDirectory() + "/Tables.html")) {
             fileOutputStream.write(result.getDocument());
         }
+
+
+
+
+
+
     }
 
 
@@ -1793,10 +1845,11 @@ public class RedactionIntegrationTest {
     @SneakyThrows
     private AnalyzeRequest prepareStorage(String file, String cvServiceResponseFile) {
 
-        ClassPathResource pdfFileResource = new ClassPathResource(file);
+        File initialFile = new File(file);
+        InputStream targetStream = new FileInputStream(initialFile);
         ClassPathResource cvServiceResponseFileResource = new ClassPathResource(cvServiceResponseFile);
 
-        return prepareStorage(pdfFileResource.getInputStream(), cvServiceResponseFileResource.getInputStream());
+        return prepareStorage(targetStream, cvServiceResponseFileResource.getInputStream());
     }
 
 }