RSS-83: Some ideas

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/service/BlockificationService.java

@@ -3,6 +3,7 @@ package com.iqser.red.service.redaction.v1.server.classification.service;
 import static java.util.stream.Collectors.toSet;
 
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.Comparator;
 import java.util.Iterator;
 import java.util.List;
@@ -15,6 +16,7 @@ import com.iqser.red.service.redaction.v1.server.classification.model.Page;
 import com.iqser.red.service.redaction.v1.server.classification.model.StringFrequencyCounter;
 import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
 import com.iqser.red.service.redaction.v1.server.classification.utils.RulingTextDirAdjustUtil;
+import com.iqser.red.service.redaction.v1.server.classification.utils.TextBlockComparator;
 import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.Ruling;
@@ -38,7 +40,7 @@ public class BlockificationService {
     public Page blockify(List<TextPositionSequence> textPositions, List<Ruling> horizontalRulingLines, List<Ruling> verticalRulingLines) {
 
         List<TextPositionSequence> chunkWords = new ArrayList<>();
-        List<AbstractTextContainer> chunkBlockList1 = new ArrayList<>();
+        List<TextBlock> chunkBlockList1 = new ArrayList<>();
 
         float minX = 1000, maxX = 0, minY = 1000, maxY = 0;
         TextPositionSequence prev = null;
@@ -50,12 +52,14 @@ public class BlockificationService {
             boolean lineSeparation = word.getMinYDirAdj() - maxY > word.getHeight() * 1.25;
             boolean startFromTop = prev != null && word.getMinYDirAdj() < prev.getMinYDirAdj() - prev.getTextHeight();
             boolean splitByX = prev != null && maxX + 50 < word.getMinXDirAdj() && prev.getMinYDirAdj() == word.getMinYDirAdj();
-            boolean xIsBeforeFirstX = prev != null && word.getMinXDirAdj() < minX;
+            boolean xIsBeforeFirstX = prev != null && word.getMinXDirAdj() < minX - 10;
             boolean newLineAfterSplit = prev != null && word.getMinYDirAdj() != prev.getMinYDirAdj() && wasSplitted && splitX1 != word.getMinXDirAdj();
             boolean isSplitByRuling = isSplitByRuling(minX, minY, maxX, maxY, word, horizontalRulingLines, verticalRulingLines);
             boolean splitByDir = prev != null && !prev.getDir().equals(word.getDir());
+            boolean splitByOtherFontAndOtherY = prev != null && prev.getMaxYDirAdj() != word.getMaxYDirAdj() && (word.getFontStyle().contains("bold") && !prev.getFontStyle().contains("bold") || prev.getFontStyle().contains("bold") && !word.getFontStyle().contains("bold"));
+            boolean isOtherFont = prev != null  && prev.getMaxYDirAdj() != word.getMaxYDirAdj() && Math.abs(Math.floor(word.getFontSize()) - Math.floor(prev.getFontSize())) > 1;
 
-            if (prev != null && (lineSeparation || startFromTop || splitByX || splitByDir || isSplitByRuling)) {
+            if (prev != null && (isOtherFont || lineSeparation  || startFromTop || splitByDir || isSplitByRuling || splitByOtherFontAndOtherY || splitByX || xIsBeforeFirstX)) {
 
                 Orientation prevOrientation = null;
                 if (!chunkBlockList1.isEmpty()) {
@@ -107,7 +111,7 @@ public class BlockificationService {
             chunkBlockList1.add(cb1);
         }
 
-        Iterator<AbstractTextContainer> itty = chunkBlockList1.iterator();
+        Iterator<TextBlock> itty = chunkBlockList1.iterator();
 
         TextBlock previousLeft = null;
         TextBlock previousRight = null;
@@ -153,7 +157,15 @@ public class BlockificationService {
             previous = block;
         }
 
-        return new Page(chunkBlockList1);
+        //-----
+
+//        Collections.sort(chunkBlockList1, new TextBlockComparator());
+
+        List<AbstractTextContainer> pl = new ArrayList<>();
+
+        pl.addAll(chunkBlockList1);
+
+        return new Page(pl);
     }
 
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/service/ClassificationService.java

@@ -1,15 +1,19 @@
 package com.iqser.red.service.redaction.v1.server.classification.service;
 
 import java.util.List;
+import java.util.Locale;
+import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
 import org.springframework.stereotype.Service;
 
 import com.iqser.red.service.redaction.v1.model.Rectangle;
 import com.iqser.red.service.redaction.v1.server.classification.model.Document;
+import com.iqser.red.service.redaction.v1.server.classification.model.Orientation;
 import com.iqser.red.service.redaction.v1.server.classification.model.Page;
 import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
 import com.iqser.red.service.redaction.v1.server.classification.utils.PositionUtils;
+import com.iqser.red.service.redaction.v1.server.redaction.utils.Patterns;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
 
 import lombok.RequiredArgsConstructor;
@@ -52,11 +56,32 @@ public class ClassificationService {
 
         var bodyTextFrame = page.getBodyTextFrame();
 
+        var pattern = Patterns.getCompiledPattern("^(\\d{1,1}\\.){1,3}\\d{1,2}\\.?\\s[0-9A-Za-z]{2,50}", true);
+        var pattern2 = Patterns.getCompiledPattern(".*\\d$", true);
+        var pattern3 = Patterns.getCompiledPattern("^(\\d{1,1}\\.){1,3}\\d{1,2}\\.?\\s[a-z]{1,2}\\/[a-z]{1,2}.*", false);
+
+        Matcher matcher = pattern.matcher(textBlock.toString());
+        Matcher matcher2 = pattern2.matcher(textBlock.toString());
+        Matcher matcher3 = pattern3.matcher(textBlock.toString());
+
         if (document.getFontSizeCounter().getMostPopular() == null) {
             textBlock.setClassification("Other");
             return;
         }
-        if (PositionUtils.isOverBodyTextFrame(bodyTextFrame, textBlock, page.getRotation()) && (document.getFontSizeCounter()
+        if (textBlock.getText().length() > 5 && (textBlock.getMostPopularWordHeight() > document.getTextHeightCounter().getMostPopular() || textBlock.getMostPopularWordFontSize() > document.getFontSizeCounter().getMostPopular()) && PositionUtils.getApproxLineCount(textBlock) < 5.9
+
+                && ((textBlock.getMostPopularWordStyle().contains("bold") && Character.isDigit(textBlock.toString().charAt(0)) && !matcher2.matches() && !textBlock.toString().contains(":")
+                || textBlock.toString().equals(textBlock.toString().toUpperCase(Locale.ROOT)) && !matcher2.matches() && !textBlock.toString().contains(":")
+                || textBlock.toString().startsWith("APPENDIX") || textBlock.toString().startsWith("TABLE")) && !textBlock.toString().endsWith(":"))) {
+            textBlock.setClassification("H 1");
+            document.setHeadlines(true);
+
+        }
+        else if(matcher.find() && PositionUtils.getApproxLineCount(textBlock) < 2.9 && !matcher2.matches() && !matcher3.matches()) {
+            textBlock.setClassification("H 2");
+            document.setHeadlines(true);
+        }
+        else if (PositionUtils.isOverBodyTextFrame(bodyTextFrame, textBlock, page.getRotation()) && (document.getFontSizeCounter()
                 .getMostPopular() == null || textBlock.getHighestFontSize() <= document.getFontSizeCounter().getMostPopular())) {
             textBlock.setClassification("Header");
 
@@ -69,32 +94,36 @@ public class ClassificationService {
             if (!Pattern.matches("[0-9]+", textBlock.toString())) {
                 textBlock.setClassification("Title");
             }
-        } else if (textBlock.getMostPopularWordFontSize() > document.getFontSizeCounter()
+        }
-                .getMostPopular() && PositionUtils.getApproxLineCount(textBlock) < 4.9 && (textBlock.getMostPopularWordStyle().equals("bold") || !document.getFontStyleCounter()
-                .getCountPerValue()
-                .containsKey("bold") && textBlock.getMostPopularWordFontSize() > document.getFontSizeCounter().getMostPopular() + 1) && textBlock.getSequences()
-                .get(0)
-                .getTextPositions()
-                .get(0)
-                .getFontSizeInPt() >= textBlock.getMostPopularWordFontSize()) {
 
-            for (int i = 1; i <= headlineFontSizes.size(); i++) {
+//        else if (textBlock.getMostPopularWordFontSize() > document.getFontSizeCounter()
-                if (textBlock.getMostPopularWordFontSize() == headlineFontSizes.get(i - 1)) {
+//                .getMostPopular() && PositionUtils.getApproxLineCount(textBlock) < 4.9 && (textBlock.getMostPopularWordStyle().equals("bold") || !document.getFontStyleCounter()
-                    textBlock.setClassification("H " + i);
+//                .getCountPerValue()
-                    document.setHeadlines(true);
+//                .containsKey("bold") && textBlock.getMostPopularWordFontSize() > document.getFontSizeCounter().getMostPopular() + 1) && textBlock.getSequences()
-                }
+//                .get(0)
-            }
+//                .getTextPositions()
-        } else if (!textBlock.getText().startsWith("Table ") && !textBlock.getText().startsWith("Figure ") && PositionUtils.isWithinBodyTextFrame(bodyTextFrame,
+//                .get(0)
-                textBlock) && textBlock.getMostPopularWordStyle().equals("bold") && !document.getFontStyleCounter()
+//                .getFontSizeInPt() >= textBlock.getMostPopularWordFontSize()) {
-                .getMostPopular()
+//
-                .equals("bold") && PositionUtils.getApproxLineCount(textBlock) < 2.9 && textBlock.getSequences()
+//            for (int i = 1; i <= headlineFontSizes.size(); i++) {
-                .get(0)
+//                if (textBlock.getMostPopularWordFontSize() == headlineFontSizes.get(i - 1)) {
-                .getTextPositions()
+//                    textBlock.setClassification("H " + i);
-                .get(0)
+//                    document.setHeadlines(true);
-                .getFontSizeInPt() >= textBlock.getMostPopularWordFontSize()) {
+//                }
-            textBlock.setClassification("H " + (headlineFontSizes.size() + 1));
+//            }
-            document.setHeadlines(true);
+//        }
-        } else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordFontSize() == document.getFontSizeCounter()
+//        else if (!textBlock.getText().startsWith("Table ") && !textBlock.getText().startsWith("Figure ") && PositionUtils.isWithinBodyTextFrame(bodyTextFrame,
+//                textBlock) && textBlock.getMostPopularWordStyle().equals("bold") && !document.getFontStyleCounter()
+//                .getMostPopular()
+//                .equals("bold") && PositionUtils.getApproxLineCount(textBlock) < 2.9 && textBlock.getSequences()
+//                .get(0)
+//                .getTextPositions()
+//                .get(0)
+//                .getFontSizeInPt() >= textBlock.getMostPopularWordFontSize()) {
+//            textBlock.setClassification("H " + (headlineFontSizes.size() + 1));
+//            document.setHeadlines(true);
+//        }
+        else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordFontSize() == document.getFontSizeCounter()
                 .getMostPopular() && textBlock.getMostPopularWordStyle().equals("bold") && !document.getFontStyleCounter().getMostPopular().equals("bold")) {
             textBlock.setClassification("TextBlock Bold");
         } else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordFont()

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/utils/TextBlockComparator.java

@@ -0,0 +1,40 @@
+package com.iqser.red.service.redaction.v1.server.classification.utils;
+
+import java.util.Comparator;
+
+import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
+
+public class TextBlockComparator implements Comparator<TextBlock> {
+
+    @Override
+    public int compare(TextBlock pos1, TextBlock pos2) {
+        // only compare text that is in the same direction
+        int cmp1 = Float.compare(pos1.getDir().getDegrees(), pos2.getDir().getDegrees());
+        if (cmp1 != 0) {
+            return cmp1;
+        }
+
+        // get the text direction adjusted coordinates
+        float x1 = pos1.getMinX();
+        float x2 = pos2.getMinX();
+
+        float pos1YBottom = pos1.getPdfMaxY();
+        float pos2YBottom = pos2.getPdfMaxY();
+
+        // note that the coordinates have been adjusted so 0,0 is in upper left
+        float pos1YTop = pos1YBottom - pos1.getHeight();
+        float pos2YTop = pos2YBottom - pos2.getHeight();
+
+        float yDifference = Math.abs(pos1YBottom - pos2YBottom);
+
+        // we will do a simple tolerance comparison
+        if (yDifference < .1 || pos2YBottom >= pos1YTop && pos2YBottom <= pos1YBottom || pos1YBottom >= pos2YTop && pos1YBottom <= pos2YBottom) {
+            return Float.compare(x1, x2);
+        } else if (pos1YBottom < pos2YBottom) {
+            return -1;
+        } else {
+            return 1;
+        }
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/utils/TextPositionSequenceComparator.java

@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.iqser.red.service.redaction.v1.server.classification.utils;
+
+import java.util.Comparator;
+import java.util.regex.Matcher;
+
+import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
+import com.iqser.red.service.redaction.v1.server.redaction.utils.Patterns;
+
+public class TextPositionSequenceComparator implements Comparator<TextPositionSequence>
+{
+    @Override
+    public int compare(TextPositionSequence pos1, TextPositionSequence pos2)
+    {
+        // only compare text that is in the same direction
+        int cmp1 = Float.compare(pos1.getDir().getDegrees(), pos2.getDir().getDegrees());
+        if (cmp1 != 0)
+        {
+            return cmp1;
+        }
+        
+        // get the text direction adjusted coordinates
+        float x1 = pos1.getMinXDirAdj();
+        float x2 = pos2.getMinXDirAdj();
+        
+        float pos1YBottom = pos1.getMaxYDirAdj();
+        float pos2YBottom = pos2.getMaxYDirAdj();
+
+        // note that the coordinates have been adjusted so 0,0 is in upper left
+        float pos1YTop = pos1YBottom - pos1.getHeight();
+        float pos2YTop = pos2YBottom - pos2.getHeight();
+
+        float yDifference = Math.abs(pos1YBottom - pos2YBottom);
+
+        // we will do a simple tolerance comparison
+        if (yDifference < .1 ||
+            pos2YBottom >= pos1YTop && pos2YBottom <= pos1YBottom ||
+            pos1YBottom >= pos2YTop && pos1YBottom <= pos2YBottom)
+        {
+            return Float.compare(x1, x2);
+        }
+        else if (pos1YBottom < pos2YBottom)
+        {
+            return -1;
+        }
+        else
+        {
+            return 1;
+        }
+    }
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/segmentation/PdfSegmentationService.java

@@ -9,10 +9,13 @@ import java.nio.file.attribute.FileAttribute;
 import java.nio.file.attribute.PosixFilePermission;
 import java.nio.file.attribute.PosixFilePermissions;
 import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.regex.Matcher;
 
 import org.apache.commons.io.IOUtils;
 import org.apache.commons.lang3.SystemUtils;
@@ -27,10 +30,12 @@ import com.iqser.red.service.redaction.v1.server.classification.model.Page;
 import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
 import com.iqser.red.service.redaction.v1.server.classification.service.BlockificationService;
 import com.iqser.red.service.redaction.v1.server.classification.service.ClassificationService;
+import com.iqser.red.service.redaction.v1.server.classification.utils.TextPositionSequenceComparator;
 import com.iqser.red.service.redaction.v1.server.parsing.PDFLinesTextStripper;
 import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
 import com.iqser.red.service.redaction.v1.server.redaction.model.PdfImage;
 import com.iqser.red.service.redaction.v1.server.redaction.model.PdfTableCell;
+import com.iqser.red.service.redaction.v1.server.redaction.utils.Patterns;
 import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.CleanRulings;
@@ -108,6 +113,7 @@ public class PdfSegmentationService {
                             stripper.getRulings(),
                             stripper.getMinCharWidth(),
                             stripper.getMaxCharHeight());
+//                    Collections.sort(stripper.getTextPositionSequences(), new TextPositionSequenceComparator());
                     Page page = blockificationService.blockify(stripper.getTextPositionSequences(), cleanRulings.getHorizontal(), cleanRulings.getVertical());
 
                     page.setRotation(rotation);
@@ -125,6 +131,28 @@ public class PdfSegmentationService {
                         imageService.findOcr(page);
                     }
 
+
+//                    var pattern = Patterns.getCompiledPattern("^\\d[\\d.]{0,5}", true);
+//                    page.getTextBlocks().sort(Comparator.comparing(a -> a.getMinY(), Comparator.reverseOrder()));
+//
+//                    AbstractTextContainer prev = null;
+//                    List<AbstractTextContainer> toRemove = new ArrayList<>();
+//                    for (AbstractTextContainer current : page.getTextBlocks()) {
+//
+//                        if(prev != null && prev instanceof TextBlock && current instanceof TextBlock && Math.abs(prev.getMinY() - current.getMinY()) <=1){
+//                            Matcher matcher = pattern.matcher(prev.getText());
+//                            if (matcher.matches()) {
+//                                ((TextBlock) current).getSequences().add(0, ((TextBlock) prev).getSequences().get(0));
+//                                toRemove.add(prev);
+//                                current.setMinX(prev.getMinX());
+//                            }
+//                        }
+//                        prev = current;
+//                    }
+//
+//                    page.getTextBlocks().removeAll(toRemove);
+
+
                     pages.add(page);
                 }
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/segmentation/SectionsBuilderService.java

@@ -68,7 +68,7 @@ public class SectionsBuilderService {
                     continue;
                 }
 
-                if (prev != null && current.getClassification().startsWith("H ") && !prev.getClassification().startsWith("H ") || !document.isHeadlines()) {
+                if (prev != null && current.getClassification().startsWith("H ") || !document.isHeadlines()) {
                     Paragraph chunkBlock = buildTextBlock(chunkWords, lastHeadline);
                     chunkBlock.setHeadline(lastHeadline);
                     if (document.isHeadlines()) {

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/visualization/service/PdfVisualisationService.java

@@ -48,7 +48,7 @@ public class PdfVisualisationService {
                     }
                     if (textBlock instanceof TextBlock) {
                         textBlock.setClassification((i + 1) + "/" + paragraph.getPageBlocks().size());
-                        visualizeTextBlock((TextBlock) textBlock, contentStream);
+                        visualizeTextBlock((TextBlock) textBlock, contentStream, 0);
                     } else if (textBlock instanceof Table) {
                         textBlock.setClassification((i + 1) + "/" + paragraph.getPageBlocks().size());
                         visualizeTable((Table) textBlock, contentStream);
@@ -71,15 +71,17 @@ public class PdfVisualisationService {
             PDPage pdPage = document.getPage(page - 1);
             PDPageContentStream contentStream = new PDPageContentStream(document, pdPage, PDPageContentStream.AppendMode.APPEND, true);
 
+            int tbnumber = 0;
             for (AbstractTextContainer textBlock : analyzedPage.getTextBlocks()) {
                 if (textBlock == null) {
                     continue;
                 }
                 if (textBlock instanceof TextBlock) {
-                    visualizeTextBlock((TextBlock) textBlock, contentStream);
+                    visualizeTextBlock((TextBlock) textBlock, contentStream, tbnumber);
                 } else if (textBlock instanceof Table) {
                     visualizeTable((Table) textBlock, contentStream);
                 }
+                tbnumber++;
             }
 
             contentStream.setStrokingColor(Color.YELLOW);
@@ -95,7 +97,7 @@ public class PdfVisualisationService {
     }
 
 
-    private void visualizeTextBlock(TextBlock textBlock, PDPageContentStream contentStream) throws IOException {
+    private void visualizeTextBlock(TextBlock textBlock, PDPageContentStream contentStream, int number) throws IOException {
 
         contentStream.setStrokingColor(Color.RED);
 
@@ -109,7 +111,7 @@ public class PdfVisualisationService {
             contentStream.setFont(PDType1Font.TIMES_ROMAN, 9f);
 
             contentStream.newLineAtOffset(textBlock.getPdfMinX(), textBlock.getPdfMaxY() + 2);
-            contentStream.showText(textBlock.getClassification() + textBlock.getOrientation() + "-->" + textBlock.getSequences().get(0).getDir());
+            contentStream.showText(number + ": " +textBlock.getClassification() + textBlock.getOrientation() + "-->" + textBlock.getSequences().get(0).getDir());
 
             contentStream.endText();
 

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/HeadlinesGoldStandardIntegrationTest.java

@@ -4,6 +4,7 @@ import static org.mockito.Mockito.when;
 
 import java.io.BufferedReader;
 import java.io.ByteArrayInputStream;
+import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.InputStreamReader;
@@ -51,9 +52,14 @@ import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.do
 import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntry;
 import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.Type;
 import com.iqser.red.service.redaction.v1.model.AnalyzeRequest;
+import com.iqser.red.service.redaction.v1.model.AnalyzeResult;
 import com.iqser.red.service.redaction.v1.model.ChangeType;
 import com.iqser.red.service.redaction.v1.model.RedactionLog;
+import com.iqser.red.service.redaction.v1.model.RedactionRequest;
+import com.iqser.red.service.redaction.v1.model.RedactionResult;
 import com.iqser.red.service.redaction.v1.model.StructureAnalyzeRequest;
+import com.iqser.red.service.redaction.v1.server.annotate.AnnotateRequest;
+import com.iqser.red.service.redaction.v1.server.annotate.AnnotateResponse;
 import com.iqser.red.service.redaction.v1.server.annotate.AnnotationService;
 import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
 import com.iqser.red.service.redaction.v1.server.client.LegalBasisClient;
@@ -61,6 +67,7 @@ import com.iqser.red.service.redaction.v1.server.client.RulesClient;
 import com.iqser.red.service.redaction.v1.server.controller.RedactionController;
 import com.iqser.red.service.redaction.v1.server.redaction.service.AnalyzeService;
 import com.iqser.red.service.redaction.v1.server.redaction.service.ManualRedactionSurroundingTextService;
+import com.iqser.red.service.redaction.v1.server.redaction.utils.OsUtils;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.ResourceLoader;
 import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
 import com.iqser.red.storage.commons.StorageAutoConfiguration;
@@ -133,6 +140,37 @@ public class HeadlinesGoldStandardIntegrationTest {
     private final static String TEST_FILE_ID = "123";
 
 
+    @Test
+    public void extractHeadlines() throws IOException {
+
+        AnalyzeRequest request = prepareStorage("files/RSS/08 - Acute Oral Toxicity Up and Down Procedur.pdf");
+
+        analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
+        analyzeService.analyze(request);
+
+        AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder().dossierId(TEST_DOSSIER_ID).fileId(TEST_FILE_ID).build());
+
+        String outputFileName = OsUtils.getTemporaryDirectory() + "/Headlines.pdf";
+
+        try (FileOutputStream fileOutputStream = new FileOutputStream(outputFileName)) {
+            fileOutputStream.write(annotateResponse.getDocument());
+        }
+
+
+        RedactionRequest redactionRequest = RedactionRequest.builder()
+                .dossierId(request.getDossierId())
+                .fileId(request.getFileId())
+                .dossierTemplateId(request.getDossierTemplateId())
+                .build();
+
+        RedactionResult result1 = redactionController.classify(redactionRequest);
+
+        try (FileOutputStream fileOutputStream = new FileOutputStream(OsUtils.getTemporaryDirectory() + "/HeadlinesClassified.pdf")) {
+            fileOutputStream.write(result1.getDocument());
+        }
+    }
+
+
     @Test
     public void testHeadlineDetection() {
 

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java

@@ -364,7 +364,7 @@ public class RedactionIntegrationTest {
     @Test
     public void titleExtraction() throws IOException {
 
-        AnalyzeRequest request = prepareStorage("files/RSS/06 - Isopyrazam - Acute Oral Toxicity Rat.pdf");
+        AnalyzeRequest request = prepareStorage("files/RSS/01 - CGA100251 - Acute Oral Toxicity (Up and Down Procedure) - Rat (1).pdf");
 
         analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
         AnalyzeResult result = analyzeService.analyze(request);